From 224d9f26a6514deb54c9d449592bda32dd57eb6a Mon Sep 17 00:00:00 2001 From: Scott Armitage Date: Wed, 24 Jan 2018 10:12:19 -0800 Subject: [PATCH 1/7] Add option to IntelHex.segments to split segments on alignment boundaries In addition to finding contiguous occupied data addresses, it is often useful to be able to further split these segments based on an integer alignment, such as a flash page size or other block size. An optional `alignment` parameter is added to IntelHex.segments which allows any segments that span an integer multiple boundary of the alignment to be split into multiple sub-segments. The semantics of the returned list of ordered tuple objects is unchanged; that is, regardless of the given alignment parameter, all addresses will be traversed as contiguous segments. --- intelhex/__init__.py | 24 ++++++++++++++++++++---- intelhex/test.py | 20 ++++++++++++++++++++ 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/intelhex/__init__.py b/intelhex/__init__.py index a631b0a..d4aefa2 100644 --- a/intelhex/__init__.py +++ b/intelhex/__init__.py @@ -873,24 +873,40 @@ def merge(self, other, overlap='error'): elif overlap == 'replace': self.start_addr = other.start_addr - def segments(self): + def segments(self, alignment=None): """Return a list of ordered tuple objects, representing contiguous occupied data addresses. Each tuple has a length of two and follows the semantics of the range and xrange objects. - The second entry of the tuple is always an integer greater than the first entry. + The second entry of the tuple is always an integer greater than the first entry. If + integer is passed as alignment, the contiguous segments are further split along + boundaries of integer multiples of the alignment. + + @param alignment integer boundary on which to split segments """ + # helper routine for splitting segments along alignment + def _align(start, end, align): + stop = (start//align + 1) * align - 1 + if stop >= end: + yield (start, end) + else: + yield (start, stop) + for (a, b) in _align(stop+1, end, align): + yield (a, b) + # get normal segments addresses = self.addresses() if not addresses: return [] elif len(addresses) == 1: return([(addresses[0], addresses[0]+1)]) + if not alignment: + alignment = len(addresses) adjacent_differences = [(b - a) for (a, b) in zip(addresses[:-1], addresses[1:])] breaks = [i for (i, x) in enumerate(adjacent_differences) if x > 1] endings = [addresses[b] for b in breaks] endings.append(addresses[-1]) beginings = [addresses[b+1] for b in breaks] beginings.insert(0, addresses[0]) - return [(a, b+1) for (a, b) in zip(beginings, endings)] - + return [(a, b+1) for (x, y) in zip(beginings, endings) for (a, b) in _align(x, y, alignment)] + def get_memory_size(self): """Returns the approximate memory footprint for data.""" n = sys.getsizeof(self) diff --git a/intelhex/test.py b/intelhex/test.py index e016e26..340b5ef 100755 --- a/intelhex/test.py +++ b/intelhex/test.py @@ -809,6 +809,26 @@ def test_segments(self): self.assertEqual(max(sg[0]), 0x102) self.assertEqual(min(sg[1]), 0x200) self.assertEqual(max(sg[1]), 0x203) + ih[0x1fe] = 5 + ih[0x1ff] = 6 + sg = ih.segments(alignment=0x200) + self.assertTrue(isinstance(sg, list)) + self.assertEqual(len(sg), 3) + self.assertTrue(isinstance(sg[0], tuple)) + self.assertTrue(len(sg[0]) == 2) + self.assertTrue(sg[0][0] < sg[0][1]) + self.assertTrue(isinstance(sg[1], tuple)) + self.assertTrue(len(sg[1]) == 2) + self.assertTrue(sg[1][0] < sg[1][1]) + self.assertTrue(isinstance(sg[2], tuple)) + self.assertTrue(len(sg[2]) == 2) + self.assertTrue(sg[2][0] < sg[2][1]) + self.assertEqual(min(sg[0]), 0x100) + self.assertEqual(max(sg[0]), 0x102) + self.assertEqual(min(sg[1]), 0x1fe) + self.assertEqual(max(sg[1]), 0x200) + self.assertEqual(min(sg[2]), 0x200) + self.assertEqual(max(sg[2]), 0x203) pass class TestIntelHexLoadBin(TestIntelHexBase): From eb9ba69deb6450a63951924c02002a8dcff5c42b Mon Sep 17 00:00:00 2001 From: Scott Armitage Date: Wed, 24 Jan 2018 11:46:57 -0800 Subject: [PATCH 2/7] Use last address instead of length to populate empty alignment --- intelhex/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/intelhex/__init__.py b/intelhex/__init__.py index d4aefa2..af91f47 100644 --- a/intelhex/__init__.py +++ b/intelhex/__init__.py @@ -898,7 +898,7 @@ def _align(start, end, align): elif len(addresses) == 1: return([(addresses[0], addresses[0]+1)]) if not alignment: - alignment = len(addresses) + alignment = addresses[-1] + 1 adjacent_differences = [(b - a) for (a, b) in zip(addresses[:-1], addresses[1:])] breaks = [i for (i, x) in enumerate(adjacent_differences) if x > 1] endings = [addresses[b] for b in breaks] From 311b4694a4604c458f2a431b6422b028fdf3327c Mon Sep 17 00:00:00 2001 From: Scott Armitage Date: Thu, 25 Jan 2018 13:30:37 -0800 Subject: [PATCH 3/7] Split _align sub-function out to module level Although this helper routine is intended for use within .segments, it is cleaner to read and test as an independent function. We rename it _align_segment to give it context, now that it is not located within the segment method. --- intelhex/__init__.py | 40 ++++++++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/intelhex/__init__.py b/intelhex/__init__.py index af91f47..ace812c 100644 --- a/intelhex/__init__.py +++ b/intelhex/__init__.py @@ -882,15 +882,6 @@ def segments(self, alignment=None): @param alignment integer boundary on which to split segments """ - # helper routine for splitting segments along alignment - def _align(start, end, align): - stop = (start//align + 1) * align - 1 - if stop >= end: - yield (start, end) - else: - yield (start, stop) - for (a, b) in _align(stop+1, end, align): - yield (a, b) # get normal segments addresses = self.addresses() if not addresses: @@ -905,7 +896,7 @@ def _align(start, end, align): endings.append(addresses[-1]) beginings = [addresses[b+1] for b in breaks] beginings.insert(0, addresses[0]) - return [(a, b+1) for (x, y) in zip(beginings, endings) for (a, b) in _align(x, y, alignment)] + return [(a, b+1) for (x, y) in zip(beginings, endings) for (a, b) in _align_segment(x, y, alignment)] def get_memory_size(self): """Returns the approximate memory footprint for data.""" @@ -1030,6 +1021,35 @@ def tobinarray(self, start=None, end=None, size=None): #/class IntelHex16bit +def _align_segment(start, end, alignment): + """Split segment into sub-segments on alignment boundaries. If the segment + spans an integer multiple of alignment it is split into two sub-segments, + each of which is further sub-divided if it spans additional bounadries. + + @param start start address of the segment + @param end end address of the segment (inclusive) + @param alignment integer alignment boundary + + @return generator of ordered tuple sub-segments representing the + start and end addresses (inclusive) of each sub-segment, + in order of increasing start address; no sub-segment will + span an integery multiple of `alignment`. + """ + if not (float(start).is_integer() and float(end).is_integer() and float(alignment).is_integer()): + raise ValueError("_align_segment: all parameters must be int") + if alignment <= 0: + raise ValueError("_align_segment: alignment must be positive") + if end < start: + raise ValueError("_align_segment: segment must be monotonic") + stop = (start//alignment + 1) * alignment - 1 + if stop >= end: + yield (start, end) + else: + yield (start, stop) + for (a, b) in _align_segment(stop+1, end, alignment): + yield (a, b) + + def hex2bin(fin, fout, start=None, end=None, size=None, pad=None): """Hex-to-Bin convertor engine. @return 0 if all OK From 1b5f02f8031592878489438ac472cb024c1c572a Mon Sep 17 00:00:00 2001 From: Scott Armitage Date: Thu, 25 Jan 2018 13:32:21 -0800 Subject: [PATCH 4/7] Add test class for _align_segment module function This should cover pretty much all the corner cases, plus some of the error handling (such as bad parameters). --- intelhex/test.py | 105 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) diff --git a/intelhex/test.py b/intelhex/test.py index 340b5ef..f5ed988 100755 --- a/intelhex/test.py +++ b/intelhex/test.py @@ -1585,6 +1585,111 @@ def test_start_linear_address(self): intelhex.Record.start_linear_address(0x12345678)) +class Test_AlignSegment(TestIntelHexBase): + + def test_singlespan(self): + # any segment between (0,3) inclusive should be returned un-changed for align=4 + # 0 --> 0 + # 0 1 --> 0 1 + # 0 1 2 --> 0 1 2 + # 0 1 2 3 --> 0 1 2 3 + # 1 --> 1 + # 1 2 --> 1 2 + # 1 2 3 --> 1 2 3 + # 2 --> 2 + # 2 3 --> 2 3 + # 3 --> 3 + for a in range(0,4): + for b in range(a,4): + self.assertEqual([(a,b)], list(intelhex._align_segment(a, b, 4))) + + def test_offset_singlespan(self): + # any segment between (8,11) inclusive should be returned un-changed for align=4 + # 8 --> 8 + # 8 9 --> 8 9 + # 8 9 a --> 8 9 a + # 8 9 a b --> 8 9 a b + # 9 --> 9 + # 9 a --> 9 a + # 9 a b --> 9 a b + # a --> a + # a b --> a b + # b --> b + for a in range(8,12): + for b in range(a,12): + self.assertEqual([(a,b)], list(intelhex._align_segment(a, b, 4))) + + def test_aligned_multispan(self): + # with align=4, segments (0,11) and (8,19) should each be split into three + # equal-length sub-segments + # 0 1 2 3 4 5 6 7 8 9 a b --> 0 1 2 3 , 4 5 6 7 , 8 9 a b + # 8 9 a b c d e f g h i j --> 8 9 a b , c d e f , g h i j + for a in [ 0, 8 ]: + aligned = [ (a, a+4-1), (a+4, a+8-1), (a+8, a+11) ] + self.assertEqual(aligned, list(intelhex._align_segment(a, a+11, 4))) + + def test_leftjustified_multispan(self): + # with align=4, if start address is aligned but end address is not, we expect + # to see n equal-length sub-segments followed by one shorter segment at the end + # 0 1 2 3 4 5 6 7 8 9 a --> 0 1 2 3 , 4 5 6 7 , 8 9 a + # 0 1 2 3 4 5 6 7 8 9 --> 0 1 2 3 , 4 5 6 7 , 8 9 + # 0 1 2 3 4 5 6 7 8 --> 0 1 2 3 , 4 5 6 7 , 8 + # 8 9 a b c d e f g h i --> 8 9 a b , c d e f , g h i + # 8 9 a b c d e f g h --> 8 9 a b , c d e f , g h + # 8 9 a b c d e f g --> 8 9 a b , c d e f , g + for a in [ 0, 8 ]: + aligned = [ (a, a+4-1), (a+4, a+8-1), (a+8, a+10) ] + self.assertEqual(aligned, list(intelhex._align_segment(a, a+10, 4))) + aligned = [ (a, a+4-1), (a+4, a+8-1), (a+8, a+ 9) ] + self.assertEqual(aligned, list(intelhex._align_segment(a, a+ 9, 4))) + aligned = [ (a, a+4-1), (a+4, a+8-1), (a+8, a+ 8) ] + self.assertEqual(aligned, list(intelhex._align_segment(a, a+ 8, 4))) + + def test_rightjustified_multispan(self): + # with align=4, if end address is aligned but start address is not, we expect + # to see one shorter segment followed by n equal-length sub-segments + # 1 2 3 4 5 6 7 8 9 a b --> 1 2 3 , 4 5 6 7 , 8 9 a b + # 2 3 4 5 6 7 8 9 a b --> 2 3 , 4 5 6 7 , 8 9 a b + # 3 4 5 6 7 8 9 a b --> 3 , 4 5 6 7 , 8 9 a b + # 9 a b c d e f g h i j --> 9 a b , c d e f , g h i j + # a b c d e f g h i j --> a b , c d e f , g h i j + # b c d e f g h i j --> b , c d e f , g h i j + for a in [ 0, 8 ]: + aligned = [ (a+1, a+4-1), (a+4, a+8-1), (a+8, a+11) ] + self.assertEqual(aligned, list(intelhex._align_segment(a+1, a+11, 4))) + aligned = [ (a+2, a+4-1), (a+4, a+8-1), (a+8, a+11) ] + self.assertEqual(aligned, list(intelhex._align_segment(a+2, a+11, 4))) + aligned = [ (a+3, a+4-1), (a+4, a+8-1), (a+8, a+11) ] + self.assertEqual(aligned, list(intelhex._align_segment(a+3, a+11, 4))) + + def test_integer_parameters_only(self): + # all parameters must evaluate to integers + with self.assertRaises(ValueError): + list(intelhex._align_segment(0, 3, 4.5)) + with self.assertRaises(ValueError): + list(intelhex._align_segment(0, 3.5, 4)) + with self.assertRaises(ValueError): + list(intelhex._align_segment(0.5, 3, 4)) + with self.assertRaises(ValueError): + list(intelhex._align_segment(0, 3, 'h')) + with self.assertRaises(ValueError): + list(intelhex._align_segment(0, 'b', 4)) + with self.assertRaises(ValueError): + list(intelhex._align_segment('a', 3, 4)) + + def test_nonpositive_alignment(self): + # alignment must be positive (>= 1) + with self.assertRaises(ValueError): + list(intelhex._align_segment(0, 3, 0)) + with self.assertRaises(ValueError): + list(intelhex._align_segment(0, 3, -4)) + + def test_negative_segment(self): + # segments must be monotonically increasing + with self.assertRaises(ValueError): + list(intelhex._align_segment(3, 0, 4)) + + class Test_GetFileAndAddrRange(TestIntelHexBase): def test_simple(self): From 60547a5d1d1883d07716943ecc63ed7dd079d06b Mon Sep 17 00:00:00 2001 From: Scott Armitage Date: Thu, 25 Jan 2018 13:43:47 -0800 Subject: [PATCH 5/7] Split out tests for segments with alignment We construct one IntelHex object with several different types of aligned and unaligned segments, then get the aligned segments in one shot and check them against the expected values. --- intelhex/test.py | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/intelhex/test.py b/intelhex/test.py index f5ed988..90ebf10 100755 --- a/intelhex/test.py +++ b/intelhex/test.py @@ -809,27 +809,27 @@ def test_segments(self): self.assertEqual(max(sg[0]), 0x102) self.assertEqual(min(sg[1]), 0x200) self.assertEqual(max(sg[1]), 0x203) - ih[0x1fe] = 5 - ih[0x1ff] = 6 - sg = ih.segments(alignment=0x200) - self.assertTrue(isinstance(sg, list)) - self.assertEqual(len(sg), 3) - self.assertTrue(isinstance(sg[0], tuple)) - self.assertTrue(len(sg[0]) == 2) - self.assertTrue(sg[0][0] < sg[0][1]) - self.assertTrue(isinstance(sg[1], tuple)) - self.assertTrue(len(sg[1]) == 2) - self.assertTrue(sg[1][0] < sg[1][1]) - self.assertTrue(isinstance(sg[2], tuple)) - self.assertTrue(len(sg[2]) == 2) - self.assertTrue(sg[2][0] < sg[2][1]) - self.assertEqual(min(sg[0]), 0x100) - self.assertEqual(max(sg[0]), 0x102) - self.assertEqual(min(sg[1]), 0x1fe) - self.assertEqual(max(sg[1]), 0x200) - self.assertEqual(min(sg[2]), 0x200) - self.assertEqual(max(sg[2]), 0x203) - pass + + def test_segments_alignment(self): + # test that address segments are correctly summarized and aligned + ih = IntelHex() + self.assertEqual([], ih.segments(alignment=4)) + # 5 input segments; 1 and 2 come out unchanged, 3, 4, and 5 come + # out split into three sub-segments each (11 total output segments) + # (using alignment=4) + ih.puts(0x000, asbytes('0123')) # aligned segment + ih.puts(0x105, asbytes( '56' )) # aligned sub-segment + ih.puts(0x200, asbytes('0123456789ab')) # aligned multi-span + ih.puts(0x300, asbytes('0123456789' )) # left-justified multi-span + ih.puts(0x402, asbytes( '23456789ab')) # right-justified multi-span + expected = [ + (0x000, 0x004), # from aligned segment + (0x105, 0x107), # from aligned sub-segment + (0x200, 0x204), (0x204, 0x208), (0x208, 0x20c), # from aligned multi-span + (0x300, 0x304), (0x304, 0x308), (0x308, 0x30a), # from left-justified multi-span + (0x402, 0x404), (0x404, 0x408), (0x408, 0x40c), # from right-justified multi-span + ] + self.assertEqual(expected, ih.segments(alignment=4)) class TestIntelHexLoadBin(TestIntelHexBase): From eba3278539536403d4f2f33f2d8bb7a1427c4882 Mon Sep 17 00:00:00 2001 From: Scott Armitage Date: Thu, 25 Jan 2018 16:41:54 -0800 Subject: [PATCH 6/7] Add another test for `_align_segment` We already had left- and right-justified test cases, so it seems trivial to add a centered test case as well, where neither the start nor the end are aligned with the boundaries. --- intelhex/test.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/intelhex/test.py b/intelhex/test.py index 90ebf10..e02fe97 100755 --- a/intelhex/test.py +++ b/intelhex/test.py @@ -1662,6 +1662,24 @@ def test_rightjustified_multispan(self): aligned = [ (a+3, a+4-1), (a+4, a+8-1), (a+8, a+11) ] self.assertEqual(aligned, list(intelhex._align_segment(a+3, a+11, 4))) + def test_centered_multispan(self): + # with align=4, if neither end address nor start address is aligned, + # we expect to see one shorter sub-segment at start and one at end, + # separated by n equal-length sub-segments. + # 1 2 3 4 5 6 7 8 9 a --> 1 2 3 , 4 5 6 7 , 8 9 a + # 2 3 4 5 6 7 8 9 --> 2 3 , 4 5 6 7 , 8 9 + # 3 4 5 6 7 8 --> 3 , 4 5 6 7 , 8 + # 9 a b c d e f g h i --> 9 a b , c d e f , g h i + # a b c d e f g h --> a b , c d e f , g h + # b c d e f g --> b , c d e f , g + for a in [ 0, 8 ]: + aligned = [ (a+1, a+4-1), (a+4, a+8-1), (a+8, a+10) ] + self.assertEqual(aligned, list(intelhex._align_segment(a+1, a+10, 4))) + aligned = [ (a+2, a+4-1), (a+4, a+8-1), (a+8, a+ 9) ] + self.assertEqual(aligned, list(intelhex._align_segment(a+2, a+ 9, 4))) + aligned = [ (a+3, a+4-1), (a+4, a+8-1), (a+8, a+ 8) ] + self.assertEqual(aligned, list(intelhex._align_segment(a+3, a+ 8, 4))) + def test_integer_parameters_only(self): # all parameters must evaluate to integers with self.assertRaises(ValueError): From a47689fcc251a7f110b55ee109a0614f28c00bcc Mon Sep 17 00:00:00 2001 From: Scott Armitage Date: Sat, 27 Jan 2018 09:47:47 -0800 Subject: [PATCH 7/7] Refactor recursive _align_segment as iterative Recursion is neat, but Python is not the best language for it. There is no real need to use it here, so we break the recursive call out into an interative loop instead. Only the `start` of the remainder changes, so we increment it same as the recursive call, and once we get to a segment that no longer needs to be further split, we `return` (raising `StopIteration`), ending the generator. The corresponding test was written to make the recursive approach fall over after the default recursion depth of 1000 was hit. In that implementation, the number of sub-segments was equal to the eventual recursion depth. The test as-written will generate 65536 sub-segments, well beyond the recursive capabilities, but handled fine by the iterative approach. If we make the segment upper bound much larger, we start slowing down the tests, so this value seems like a good compromise. --- intelhex/__init__.py | 15 ++++++++------- intelhex/test.py | 6 ++++++ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/intelhex/__init__.py b/intelhex/__init__.py index ace812c..0ae989e 100644 --- a/intelhex/__init__.py +++ b/intelhex/__init__.py @@ -1041,13 +1041,14 @@ def _align_segment(start, end, alignment): raise ValueError("_align_segment: alignment must be positive") if end < start: raise ValueError("_align_segment: segment must be monotonic") - stop = (start//alignment + 1) * alignment - 1 - if stop >= end: - yield (start, end) - else: - yield (start, stop) - for (a, b) in _align_segment(stop+1, end, alignment): - yield (a, b) + while True: + stop = (start//alignment + 1) * alignment - 1 + if stop >= end: + yield (start, end) + return + else: + yield (start, stop) + start = stop+1 def hex2bin(fin, fout, start=None, end=None, size=None, pad=None): diff --git a/intelhex/test.py b/intelhex/test.py index e02fe97..cc8a396 100755 --- a/intelhex/test.py +++ b/intelhex/test.py @@ -1707,6 +1707,12 @@ def test_negative_segment(self): with self.assertRaises(ValueError): list(intelhex._align_segment(3, 0, 4)) + def test_many_splits(self): + # this will produce 65536 sub-segments; previous recursive + # implementation would fall over at 1000 + for subsegment in intelhex._align_segment(0, 2**18, 4): + pass + class Test_GetFileAndAddrRange(TestIntelHexBase):