diff --git a/libs/sm/VDI.py b/libs/sm/VDI.py index 03c92737b..c2e679693 100644 --- a/libs/sm/VDI.py +++ b/libs/sm/VDI.py @@ -101,6 +101,7 @@ def __init__(self, sr, uuid): self.description = '' self.vbds = [] self.size = 0 + self._block_size = -1 self.utilisation = 0 self.vdi_type = '' self.has_child = 0 @@ -120,6 +121,12 @@ def __init__(self, sr, uuid): self.load(uuid) + @property + def block_size(self): + if self._block_size < 0: + self._block_size = vhdutil.getBlockSize(self.path) + return self._block_size + @staticmethod def from_uuid(session, vdi_uuid): diff --git a/libs/sm/cleanup.py b/libs/sm/cleanup.py index 850086acd..a6bcbedc7 100644 --- a/libs/sm/cleanup.py +++ b/libs/sm/cleanup.py @@ -535,12 +535,19 @@ def __init__(self, sr, uuid, raw): self.sizeVirt = -1 self._sizeVHD = -1 self._sizeAllocated = -1 + self._block_size = -1 self._hidden = False self.parent = None self.children = [] self._vdiRef = None self._clearRef() + @property + def block_size(self): + if self._block_size < 0: + self._block_size = vhdutil.getBlockSize(self.path) + return self._block_size + @staticmethod def extractUuid(path): raise NotImplementedError("Implement in sub class") @@ -1075,14 +1082,16 @@ def _getCoalescedSizeData(self): blocksParent = self.parent.getVHDBlocks() numBlocks = Util.countBits(blocksChild, blocksParent) Util.log("Num combined blocks = %d" % numBlocks) - sizeData = numBlocks * vhdutil.VHD_BLOCK_SIZE + sizeData = numBlocks * self.block_size assert(sizeData <= self.sizeVirt) return sizeData def _calcExtraSpaceForCoalescing(self): sizeData = self._getCoalescedSizeData() - sizeCoalesced = sizeData + vhdutil.calcOverheadBitmap(sizeData) + \ - vhdutil.calcOverheadEmpty(self.sizeVirt) + sizeCoalesced = sizeData + vhdutil.calcOverheadBitmap( + sizeData, + self.block_size + ) + vhdutil.calcOverheadEmpty(self.sizeVirt) Util.log("Coalesced size = %s" % Util.num2str(sizeCoalesced)) return sizeCoalesced - self.parent.getSizeVHD() @@ -1238,7 +1247,7 @@ def deflate(self): self._sizeAllocated = -1 def inflateFully(self): - self.inflate(lvhdutil.calcSizeVHDLV(self.sizeVirt)) + self.inflate(lvhdutil.calcSizeVHDLV(self.sizeVirt, self.block_size)) def inflateParentForCoalesce(self): """Inflate the parent only as much as needed for the purposes of @@ -1461,7 +1470,10 @@ def _queryVHDBlocks(self): def _calcExtraSpaceForCoalescing(self): if self.parent.raw: return 0 # raw parents are never deflated in the first place - sizeCoalesced = lvhdutil.calcSizeVHDLV(self._getCoalescedSizeData()) + sizeCoalesced = lvhdutil.calcSizeVHDLV( + self._getCoalescedSizeData(), + self.block_size + ) Util.log("Coalesced size = %s" % Util.num2str(sizeCoalesced)) return sizeCoalesced - self.parent.sizeLV @@ -2775,7 +2787,7 @@ def _finishCoalesceLeaf(self, parent): parent.deflate() def _calcExtraSpaceNeeded(self, child, parent): - return lvhdutil.calcSizeVHDLV(parent.sizeVirt) - parent.sizeLV + return lvhdutil.calcSizeVHDLV(parent.sizeVirt, parent.block_size) - parent.sizeLV def _handleInterruptedCoalesceLeaf(self): entries = self.journaler.getAll(VDI.JRN_LEAF) diff --git a/libs/sm/drivers/FileSR.py b/libs/sm/drivers/FileSR.py index 75f7f9d15..5cb4601d3 100644 --- a/libs/sm/drivers/FileSR.py +++ b/libs/sm/drivers/FileSR.py @@ -561,7 +561,10 @@ def create(self, sr_uuid, vdi_uuid, size): if self.vdi_type == vhdutil.VDI_TYPE_VHD: try: - size = vhdutil.validate_and_round_vhd_size(int(size)) + size = vhdutil.validate_and_round_vhd_size( + int(size), + vhdutil.DEFAULT_VHD_BLOCK_SIZE + ) mb = 1024 * 1024 size_mb = size // mb util.ioretry(lambda: self._create(str(size_mb), self.path)) @@ -656,7 +659,7 @@ def resize(self, sr_uuid, vdi_uuid, size): return VDI.VDI.get_params(self) # We already checked it is a VDI_TYPE_VHD - size = vhdutil.validate_and_round_vhd_size(int(size)) + size = vhdutil.validate_and_round_vhd_size(int(size), self.block_size) jFile = JOURNAL_FILE_PREFIX + self.uuid try: diff --git a/libs/sm/drivers/LVHDSR.py b/libs/sm/drivers/LVHDSR.py index f3ef89727..44bcbb4c0 100644 --- a/libs/sm/drivers/LVHDSR.py +++ b/libs/sm/drivers/LVHDSR.py @@ -716,7 +716,10 @@ def scan(self, uuid): util.roundup(lvutil.LVM_SIZE_INCREMENT, vhdutil.calcOverheadEmpty(lvhdutil.MSIZE)) else: - utilisation = lvhdutil.calcSizeVHDLV(int(size)) + utilisation = lvhdutil.calcSizeVHDLV( + int(size), + vhdutil.getBlockSize(lvPath) + ) vdi_ref = self.session.xenapi.VDI.db_introduce( vdi_uuid, @@ -984,7 +987,10 @@ def _undoCloneOp(self, lvs, origUuid, baseUuid, clonUuid): # inflate the parent to fully-allocated size if base.vdiType == vhdutil.VDI_TYPE_VHD: - fullSize = lvhdutil.calcSizeVHDLV(vhdInfo.sizeVirt) + fullSize = lvhdutil.calcSizeVHDLV( + vhdInfo.sizeVirt, + vhdutil.getBlockSize(basePath) + ) lvhdutil.inflate(self.journaler, self.uuid, baseUuid, fullSize) # rename back @@ -1173,7 +1179,7 @@ def _undoAllVHDJournals(self): util.SMlog("Found VHD journal %s, reverting %s" % (uuid, vdi.path)) self.lvActivator.activate(uuid, vdi.lvname, False) self.lvmCache.activateNoRefcount(jlvName) - fullSize = lvhdutil.calcSizeVHDLV(vdi.size) + fullSize = lvhdutil.calcSizeVHDLV(vdi.size, vdi.block_size) lvhdutil.inflate(self.journaler, self.uuid, vdi.uuid, fullSize) try: jFile = os.path.join(self.path, jlvName) @@ -1184,7 +1190,7 @@ def _undoAllVHDJournals(self): util.SMlog("VHD revert failed but VHD ok: removing journal") # Attempt to reclaim unused space vhdInfo = vhdutil.getVHDInfo(vdi.path, lvhdutil.extractUuid, False) - NewSize = lvhdutil.calcSizeVHDLV(vhdInfo.sizeVirt) + NewSize = lvhdutil.calcSizeVHDLV(vhdInfo.sizeVirt, vdi.block_size) if NewSize < fullSize: lvhdutil.deflate(self.lvmCache, vdi.lvname, int(NewSize)) lvhdutil.lvRefreshOnAllSlaves(self.session, self.uuid, @@ -1343,7 +1349,10 @@ def create(self, sr_uuid, vdi_uuid, size): if self.exists: raise xs_errors.XenError('VDIExists') - size = vhdutil.validate_and_round_vhd_size(int(size)) + size = vhdutil.validate_and_round_vhd_size( + int(size), + vhdutil.DEFAULT_VHD_BLOCK_SIZE + ) util.SMlog("LVHDVDI.create: type = %s, %s (size=%s)" % \ (self.vdi_type, self.path, size)) @@ -1356,7 +1365,10 @@ def create(self, sr_uuid, vdi_uuid, size): lvSize = util.roundup(lvutil.LVM_SIZE_INCREMENT, vhdutil.calcOverheadEmpty(lvhdutil.MSIZE)) elif self.sr.provision == "thick": - lvSize = lvhdutil.calcSizeVHDLV(int(size)) + lvSize = lvhdutil.calcSizeVHDLV( + int(size), + vhdutil.DEFAULT_VHD_BLOCK_SIZE + ) self.sr._ensureSpaceAvailable(lvSize) @@ -1459,7 +1471,10 @@ def attach(self, sr_uuid, vdi_uuid): needInflate = False else: self._loadThis() - if self.utilisation >= lvhdutil.calcSizeVHDLV(self.size): + if ( + self.utilisation >= + lvhdutil.calcSizeVHDLV(self.size, self.block_size) + ): needInflate = False if needInflate: @@ -1479,7 +1494,7 @@ def detach(self, sr_uuid, vdi_uuid): util.SMlog("LVHDVDI.detach for %s" % self.uuid) self._loadThis() already_deflated = (self.utilisation < \ - lvhdutil.calcSizeVHDLV(self.size)) + lvhdutil.calcSizeVHDLV(self.size, self.block_size)) needDeflate = True if self.vdi_type == vhdutil.VDI_TYPE_RAW or already_deflated: needDeflate = False @@ -1520,7 +1535,7 @@ def resize(self, sr_uuid, vdi_uuid, size): '(current size: %d, new size: %d)' % (self.size, size)) raise xs_errors.XenError('VDISize', opterr='shrinking not allowed') - size = vhdutil.validate_and_round_vhd_size(int(size)) + size = vhdutil.validate_and_round_vhd_size(int(size), self.block_size) if size == self.size: return VDI.VDI.get_params(self) @@ -1530,7 +1545,7 @@ def resize(self, sr_uuid, vdi_uuid, size): lvSizeNew = util.roundup(lvutil.LVM_SIZE_INCREMENT, size) else: lvSizeOld = self.utilisation - lvSizeNew = lvhdutil.calcSizeVHDLV(size) + lvSizeNew = lvhdutil.calcSizeVHDLV(size, self.block_size) if self.sr.provision == "thin": # VDI is currently deflated, so keep it deflated lvSizeNew = lvSizeOld @@ -1696,7 +1711,7 @@ def _snapshot(self, snapType, cloneOp=False, cbtlog=None, cbt_consistency=None): self.issnap = self.session.xenapi.VDI.get_is_a_snapshot( \ self.sr.srcmd.params['vdi_ref']) - fullpr = lvhdutil.calcSizeVHDLV(self.size) + fullpr = lvhdutil.calcSizeVHDLV(self.size, self.block_size) thinpr = util.roundup(lvutil.LVM_SIZE_INCREMENT, \ vhdutil.calcOverheadEmpty(lvhdutil.MSIZE)) lvSizeOrig = thinpr diff --git a/libs/sm/lvhdutil.py b/libs/sm/lvhdutil.py index 71b5cbbb1..337f4aa4a 100644 --- a/libs/sm/lvhdutil.py +++ b/libs/sm/lvhdutil.py @@ -95,11 +95,11 @@ def calcSizeLV(sizeVHD): return util.roundup(LVM_SIZE_INCREMENT, sizeVHD) -def calcSizeVHDLV(sizeVirt): +def calcSizeVHDLV(sizeVirt, block_size): # all LVHD VDIs have the metadata area preallocated for the maximum # possible virtual size (for fast online VDI.resize) metaOverhead = vhdutil.calcOverheadEmpty(MSIZE) - bitmapOverhead = vhdutil.calcOverheadBitmap(sizeVirt) + bitmapOverhead = vhdutil.calcOverheadBitmap(sizeVirt, block_size) return calcSizeLV(sizeVirt + metaOverhead + bitmapOverhead) @@ -208,7 +208,12 @@ def setSizeVirt(journaler, srUuid, vdiUuid, size, jFile): lvName = LV_PREFIX[vhdutil.VDI_TYPE_VHD] + vdiUuid vgName = VG_PREFIX + srUuid path = os.path.join(VG_LOCATION, vgName, lvName) - inflate(journaler, srUuid, vdiUuid, calcSizeVHDLV(size)) + inflate( + journaler, + srUuid, + vdiUuid, + calcSizeVHDLV(size, vhdutil.getBlockSize(path)) + ) vhdutil.setSizeVirt(path, size, jFile) @@ -233,7 +238,8 @@ def attachThin(journaler, srUuid, vdiUuid): _tryAcquire(lock) lvmCache.refresh() vhdInfo = vhdutil.getVHDInfoLVM(lvName, extractUuid, vgName) - newSize = calcSizeVHDLV(vhdInfo.sizeVirt) + path = os.path.join(VG_LOCATION, vgName, lvName) + newSize = calcSizeVHDLV(vhdInfo.sizeVirt, vhdutil.getBlockSize(path)) currSizeLV = lvmCache.getSize(lvName) if newSize <= currSizeLV: return diff --git a/libs/sm/vhdutil.py b/libs/sm/vhdutil.py index ccb5e57b4..f6348dda6 100644 --- a/libs/sm/vhdutil.py +++ b/libs/sm/vhdutil.py @@ -30,7 +30,7 @@ MAX_CHAIN_SIZE = 30 # max VHD parent chain size VHD_UTIL = "/usr/bin/vhd-util" OPT_LOG_ERR = "--debug" -VHD_BLOCK_SIZE = 2 * 1024 * 1024 +DEFAULT_VHD_BLOCK_SIZE = 2 * 1024 * 1024 VHD_FOOTER_SIZE = 512 # lock to lock the entire SR for short ops @@ -82,9 +82,9 @@ def calcOverheadEmpty(virtual_size): return overhead -def calcOverheadBitmap(virtual_size): - num_blocks = virtual_size // VHD_BLOCK_SIZE - if virtual_size % VHD_BLOCK_SIZE: +def calcOverheadBitmap(virtual_size, block_size): + num_blocks = virtual_size // block_size + if virtual_size % block_size: num_blocks += 1 return num_blocks * 4096 @@ -93,10 +93,18 @@ def ioretry(cmd, text=True): return util.ioretry(lambda: util.pread2(cmd, text=text), errlist=[errno.EIO, errno.EAGAIN]) +def getBlockSize(path): + cmd = [VHD_UTIL, "read", "-pn", path] + ret = ioretry(cmd) + for field in ret.split('\n'): + field = field.lstrip() + if not field.startswith("Block size"): continue + return int(field.split(':')[1].lstrip().split()[0]) + raise util.SMException("Unable to find block size in VHD with path: {}".format(path)) + -def convertAllocatedSizeToBytes(size): - # Assume we have standard 2MB allocation blocks - return size * 2 * 1024 * 1024 +def convertAllocatedSizeToBytes(size, block_size): + return size * block_size def getVHDInfo(path, extractUuidFunction, includeParent=True): @@ -120,7 +128,10 @@ def getVHDInfo(path, extractUuidFunction, includeParent=True): vhdInfo.parentUuid = extractUuidFunction(fields[nextIndex]) nextIndex += 1 vhdInfo.hidden = int(fields[nextIndex].replace("hidden: ", "")) - vhdInfo.sizeAllocated = convertAllocatedSizeToBytes(int(fields[nextIndex+1])) + vhdInfo.sizeAllocated = convertAllocatedSizeToBytes( + int(fields[nextIndex+1]), + getBlockSize(path) + ) vhdInfo.path = path return vhdInfo @@ -279,7 +290,7 @@ def setSizePhys(path, size, debug=True): def getAllocatedSize(path): cmd = [VHD_UTIL, "query", OPT_LOG_ERR, '-a', '-n', path] ret = ioretry(cmd) - return convertAllocatedSizeToBytes(int(ret)) + return convertAllocatedSizeToBytes(int(ret), getBlockSize(path)) def killData(path): "zero out the disk (kill all data inside the VHD file)" @@ -406,7 +417,7 @@ def repair(path): ioretry([VHD_UTIL, 'repair', '-n', path]) -def validate_and_round_vhd_size(size): +def validate_and_round_vhd_size(size, block_size): """ Take the supplied vhd size, in bytes, and check it is positive and less that the maximum supported size, rounding up to the next block boundary """ @@ -419,7 +430,7 @@ def validate_and_round_vhd_size(size): if size < MIN_VHD_SIZE: size = MIN_VHD_SIZE - size = util.roundup(VHD_BLOCK_SIZE, size) + size = util.roundup(block_size, size) return size diff --git a/tests/test_FileSR.py b/tests/test_FileSR.py index 914549982..5b0913913 100644 --- a/tests/test_FileSR.py +++ b/tests/test_FileSR.py @@ -327,6 +327,7 @@ def test_create_vdi_vhd(self, mock_vhdutil): vdi = FakeFileVDI(sr, vdi_uuid) vdi.vdi_type = vhdutil.VDI_TYPE_VHD mock_vhdutil.validate_and_round_vhd_size.side_effect = vhdutil.validate_and_round_vhd_size + mock_vhdutil.DEFAULT_VHD_BLOCK_SIZE = vhdutil.DEFAULT_VHD_BLOCK_SIZE # Act vdi.create(sr_uuid, vdi_uuid, 20 * 1024 * 1024) diff --git a/tests/test_LVHDSR.py b/tests/test_LVHDSR.py index 20fda852d..0f4132816 100644 --- a/tests/test_LVHDSR.py +++ b/tests/test_LVHDSR.py @@ -307,9 +307,13 @@ def setUp(self): self.mock_lvhdutil.LV_PREFIX = lvhdutil.LV_PREFIX vhdutil_patcher = mock.patch('sm.drivers.LVHDSR.vhdutil', autospec=True) self.mock_vhdutil = vhdutil_patcher.start() + self.mock_vhdutil.getBlockSize.return_value = vhdutil.DEFAULT_VHD_BLOCK_SIZE self.mock_vhdutil.VDI_TYPE_VHD = vhdutil.VDI_TYPE_VHD self.mock_vhdutil.VDI_TYPE_RAW = vhdutil.VDI_TYPE_RAW self.mock_vhdutil.MAX_CHAIN_SIZE = vhdutil.MAX_CHAIN_SIZE + vdi_vhdutil_patcher = mock.patch('sm.VDI.vhdutil', autospec=True) + self.mock_vdi_vhdutil = vdi_vhdutil_patcher.start() + self.mock_vdi_vhdutil.getBlockSize.return_value = vhdutil.DEFAULT_VHD_BLOCK_SIZE lvutil_patcher = mock.patch('sm.drivers.LVHDSR.lvutil', autospec=True) self.mock_lvutil = lvutil_patcher.start() vdi_util_patcher = mock.patch('sm.VDI.util', autospec=True) diff --git a/tests/test_vhdutil.py b/tests/test_vhdutil.py index 0af601397..d09abcd1e 100644 --- a/tests/test_vhdutil.py +++ b/tests/test_vhdutil.py @@ -20,27 +20,38 @@ class TestVhdUtil(unittest.TestCase): def test_validate_and_round_min_size(self): - size = vhdutil.validate_and_round_vhd_size(2 * 1024 * 1024) + size = vhdutil.validate_and_round_vhd_size( + 2 * 1024 * 1024, + vhdutil.DEFAULT_VHD_BLOCK_SIZE + ) self.assertTrue(size == 2 * 1024 * 1024) def test_validate_and_round_max_size(self): - size = vhdutil.validate_and_round_vhd_size(vhdutil.MAX_VHD_SIZE) + size = vhdutil.validate_and_round_vhd_size( + vhdutil.MAX_VHD_SIZE, + vhdutil.DEFAULT_VHD_BLOCK_SIZE + ) self.assertTrue(size == vhdutil.MAX_VHD_SIZE) def test_validate_and_round_odd_size_up_to_next_boundary(self): - size = vhdutil.validate_and_round_vhd_size(vhdutil.MAX_VHD_SIZE - 1) + size = vhdutil.validate_and_round_vhd_size( + vhdutil.MAX_VHD_SIZE - 1, + vhdutil.DEFAULT_VHD_BLOCK_SIZE) self.assertTrue(size == vhdutil.MAX_VHD_SIZE) def test_validate_and_round_negative(self): with self.assertRaises(xs_errors.SROSError): - vhdutil.validate_and_round_vhd_size(-1) + vhdutil.validate_and_round_vhd_size(-1, vhdutil.DEFAULT_VHD_BLOCK_SIZE) def test_validate_and_round_too_large(self): with self.assertRaises(xs_errors.SROSError): - vhdutil.validate_and_round_vhd_size(vhdutil.MAX_VHD_SIZE + 1) + vhdutil.validate_and_round_vhd_size( + vhdutil.MAX_VHD_SIZE + 1, + vhdutil.DEFAULT_VHD_BLOCK_SIZE + ) @testlib.with_context def test_calc_overhead_empty_small(self, context): @@ -65,14 +76,20 @@ def test_calc_overhead_empty_max(self, context): def test_calc_overhead_bitmap_round_blocks(self, context): virtual_size = 24 * 1024 * 1024 - result = vhdutil.calcOverheadBitmap(virtual_size) + result = vhdutil.calcOverheadBitmap( + virtual_size, + vhdutil.DEFAULT_VHD_BLOCK_SIZE + ) self.assertEqual(49152, result) @testlib.with_context def test_calc_overhead_bitmap_extra_block(self, context): virtual_size = 25 * 1024 * 1024 - result = vhdutil.calcOverheadBitmap(virtual_size) + result = vhdutil.calcOverheadBitmap( + virtual_size, + vhdutil.DEFAULT_VHD_BLOCK_SIZE + ) self.assertEqual(53248, result) @@ -376,10 +393,14 @@ def test_function(args, inp): context.add_executable(VHD_UTIL, test_function) from sm.drivers import FileSR - vhdinfo = vhdutil.getVHDInfo(TEST_VHD_PATH, FileSR.FileVDI.extractUuid) + with unittest.mock.patch( + "sm.vhdutil.getBlockSize", + return_value=vhdutil.DEFAULT_VHD_BLOCK_SIZE + ): + vhdinfo = vhdutil.getVHDInfo(TEST_VHD_PATH, FileSR.FileVDI.extractUuid) - # Act/Assert - self.assertEqual(18856*2*1024*1024 , vhdinfo.sizeAllocated) + # Act/Assert + self.assertEqual(18856*2*1024*1024 , vhdinfo.sizeAllocated) @testlib.with_context def test_get_allocated_size(self, context): @@ -396,7 +417,12 @@ def test_function(args, inp): context.add_executable(VHD_UTIL, test_function) # Act - result = vhdutil.getAllocatedSize(TEST_VHD_NAME) + result = 0 + with unittest.mock.patch( + "sm.vhdutil.getBlockSize", + return_value=vhdutil.DEFAULT_VHD_BLOCK_SIZE + ): + result = vhdutil.getAllocatedSize(TEST_VHD_NAME) # Assert self.assertEqual(18856*2*1024*1024, result) @@ -404,3 +430,22 @@ def test_function(args, inp): [VHD_UTIL, "query", "--debug", "-a", "-n", TEST_VHD_NAME], call_args) + + @testlib.with_context + def test_get_block_size(self, context): + """ + Test that vhdutil.getBlockSize returns the block size in bytes + """ + + # Arrange + call_args = None + + def test_function(args, inp): + nonlocal call_args + call_args = args + return 0, "Header version: 0x00010000\nBlock size: {} (2 MB)".format(vhdutil.DEFAULT_VHD_BLOCK_SIZE), "" + + context.add_executable(VHD_UTIL, test_function) + + # Act/Assert + self.assertEqual(vhdutil.DEFAULT_VHD_BLOCK_SIZE, vhdutil.getBlockSize(TEST_VHD_NAME))