From cba44d9bf98018ce0f69daf8c7a59ccb152dfe60 Mon Sep 17 00:00:00 2001 From: Damien Thenot Date: Fri, 16 Feb 2024 17:01:03 +0100 Subject: [PATCH] feat(LargeBlock): introduce largeblocksr A SR inheriting from a EXTSR allowing to use a 4KiB blocksize device as SR. Create a 512 bytes loop device on top of a 4KiB device then give it to EXTSR code. It uses the same device-config as a normal local SR, i.e. `device-config:device=/dev/nvme0n1` After creation, the driver find the device under the VG to identify the correct disk. It means that creating the SR with a non-stable disk identifier is doable and it will work as EXTSR would by ignoring the device-config after creation. Identifying the correct disk by using LVM infos. The VG is created using a different prefix name from EXTSR. It is `XSLocalLargeBlock-`. The SR artificially limits the creation to disk not being 512b. It will throw an error if a disk whose blocksize is 512 is given. We currently don't support multi devices, it fails at the EXTSR creation. Sometimes the pvremove from EXTSR using the loop device fails. In this case, we need to remove the real device from PV list ourself in the error handling. --- Makefile | 1 + drivers/LargeBlockSR.py | 246 +++++++++++++++++++++++++++++++++++ drivers/XE_SR_ERRORCODES.xml | 22 ++++ drivers/cleanup.py | 2 +- drivers/lvutil.py | 8 +- drivers/on_slave.py | 1 + 6 files changed, 277 insertions(+), 3 deletions(-) create mode 100644 drivers/LargeBlockSR.py diff --git a/Makefile b/Makefile index 701dd1f4c..24ba532ab 100755 --- a/Makefile +++ b/Makefile @@ -20,6 +20,7 @@ SM_DRIVERS += GlusterFS SM_DRIVERS += XFS SM_DRIVERS += ZFS SM_DRIVERS += MooseFS +SM_DRIVERS += LargeBlock SM_LIBS := SR SM_LIBS += SRCommand diff --git a/drivers/LargeBlockSR.py b/drivers/LargeBlockSR.py new file mode 100644 index 000000000..8fe42b48c --- /dev/null +++ b/drivers/LargeBlockSR.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2024 Vates SAS - damien.thenot@vates.tech +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import SR +from SR import deviceCheck +import SRCommand +import EXTSR +import util +import xs_errors +import os +import re +import lvutil + +CAPABILITIES = ["SR_PROBE", "SR_UPDATE", "SR_SUPPORTS_LOCAL_CACHING", + "VDI_CREATE", "VDI_DELETE", "VDI_ATTACH", "VDI_DETACH", + "VDI_UPDATE", "VDI_CLONE", "VDI_SNAPSHOT", "VDI_RESIZE", "VDI_MIRROR", + "VDI_GENERATE_CONFIG", + "VDI_RESET_ON_BOOT/2", "ATOMIC_PAUSE", "VDI_CONFIG_CBT", + "VDI_ACTIVATE", "VDI_DEACTIVATE", "THIN_PROVISIONING", "VDI_READ_CACHING"] + +CONFIGURATION = [['device', 'local device path (required) (e.g. /dev/sda3)']] + +DRIVER_INFO = { + 'name': 'Large Block SR', + 'description': 'SR plugin which emulates a 512 bytes disk on top of a 4KiB device then create a EXT SR', + 'vendor': 'Vates', + 'copyright': '(C) 2024 Vates', + 'driver_version': '1.0', + 'required_api_version': '1.0', + 'capabilities': CAPABILITIES, + 'configuration': CONFIGURATION +} + +LARGEBLOCK_PREFIX = "XSLocalLargeBlock-" + +class LargeBlockSR(EXTSR.EXTSR): + """Emulating 512b drives for EXT storage repository""" + + DRIVER_TYPE = "largeblock" + LOOP_SECTOR_SIZE = 512 + + @staticmethod + def handles(srtype): + return srtype == LargeBlockSR.DRIVER_TYPE + + def load(self, sr_uuid): + super(LargeBlockSR, self).load(sr_uuid) + self.is_deleting = False + self.vgname = LARGEBLOCK_PREFIX + sr_uuid + self.remotepath = os.path.join("/dev", self.vgname, sr_uuid) + + def attach(self, sr_uuid): + if not self.is_deleting: + vg_device = self._get_device() + self.dconf["device"] = ",".join(vg_device) + self._create_emulated_device() + if not self._is_vg_connection_correct(): # Check if we need to redo the connection by parsing `vgs -o vg_name,devices self.vgname` + self._redo_vg_connection() # Call redo VG connection to connect it correctly to the loop device instead of the real 4KiB block device + super(LargeBlockSR, self).attach(sr_uuid) + + def detach(self, sr_uuid): + if not self.is_deleting: + vg_device = self._get_device() + self.dconf["device"] = ",".join(vg_device) + super(LargeBlockSR, self).detach(sr_uuid) + if not self.is_deleting: + self._destroy_emulated_device() + + @deviceCheck + def create(self, sr_uuid, size): + base_devices = self.dconf["device"].split(",") + for dev in base_devices: + logical_blocksize = util.pread2(["blockdev", "--getss", dev]).strip() + if logical_blocksize == "512": + raise xs_errors.XenError("LargeBlockIncorrectBlocksize", opterr="The logical blocksize of the device {} is compatible with normal SR types".format(dev)) + + try: + self._create_emulated_device() + super(LargeBlockSR, self).create(sr_uuid, size) + finally: + self._destroy_emulated_device(base_devices) + + def delete(self, sr_uuid): + base_devices = self._get_device() + self.dconf["device"] = ",".join(self._get_loopdev_from_device(base_devices)) + + self.is_deleting = True + try: + super(LargeBlockSR, self).delete(sr_uuid) + except SR.SROSError: + # In case, the lvremove doesn't like the loop device, it will throw an error. + # We need to remove the device ourselves using the real device in this case. + for dev in base_devices: + util.pread2(["pvremove", dev]) + finally: + self._destroy_emulated_device(base_devices) + self.is_deleting = False + + @deviceCheck + def probe(self): + # We override EXTSR.probe because it uses EXT_PREFIX in this call + return lvutil.srlist_toxml( + lvutil.scan_srlist(LARGEBLOCK_PREFIX, self.dconf['device']), + LARGEBLOCK_PREFIX + ) + + def _create_loopdev(self, dev, emulated_path): + cmd = ["losetup", "-f", "-v", "--show", "--sector-size", str(self.LOOP_SECTOR_SIZE), dev] + loopdev = util.pread2(cmd).rstrip() + + if os.path.exists(emulated_path) and os.path.islink(emulated_path): + os.unlink(emulated_path) + + try: + os.symlink(loopdev, emulated_path) + except OSError: + raise xs_errors.XenError("LargeBlockSymlinkExist", opterr="Symlink {} couldn't be created".format(emulated_path)) + + def _delete_loopdev(self, dev, emulated_path): + if os.path.exists(emulated_path) and os.path.islink(emulated_path): + os.unlink(emulated_path) + + # The backing file isn't a symlink if given by ID in device-config but the real device + dev = os.path.realpath(dev) + loopdevs = self._get_loopdev_from_device(dev) + + if loopdevs != None: + try: + for lp in loopdevs: + cmd = ["losetup", "-d", lp] # Remove the loop device + util.pread2(cmd) + except SR.SROSError: + util.SMlog("Couldn't removed losetup devices: {}".format(loopdevs)) + else: + xs_errors.XenError("LargeBlockNoLosetup", opterr="Couldn't find loop device for {}".format(dev)) + + @staticmethod + def _get_loopdev_from_device(device): + lpdevs = [] + output = util.pread2(["losetup", "--list"]).rstrip() + if output: + for line in output.split("\n"): + line = line.split() + loopdev = line[0] + dev = line[5].strip() + if dev in device: + lpdevs.append(loopdev) + return lpdevs + + @staticmethod + def _get_device_from_loopdev(loopdevs): + devices = [] + output = util.pread2(["losetup", "--list"]).rstrip() + if output: + for line in output.split("\n"): + line = line.split() + lpdev = line[0] + dev = line[5] + if lpdev in loopdevs: + devices.append(dev) + return devices + + def _get_device_from_vg(self): + devices = [] + output = util.pread2(["vgs", "--noheadings", "-o", "vg_name,devices", self.vgname]).splitlines() + for line in output: + line = line.split() + dev = line[1].split("(")[0] + if os.path.islink(dev): + dev = os.path.realpath(dev) + devices.append(dev) + return devices + + def _get_device(self): + vg_device = self._get_device_from_vg() + for dev in vg_device: + if re.match(r"(.*\.512)|(/dev/loop[0-9]+)", dev): + lpdev = os.path.realpath(dev) + realdev = self._get_device_from_loopdev(lpdev)[0] + vg_device.remove(dev) + vg_device.append(realdev) + + return vg_device + + def _is_vg_connection_correct(self): + output = util.pread2(["vgs", "--noheadings", "-o", "vg_name,devices", self.vgname]).split() + output[1] = output[1].split("(")[0] + return bool(re.match(r"(.*\.512)|(/dev/loop[0-9]+)", output[1])) + + def _redo_vg_connection(self): + """ + In case of using a LargeBlockSR, the LVM scan at boot will find the LogicalVolume on the real block device. + And when the PBD is connecting, it will mount from the original device instead of the loop device since LVM prefers real devices it has seen first. + The PBD plug will succeed but then the SR will be accessed through the 4KiB device, returning to the erroneous behaviour on 4KiB device. + VM won't be able to run because vhd-util will fail to scan the VDI. + This function force the LogicalVolume to be mounted on top of our emulation layer by disabling the VolumeGroup and re-enabling while applying a filter. + """ + + util.SMlog("Reconnecting VG {} to use emulated device".format(self.vgname)) + try: + lvutil.setActiveVG(self.vgname, False) + lvutil.setActiveVG(self.vgname, True, config="devices{ global_filter = [ \"r|^/dev/nvme.*|\", \"a|/dev/loop.*|\" ] }") + except util.CommandException as e: + xs_errors.XenError("LargeBlockVGReconnectFailed", opterr="Failed to reconnect the VolumeGroup {}, error: {}".format(self.vgname, e)) + + + @classmethod + def _get_emulated_device_path(cls, dev): + return "{dev}.{bs}".format(dev=dev, bs=cls.LOOP_SECTOR_SIZE) + + def _create_emulated_device(self): + base_devices = self.dconf["device"].split(",") + emulated_devices = [] + for dev in base_devices: + emulated_path = self._get_emulated_device_path(dev) + self._create_loopdev(dev, emulated_path) + emulated_devices.append(emulated_path) + + emulated_devices = ",".join(emulated_devices) + self.dconf["device"] = emulated_devices + + def _destroy_emulated_device(self, devices=None): + if devices is None: + devices = self.dconf["device"].split(",") + + for dev in devices: + emulated_path = self._get_emulated_device_path(dev) + self._delete_loopdev(dev, emulated_path) + +if __name__ == '__main__': + SRCommand.run(LargeBlockSR, DRIVER_INFO) +else: + SR.registerSR(LargeBlockSR) diff --git a/drivers/XE_SR_ERRORCODES.xml b/drivers/XE_SR_ERRORCODES.xml index 0dac0613a..b24bd4c2f 100755 --- a/drivers/XE_SR_ERRORCODES.xml +++ b/drivers/XE_SR_ERRORCODES.xml @@ -947,4 +947,26 @@ LINSTOR SR delete error 5007 + + + LargeBlockSymlinkExist + Symlink already exists + 5008 + + + + LargeBlockNoLosetup + Couldn't find loop device + 5009 + + + LargeBlockIncorrectBlocksize + Blocksize isn't compatible with the driver + 5010 + + + LargeBlockVGReconnectFailed + Failed to reconnect the VolumeGroup + 5011 + diff --git a/drivers/cleanup.py b/drivers/cleanup.py index e51fa8d97..2aec5a748 100755 --- a/drivers/cleanup.py +++ b/drivers/cleanup.py @@ -3431,7 +3431,7 @@ def normalizeType(type): type = SR.TYPE_LVHD if type in [ "ext", "nfs", "ocfsoiscsi", "ocfsohba", "smb", "cephfs", "glusterfs", - "moosefs", "xfs", "zfs" + "moosefs", "xfs", "zfs", "largeblock" ]: type = SR.TYPE_FILE if type in ["linstor"]: diff --git a/drivers/lvutil.py b/drivers/lvutil.py index f1d1c5bd3..a09020ef6 100755 --- a/drivers/lvutil.py +++ b/drivers/lvutil.py @@ -571,12 +571,16 @@ def resizePV(dev): util.SMlog("Failed to grow the PV, non-fatal") -def setActiveVG(path, active): +def setActiveVG(path, active, config=None): "activate or deactivate VG 'path'" val = "n" if active: val = "y" - text = cmd_lvm([CMD_VGCHANGE, "-a" + val, path]) + cmd = [CMD_VGCHANGE, "-a" + val, path] + if config: + cmd.append("--config") + cmd.append(config) + text = cmd_lvm(cmd) @lvmretry diff --git a/drivers/on_slave.py b/drivers/on_slave.py index 524424f61..2f58281ae 100755 --- a/drivers/on_slave.py +++ b/drivers/on_slave.py @@ -78,6 +78,7 @@ def _is_open(session, args): import SR import CephFSSR import EXTSR + import LargeBlockSR import GlusterFSSR import LinstorSR import LVHDSR