From 16aaecd202aaf1f774986bf8280c06dad39fa414 Mon Sep 17 00:00:00 2001 From: Yihuang Yu Date: Thu, 10 Aug 2023 16:00:22 +0800 Subject: [PATCH 1/2] provider: Add hostdev module Add the hostdev module to set up host devices for testing. 1. hostdev: PF/VF instance to manage host device 2. hostdev.dev_setup: Define a contextmanager to set up host devices 3. hostdev.utils: Utilities during the hostdev test life cycle Signed-off-by: Yihuang Yu --- provider/hostdev/__init__.py | 305 ++++++++++++++++++++++++++++++++++ provider/hostdev/dev_setup.py | 65 ++++++++ provider/hostdev/utils.py | 204 +++++++++++++++++++++++ 3 files changed, 574 insertions(+) create mode 100644 provider/hostdev/__init__.py create mode 100644 provider/hostdev/dev_setup.py create mode 100644 provider/hostdev/utils.py diff --git a/provider/hostdev/__init__.py b/provider/hostdev/__init__.py new file mode 100644 index 0000000000..f092fde040 --- /dev/null +++ b/provider/hostdev/__init__.py @@ -0,0 +1,305 @@ +import logging +from pathlib import Path + +from avocado.utils import process, wait +from virttest import utils_net + +LOG_JOB = logging.getLogger("avocado.test") + +PCI_PATH = Path("/sys/bus/pci/") +PCI_DEV_PATH = PCI_PATH / "devices" +PCI_DRV_PATH = PCI_PATH / "drivers" + +# Refer to driverctl +DEV_CLASSES = { + "storage": "01", + "network": "02", + "display": "03", + "multimedia": "04", + "memory": "05", + "bridge": "06", + "communication": "07", + "system": "08", + "input": "09", + "docking": "0a", + "processor": "0b", + "serial": "0c", +} + + +class HostDeviceError(Exception): + pass + + +class HostDeviceBindError(HostDeviceError): + def __init__(self, slot_id, driver, error): + self.slot_id = slot_id + self.driver = driver + self.error = error + + def __str__(self): + return ( + f'Cannot bind "{self.slot_id}" to driver "{self.driver}": ' f"{self.error}" + ) + + +class HostDeviceUnbindError(HostDeviceBindError): + def __init__(self, slot_id, driver, error): + super().__init__(slot_id, driver, error) + + def __str__(self): + return ( + f'Cannot unbind "{self.slot_id}" from driver "{self.driver}": ' + f"{self.error}" + ) + + +class VFCreateError(HostDeviceError): + def __init__(self, slot_id, error): + self.slot_id = slot_id + self.error = error + + def __str__(self): + return f"Failed to create VF devices for {self.slot_id}: {self.error}" + + +class PFDevice: + def __init__(self, slot_id, driver): + """ + Manages a physical function device for a given slot ID, configure its + cycle via sysfs interface. + + Args: + slot_id (str): The device slot ID, e.g.: '0000:01:00.0' + driver (str): Driver to bind, e.g.: 'vfio-pci' + """ + self.driver = driver + self.slot_id = slot_id + self.slot_path = PCI_DEV_PATH / self.slot_id + self.origin_driver = self._get_current_driver(slot_id) + self.pci_class = (self.slot_path / "class").read_text().strip()[2:4] + self.device_id = (self.slot_path / "device").read_text().strip()[2:] + self.vendor_id = (self.slot_path / "vendor").read_text().strip()[2:] + if ( + self.pci_class == DEV_CLASSES["network"] + and self.origin_driver != "vfio-pci" + ): + self.mac_addresses = [ + (nic / "address").read_text().strip() + for nic in (self.slot_path / "net").iterdir() + ] + + @staticmethod + def _get_current_driver(slot_id): + """ + Get the current used driver for the given slot + + Args: + slot_id (str): The device slot ID + + Returns: The current driver name if exists, None otherwise + """ + current_driver = PCI_DEV_PATH / slot_id / "driver" + if current_driver.exists(): + return current_driver.resolve().name + + @property + def same_iommu_group_devices(self): + """ + Get all devices in the same iommu group + + Returns: All devices in the same iommu group from managed slot ID + """ + return (self.slot_path / "iommu_group" / "devices").iterdir() + + def bind_all(self, driver): + """ + Bind all devices to the driver, takes a list of device locations + """ + group_devices = self.same_iommu_group_devices + for dev in group_devices: + self.bind_one(dev.name, driver) + + def bind_one(self, slot_id, driver): + """ + Bind the device given by "slot_id" to the driver "driver". If the + device is already bound to a different driver, it will be unbound first + """ + current_driver = self._get_current_driver(slot_id) + if current_driver: + if current_driver == driver: + LOG_JOB.info( + f"Notice: {slot_id} already bound to driver " f"{driver}, skipping" + ) + return + self.unbind_one(slot_id) + LOG_JOB.info(f"Binding driver for device {slot_id}") + # For kernels >= 3.15 driver_override can be used to specify the driver + # for a device rather than relying on the driver to provide a positive + # match of the device. + override_file = PCI_DEV_PATH / slot_id / "driver_override" + if override_file.exists(): + try: + override_file.write_text(driver) + except OSError as e: + raise HostDeviceError( + f"Failed to set the driver " f"override for {slot_id}: {str(e)}" + ) + # For kernels < 3.15 use new_id to add PCI id's to the driver + else: + id_file = PCI_DRV_PATH / driver / "new_id" + try: + id_file.write_text(f"{self.vendor_id} {self.device_id}") + except OSError as e: + raise HostDeviceError( + f"Failed to assign the new ID of {slot_id} for driver " + f"{driver}: {str(e)}" + ) + + # Bind to the driver + try: + with (PCI_DRV_PATH / driver / "bind").open("a") as bind_f: + bind_f.write(slot_id) + except OSError as e: + raise HostDeviceBindError(slot_id, driver, str(e)) + + # Before unbinding it, overwrite driver_override with empty string so + # that the device can be bound to any other driver + if override_file.exists(): + try: + override_file.write_text("\00") + except OSError as e: + raise HostDeviceError( + f'{slot_id} refused to restore "driver_override" to empty ' + f"string: {str(e)}" + ) + + def config(self, params): + """ + Load the driver module and bind all devices to the driver. If users + want to customize the module parameters, they should reload it + themselves. + + Args: + params (virttest.utils_params.Params): params to configure module + """ + self.bind_all(self.driver) + + def restore(self): + """ + Unload the module and restore the device at the end of the test cycle + """ + if self.origin_driver: + self.bind_all(self.origin_driver) + + def unbind_one(self, slot_id): + """ + Unbind the device identified by "slot_id" from its current driver + """ + current_driver = self._get_current_driver(slot_id) + if current_driver: + LOG_JOB.info(f'Unbinding current driver "{current_driver}"') + driver_path = PCI_DRV_PATH / current_driver + try: + with (driver_path / "unbind").open("a") as unbind_f: + unbind_f.write(slot_id) + except OSError as e: + HostDeviceUnbindError(slot_id, current_driver, str(e)) + + +class VFDevice(PFDevice): + def __init__(self, slot_id, driver): + """ + Manages a virtual function device for a given slot ID, configure its + cycle via sysfs interface. + + Args: + slot_id (str): The device slot ID, e.g.: '0000:01:00.0' + driver (str): Driver to bind, e.g.: 'vfio-pci' + """ + super().__init__(slot_id, driver) + self.num_vfs = 0 + self.vfs = [] + self.sriov_numvfs_path = self.slot_path / "sriov_numvfs" + + def _config_net_vfs(self, params): + """ + Configure all VFs after created, assigning them to have a specific MAC + address instead of using the default "00:00:00:00:00:00" + Args: + params (virttest.utils_params.Params): params to configure VFs + """ + self.mac_addresses = [] + dev_name = next((self.slot_path / "net").iterdir()).name + for idx, vf in enumerate(self.vfs): + mac = params.get( + f"hostdev_vf{idx}_mac", utils_net.generate_mac_address_simple() + ) + self.mac_addresses.append(mac) + LOG_JOB.info(f'Assigning MAC address "{mac}" to VF "{vf}"') + process.run(f"ip link set dev {dev_name} vf {idx} mac {mac}") + + def bind_all(self, driver): + """ + Override this method to bind all VFs to the driver + """ + for dev in self.vfs: + self.bind_one(dev, driver) + + def config(self, params): + """ + Override this method to create VFs first, and if the PF is NIC, then + configure all VFs with fixed MAC address + """ + vf_counts = params.get_numeric("hostdev_vf_counts") + self.create_vfs(vf_counts) + super().config(params) + if (self.slot_path / "class").read_text()[2:4] == "02": + LOG_JOB.info( + f'Device "{self.slot_id}" is a network device, configure MAC address ' + f"for VFs" + ) + self._config_net_vfs(params) + + def create_vfs(self, counts): + """ + Create a set number of VFs by "counts" + + Args: + counts (int): Number of VFs to create + """ + if self._get_current_driver(self.slot_id) == "vfio-pci": + raise VFCreateError( + self.slot_id, + f'Slot "{self.slot_id}" is bound to ' + f'"vfio-pci", please fall back to kernel driver ' + f"to create VFs", + ) + try: + if counts > int((self.slot_path / "sriov_totalvfs").read_text().strip()): + raise VFCreateError( + self.slot_id, + "Count of VF to be created is " 'larger than "sriov_totalvfs"', + ) + self.num_vfs = int(self.sriov_numvfs_path.read_text().strip()) + with self.sriov_numvfs_path.open("w") as numvfs_f: + if self.num_vfs != 0: + numvfs_f.write("0") + numvfs_f.flush() + numvfs_f.write(str(counts)) + except OSError as e: + raise VFCreateError(self.slot_id, str(e)) + wait.wait_for( + lambda: len(list(self.slot_path.glob("virtfn*"))) == counts, + timeout=(counts * 10), + ) + + vfs = sorted(list(self.slot_path.glob("virtfn*"))) + self.vfs = [vf.resolve().name for vf in vfs] + + def restore(self): + """ + Unload the module and restore the device at the end of the test cycle + """ + super().restore() + self.create_vfs(self.num_vfs) diff --git a/provider/hostdev/dev_setup.py b/provider/hostdev/dev_setup.py new file mode 100644 index 0000000000..a83e88c9a5 --- /dev/null +++ b/provider/hostdev/dev_setup.py @@ -0,0 +1,65 @@ +import logging +from contextlib import contextmanager + +from virttest import utils_kernel_module + +from provider import hostdev +from provider.hostdev.utils import PCI_DEV_PATH + +LOG_JOB = logging.getLogger("avocado.test") + + +def config_hostdev(slot_id, params): + """ + Configure a host device with given parameters. + + Args: + slot_id (str): The device slot ID, e.g.: '0000:01:00.0' + params (virttest.utils_params.Params): params to configure the hostdev + + Returns: The hostdev object + """ + bind_driver = params.get("hostdev_bind_driver") + driver_module = utils_kernel_module.KernelModuleHandler(bind_driver) + module_params = params.get("hostdev_driver_module_params", "") + if not driver_module.was_loaded: + driver_module.reload_module(True, module_params) + dev_type = params.get("hostdev_assignment_type", "pf") + if dev_type == "pf": + host_dev = hostdev.PFDevice(slot_id, bind_driver) + elif dev_type == "vf": + host_dev = hostdev.VFDevice(slot_id, bind_driver) + else: + raise NotImplementedError(f'Device type "{dev_type}" is not supported') + host_dev.config(params) + return host_dev + + +@contextmanager +def hostdev_setup(params): + """ + Set up all host devices to prepare the test environment. + + Args: + params (virttest.utils_params.Params): Dict of the test parameters + """ + # Get all host pci slots that need to be set up + host_pci_slots = params.objects("setup_hostdev_slots") + host_devs = [] + # Set up host devices + for slot in host_pci_slots: + if not (PCI_DEV_PATH / slot).exists(): + LOG_JOB.warning( + f"The provided slot({slot}) does not exist, " f"skipping setup it." + ) + continue + hostdev_params = params.object_params(slot) + host_dev = config_hostdev(slot, hostdev_params) + host_devs.append(host_dev) + params[f"hostdev_manager_{slot}"] = host_dev + + try: + yield params + finally: + for dev in reversed(host_devs): + dev.restore() diff --git a/provider/hostdev/utils.py b/provider/hostdev/utils.py new file mode 100644 index 0000000000..e19733b85b --- /dev/null +++ b/provider/hostdev/utils.py @@ -0,0 +1,204 @@ +import json +import logging +import time + +from aexpect.remote import wait_for_login +from virttest import utils_misc, utils_net + +from provider.hostdev import DEV_CLASSES, PCI_DEV_PATH, PCI_DRV_PATH + +LOG_JOB = logging.getLogger("avocado.test") + + +def get_pci_by_class(dev_class, driver=None): + """ + Get device pci slots by given device type and driver + + Args: + dev_class (str): The device class type, e.g.: "network" + driver (str): Driver used by devices + + Returns: A list of all matched devices + """ + pci_ids = set() + class_id = DEV_CLASSES.get(dev_class) + for dev_path in PCI_DEV_PATH.iterdir(): + if dev_path.joinpath("class").read_text()[2:4] != class_id: + continue + pci_ids.add(dev_path.name) + if driver: + pci_ids &= set(get_pci_by_driver(driver)) + return sorted(pci_ids) + + +def get_pci_by_driver(driver): + """ + Get device pci slots by given driver + + Args: + driver (str): Driver used by devices + + Returns: A list of all matched devices + """ + driver_path = PCI_DRV_PATH / driver + pci_ids = { + pci_path.name + for pci_path in driver_path.glob("**/[0-9a-z]*:[0-9a-z]*:[0-9a-z]*.[0-9a-z]*") + } + return sorted(pci_ids) + + +def get_pci_by_dev_type(dev_type, dev_class, driver=None): + """ + + Args: + dev_type (str): "pf" or "vf" you want to filter + dev_class: The device class type, e.g.: "network" + driver: Driver used by devices + + Returns: A list of all matched devices + + """ + dev_type = dev_type.lower() + if dev_type not in ["pf", "vf"]: + raise ValueError(f'Device type({dev_type}) must be "pf" or "vf"') + pf_pci_ids = [] + vf_pci_ids = [] + pci_ids = get_pci_by_class(dev_class, driver) + for pci in pci_ids: + if PCI_DEV_PATH.joinpath(pci, "physfn").exists(): + vf_pci_ids.append(pci) + else: + pf_pci_ids.append(pci) + + return pf_pci_ids if dev_type == "pf" else vf_pci_ids + + +def get_parent_slot(slot_id): + """ + Get the device parent id. If it's a VF device, return the physical parent + device ID. Otherwise, return the slot_id itself. + + Args: + slot_id (str): The device slot ID, e.g.: '0000:01:00.0' + + Returns: The parent slot id + """ + physfn_path = PCI_DEV_PATH / slot_id / "physfn" + if physfn_path.exists(): + return physfn_path.resolve().name + return slot_id + + +def get_ifname_from_pci(pci_slot): + """ + Get the NIC device name from its pci slot id. + + Args: + pci_slot (str): The slot id of the NIC device + + Returns: The NIC name from its pci slot + + """ + pci_net_path = PCI_DEV_PATH / pci_slot / "net" + if pci_net_path.exists(): + try: + return next((PCI_DEV_PATH / pci_slot / "net").iterdir()).name + except OSError as e: + LOG_JOB.error(f"Cannot get the NIC name of {pci_slot}: str({e})") + return "" + + +def get_guest_ip_from_mac(vm, mac, ip_version=4): + """ + Get IP address from MAC address with selected IP version + + Args: + vm (virttest.qemu_vm.VM): The vm object + mac (str): The MAC address of the VM's network interface + ip_version (int): IP version to use for connecting (default is IPv4) + + Returns: The IP address if found + """ + if ip_version not in [4, 6]: + raise ValueError(f"Unsupported IP version: {ip_version}") + ip_addr = "" + os_type = vm.params["os_type"] + serial_session = vm.wait_for_serial_login() + + try: + if os_type == "linux": + addr_family = "inet" if ip_version == 4 else "inet6" + for ifname in utils_net.get_linux_ifname(serial_session): + nic_info = json.loads( + serial_session.cmd_output_safe(f"ip -j link show {ifname}") + )[0] + if nic_info["address"] == mac: + if "(disconnected)" in serial_session.cmd_output_safe( + f"nmcli -g GENERAL.STATE device show {ifname}" + ): + serial_session.cmd(f"nmcli device up {ifname}") + ip_info = json.loads( + serial_session.cmd_output_safe(f"ip -j addr show {ifname}") + )[0] + for addr_info in ip_info["addr_info"]: + if ( + addr_info["family"] == addr_family + and addr_info["scope"] == "global" + ): + ip_addr = addr_info["local"] + else: + if "(connected)" in serial_session.cmd_output_safe( + f"nmcli -g GENERAL.STATE device show {ifname}" + ): + serial_session.cmd(f"nmcli device down {ifname}") + elif os_type == "windows": + ifname = utils_net.get_windows_nic_attribute( + serial_session, "macaddress", mac, "netconnectionid" + ) + utils_net.enable_windows_guest_network(serial_session, ifname) + nic_info = utils_net.get_net_if_addrs_win(serial_session, mac) + ip_addr = nic_info["ipv4"] if ip_version == 4 else nic_info["ipv6"] + else: + raise ValueError("Unknown os type") + finally: + serial_session.close() + LOG_JOB.info(f"IP address of MAC address({mac}) is: {ip_addr}") + return ip_addr + + +def ssh_login_from_mac(vm, mac, ip_version=4): + """ + Establish an SSH login session to a VM using its MAC address. + + Args: + vm (virttest.qemu_vm.VM): The vm object + mac (str): The MAC address of the VM's network interface + ip_version (int): IP version to use for connecting (default is IPv4) + + Returns: The ssh session object + """ + ip_addr = get_guest_ip_from_mac(vm, mac, ip_version) + if ip_addr: + username = vm.params.get("username", "") + password = vm.params.get("password", "") + prompt = vm.params.get("shell_prompt", r"[\#\$]\s*$") + port = vm.params.get("shell_port") + linesep = vm.params.get("shell_linesep", "\n").encode().decode("unicode_escape") + log_filename = ( + f'session-{vm.name}-{time.strftime("%m-%d-%H-%M-%S")}-' + f"{utils_misc.generate_random_string(4)}.log" + ) + log_filename = utils_misc.get_log_filename(log_filename) + log_function = utils_misc.log_line + return wait_for_login( + "ssh", + ip_addr, + port, + username, + password, + prompt, + linesep, + log_filename, + log_function, + ) From 2ce3bba1807181af186a85ec72ec77f23d3c4b0a Mon Sep 17 00:00:00 2001 From: Yihuang Yu Date: Thu, 10 Aug 2023 16:00:39 +0800 Subject: [PATCH 2/2] vfio_net_boot: Basic test case to test host NIC passthrough This set of test cases is to test host NIC passthrough in to the guest. Passthrough some NICs to vm, and then login via ssh, then ping external IP address from guest. Signed-off-by: Yihuang Yu --- qemu/tests/cfg/vfio_net_boot.cfg | 49 +++++++++++++++++++++ qemu/tests/vfio_net_boot.py | 73 ++++++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+) create mode 100644 qemu/tests/cfg/vfio_net_boot.cfg create mode 100644 qemu/tests/vfio_net_boot.py diff --git a/qemu/tests/cfg/vfio_net_boot.cfg b/qemu/tests/cfg/vfio_net_boot.cfg new file mode 100644 index 0000000000..1e981ad586 --- /dev/null +++ b/qemu/tests/cfg/vfio_net_boot.cfg @@ -0,0 +1,49 @@ +- vfio_net_boot: + virt_test_type = qemu + type = vfio_net_boot + start_vm = no + nics = "" + # Special host pci slots to be configured + # setup_hostdev_slots = 0000:00:00.1 + hostdev_bind_driver = vfio-pci + vm_hostdev_driver = vfio-pci + ext_host = www.redhat.com + variants: + - single_vm: + vms = "vm1" + - multi_vms: + vms = "vm1 vm2" + image_snapshot = yes + variants: + - one_nic: + vm_hostdevs = hostdev1 + - multi_nics: + vm_hostdevs = hostdev1 hostdev2 + variants: + - pf: + hostdev_assignment_type = pf + - vf: + hostdev_assignment_type = vf + hostdev_vf_counts = 4 + variants ip_version: + - ipv4: + - ipv6: + variants: + - @default: + - virtio_iommu: + required_qemu= [7.0.0,) + only x86_64, aarch64 + x86_64: + only q35 + enable_guest_iommu = yes + virtio_iommu = yes + - intel_iommu: + only q35 + only HostCpuVendor.intel + machine_type_extra_params = "kernel-irqchip=split" + intel_iommu = yes + enable_guest_iommu = yes + variants: + - @default: + - hugepage: + hugepage = yes diff --git a/qemu/tests/vfio_net_boot.py b/qemu/tests/vfio_net_boot.py new file mode 100644 index 0000000000..641c8f8cc3 --- /dev/null +++ b/qemu/tests/vfio_net_boot.py @@ -0,0 +1,73 @@ +from virttest import error_context, utils_net + +from provider import hostdev +from provider.hostdev import utils as hostdev_utils +from provider.hostdev.dev_setup import hostdev_setup + + +@error_context.context_aware +def run(test, params, env): + """ + Assign host devices to VM and do ping test. + + :param test: QEMU test object. + :type test: avocado_vt.test.VirtTest + :param params: Dictionary with the test parameters. + :type params: virttest.utils_params.Params + :param env: Dictionary with test environment. + :type env: virttest.utils_env.Env + """ + ip_version = params["ip_version"] + with hostdev_setup(params) as params: + hostdev_driver = params.get("vm_hostdev_driver", "vfio-pci") + assignment_type = params.get("hostdev_assignment_type") + ext_host = params.get( + "ext_host", utils_net.get_host_ip_address(ip_ver=ip_version) + ) + available_pci_slots = hostdev_utils.get_pci_by_dev_type( + assignment_type, "network", hostdev_driver + ) + # Create all VMs first + for vm in env.get_all_vms(): + vm_hostdevs = vm.params.objects("vm_hostdevs") + pci_slots = [] + error_context.base_context(f"Setting hostdevs for {vm.name}", test.log.info) + for dev in vm_hostdevs: + pci_slot = available_pci_slots.pop(0) + vm.params[f"vm_hostdev_host_{dev}"] = pci_slot + pci_slots.append(pci_slot) + vm.create(params=vm.params) + vm.verify_alive() + vm.params["vm_hostdev_slots"] = pci_slots + + # Login and ping + for vm in env.get_all_vms(): + try: + error_context.context("Log into guest via MAC address", test.log.info) + for slot in vm.params["vm_hostdev_slots"]: + parent_slot = hostdev_utils.get_parent_slot(slot) + slot_manager = params[f"hostdev_manager_{parent_slot}"] + if type(slot_manager) is hostdev.PFDevice: + mac_addr = slot_manager.mac_addresses[0] + else: + slot_index = slot_manager.vfs.index(slot) + mac_addr = slot_manager.mac_addresses[slot_index] + session = hostdev_utils.ssh_login_from_mac( + vm, mac_addr, int(ip_version[-1]) + ) + if session: + error_context.context( + f"Ping {ext_host} from {vm.name}", test.log.info + ) + s_ping, _ = utils_net.ping( + ext_host, + 10, + session=session, + timeout=30, + force_ipv4=(ip_version == "ipv4"), + ) + session.close() + if s_ping: + test.fail(f"Fail to ping {ext_host} from {vm.name}") + finally: + vm.destroy()