diff --git a/provider/hostdev/__init__.py b/provider/hostdev/__init__.py new file mode 100644 index 0000000000..f092fde040 --- /dev/null +++ b/provider/hostdev/__init__.py @@ -0,0 +1,305 @@ +import logging +from pathlib import Path + +from avocado.utils import process, wait +from virttest import utils_net + +LOG_JOB = logging.getLogger("avocado.test") + +PCI_PATH = Path("/sys/bus/pci/") +PCI_DEV_PATH = PCI_PATH / "devices" +PCI_DRV_PATH = PCI_PATH / "drivers" + +# Refer to driverctl +DEV_CLASSES = { + "storage": "01", + "network": "02", + "display": "03", + "multimedia": "04", + "memory": "05", + "bridge": "06", + "communication": "07", + "system": "08", + "input": "09", + "docking": "0a", + "processor": "0b", + "serial": "0c", +} + + +class HostDeviceError(Exception): + pass + + +class HostDeviceBindError(HostDeviceError): + def __init__(self, slot_id, driver, error): + self.slot_id = slot_id + self.driver = driver + self.error = error + + def __str__(self): + return ( + f'Cannot bind "{self.slot_id}" to driver "{self.driver}": ' f"{self.error}" + ) + + +class HostDeviceUnbindError(HostDeviceBindError): + def __init__(self, slot_id, driver, error): + super().__init__(slot_id, driver, error) + + def __str__(self): + return ( + f'Cannot unbind "{self.slot_id}" from driver "{self.driver}": ' + f"{self.error}" + ) + + +class VFCreateError(HostDeviceError): + def __init__(self, slot_id, error): + self.slot_id = slot_id + self.error = error + + def __str__(self): + return f"Failed to create VF devices for {self.slot_id}: {self.error}" + + +class PFDevice: + def __init__(self, slot_id, driver): + """ + Manages a physical function device for a given slot ID, configure its + cycle via sysfs interface. + + Args: + slot_id (str): The device slot ID, e.g.: '0000:01:00.0' + driver (str): Driver to bind, e.g.: 'vfio-pci' + """ + self.driver = driver + self.slot_id = slot_id + self.slot_path = PCI_DEV_PATH / self.slot_id + self.origin_driver = self._get_current_driver(slot_id) + self.pci_class = (self.slot_path / "class").read_text().strip()[2:4] + self.device_id = (self.slot_path / "device").read_text().strip()[2:] + self.vendor_id = (self.slot_path / "vendor").read_text().strip()[2:] + if ( + self.pci_class == DEV_CLASSES["network"] + and self.origin_driver != "vfio-pci" + ): + self.mac_addresses = [ + (nic / "address").read_text().strip() + for nic in (self.slot_path / "net").iterdir() + ] + + @staticmethod + def _get_current_driver(slot_id): + """ + Get the current used driver for the given slot + + Args: + slot_id (str): The device slot ID + + Returns: The current driver name if exists, None otherwise + """ + current_driver = PCI_DEV_PATH / slot_id / "driver" + if current_driver.exists(): + return current_driver.resolve().name + + @property + def same_iommu_group_devices(self): + """ + Get all devices in the same iommu group + + Returns: All devices in the same iommu group from managed slot ID + """ + return (self.slot_path / "iommu_group" / "devices").iterdir() + + def bind_all(self, driver): + """ + Bind all devices to the driver, takes a list of device locations + """ + group_devices = self.same_iommu_group_devices + for dev in group_devices: + self.bind_one(dev.name, driver) + + def bind_one(self, slot_id, driver): + """ + Bind the device given by "slot_id" to the driver "driver". If the + device is already bound to a different driver, it will be unbound first + """ + current_driver = self._get_current_driver(slot_id) + if current_driver: + if current_driver == driver: + LOG_JOB.info( + f"Notice: {slot_id} already bound to driver " f"{driver}, skipping" + ) + return + self.unbind_one(slot_id) + LOG_JOB.info(f"Binding driver for device {slot_id}") + # For kernels >= 3.15 driver_override can be used to specify the driver + # for a device rather than relying on the driver to provide a positive + # match of the device. + override_file = PCI_DEV_PATH / slot_id / "driver_override" + if override_file.exists(): + try: + override_file.write_text(driver) + except OSError as e: + raise HostDeviceError( + f"Failed to set the driver " f"override for {slot_id}: {str(e)}" + ) + # For kernels < 3.15 use new_id to add PCI id's to the driver + else: + id_file = PCI_DRV_PATH / driver / "new_id" + try: + id_file.write_text(f"{self.vendor_id} {self.device_id}") + except OSError as e: + raise HostDeviceError( + f"Failed to assign the new ID of {slot_id} for driver " + f"{driver}: {str(e)}" + ) + + # Bind to the driver + try: + with (PCI_DRV_PATH / driver / "bind").open("a") as bind_f: + bind_f.write(slot_id) + except OSError as e: + raise HostDeviceBindError(slot_id, driver, str(e)) + + # Before unbinding it, overwrite driver_override with empty string so + # that the device can be bound to any other driver + if override_file.exists(): + try: + override_file.write_text("\00") + except OSError as e: + raise HostDeviceError( + f'{slot_id} refused to restore "driver_override" to empty ' + f"string: {str(e)}" + ) + + def config(self, params): + """ + Load the driver module and bind all devices to the driver. If users + want to customize the module parameters, they should reload it + themselves. + + Args: + params (virttest.utils_params.Params): params to configure module + """ + self.bind_all(self.driver) + + def restore(self): + """ + Unload the module and restore the device at the end of the test cycle + """ + if self.origin_driver: + self.bind_all(self.origin_driver) + + def unbind_one(self, slot_id): + """ + Unbind the device identified by "slot_id" from its current driver + """ + current_driver = self._get_current_driver(slot_id) + if current_driver: + LOG_JOB.info(f'Unbinding current driver "{current_driver}"') + driver_path = PCI_DRV_PATH / current_driver + try: + with (driver_path / "unbind").open("a") as unbind_f: + unbind_f.write(slot_id) + except OSError as e: + HostDeviceUnbindError(slot_id, current_driver, str(e)) + + +class VFDevice(PFDevice): + def __init__(self, slot_id, driver): + """ + Manages a virtual function device for a given slot ID, configure its + cycle via sysfs interface. + + Args: + slot_id (str): The device slot ID, e.g.: '0000:01:00.0' + driver (str): Driver to bind, e.g.: 'vfio-pci' + """ + super().__init__(slot_id, driver) + self.num_vfs = 0 + self.vfs = [] + self.sriov_numvfs_path = self.slot_path / "sriov_numvfs" + + def _config_net_vfs(self, params): + """ + Configure all VFs after created, assigning them to have a specific MAC + address instead of using the default "00:00:00:00:00:00" + Args: + params (virttest.utils_params.Params): params to configure VFs + """ + self.mac_addresses = [] + dev_name = next((self.slot_path / "net").iterdir()).name + for idx, vf in enumerate(self.vfs): + mac = params.get( + f"hostdev_vf{idx}_mac", utils_net.generate_mac_address_simple() + ) + self.mac_addresses.append(mac) + LOG_JOB.info(f'Assigning MAC address "{mac}" to VF "{vf}"') + process.run(f"ip link set dev {dev_name} vf {idx} mac {mac}") + + def bind_all(self, driver): + """ + Override this method to bind all VFs to the driver + """ + for dev in self.vfs: + self.bind_one(dev, driver) + + def config(self, params): + """ + Override this method to create VFs first, and if the PF is NIC, then + configure all VFs with fixed MAC address + """ + vf_counts = params.get_numeric("hostdev_vf_counts") + self.create_vfs(vf_counts) + super().config(params) + if (self.slot_path / "class").read_text()[2:4] == "02": + LOG_JOB.info( + f'Device "{self.slot_id}" is a network device, configure MAC address ' + f"for VFs" + ) + self._config_net_vfs(params) + + def create_vfs(self, counts): + """ + Create a set number of VFs by "counts" + + Args: + counts (int): Number of VFs to create + """ + if self._get_current_driver(self.slot_id) == "vfio-pci": + raise VFCreateError( + self.slot_id, + f'Slot "{self.slot_id}" is bound to ' + f'"vfio-pci", please fall back to kernel driver ' + f"to create VFs", + ) + try: + if counts > int((self.slot_path / "sriov_totalvfs").read_text().strip()): + raise VFCreateError( + self.slot_id, + "Count of VF to be created is " 'larger than "sriov_totalvfs"', + ) + self.num_vfs = int(self.sriov_numvfs_path.read_text().strip()) + with self.sriov_numvfs_path.open("w") as numvfs_f: + if self.num_vfs != 0: + numvfs_f.write("0") + numvfs_f.flush() + numvfs_f.write(str(counts)) + except OSError as e: + raise VFCreateError(self.slot_id, str(e)) + wait.wait_for( + lambda: len(list(self.slot_path.glob("virtfn*"))) == counts, + timeout=(counts * 10), + ) + + vfs = sorted(list(self.slot_path.glob("virtfn*"))) + self.vfs = [vf.resolve().name for vf in vfs] + + def restore(self): + """ + Unload the module and restore the device at the end of the test cycle + """ + super().restore() + self.create_vfs(self.num_vfs) diff --git a/provider/hostdev/dev_setup.py b/provider/hostdev/dev_setup.py new file mode 100644 index 0000000000..a83e88c9a5 --- /dev/null +++ b/provider/hostdev/dev_setup.py @@ -0,0 +1,65 @@ +import logging +from contextlib import contextmanager + +from virttest import utils_kernel_module + +from provider import hostdev +from provider.hostdev.utils import PCI_DEV_PATH + +LOG_JOB = logging.getLogger("avocado.test") + + +def config_hostdev(slot_id, params): + """ + Configure a host device with given parameters. + + Args: + slot_id (str): The device slot ID, e.g.: '0000:01:00.0' + params (virttest.utils_params.Params): params to configure the hostdev + + Returns: The hostdev object + """ + bind_driver = params.get("hostdev_bind_driver") + driver_module = utils_kernel_module.KernelModuleHandler(bind_driver) + module_params = params.get("hostdev_driver_module_params", "") + if not driver_module.was_loaded: + driver_module.reload_module(True, module_params) + dev_type = params.get("hostdev_assignment_type", "pf") + if dev_type == "pf": + host_dev = hostdev.PFDevice(slot_id, bind_driver) + elif dev_type == "vf": + host_dev = hostdev.VFDevice(slot_id, bind_driver) + else: + raise NotImplementedError(f'Device type "{dev_type}" is not supported') + host_dev.config(params) + return host_dev + + +@contextmanager +def hostdev_setup(params): + """ + Set up all host devices to prepare the test environment. + + Args: + params (virttest.utils_params.Params): Dict of the test parameters + """ + # Get all host pci slots that need to be set up + host_pci_slots = params.objects("setup_hostdev_slots") + host_devs = [] + # Set up host devices + for slot in host_pci_slots: + if not (PCI_DEV_PATH / slot).exists(): + LOG_JOB.warning( + f"The provided slot({slot}) does not exist, " f"skipping setup it." + ) + continue + hostdev_params = params.object_params(slot) + host_dev = config_hostdev(slot, hostdev_params) + host_devs.append(host_dev) + params[f"hostdev_manager_{slot}"] = host_dev + + try: + yield params + finally: + for dev in reversed(host_devs): + dev.restore() diff --git a/provider/hostdev/utils.py b/provider/hostdev/utils.py new file mode 100644 index 0000000000..e19733b85b --- /dev/null +++ b/provider/hostdev/utils.py @@ -0,0 +1,204 @@ +import json +import logging +import time + +from aexpect.remote import wait_for_login +from virttest import utils_misc, utils_net + +from provider.hostdev import DEV_CLASSES, PCI_DEV_PATH, PCI_DRV_PATH + +LOG_JOB = logging.getLogger("avocado.test") + + +def get_pci_by_class(dev_class, driver=None): + """ + Get device pci slots by given device type and driver + + Args: + dev_class (str): The device class type, e.g.: "network" + driver (str): Driver used by devices + + Returns: A list of all matched devices + """ + pci_ids = set() + class_id = DEV_CLASSES.get(dev_class) + for dev_path in PCI_DEV_PATH.iterdir(): + if dev_path.joinpath("class").read_text()[2:4] != class_id: + continue + pci_ids.add(dev_path.name) + if driver: + pci_ids &= set(get_pci_by_driver(driver)) + return sorted(pci_ids) + + +def get_pci_by_driver(driver): + """ + Get device pci slots by given driver + + Args: + driver (str): Driver used by devices + + Returns: A list of all matched devices + """ + driver_path = PCI_DRV_PATH / driver + pci_ids = { + pci_path.name + for pci_path in driver_path.glob("**/[0-9a-z]*:[0-9a-z]*:[0-9a-z]*.[0-9a-z]*") + } + return sorted(pci_ids) + + +def get_pci_by_dev_type(dev_type, dev_class, driver=None): + """ + + Args: + dev_type (str): "pf" or "vf" you want to filter + dev_class: The device class type, e.g.: "network" + driver: Driver used by devices + + Returns: A list of all matched devices + + """ + dev_type = dev_type.lower() + if dev_type not in ["pf", "vf"]: + raise ValueError(f'Device type({dev_type}) must be "pf" or "vf"') + pf_pci_ids = [] + vf_pci_ids = [] + pci_ids = get_pci_by_class(dev_class, driver) + for pci in pci_ids: + if PCI_DEV_PATH.joinpath(pci, "physfn").exists(): + vf_pci_ids.append(pci) + else: + pf_pci_ids.append(pci) + + return pf_pci_ids if dev_type == "pf" else vf_pci_ids + + +def get_parent_slot(slot_id): + """ + Get the device parent id. If it's a VF device, return the physical parent + device ID. Otherwise, return the slot_id itself. + + Args: + slot_id (str): The device slot ID, e.g.: '0000:01:00.0' + + Returns: The parent slot id + """ + physfn_path = PCI_DEV_PATH / slot_id / "physfn" + if physfn_path.exists(): + return physfn_path.resolve().name + return slot_id + + +def get_ifname_from_pci(pci_slot): + """ + Get the NIC device name from its pci slot id. + + Args: + pci_slot (str): The slot id of the NIC device + + Returns: The NIC name from its pci slot + + """ + pci_net_path = PCI_DEV_PATH / pci_slot / "net" + if pci_net_path.exists(): + try: + return next((PCI_DEV_PATH / pci_slot / "net").iterdir()).name + except OSError as e: + LOG_JOB.error(f"Cannot get the NIC name of {pci_slot}: str({e})") + return "" + + +def get_guest_ip_from_mac(vm, mac, ip_version=4): + """ + Get IP address from MAC address with selected IP version + + Args: + vm (virttest.qemu_vm.VM): The vm object + mac (str): The MAC address of the VM's network interface + ip_version (int): IP version to use for connecting (default is IPv4) + + Returns: The IP address if found + """ + if ip_version not in [4, 6]: + raise ValueError(f"Unsupported IP version: {ip_version}") + ip_addr = "" + os_type = vm.params["os_type"] + serial_session = vm.wait_for_serial_login() + + try: + if os_type == "linux": + addr_family = "inet" if ip_version == 4 else "inet6" + for ifname in utils_net.get_linux_ifname(serial_session): + nic_info = json.loads( + serial_session.cmd_output_safe(f"ip -j link show {ifname}") + )[0] + if nic_info["address"] == mac: + if "(disconnected)" in serial_session.cmd_output_safe( + f"nmcli -g GENERAL.STATE device show {ifname}" + ): + serial_session.cmd(f"nmcli device up {ifname}") + ip_info = json.loads( + serial_session.cmd_output_safe(f"ip -j addr show {ifname}") + )[0] + for addr_info in ip_info["addr_info"]: + if ( + addr_info["family"] == addr_family + and addr_info["scope"] == "global" + ): + ip_addr = addr_info["local"] + else: + if "(connected)" in serial_session.cmd_output_safe( + f"nmcli -g GENERAL.STATE device show {ifname}" + ): + serial_session.cmd(f"nmcli device down {ifname}") + elif os_type == "windows": + ifname = utils_net.get_windows_nic_attribute( + serial_session, "macaddress", mac, "netconnectionid" + ) + utils_net.enable_windows_guest_network(serial_session, ifname) + nic_info = utils_net.get_net_if_addrs_win(serial_session, mac) + ip_addr = nic_info["ipv4"] if ip_version == 4 else nic_info["ipv6"] + else: + raise ValueError("Unknown os type") + finally: + serial_session.close() + LOG_JOB.info(f"IP address of MAC address({mac}) is: {ip_addr}") + return ip_addr + + +def ssh_login_from_mac(vm, mac, ip_version=4): + """ + Establish an SSH login session to a VM using its MAC address. + + Args: + vm (virttest.qemu_vm.VM): The vm object + mac (str): The MAC address of the VM's network interface + ip_version (int): IP version to use for connecting (default is IPv4) + + Returns: The ssh session object + """ + ip_addr = get_guest_ip_from_mac(vm, mac, ip_version) + if ip_addr: + username = vm.params.get("username", "") + password = vm.params.get("password", "") + prompt = vm.params.get("shell_prompt", r"[\#\$]\s*$") + port = vm.params.get("shell_port") + linesep = vm.params.get("shell_linesep", "\n").encode().decode("unicode_escape") + log_filename = ( + f'session-{vm.name}-{time.strftime("%m-%d-%H-%M-%S")}-' + f"{utils_misc.generate_random_string(4)}.log" + ) + log_filename = utils_misc.get_log_filename(log_filename) + log_function = utils_misc.log_line + return wait_for_login( + "ssh", + ip_addr, + port, + username, + password, + prompt, + linesep, + log_filename, + log_function, + ) diff --git a/qemu/tests/cfg/vfio_net_boot.cfg b/qemu/tests/cfg/vfio_net_boot.cfg new file mode 100644 index 0000000000..1e981ad586 --- /dev/null +++ b/qemu/tests/cfg/vfio_net_boot.cfg @@ -0,0 +1,49 @@ +- vfio_net_boot: + virt_test_type = qemu + type = vfio_net_boot + start_vm = no + nics = "" + # Special host pci slots to be configured + # setup_hostdev_slots = 0000:00:00.1 + hostdev_bind_driver = vfio-pci + vm_hostdev_driver = vfio-pci + ext_host = www.redhat.com + variants: + - single_vm: + vms = "vm1" + - multi_vms: + vms = "vm1 vm2" + image_snapshot = yes + variants: + - one_nic: + vm_hostdevs = hostdev1 + - multi_nics: + vm_hostdevs = hostdev1 hostdev2 + variants: + - pf: + hostdev_assignment_type = pf + - vf: + hostdev_assignment_type = vf + hostdev_vf_counts = 4 + variants ip_version: + - ipv4: + - ipv6: + variants: + - @default: + - virtio_iommu: + required_qemu= [7.0.0,) + only x86_64, aarch64 + x86_64: + only q35 + enable_guest_iommu = yes + virtio_iommu = yes + - intel_iommu: + only q35 + only HostCpuVendor.intel + machine_type_extra_params = "kernel-irqchip=split" + intel_iommu = yes + enable_guest_iommu = yes + variants: + - @default: + - hugepage: + hugepage = yes diff --git a/qemu/tests/vfio_net_boot.py b/qemu/tests/vfio_net_boot.py new file mode 100644 index 0000000000..641c8f8cc3 --- /dev/null +++ b/qemu/tests/vfio_net_boot.py @@ -0,0 +1,73 @@ +from virttest import error_context, utils_net + +from provider import hostdev +from provider.hostdev import utils as hostdev_utils +from provider.hostdev.dev_setup import hostdev_setup + + +@error_context.context_aware +def run(test, params, env): + """ + Assign host devices to VM and do ping test. + + :param test: QEMU test object. + :type test: avocado_vt.test.VirtTest + :param params: Dictionary with the test parameters. + :type params: virttest.utils_params.Params + :param env: Dictionary with test environment. + :type env: virttest.utils_env.Env + """ + ip_version = params["ip_version"] + with hostdev_setup(params) as params: + hostdev_driver = params.get("vm_hostdev_driver", "vfio-pci") + assignment_type = params.get("hostdev_assignment_type") + ext_host = params.get( + "ext_host", utils_net.get_host_ip_address(ip_ver=ip_version) + ) + available_pci_slots = hostdev_utils.get_pci_by_dev_type( + assignment_type, "network", hostdev_driver + ) + # Create all VMs first + for vm in env.get_all_vms(): + vm_hostdevs = vm.params.objects("vm_hostdevs") + pci_slots = [] + error_context.base_context(f"Setting hostdevs for {vm.name}", test.log.info) + for dev in vm_hostdevs: + pci_slot = available_pci_slots.pop(0) + vm.params[f"vm_hostdev_host_{dev}"] = pci_slot + pci_slots.append(pci_slot) + vm.create(params=vm.params) + vm.verify_alive() + vm.params["vm_hostdev_slots"] = pci_slots + + # Login and ping + for vm in env.get_all_vms(): + try: + error_context.context("Log into guest via MAC address", test.log.info) + for slot in vm.params["vm_hostdev_slots"]: + parent_slot = hostdev_utils.get_parent_slot(slot) + slot_manager = params[f"hostdev_manager_{parent_slot}"] + if type(slot_manager) is hostdev.PFDevice: + mac_addr = slot_manager.mac_addresses[0] + else: + slot_index = slot_manager.vfs.index(slot) + mac_addr = slot_manager.mac_addresses[slot_index] + session = hostdev_utils.ssh_login_from_mac( + vm, mac_addr, int(ip_version[-1]) + ) + if session: + error_context.context( + f"Ping {ext_host} from {vm.name}", test.log.info + ) + s_ping, _ = utils_net.ping( + ext_host, + 10, + session=session, + timeout=30, + force_ipv4=(ip_version == "ipv4"), + ) + session.close() + if s_ping: + test.fail(f"Fail to ping {ext_host} from {vm.name}") + finally: + vm.destroy()