Merge pull request #4942 from Yingshun/viommu_start

sriov: Add a case about iommu_device_settings

libvirt/tests/cfg/sriov/vIOMMU/iommu_device_settings.cfg

@@ -0,0 +1,65 @@
+- vIOMMU.iommu_device_settings:
+    type = iommu_device_settings
+    start_vm = "no"
+    ping_dest = '8.8.8.8'
+    disk_driver = {'name': 'qemu', 'type': 'qcow2', 'iommu': 'on'}
+    variants:
+        - virtio:
+            only q35, aarch64
+            func_supported_since_libvirt_ver = (8, 3, 0)
+            iommu_dict = {'model': 'virtio'}
+        - intel:
+            only q35
+            start_vm = "yes"
+            enable_guest_iommu = "yes"
+            iommu_dict = {'model': 'intel', 'driver': {'intremap': 'on', 'caching_mode': 'on', 'eim': 'on', 'iotlb': 'on', 'aw_bits': '48'}}
+        - smmuv3:
+            only aarch64
+            func_supported_since_libvirt_ver = (5, 5, 0)
+            iommu_dict = {'model': 'smmuv3'}
+    variants:
+        - e1000e:
+            no smmuv3
+            iface_model = 'e1000e'
+            iface_dict = {'type_name': 'network', 'model': '${iface_model}', 'source': {'network': 'default'}}
+        - rtl8139:
+            no smmuv3        
+            iface_model = 'rtl8139'
+            iface_dict = {'type_name': 'network', 'model': '${iface_model}', 'source': {'network': 'default'}}
+        - virtio_muti_devices:
+            disk_dict = {'target': {'dev': 'vda', 'bus': 'virtio'}, 'device': 'disk', 'driver': ${disk_driver}}
+            video_dict = {'primary': 'yes', 'model_heads': '1', 'model_type': 'virtio', 'driver': {'iommu': 'on'}}
+            test_devices = ["Eth", "block", "gpu"]
+            variants:
+                - vhost_on:
+                    interface_driver_name = "vhost"
+                - vhost_off:
+                    interface_driver_name = "qemu"
+            interface_driver = {'driver_attr': {'name': '${interface_driver_name}', 'iommu': 'on'}}
+            iface_dict = {'type_name': 'network', 'model': 'virtio', 'driver': ${interface_driver}, 'source': {'network': 'default'}}
+        - hostdev_iface:
+            only virtio
+            ping_dest = ''
+            test_devices = ["Eth"]
+            iface_dict = {'managed': 'yes', 'type_name': 'hostdev', 'hostdev_address': {'type_name': 'pci', 'attrs': vf_pci_addr}}
+        - scsi_controller:
+            test_devices = ["scsi"]
+            controller_dicts = [{'type': 'scsi', 'model': 'virtio-scsi','driver': {'iommu': 'on'}}]
+            disk_dict = {'target': {'dev': 'sda', 'bus': 'scsi'}}
+            cleanup_ifaces = no
+        - pcie_to_pci_bridge_controller:
+            test_devices = ["Eth", "block"]
+            controller_dicts = [{'type': 'pci', 'model': 'pcie-to-pci-bridge', 'pre_controller': 'pcie-root-port'}]
+            disk_dict = {'target': {'dev': 'vda', 'bus': 'virtio'}, 'device': 'disk', 'driver': ${disk_driver}, 'model': 'virtio-non-transitional'}
+            variants:
+                - virtio_non_transitional:
+                    iface_dict = {'source': {'network': 'default'}, 'driver': {'driver_attr': {'iommu': 'on'}},'type_name': 'network', 'model': 'virtio-non-transitional'}
+                - hostdev_iface:
+                    only virtio
+                    ping_dest = ''
+                    iface_dict = {'managed': 'yes', 'type_name': 'hostdev', 'hostdev_address': {'type_name': 'pci', 'attrs': vf_pci_addr}}
+        - pcie_root_port_from_expander_bus:
+            test_devices = ["Eth", "block"]
+            root_port = {'type': 'pci', 'model': 'pcie-root-port', 'pre_controller': 'pcie-expander-bus'}
+            controller_dicts = [{'type': 'pci', 'model': 'pcie-expander-bus', 'pre_controller': 'pcie-root'}, ${root_port}, ${root_port}]
+            disk_dict = {'target': {'dev': 'vda', 'bus': 'virtio'}, 'device': 'disk', 'driver': ${disk_driver}}

libvirt/tests/src/sriov/vIOMMU/iommu_device_settings.py

@@ -0,0 +1,56 @@
+from virttest import libvirt_version
+from virttest import utils_disk
+
+from virttest.libvirt_xml import vm_xml
+from virttest.utils_libvirt import libvirt_vmxml
+
+from provider.sriov import sriov_base
+from provider.sriov import check_points
+
+
+def run(test, params, env):
+    """
+    Start vm with iommu device and kinds of virtio devices with iommu=on, and
+    check network and disk function.
+    """
+    libvirt_version.is_libvirt_feature_supported(params)
+    cleanup_ifaces = "yes" == params.get("cleanup_ifaces", "yes")
+    ping_dest = params.get('ping_dest')
+    iommu_dict = eval(params.get('iommu_dict', '{}'))
+    test_devices = eval(params.get("test_devices", "[]"))
+
+    vm_name = params.get("main_vm", "avocado-vt-vm1")
+    vm = env.get_vm(vm_name)
+    test_obj = sriov_base.SRIOVTest(vm, test, params)
+    try:
+        test_obj.setup_iommu_test(iommu_dict=iommu_dict, cleanup_ifaces=cleanup_ifaces)
+        test_obj.prepare_controller()
+        for dev in ["disk", "video"]:
+            dev_dict = eval(params.get('%s_dict' % dev, '{}'))
+            if dev == "disk":
+                dev_dict = test_obj.update_disk_addr(dev_dict)
+            libvirt_vmxml.modify_vm_device(
+                vm_xml.VMXML.new_from_dumpxml(vm.name), dev, dev_dict)
+        iface_dict = test_obj.parse_iface_dict()
+        if cleanup_ifaces:
+            libvirt_vmxml.modify_vm_device(
+                    vm_xml.VMXML.new_from_dumpxml(vm.name),
+                    "interface", iface_dict)
+
+        test.log.info("TEST_STEP: Start the VM.")
+        vm.start()
+        vm.cleanup_serial_console()
+        vm.create_serial_console()
+        vm_session = vm.wait_for_serial_login(
+            timeout=int(params.get('login_timeout')))
+        test.log.debug(vm_xml.VMXML.new_from_dumpxml(vm.name))
+
+        test.log.info("TEST_STEP: Check dmesg message about iommu inside the vm.")
+        vm_session.cmd("dmesg | grep -i 'Adding to iommu group'")
+        check_points.check_vm_iommu_group(vm_session, test_devices)
+
+        test.log.info("TEST_STEP: Check if the VM disk and network are woring well.")
+        utils_disk.dd_data_to_vm_disk(vm_session, "/mnt/test")
+        check_points.check_vm_network_accessed(vm_session, ping_dest=ping_dest)
+    finally:
+        test_obj.teardown_iommu_test()

provider/sriov/check_points.py

@@ -124,7 +124,8 @@ def check_vlan(dev_name, iface_dict, status_error=False):
 
 
 def check_vm_network_accessed(vm_session, ping_count=3, ping_timeout=5,
-                              tcpdump_iface=None, tcpdump_status_error=False):
+                              tcpdump_iface=None, tcpdump_status_error=False,
+                              ping_dest=None):
     """
     Test VM's network accessibility
 
@@ -134,9 +135,11 @@ def check_vm_network_accessed(vm_session, ping_count=3, ping_timeout=5,
     :param tcpdump_iface: The interface to check
     :param tcpdump_status_error: Whether the tcpdump's output should include
         the string "ICMP echo request"
+    :param ping_dest: Ping destination address
     :raise: test.fail when ping fails.
     """
-    ping_dest = sriov_base.get_ping_dest(vm_session)
+    if not ping_dest:
+        ping_dest = sriov_base.get_ping_dest(vm_session)
     if tcpdump_iface:
         cmd = "tcpdump  -i %s icmp" % tcpdump_iface
         tcpdump_session = aexpect.ShellSession('bash')
@@ -181,3 +184,26 @@ def check_vm_iface_num(session, exp_num, timeout=10, first=0.0):
     if len(p_iface) != exp_num:
         exceptions.TestFail("%d interface(s) should be found on the vm."
                             % exp_num)
+
+
+def check_vm_iommu_group(vm_session, test_devices):
+    """
+    Check the devices with iommu enabled are located in a separate iommu group
+
+    :param vm_session: VM session
+    :param test_devices: test devices to check
+    """
+    for dev in test_devices:
+        s, o = vm_session.cmd_status_output(
+            "lspci | awk 'BEGIN{IGNORECASE=1} /%s/ {print $1}'" % dev)
+        if s:
+            exceptions.TestFail("Failed to get pci address!")
+
+        cmd = ("find /sys/kernel/iommu_groups/ -type l | xargs ls -l | awk -F "
+               "'/' '/%s / {print(\"\",$2,$3,$4,$5,$6)}' OFS='/' " % o.strip())
+        s, o = vm_session.cmd_status_output(cmd)
+        if s:
+            exceptions.TestFail("Failed to find iommu group!")
+        s, o = vm_session.cmd_status_output("ls %s" % o)
+        if s:
+            exceptions.TestFail("Failed to get the device in the iommu group!")

provider/sriov/sriov_base.py

@@ -17,6 +17,7 @@
 from virttest.utils_libvirt import libvirt_vmxml
 from virttest.utils_libvirt import libvirt_virtio
 from virttest.utils_libvirt import libvirt_network
+from virttest.utils_libvirt import libvirt_pcicontr
 
 from provider.interface import interface_base
 
@@ -142,13 +143,35 @@ def __init__(self, vm, test, params, session=None):
         self.default_vf_mac = utils_sriov.get_vf_mac(
             self.pf_name, session=self.session)
         self.vf_mac = ""
-
+        self.dev_slot = None
+        self.controller_dicts = eval(self.params.get("controller_dicts", "[]"))
         new_xml = vm_xml.VMXML.new_from_inactive_dumpxml(self.vm.name)
         self.orig_config_xml = new_xml.copy()
 
     def __del__(self):
         recover_vf(self.pf_pci, self.params, 0, session=self.session)
 
+    def update_disk_addr(self, disk_dict):
+        """
+        Update disk address
+
+        :param disk_dict: The original disk attrs
+        :return: The updated disk attrs
+        """
+        if self.controller_dicts:
+            dev_bus = self.controller_dicts[-1].get('index')
+            dev_attrs = {'bus': dev_bus}
+            if disk_dict['target']['bus'] != "scsi":
+                dev_attrs.update({'type': self.controller_dicts[-1].get('type')})
+                if self.controller_dicts[-1].get('model') == 'pcie-to-pci-bridge':
+                    self.dev_slot = 1
+                    dev_attrs.update({'slot': self.dev_slot})
+
+            disk_dict.update({"address": {'attrs': dev_attrs}})
+            if self.controller_dicts[-1]['model'] == 'pcie-root-port':
+                self.controller_dicts.pop()
+        return disk_dict
+
     def parse_iface_dict(self):
         """
         Parse iface_dict from params
@@ -172,6 +195,13 @@ def parse_iface_dict(self):
                 del vf_pci_addr2['type']
             iface_dict = eval(self.params.get('hostdev_dict', '{}'))
 
+        if self.controller_dicts and iface_dict:
+            iface_bus = "%0#4x" % int(self.controller_dicts[-1].get('index'))
+            iface_attrs = {'bus': iface_bus}
+            if isinstance(self.dev_slot, int):
+                self.dev_slot += 1
+                iface_attrs.update({'slot': self.dev_slot})
+            iface_dict.update({"address": {'attrs': iface_attrs}})
         self.test.log.debug("iface_dict: %s.", iface_dict)
         return iface_dict
 
@@ -210,6 +240,40 @@ def parse_iommu_test_params(self):
                         "br_dict": br_dict, "dev_type": dev_type}
         return iommu_params
 
+    def prepare_controller(self):
+        """
+        Prepare controller(s)
+
+        :return: Updated controller attrs
+        """
+        if not self.controller_dicts:
+            return
+
+        for contr_dict in self.controller_dicts:
+            pre_controller = contr_dict.get("pre_controller")
+            if pre_controller:
+                pre_contrs = list(
+                    filter(None, [c.get('index') for c in self.controller_dicts
+                                  if c['type'] == contr_dict['type'] and
+                                  c['model'] == pre_controller]))
+                if pre_contrs:
+                    pre_idx = pre_contrs[0]
+                else:
+                    pre_idx = libvirt_pcicontr.get_max_contr_indexes(
+                        vm_xml.VMXML.new_from_dumpxml(self.vm.name),
+                        contr_dict['type'], pre_controller)
+                if not pre_idx:
+                    self.test.error(
+                        f"Unable to get index of {pre_controller} controller!")
+                contr_dict.pop("pre_controller")
+            libvirt_vmxml.modify_vm_device(
+                vm_xml.VMXML.new_from_dumpxml(self.vm.name), 'controller',
+                contr_dict, 100)
+            contr_dict['index'] = libvirt_pcicontr.get_max_contr_indexes(
+                vm_xml.VMXML.new_from_dumpxml(self.vm.name),
+                contr_dict['type'], contr_dict['model'])[-1]
+        return self.controller_dicts
+
     def get_dev_name(self):
         """
         Get device name
@@ -270,10 +334,12 @@ def setup_default(self, **dargs):
         dev_name = dargs.get('dev_name')
         managed_disabled = dargs.get('managed_disabled', False)
         network_dict = dargs.get("network_dict", {})
+        cleanup_ifaces = "yes" == dargs.get("cleanup_ifaces", "yes")
         self.test.log.info("TEST_SETUP: Clear up the existing VM "
                            "interface(s) before testing.")
-        libvirt_vmxml.remove_vm_devices_by_type(self.vm, 'interface')
-        libvirt_vmxml.remove_vm_devices_by_type(self.vm, 'hostdev')
+        if cleanup_ifaces:
+            libvirt_vmxml.remove_vm_devices_by_type(self.vm, 'interface')
+            libvirt_vmxml.remove_vm_devices_by_type(self.vm, 'hostdev')
         if network_dict:
             self.test.log.info("TEST_SETUP: Create new network.")
             libvirt_network.create_or_del_network(network_dict)