Merge pull request autotest#6191 from Yingshun/viommu_expr

vIOMMU: Add tests for downstream ports

libvirt/tests/cfg/sriov/vIOMMU/hotplug_device_with_iommu_enabled.cfg

@@ -57,3 +57,18 @@
             disk_dict = {'target': {'dev': 'vdb', 'bus': 'virtio'}, 'device': 'disk', 'driver': ${disk_driver}}
             interface_driver = {'driver_attr': {'name': 'vhost', 'iommu': 'on', 'ats': 'on'}}
             iface_dict = {'type_name': 'network', 'model': 'virtio', 'driver': ${interface_driver}, 'source': {'network': 'default'}}
+        - pcie_downstream_port:
+            test_devices = ["Eth", "block"]
+            upstream_port = {'type': 'pci', 'model': 'pcie-switch-upstream-port', 'pre_controller': 'pcie-root-port'}
+            downstream_port1 = {'type': 'pci', 'model': 'pcie-switch-downstream-port', 'target': {'chassis': '20', 'port': 0x0}, 'pre_controller': 'pcie-switch-upstream-port'}
+            downstream_port2 = {'type': 'pci', 'model': 'pcie-switch-downstream-port', 'target': {'chassis': '21', 'port': 0x0}, 'pre_controller': 'pcie-switch-upstream-port'}
+            iface_dict = {'type_name': 'network', 'model': 'virtio', 'driver': {'driver_attr': {'name': 'vhost', 'iommu': 'on'}}, 'source': {'network': 'default'}}
+            disk_dict = {'target': {'dev': 'vdb', 'bus': 'virtio'}, 'device': 'disk', 'driver': ${disk_driver}}
+            variants:
+                - from_pcie_root_port:
+                    root_port = {'type': 'pci', 'model': 'pcie-root-port', 'pre_controller': 'pcie-root'}
+                    controller_dicts = [${root_port}, ${upstream_port}, ${upstream_port}, ${downstream_port1}, ${downstream_port2}]
+                - from_expander_bus:
+                    downstream_port = {'type': 'pci', 'model': 'pcie-switch-downstream-port', 'pre_controller': 'pcie-switch-upstream-port'}
+                    root_port = {'type': 'pci', 'model': 'pcie-root-port', 'pre_controller': 'pcie-expander-bus'}
+                    controller_dicts = [{'type': 'pci', 'model': 'pcie-expander-bus', 'target': {'busNr': '250'}, 'pre_controller': 'pcie-root'}, ${root_port}, ${upstream_port}, ${upstream_port}, ${downstream_port}, ${downstream_port}]

libvirt/tests/cfg/sriov/vIOMMU/iommu_device_settings.cfg

@@ -67,3 +67,19 @@
             controller_dicts = [{'type': 'pci', 'model': 'pcie-expander-bus', 'target': {'busNr': '252'}, 'pre_controller': 'pcie-root'}, ${root_port}, ${root_port}]
             iface_dict = {'source': {'network': 'default'}}
             disk_dict = {'target': {'dev': 'vda', 'bus': 'virtio'}, 'device': 'disk', 'driver': ${disk_driver}}
+        - pcie_downstream_port:
+            test_devices = ["Eth", "block"]
+            dev_in_same_iommu_group = ["XIO3130", *${test_devices}]
+            upstream_port = {'type': 'pci', 'model': 'pcie-switch-upstream-port', 'pre_controller': 'pcie-root-port'}
+            downstream_port1 = {'type': 'pci', 'model': 'pcie-switch-downstream-port', 'target': {'chassis': '20', 'port': 0x0}, 'pre_controller': 'pcie-switch-upstream-port'}
+            downstream_port2 = {'type': 'pci', 'model': 'pcie-switch-downstream-port', 'target': {'chassis': '21', 'port': 0x0}, 'pre_controller': 'pcie-switch-upstream-port'}
+            iface_dict = {'type_name': 'network', 'model': 'virtio', 'driver': {'driver_attr': {'name': 'vhost', 'iommu': 'on'}}, 'source': {'network': 'default'}}
+            disk_dict = {'target': {'dev': 'vdb', 'bus': 'virtio'}, 'device': 'disk', 'driver': ${disk_driver}}
+            variants:
+                - from_pcie_root_port:
+                    root_port = {'type': 'pci', 'model': 'pcie-root-port', 'pre_controller': 'pcie-root'}
+                    controller_dicts = [${root_port}, ${upstream_port}, ${upstream_port}, ${downstream_port1}, ${downstream_port2}]
+                - from_expander_bus:
+                    downstream_port = {'type': 'pci', 'model': 'pcie-switch-downstream-port', 'pre_controller': 'pcie-switch-upstream-port'}
+                    root_port = {'type': 'pci', 'model': 'pcie-root-port', 'pre_controller': 'pcie-expander-bus'}
+                    controller_dicts = [{'type': 'pci', 'model': 'pcie-expander-bus', 'target': {'busNr': '250'}, 'pre_controller': 'pcie-root'}, ${root_port}, ${upstream_port}, ${upstream_port}, ${downstream_port}, ${downstream_port}]

libvirt/tests/src/sriov/vIOMMU/hotplug_device_with_iommu_enabled.py

@@ -30,13 +30,16 @@ def detach_dev(device_type, dev_iommu_info, vm_session):
         dev_obj = vm_xml.VMXML.new_from_dumpxml(vm_name).get_devices(device_type)[-1]
         virsh.detach_device(vm_name, dev_obj.xml, wait_for_event=True,
                             debug=True, ignore_status=False)
-        if dev_iommu_info:
-            res = vm_session.cmd_status_output('ifconfig')
-            test.log.debug(res)
-            s, o = vm_session.cmd_status_output("ls %s" % dev_iommu_info)
-            test.log.debug(f"{device_type} iommu check: {o}")
-            if not s:
-                test.fail("The %s should be removed from the iommu group" % device_type)
+        if not dev_iommu_info:
+            return
+        for dev_info in dev_iommu_info:
+            if dev_info:
+                res = vm_session.cmd_status_output('ifconfig')
+                test.log.debug(res)
+                s, o = vm_session.cmd_status_output("ls %s" % dev_info)
+                test.log.debug(f"{device_type} iommu check: {o}")
+                if not s:
+                    test.fail("The %s should be removed from the iommu group" % device_type)
 
     cleanup_ifaces = params.get("cleanup_ifaces", "yes")
     disk_dict = eval(params.get('disk_dict', '{}'))
@@ -65,7 +68,7 @@ def detach_dev(device_type, dev_iommu_info, vm_session):
         vm.create_serial_console()
         vm_session = vm.wait_for_serial_login(
             timeout=int(params.get('login_timeout')))
-
+        pre_devices = viommu_base.get_devices_pci(vm_session, test_devices)
         if disk_dict:
             test.log.info("TEST_STEP: Attach a disk device to VM.")
             disk_dict = test_obj.update_disk_addr(disk_dict)
@@ -88,7 +91,7 @@ def detach_dev(device_type, dev_iommu_info, vm_session):
 
         test.log.info("TEST_STEP: Check dmesg message about iommu inside the vm.")
         vm_session.cmd("dmesg | grep -i 'Adding to iommu group'", timeout=300)
-        dev_iommu_groups = viommu_base.check_vm_iommu_group(vm_session, test_devices)
+        dev_iommu_groups = viommu_base.check_vm_iommu_group(vm_session, test_devices, pre_devices)
         test.log.debug(f"Device iommu groups info: {dev_iommu_groups}")
 
         if "block" in test_devices:

libvirt/tests/src/sriov/vIOMMU/iommu_device_settings.py

@@ -19,6 +19,7 @@ def run(test, params, env):
     ping_dest = params.get('ping_dest')
     iommu_dict = eval(params.get('iommu_dict', '{}'))
     test_devices = eval(params.get("test_devices", "[]"))
+    dev_in_same_iommu_group = eval(params.get("dev_in_same_iommu_group", "[]"))
     need_sriov = "yes" == params.get("need_sriov", "no")
 
     vm_name = params.get("main_vm", "avocado-vt-vm1")
@@ -32,6 +33,13 @@ def run(test, params, env):
     try:
         test_obj.setup_iommu_test(iommu_dict=iommu_dict, cleanup_ifaces=cleanup_ifaces)
         test_obj.prepare_controller()
+        vm.start()
+        vm.cleanup_serial_console()
+        vm.create_serial_console()
+        vm_session = vm.wait_for_serial_login(
+            timeout=int(params.get('login_timeout')))
+        pre_devices = viommu_base.get_devices_pci(vm_session, test_devices)
+        vm.destroy()
 
         for dev in ["disk", "video"]:
             dev_dict = eval(params.get('%s_dict' % dev, '{}'))
@@ -56,15 +64,19 @@ def run(test, params, env):
 
         test.log.info("TEST_STEP: Start the VM.")
         vm.start()
-        vm.cleanup_serial_console()
-        vm.create_serial_console()
         vm_session = vm.wait_for_serial_login(
             timeout=int(params.get('login_timeout')))
         test.log.debug(vm_xml.VMXML.new_from_dumpxml(vm.name))
 
         test.log.info("TEST_STEP: Check dmesg message about iommu inside the vm.")
         vm_session.cmd("dmesg | grep -i 'Adding to iommu group'")
-        viommu_base.check_vm_iommu_group(vm_session, test_devices)
+        viommu_base.check_vm_iommu_group(vm_session, test_devices, pre_devices)
+        if dev_in_same_iommu_group:
+            devices_pci_info = viommu_base.get_devices_pci(vm_session, dev_in_same_iommu_group)
+            devices_pci = [x for y in devices_pci_info.values() for x in y]
+            dev_dir = str(viommu_base.get_iommu_dev_dir(vm_session, devices_pci[0]))
+            for dev in devices_pci:
+                vm_session.cmd(f"ls {dev_dir} |grep {dev}")
 
         test.log.info("TEST_STEP: Check if the VM disk and network are woring well.")
         utils_disk.dd_data_to_vm_disk(vm_session, "/mnt/test")

provider/viommu/viommu_base.py

@@ -1,4 +1,5 @@
 import os
+import logging
 
 from avocado.core import exceptions
 
@@ -10,31 +11,88 @@
 from virttest.utils_libvirt import libvirt_virtio
 from virttest.utils_libvirt import libvirt_pcicontr
 
+LOG = logging.getLogger('avocado.' + __name__)
 
-def check_vm_iommu_group(vm_session, test_devices):
+
+def get_devices_pci(vm_session, test_devices):
     """
-    Check the devices with iommu enabled are located in a separate iommu group
+    Get the devices pci
 
     :param vm_session: VM session
     :param test_devices: test devices to check
+    :return: The test devices' pci
     """
     result = {}
     for dev in test_devices:
         s, o = vm_session.cmd_status_output(
             "lspci | awk 'BEGIN{IGNORECASE=1} /%s/ {print $1}'" % dev)
+        LOG.debug(o)
         if s:
             exceptions.TestFail("Failed to get pci address!")
+        result.update({dev: o.strip().splitlines()})
+    return result
 
-        cmd = ("find /sys/kernel/iommu_groups/ -type l | xargs ls -l | awk -F "
-               "'/' '/%s / {print(\"\",$2,$3,$4,$5,$6)}' OFS='/' " % o.splitlines()[0])
-        s, o = vm_session.cmd_status_output(cmd)
-        if s:
-            exceptions.TestFail("Failed to find iommu group!")
-        device_dir = o.strip().splitlines()[-1]
-        s, dev_pci = vm_session.cmd_status_output("ls %s" % device_dir)
-        if s:
-            exceptions.TestFail("Failed to get the device in the iommu group!")
-        result.update({dev: os.path.join(device_dir, dev_pci.strip())})
+
+def get_added_devices_pci(vm_session, test_devices, orig_devices=None):
+    """
+    Get the newly added devices pci
+
+    :param vm_session: VM session
+    :param test_devices: test devices to check
+    :param orig_devices: The original devices
+    :return: The pci of the added device
+    """
+    result = {}
+    act_devices = get_devices_pci(vm_session, test_devices)
+    if not orig_devices:
+        return act_devices
+    for k, v in act_devices.items():
+        added_dev = list(filter(lambda x: x not in orig_devices.get(k),  v))
+        LOG.debug(f"added dev: {added_dev}")
+        if not added_dev:
+            exceptions.TestFail("Failed to get the added devices' pci address!")
+        result[k] = added_dev
+    return result
+
+
+def get_iommu_dev_dir(vm_session, pci_addr):
+    """
+    Get iommu group's devices
+
+    :param vm_session: VM session
+    :param pci_addr: Device pci address
+    :return: The directory of iommu group devices
+    """
+    cmd = ("find /sys/kernel/iommu_groups/ -type l | xargs ls -l | awk -F "
+           "'/' '/%s / {print(\"\",$2,$3,$4,$5,$6)}' OFS='/' " % pci_addr)
+    s, o = vm_session.cmd_status_output(cmd)
+    LOG.debug(o)
+    if s:
+        exceptions.TestFail("Failed to find iommu group!")
+    return o.strip().splitlines()[-1]
+
+
+def check_vm_iommu_group(vm_session, test_devices, orig_devices=None):
+    """
+    Check the devices with iommu enabled are located in a separate iommu group
+
+    :param vm_session: VM session
+    :param test_devices: test devices to check
+    :param orig_devices: The original devices
+    :return: The test devices' iommu group info
+    """
+    act_devices = get_added_devices_pci(vm_session, test_devices, orig_devices)
+    result = {}
+    for dev in test_devices:
+        tmp_path = []
+        for pci_addr in act_devices[dev]:
+            device_dir = get_iommu_dev_dir(vm_session, pci_addr)
+            s, dev_pci = vm_session.cmd_status_output("ls %s" % device_dir)
+            LOG.debug(f"dev pci: {dev_pci}")
+            if s:
+                exceptions.TestFail("Failed to get the device in the iommu group!")
+            tmp_path.append(os.path.join(device_dir, pci_addr))
+        result.update({dev: tmp_path})
     return result
 
 
@@ -100,9 +158,24 @@ def prepare_controller(self):
                 contr_slot = libvirt_pcicontr.get_free_pci_slot(
                         vm_xml.VMXML.new_from_dumpxml(self.vm.name),
                         controller_bus=pre_contr_bus)
-                if contr_dict.get("model") == "pcie-to-pci-bridge":
+                if contr_dict.get("model") in ["pcie-to-pci-bridge", "pcie-switch-upstream-port", "pcie-switch-downstream-port'"]:
                     contr_slot = "0"
                 contr_attrs.update({"slot": contr_slot})
+                self.test.log.debug(contr_slot)
+                if contr_dict.get("model") in ["pcie-switch-upstream-port"]:
+                    pci_devices = list(vm_xml.VMXML.new_from_dumpxml(self.vm.name).xmltreefile.find("devices"))
+                    used_function = []
+                    for dev in pci_devices:
+                        address = dev.find("address")
+                        if address is not None and address.get("bus") == pre_contr_bus:
+                            used_function.append(address.get("function"))
+                    self.test.log.debug(contr_dict)
+                    self.test.log.debug(f"Used function: {used_function}")
+                    available_function = sorted(list(filter(lambda x: x not in used_function,  ["%0#x" % d for d in range(8)])))
+                    self.test.log.debug(f"available_function: {available_function}")
+                    contr_function = available_function[0]
+                    contr_attrs.update({"function": contr_function})
+
                 contr_dict.update({"address": {"attrs": contr_attrs}})
                 contr_dict.pop("pre_controller")
             libvirt_vmxml.modify_vm_device(
@@ -139,7 +212,7 @@ def update_disk_addr(self, disk_dict):
             if disk_dict["target"]["bus"] == "scsi":
                 disk_dict["address"]["attrs"].update({"type": "drive", "controller": dev_bus})
 
-            if self.controller_dicts[-1]["model"] == "pcie-root-port":
+            if self.controller_dicts[-1]["model"] in ["pcie-root-port", "pcie-switch-downstream-port"]:
                 self.controller_dicts.pop()
         return disk_dict