Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

test_setup: refactor PciAssignable() to configure remote host for migration #1985

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
+79 −50
Open

test_setup: refactor PciAssignable() to configure remote host for migration #1985

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
107 changes: 60 additions & 47 deletions virttest/test_setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1041,7 +1041,7 @@ class PciAssignable(object):
def __init__(self, driver=None, driver_option=None, host_set_flag=None,
kvm_params=None, vf_filter_re=None, pf_filter_re=None,
device_driver=None, nic_name_re=None, static_ip=None,
net_mask=None, start_addr_PF=None, pa_type=None):
net_mask=None, start_addr_PF=None, pa_type=None, session=None):
"""
Initialize parameter 'type' which could be:
vf: Virtual Functions
Expand Down Expand Up @@ -1076,6 +1076,7 @@ def __init__(self, driver=None, driver_option=None, host_set_flag=None,
:param static_ip: Flag to be set if the test should assign static IP
:param start_addr_PF: Starting private IPv4 address for the PF interface
:param pa_type: pci_assignable type, pf or vf
:param session: ShellSession object of VM/Remote Host
"""
self.devices = []
self.driver = driver
Expand All @@ -1091,7 +1092,12 @@ def __init__(self, driver=None, driver_option=None, host_set_flag=None,
self.net_mask = net_mask
self.start_addr_PF = start_addr_PF
self.pa_type = pa_type

self.session = session
self.cmd_output = process.getoutput
self.cmd_status_output = process.getstatusoutput
if self.session:
self.cmd_output = self.session.cmd_output
self.cmd_status_output = self.session.cmd_status_output
if nic_name_re:
self.nic_name_re = nic_name_re
else:
Expand Down Expand Up @@ -1208,7 +1214,7 @@ def _release_dev(self, pci_id):
"""
base_dir = "/sys/bus/pci"
drv_path = os.path.join(base_dir, "devices/%s/driver" % pci_id)
if self.device_driver in os.readlink(drv_path):
if self.device_driver in utils_misc.readlink(drv_path, session=self.session):
error_context.context(
"Release device %s to host" % pci_id, logging.info)

Expand All @@ -1218,7 +1224,7 @@ def _release_dev(self, pci_id):
logging.info("Run command in host: %s" % cmd)
try:
output = None
output = decode_to_text(process.system_output(cmd, shell=True, timeout=60))
output = self.cmd_output(cmd, timeout=60)
except Exception:
msg = "Command %s fail with output %s" % (cmd, output)
logging.error(msg)
Expand All @@ -1229,7 +1235,7 @@ def _release_dev(self, pci_id):
logging.info("Run command in host: %s" % cmd)
try:
output = None
output = decode_to_text(process.system_output(cmd, shell=True, timeout=60))
output = self.cmd_output(cmd, timeout=60)
except Exception:
msg = "Command %s fail with output %s" % (cmd, output)
logging.error(msg)
Expand Down Expand Up @@ -1257,7 +1263,7 @@ def get_vf_status(self, vf_id):
tub_path = os.path.join(base_dir, "drivers/pci-stub")
vf_res_path = os.path.join(tub_path, "%s/resource*" % vf_id)
cmd = "lsof %s" % vf_res_path
output = decode_to_text(process.system_output(cmd, timeout=60, ignore_status=True))
output = self.cmd_output(cmd, timeout=60)
if 'qemu' in output:
return True
else:
Expand Down Expand Up @@ -1303,27 +1309,28 @@ def get_pf_vf_info(self):
for pf_id in pf_ids:
pf_info = {}
vf_ids = []
full_id = utils_misc.get_full_pci_id(pf_id)
full_id = utils_misc.get_full_pci_id(pf_id, session=self.session)
pf_info["pf_id"] = full_id
pf_info["occupied"] = False
d_link = os.path.join("/sys/bus/pci/devices", full_id)
txt = decode_to_text(process.system_output("ls %s" % d_link))
txt = self.cmd_output("ls %s" % d_link)
re_vfn = "(virtfn\d+)"
paths = re.findall(re_vfn, txt)
for path in paths:
f_path = os.path.join(d_link, path)
vf_id = os.path.basename(os.path.realpath(f_path))
vf_id = os.path.basename(utils_misc.readlink(f_path,
session=self.session))
vf_info = {}
vf_info["vf_id"] = vf_id
vf_info["occupied"] = False
vf_ids.append(vf_info)
pf_info["vf_ids"] = vf_ids
pf_vf_dict.append(pf_info)
if_out = decode_to_text(process.system_output("ifconfig -a"))
if_out = self.cmd_output("ifconfig -a")
ethnames = re.findall(self.nic_name_re, if_out)
for eth in ethnames:
cmd = "ethtool -i %s | awk '/bus-info/ {print $2}'" % eth
pci_id = decode_to_text(process.system_output(cmd, shell=True)).strip()
pci_id = self.cmd_output(cmd).strip()
if not pci_id:
continue
for pf in pf_vf_dict:
Expand Down Expand Up @@ -1408,13 +1415,13 @@ def get_devs(self, devices=None):
set_mac_cmd = "ip link set dev %s vf %s mac %s " % (ethname,
vf_num,
device["mac"])
process.run(set_mac_cmd)
self.cmd_output(set_mac_cmd)

elif d_type == "pf":
dev_id = pf_ids.pop(0)
dev_ids.append(dev_id)
unbind_driver = os.path.realpath(os.path.join(base_dir,
"devices/%s/driver" % dev_id))
unbind_driver_path = os.path.join(base_dir, "devices/%s/driver" % dev_id)
unbind_driver = utils_misc.readlink(unbind_driver_path, session=self.session)
self.dev_unbind_drivers[dev_id] = unbind_driver
if len(dev_ids) != len(devices):
logging.error("Did not get enough PCI Device")
Expand All @@ -1428,7 +1435,7 @@ def get_vfs_count(self):
# 'virtual function' belongs to which physical card considering
# that if the host has more than one 82576 card. PCI_ID?
cmd = "lspci | grep '%s' | wc -l" % self.vf_filter_re
vf_num = int(decode_to_text(process.system_output(cmd, shell=True, verbose=False)))
vf_num = int(self.cmd_output(cmd))
logging.info("Found %s vf in host", vf_num)
return vf_num

Expand All @@ -1445,7 +1452,7 @@ def get_same_group_devs(self, pci_id):
base_dir = "/sys/bus/pci/devices"
devices_link = os.path.join(base_dir,
"%s/iommu_group/devices/" % pci_id)
out = decode_to_text(process.system_output("ls %s" % devices_link))
out = self.cmd_output("ls %s" % devices_link)

if out:
pci_ids = out.split()
Expand All @@ -1456,13 +1463,12 @@ def get_pf_ids(self):
Get the id of PF devices
"""
cmd = "lspci | grep -v 'Virtual Function' |awk '/%s/ {print $1}'" % self.pf_filter_re
PF_devices = [i for i in decode_to_text(process.system_output(
cmd, shell=True)).splitlines()]
PF_devices = [i for i in self.cmd_output(cmd).splitlines()]
if not PF_devices:
raise exceptions.TestSkipError("No specified pf found in the host!")
pf_ids = []
for pf in PF_devices:
pf_id = utils_misc.get_full_pci_id(pf)
pf_id = utils_misc.get_full_pci_id(pf, session=self.session)
pf_ids.append(pf_id)
return pf_ids

Expand All @@ -1477,12 +1483,13 @@ def assign_static_ip(self):
"No IP / netmask found, please populate starting IP address for PF devices in configuration file")
ip_addr = netaddr.IPAddress(self.start_addr_PF)
for PF in pf_devices:
ifname = utils_misc.get_interface_from_pci_id(PF)
ifname = utils_misc.get_interface_from_pci_id(PF,
session=self.session)
ip_assign = "ifconfig %s %s netmask %s up" % (
ifname, ip_addr, self.net_mask)
logging.info("assign IP to PF device %s : %s", PF,
ip_assign)
cmd = process.system(ip_assign, shell=True, ignore_status=True)
cmd, _ = self.cmd_status_output(ip_assign)
if cmd:
raise exceptions.TestSetupFail("Failed to assign IP : %s"
% cmd)
Expand All @@ -1504,7 +1511,7 @@ def get_controller_type(self):
"""
try:
cmd = "lspci | grep '%s'| grep -o '\s[A-Z].*:\s'" % self.pf_filter_re
return decode_to_text(process.system_output(cmd, shell=True)).split("\n")[-1].strip().strip(':')
return self.cmd_output(cmd, shell=True).split("\n")[-1].strip().strip(':')
except IndexError:
logging.debug("Unable to fetch the controller details")
return None
Expand All @@ -1518,7 +1525,8 @@ def is_binded_to_stub(self, full_id):
"""
base_dir = "/sys/bus/pci/"
stub_path = os.path.join(base_dir, "drivers/%s" % self.device_driver)
return os.path.exists(os.path.join(stub_path, full_id))
return utils_misc.check_exists(os.path.join(stub_path, full_id),
session=self.session)

def generate_ib_port_id(self):
"""
Expand All @@ -1535,8 +1543,8 @@ def set_linkvf_ib(self):
pids = {}
try:
cmd = "ls -R /sys/class/infiniband/*/device/sriov/*"
dev = decode_to_text(process.system_output(cmd, shell=True))
except process.CmdError as detail:
dev = self.cmd_output(cmd)
except Exception as detail:
logging.error("No VF's found for set-up, command-failed as: %s",
str(detail))
return False
Expand All @@ -1552,8 +1560,9 @@ def set_linkvf_ib(self):
for key in pids.keys():
logging.info("The key %s corresponds to %s", key, pids[key])
for subkey in pids[key].keys():
status = process.system("echo %s > %s" % (pids[key][subkey],
os.path.join(key, subkey)), shell=True)
status, _ = self.cmd_status_output("echo %s > %s" %
(pids[key][subkey],
os.path.join(key, subkey)))
if status != 0:
return False
return True
Expand All @@ -1567,7 +1576,8 @@ def set_vf(self, pci_pf, vf_no="0"):
:return: True on success, False on failure
"""
cmd = "echo %s > /sys/bus/pci/devices/%s/sriov_numvfs" % (vf_no, pci_pf)
if process.system(cmd, shell=True, ignore_status=True):
status, _ = self.cmd_status_output(cmd)
if status:
logging.debug("Failed to set %s vfs in %s", vf_no, pci_pf)
return False
# When the VFs loaded on a PF are > 10 [I have tested till 63 which is
Expand Down Expand Up @@ -1598,7 +1608,8 @@ def remove_driver(self, driver=None):
if not self.set_vf(PF):
return False
cmd = "rmmod mlx5_ib;modprobe -r mlx5_core;modprobe mlx5_ib"
if process.system(cmd, ignore_status=True, shell=True):
status, _ = self.cmd_status_output(cmd)
if status:
logging.debug("Failed to remove driver: %s", driver)
return False
return True
Expand All @@ -1616,7 +1627,8 @@ def modprobe_driver(self, driver=None):
msg = "Loading the driver '%s'" % driver
error_context.context(msg, logging.info)
cmd = "modprobe %s" % driver
if process.system(cmd, ignore_status=True, shell=True):
status, _ = self.cmd_status_output(cmd)
if status:
logging.debug("Failed to modprobe driver: %s", driver)
return False
return True
Expand All @@ -1638,7 +1650,7 @@ def sr_iov_setup(self):
logging.info)
if ARCH != 'ppc64le':
kvm_re_probe = True
dmesg = decode_to_text(process.system_output("dmesg", verbose=False))
dmesg = self.cmd_output("dmesg")
ecap = re.findall("ecap\s+(.\w+)", dmesg)
if not ecap:
logging.error("Fail to check host interrupt remapping support")
Expand All @@ -1656,32 +1668,32 @@ def sr_iov_setup(self):
error_context.context("enable PCI passthrough with '%s'" % cmd,
logging.info)
try:
process.system(cmd)
self.cmd_output(cmd)
except Exception:
logging.debug(
"Can not enable the interrupt remapping support")
lnk = "/sys/module/vfio_iommu_type1/parameters/allow_unsafe_interrupts"
if self.device_driver == "vfio-pci":
# If driver is not available modprobe it
if process.system('lsmod | grep vfio_pci', ignore_status=True,
shell=True):
status, _ = self.cmd_status_output('lsmod | grep vfio_pci')
if status:
self.modprobe_driver(driver="vfio-pci")
time.sleep(3)
if not ecap or (int(ecap[0], 16) & 8 != 8):
cmd = "echo Y > %s" % lnk
error_context.context("enable PCI passthrough with '%s'" % cmd,
logging.info)
process.run(cmd)
self.cmd_output(cmd)
else:
if self.device_driver == "vfio-pci":
if process.system('lsmod | grep vfio_pci', ignore_status=True,
shell=True):
status, _ = self.cmd_status_output('lsmod | grep vfio_pci')
if status:
self.modprobe_driver(driver="vfio-pci")
time.sleep(3)
re_probe = False
# If driver not available after modprobe try to remove it and reprobe
if process.system("lsmod | grep -w %s" % self.driver, ignore_status=True,
shell=True):
status, _ = self.cmd_status_output("lsmod | grep -w %s" % self.driver)
if status:
re_probe = True
# If driver is available and pa_type is vf then set VFs
elif self.pa_type == 'vf':
Expand Down Expand Up @@ -1712,9 +1724,7 @@ def sr_iov_setup(self):
if not self.check_vfs_count():
# Even after re-probe there are no VFs created
return False
dmesg = decode_to_text(process.system_output("dmesg", timeout=60,
ignore_status=True,
verbose=False))
_, dmesg = self.cmd_status_output("dmesg", timeout=60)
file_name = "host_dmesg_after_load_%s.txt" % self.driver
logging.info("Log dmesg after loading '%s' to '%s'.", self.driver,
file_name)
Expand Down Expand Up @@ -1743,15 +1753,15 @@ def sr_iov_cleanup(self):
cmd = "echo %s > %s" % (value, kvm_param)
logging.info("Write '%s' to '%s'", value, kvm_param)
try:
process.system(cmd)
self.cmd_output(cmd)
except Exception:
logging.error("Failed to write '%s' to '%s'", value,
kvm_param)

re_probe = False
# if lsmod lists the driver then remove it to clean up
if not process.system('lsmod | grep %s' % self.driver,
ignore_status=True, shell=True):
status, _ = self.cmd_status_output('lsmod | grep %s' % self.driver)
if not status:
if not self.remove_driver():
re_probe = True

Expand Down Expand Up @@ -1799,8 +1809,11 @@ def request_devs(self, devices=None):
if not os.path.exists(drv_path):
dev_prev_driver = ""
else:
dev_prev_driver = os.path.realpath(os.path.join(drv_path,
os.readlink(drv_path)))
dev_prev_driver = os.path.join(drv_path,
utils_misc.readlink(drv_path,
session=self.session))
dev_prev_driver = utils_misc.readlink(dev_prev_driver,
session=self.session)
self.dev_drivers[pci_id] = dev_prev_driver

# Judge whether the device driver has been binded to stub
Expand Down
22 changes: 19 additions & 3 deletions virttest/utils_misc.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -722,16 +722,20 @@ def display_attributes(instance):
logging.debug(" %s: %s", name, value)


def get_full_pci_id(pci_id):
def get_full_pci_id(pci_id, session=None):
"""
Get full PCI ID of pci_id.
:param pci_id: PCI ID of a device.
"""
func = process.getoutput
if session:
func = session.cmd_output
cmd = "lspci -D | awk '/%s/ {print $1}'" % pci_id
try:
return decode_to_text(process.system_output(cmd, shell=True))
except process.CmdError:
return func(cmd)
except Exception as info:
logging.error(info)
return None


Expand Down Expand Up @@ -973,6 +977,18 @@ def check_exists(path, session=None):
return os.path.exists(path)


def readlink(path, session=None):
"""
wrapper method to symbolic link of given path in local/remote host/VM
:param path: path to get symbolic link
:param session: ShellSession object of VM/remote host
"""
if session:
cmd = "readlink -f %s" % path
return session.cmd_output(cmd)
return os.readlink(path)


def safe_rmdir(path, timeout=10, session=None):
"""
Try to remove a directory safely, even on NFS filesystems.
Expand Down