From 7309b7adf7e5ae5c96124c17573b3429e9750481 Mon Sep 17 00:00:00 2001 From: Pavel Boldin Date: Thu, 8 Aug 2024 01:54:00 +0300 Subject: [PATCH 1/9] kernel_source_installer: fix mirrorlist on centos7 Signed-off-by: Pavel Boldin --- lisa/transformers/kernel_source_installer.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lisa/transformers/kernel_source_installer.py b/lisa/transformers/kernel_source_installer.py index 577e3975e3..77efa27610 100644 --- a/lisa/transformers/kernel_source_installer.py +++ b/lisa/transformers/kernel_source_installer.py @@ -302,9 +302,18 @@ def _build_code(self, node: Node, code_path: PurePath, kconfig_file: str) -> Non # set timeout to 2 hours make.make(arguments="", cwd=code_path, timeout=60 * 60 * 2) + def _fix_mirrorlist_to_vault(self, node: Node) -> None: + node.execute("sed -i '\ + s/^mirrorlist=/#mirrorlist=/;\ + s/^#baseurl=/baseurl=/;\ + /^baseurl=/ s/mirror/vault/\ + ' /etc/yum.repos.d/CentOS-*.repo", shell=True, sudo=True) + def _install_build_tools(self, node: Node) -> None: os = node.os self._log.info("installing build tools") + if isinstance(node.os, Redhat) and node.os.information.version < "8.0.0": + self._fix_mirrorlist_to_vault(node) if isinstance(os, Redhat): for package in list( ["elfutils-libelf-devel", "openssl-devel", "dwarves", "bc"] From 638a2828501f88312d21d61692f714da73ece872 Mon Sep 17 00:00:00 2001 From: Pavel Boldin Date: Thu, 8 Aug 2024 01:53:54 +0300 Subject: [PATCH 2/9] dpdk: build via devtools on RHEL<8.0.0 Signed-off-by: Pavel Boldin --- lisa/base_tools/mv.py | 7 ++++-- microsoft/testsuites/dpdk/dpdktestpmd.py | 28 ++++++++++++++++++++++-- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/lisa/base_tools/mv.py b/lisa/base_tools/mv.py index 3102bf6aac..bf79259ba5 100644 --- a/lisa/base_tools/mv.py +++ b/lisa/base_tools/mv.py @@ -2,6 +2,7 @@ # Licensed under the MIT license. from lisa.executable import Tool +from typing import Union class Mv(Tool): @@ -14,13 +15,15 @@ def can_install(self) -> bool: return False def move( - self, src_path: str, dest_path: str, overwrite: bool = False, sudo: bool = False + self, src_path: str, dest_path: str, overwrite: bool = False, + sudo: bool = False, ignore_error: bool = False ) -> None: args = "-f" if overwrite else "" + expected_exit_code = None if ignore_error else 0 self.run( f"{args} {src_path} {dest_path}", sudo=sudo, shell=True, force_run=True, - expected_exit_code=0, + expected_exit_code=expected_exit_code, ) diff --git a/microsoft/testsuites/dpdk/dpdktestpmd.py b/microsoft/testsuites/dpdk/dpdktestpmd.py index facea07d82..5991f0ee97 100644 --- a/microsoft/testsuites/dpdk/dpdktestpmd.py +++ b/microsoft/testsuites/dpdk/dpdktestpmd.py @@ -13,7 +13,7 @@ from lisa.executable import ExecutableResult, Tool from lisa.features import Disk from lisa.nic import NicInfo -from lisa.operating_system import Debian, Fedora, Suse, Ubuntu +from lisa.operating_system import Debian, Fedora, Suse, Ubuntu, Redhat from lisa.tools import ( Chmod, Dmesg, @@ -531,7 +531,8 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: ) # if dpdk is already installed, find the binary and check the version - if self.find_testpmd_binary(assert_on_fail=False): + if self.find_testpmd_binary(assert_on_fail=False) or \ + self.find_testpmd_binary(check_path='/usr/local/bin', assert_on_fail=False): pkgconfig = self.node.tools[Pkgconfig] if pkgconfig.package_info_exists( self._dpdk_lib_name, @@ -610,6 +611,29 @@ def _install(self) -> bool: ) ) + if isinstance(node.os, Redhat) and node.os.information.version < "8.0.0": + node.os.install_packages(["centos-release-scl"]) + devtoolset_version = 8 + devtoolset_pkg = f"devtoolset-{devtoolset_version}" + node.os.install_packages([devtoolset_pkg]) + links = { + "gcc": ("gcc", "cc"), + "g++": ("g++", "c++"), + } + for binary in [alias + for aliases in links.values() + for alias in aliases + ]: + node.tools[Mv].move(f"/bin/{binary}", f"/bin/{binary}_back", + overwrite=True, sudo=True, ignore_error=None) + devtoolset_binpath = f"/opt/rh/{devtoolset_pkg}/root/bin" + for binary, aliases in links.items(): + for alias in aliases: + result = node.execute( + f"ln -s {devtoolset_binpath}/{binary} /bin/{alias}", sudo=True + ) + result.assert_exit_code() + # before doing anything: determine if backport repo needs to be enabled self._set_backport_repo_args() From 7edabb1bf5b311bf0b33d9db91cec3d98736e21d Mon Sep 17 00:00:00 2001 From: Pavel Boldin Date: Thu, 8 Aug 2024 01:53:56 +0300 Subject: [PATCH 3/9] dpdk: control kernel update via dpdk_kernel_update Signed-off-by: Pavel Boldin --- microsoft/testsuites/dpdk/dpdktestpmd.py | 28 ++++++++++++++---------- microsoft/testsuites/dpdk/dpdkutil.py | 2 ++ microsoft/testsuites/dpdk/rdma_core.py | 20 ++++++++++------- 3 files changed, 30 insertions(+), 20 deletions(-) diff --git a/microsoft/testsuites/dpdk/dpdktestpmd.py b/microsoft/testsuites/dpdk/dpdktestpmd.py index 5991f0ee97..e531b5eac8 100644 --- a/microsoft/testsuites/dpdk/dpdktestpmd.py +++ b/microsoft/testsuites/dpdk/dpdktestpmd.py @@ -488,6 +488,8 @@ def get_dpdk_portmask(self, ports: List[int]) -> str: def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) + # do not update kernel on backporting + self.update_kernel = kwargs.pop("update_kernel", True) # set source args for builds if needed, first for dpdk self.dpdk_build_path: Optional[PurePath] = None self._dpdk_source: str = kwargs.pop("dpdk_source", PACKAGE_MANAGER_SOURCE) @@ -499,6 +501,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: node=self.node, rdma_core_source=rdma_core_source, rdma_core_ref=rdma_core_ref, + update_kernel=self.update_kernel, ) self._sample_apps_to_build = kwargs.pop("sample_apps", list()) self._dpdk_version_info = VersionInfo(0, 0) @@ -885,7 +888,7 @@ def _load_drivers_for_dpdk(self) -> None: if isinstance(self.node.os, (Ubuntu, Suse)): # Ubuntu shouldn't need any special casing, skip to loading rdma/ib pass - elif isinstance(self.node.os, Debian): + elif self.update_kernel and isinstance(self.node.os, Debian): # NOTE: debian buster doesn't include rdma and ib drivers # on 5.4 specifically for linux-image-cloud: # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1012639 @@ -984,8 +987,9 @@ def _install_ubuntu_dependencies(self) -> None: return # appease the type checker # apply update to latest first - ubuntu.update_packages("linux-azure") - node.reboot() + if self.update_kernel: + ubuntu.update_packages("linux-azure") + node.reboot() if ubuntu.information.version < "18.4.0": raise SkippedException( f"Ubuntu {str(ubuntu.information.version)} is not supported. " @@ -1004,9 +1008,8 @@ def _install_ubuntu_dependencies(self) -> None: extra_args=self._backport_repo_args, ) # MANA tests use linux-modules-extra-azure, install if it's available. - if self.vf_helper.is_mana() and ubuntu.is_package_in_repo( - "linux-modules-extra-azure" - ): + if self.update_kernel and self.vf_helper.is_mana() and \ + ubuntu.is_package_in_repo("linux-modules-extra-azure"): ubuntu.install_packages("linux-modules-extra-azure") def _install_fedora_dependencies(self) -> None: @@ -1021,12 +1024,13 @@ def _install_fedora_dependencies(self) -> None: # DPDK is very sensitive to rdma-core/kernel mismatches # update to latest kernel before installing dependencies - rhel.install_packages(["kernel", "kernel-modules-extra", "kernel-headers"]) - node.reboot() - try: - rhel.install_packages("kernel-devel") - except MissingPackagesException: - node.log.debug("Fedora: kernel-devel not found, attempting to continue") + if self.update_kernel: + rhel.install_packages(["kernel", "kernel-modules-extra", "kernel-headers"]) + node.reboot() + try: + rhel.install_packages("kernel-devel") + except MissingPackagesException: + node.log.debug("Fedora: kernel-devel not found, attempting to continue") if rhel.information.version.major == 7: # Add packages for rhel7 diff --git a/microsoft/testsuites/dpdk/dpdkutil.py b/microsoft/testsuites/dpdk/dpdkutil.py index a425266b27..07f8b64784 100644 --- a/microsoft/testsuites/dpdk/dpdkutil.py +++ b/microsoft/testsuites/dpdk/dpdkutil.py @@ -333,6 +333,7 @@ def initialize_node_resources( dpdk_branch = variables.get("dpdk_branch", "") rdma_core_source = variables.get("rdma_core_source", "") rdma_core_ref = variables.get("rdma_core_git_ref", "") + update_kernel = variables.get("dpdk_update_kernel", True) force_net_failsafe_pmd = variables.get("dpdk_force_net_failsafe_pmd", False) enforce_strict_threshold = variables.get("dpdk_enforce_strict_threshold", False) log.info( @@ -374,6 +375,7 @@ def initialize_node_resources( rdma_core_ref=rdma_core_ref, enforce_strict_threshold=enforce_strict_threshold, build_release=build_release, + update_kernel=update_kernel, ) # init and enable hugepages (required by dpdk) diff --git a/microsoft/testsuites/dpdk/rdma_core.py b/microsoft/testsuites/dpdk/rdma_core.py index d5e6334978..2dcf1485f0 100644 --- a/microsoft/testsuites/dpdk/rdma_core.py +++ b/microsoft/testsuites/dpdk/rdma_core.py @@ -14,7 +14,8 @@ class RdmaCoreManager: - def __init__(self, node: Node, rdma_core_source: str, rdma_core_ref: str) -> None: + def __init__(self, node: Node, rdma_core_source: str, rdma_core_ref: str, + update_kernel: bool) -> None: self.is_installed_from_source = False self.node = node self._rdma_core_source = rdma_core_source @@ -27,6 +28,8 @@ def __init__(self, node: Node, rdma_core_source: str, rdma_core_ref: str) -> Non assert build_location, "Could not find a location to build rdma-core" self._build_location = node.get_pure_path(build_location).joinpath("rdma") + self.update_kernel = update_kernel + def get_missing_distro_packages(self) -> str: distro = self.node.os package = "" @@ -186,13 +189,14 @@ def do_source_install(self) -> None: "libbpf-devel", ] ) - distro.install_packages( - [ - "kernel-devel", - "kernel-modules-extra", - "kernel-headers", - ] - ) + if self.update_kernel: + distro.install_packages( + [ + "kernel-devel", + "kernel-modules-extra", + "kernel-headers", + ] + ) else: # no-op, throw for invalid distro is before this function return From d2c5d1e738c6956b7d0fafe69899110cc22f9bca Mon Sep 17 00:00:00 2001 From: Pavel Boldin Date: Thu, 8 Aug 2024 01:53:59 +0300 Subject: [PATCH 4/9] dpdk: force dpdk build on RHEL<8.0 Signed-off-by: Pavel Boldin --- microsoft/testsuites/dpdk/dpdkutil.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/microsoft/testsuites/dpdk/dpdkutil.py b/microsoft/testsuites/dpdk/dpdkutil.py index 07f8b64784..fa5b476d8c 100644 --- a/microsoft/testsuites/dpdk/dpdkutil.py +++ b/microsoft/testsuites/dpdk/dpdkutil.py @@ -23,7 +23,7 @@ from lisa.base_tools.uname import Uname from lisa.features import NetworkInterface from lisa.nic import NicInfo -from lisa.operating_system import Fedora, OperatingSystem, Ubuntu +from lisa.operating_system import Fedora, OperatingSystem, Ubuntu, CentOs from lisa.tools import ( Dmesg, Echo, @@ -178,8 +178,15 @@ def _set_forced_source_by_distro( # Default to 20.11 unless another version is provided by the # user. 20.11 is the latest dpdk version for 18.04. if ( - isinstance(node.os, Ubuntu) - and node.os.information.version < "20.4.0" + ( + isinstance(node.os, Ubuntu) + and node.os.information.version < "20.4.0" + ) + or + ( + isinstance(node.os, CentOs) + and node.os.information.version < "8.0.0" + ) or examples != None ): variables["dpdk_source"] = variables.get("dpdk_source", DPDK_STABLE_GIT_REPO) From b981aee642a598c3ade809f840fd94f7ddd91427 Mon Sep 17 00:00:00 2001 From: Pavel Boldin Date: Thu, 8 Aug 2024 01:54:01 +0300 Subject: [PATCH 5/9] dpdtestpmd: fix mirrorlist on centos7 Signed-off-by: Pavel Boldin --- microsoft/testsuites/dpdk/dpdktestpmd.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/microsoft/testsuites/dpdk/dpdktestpmd.py b/microsoft/testsuites/dpdk/dpdktestpmd.py index e531b5eac8..2e385c59f2 100644 --- a/microsoft/testsuites/dpdk/dpdktestpmd.py +++ b/microsoft/testsuites/dpdk/dpdktestpmd.py @@ -596,6 +596,13 @@ def _set_backport_repo_args(self) -> None: else: self._backport_repo_args = [] + def _fix_mirrorlist_to_vault(self, node) -> None: + node.execute("""sed -i ' + s/^mirrorlist=/#mirrorlist=/; + s/^#[ ]*baseurl=/baseurl=/; + /^baseurl=/ s/mirror/vault/; + ' /etc/yum.repos.d/CentOS-*.repo""", shell=True, sudo=True) + def _install(self) -> bool: self._testpmd_output_after_reenable = "" self._testpmd_output_before_rescind = "" @@ -615,7 +622,11 @@ def _install(self) -> bool: ) if isinstance(node.os, Redhat) and node.os.information.version < "8.0.0": + self._fix_mirrorlist_to_vault(node) node.os.install_packages(["centos-release-scl"]) + + # Fix CentOS-SCL's paths to mirrorlist + self._fix_mirrorlist_to_vault(node) devtoolset_version = 8 devtoolset_pkg = f"devtoolset-{devtoolset_version}" node.os.install_packages([devtoolset_pkg]) From 30a2987d1955e7f31b2e6dfca98e80a99b39b45a Mon Sep 17 00:00:00 2001 From: Pavel Boldin Date: Thu, 8 Aug 2024 01:55:36 +0300 Subject: [PATCH 6/9] dpdk: check enable_{drivers,app} support Signed-off-by: Pavel Boldin --- microsoft/testsuites/dpdk/dpdktestpmd.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/microsoft/testsuites/dpdk/dpdktestpmd.py b/microsoft/testsuites/dpdk/dpdktestpmd.py index 2e385c59f2..61f539b2a1 100644 --- a/microsoft/testsuites/dpdk/dpdktestpmd.py +++ b/microsoft/testsuites/dpdk/dpdktestpmd.py @@ -15,6 +15,7 @@ from lisa.nic import NicInfo from lisa.operating_system import Debian, Fedora, Suse, Ubuntu, Redhat from lisa.tools import ( + Cat, Chmod, Dmesg, Echo, @@ -794,11 +795,20 @@ def _install(self) -> bool: # add mana driver to build if needed if self.vf_helper.is_mana(): drivers_to_build += ",net/mana" - # shrink build - build_flags += [ - f"-Denable_drivers={drivers_to_build}", - "-Denable_apps=app/test-pmd", - ] + + # shrink build, if supported + cat = node.tools[Cat] + meson_options_content = cat.run("meson_options.txt", + cwd=self.dpdk_path, shell=True).stdout + if "enable_drivers" in meson_options_content: + build_flags += [ + f"-Denable_drivers={drivers_to_build}" + ] + + if "enable_apps" in meson_options_content: + build_flags += [ + "-Denable_apps=app/test-pmd" + ] node.execute( f"meson setup {' '.join(build_flags)} build", From 22acf02b6a3dde5a665b7d63acbf40fd0e7d981a Mon Sep 17 00:00:00 2001 From: Pavel Boldin Date: Thu, 8 Aug 2024 01:56:19 +0300 Subject: [PATCH 7/9] dpdk: fix build of v19.11 Signed-off-by: Pavel Boldin --- microsoft/testsuites/dpdk/dpdktestpmd.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/microsoft/testsuites/dpdk/dpdktestpmd.py b/microsoft/testsuites/dpdk/dpdktestpmd.py index 61f539b2a1..dfb7d3acf8 100644 --- a/microsoft/testsuites/dpdk/dpdktestpmd.py +++ b/microsoft/testsuites/dpdk/dpdktestpmd.py @@ -326,6 +326,7 @@ def generate_testpmd_command( ).is_greater_than(0) return ( + f"env LD_LIBRARY_PATH=/usr/local/lib64 " f"{self._testpmd_install_path} {core_list} " f"{nic_include_info} {log_level_args}" f" -- --forward-mode={mode} " @@ -711,11 +712,11 @@ def _install(self) -> bool: if ( isinstance(distro, Debian) or isinstance(distro, (Fedora, Suse)) - and distro.package_exists("dpdk") ): # if not using package manager and dpdk is already installed, uninstall it # in preperation for source build - distro.uninstall_packages("dpdk") + if distro.package_exists("dpdk"): + distro.uninstall_packages("dpdk") else: raise NotImplementedError( "Dpdk package names are missing in dpdktestpmd.install" @@ -1064,6 +1065,7 @@ def _install_fedora_dependencies(self) -> None: rhel.group_install_packages("Development Tools") rhel.install_packages(self._fedora_packages) + rhel.uninstall_packages(["doxygen"]) # ensure RDMA service is started if present. From 0e40c4d8d5c1137b1c0ad4793b82e725be9602a6 Mon Sep 17 00:00:00 2001 From: Pavel Boldin Date: Wed, 30 Oct 2024 17:04:35 +0200 Subject: [PATCH 8/9] kernel_source_installer: fix deps Signed-off-by: Pavel Boldin --- lisa/transformers/kernel_source_installer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lisa/transformers/kernel_source_installer.py b/lisa/transformers/kernel_source_installer.py index 77efa27610..90c4272383 100644 --- a/lisa/transformers/kernel_source_installer.py +++ b/lisa/transformers/kernel_source_installer.py @@ -338,6 +338,7 @@ def _install_build_tools(self, node: Node) -> None: "git", "build-essential", "bison", + "cpio", "flex", "libelf-dev", "libncurses5-dev", @@ -345,6 +346,7 @@ def _install_build_tools(self, node: Node) -> None: "libssl-dev", "bc", "ccache", + "zstd" ] ) elif isinstance(os, CBLMariner): From af1fef1fcbce19b935ea9b4a92c5731da3c33dd9 Mon Sep 17 00:00:00 2001 From: Pavel Boldin Date: Tue, 5 Nov 2024 17:02:02 +0200 Subject: [PATCH 9/9] kernel_source_installer: optional devtoolset for <3.10 Signed-off-by: Pavel Boldin --- lisa/transformers/kernel_source_installer.py | 30 ++++++++++++++------ 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/lisa/transformers/kernel_source_installer.py b/lisa/transformers/kernel_source_installer.py index 90c4272383..064b12baff 100644 --- a/lisa/transformers/kernel_source_installer.py +++ b/lisa/transformers/kernel_source_installer.py @@ -14,7 +14,7 @@ from lisa.tools import Cp, Echo, Git, Make, Sed, Uname from lisa.tools.gcc import Gcc from lisa.tools.lscpu import Lscpu -from lisa.util import LisaException, field_metadata, subclasses +from lisa.util import LisaException, VersionInfo, field_metadata, subclasses, parse_version from lisa.util.logger import Logger, get_logger from .kernel_installer import BaseInstaller, BaseInstallerSchema @@ -158,11 +158,23 @@ def install(self) -> str: # modify code self._modify_code(node=node, code_path=self._code_path) + + result = node.execute( + "make kernelversion 2>/dev/null", + cwd=self._code_path, + shell=True, + ) + result.assert_exit_code( + 0, + f"failed on get kernel version: {result.stdout}", + ) + kernel_version = parse_version(result.stdout) + kconfig_file = runbook.kernel_config_file self._build_code( - node=node, code_path=self._code_path, kconfig_file=kconfig_file + node=node, code_path=self._code_path, kconfig_file=kconfig_file, + kernel_version=kernel_version ) - self._install_build(node=node, code_path=self._code_path) result = node.execute( @@ -170,16 +182,15 @@ def install(self) -> str: cwd=self._code_path, shell=True, ) - - kernel_version = result.stdout result.assert_exit_code( 0, - f"failed on get kernel version: {kernel_version}", + f"failed on get kernel release: {result.stdout}", ) + kernel_release = result.stdout # copy current config back to system folder. result = node.execute( - f"cp .config /boot/config-{kernel_version}", + f"cp .config /boot/config-{kernel_release}", cwd=self._code_path, sudo=True, ) @@ -227,7 +238,7 @@ def _modify_code(self, node: Node, code_path: PurePath) -> None: self._log.debug(f"modifying code by {modifier.type_name()}") modifier.modify() - def _build_code(self, node: Node, code_path: PurePath, kconfig_file: str) -> None: + def _build_code(self, node: Node, code_path: PurePath, kconfig_file: str, kernel_version: VersionInfo) -> None: self._log.info("building code...") uname = node.tools[Uname] @@ -287,7 +298,8 @@ def _build_code(self, node: Node, code_path: PurePath, kconfig_file: str) -> Non result.assert_exit_code() # the gcc version of Redhat 7.x is too old. Upgrade it. - if isinstance(node.os, Redhat) and node.os.information.version < "8.0.0": + if kernel_version > "3.10.0" and isinstance(node.os, Redhat) and node.os.information.version < "8.0.0": + node.os.install_packages(["centos-release-scl"]) node.os.install_packages(["devtoolset-8"]) node.tools[Mv].move("/bin/gcc", "/bin/gcc_back", overwrite=True, sudo=True) result.assert_exit_code()