diff --git a/lisa/tools/meson.py b/lisa/tools/meson.py index 9ca4d37ee6..05691eb069 100644 --- a/lisa/tools/meson.py +++ b/lisa/tools/meson.py @@ -2,7 +2,7 @@ # Licensed under the MIT license. from pathlib import PurePath -from typing import cast +from typing import Dict, Optional, cast from semver import VersionInfo @@ -50,9 +50,15 @@ def _install(self) -> bool: ) return self._check_exists() - def setup(self, args: str, cwd: PurePath, build_dir: str = "build") -> PurePath: + def setup( + self, + args: str, + cwd: PurePath, + build_dir: str = "build", + update_envs: Optional[Dict[str, str]] = None, + ) -> PurePath: self.run( - f"{args} {build_dir}", + parameters=f"{args} {build_dir}", force_run=True, shell=True, cwd=cwd, @@ -60,5 +66,6 @@ def setup(self, args: str, cwd: PurePath, build_dir: str = "build") -> PurePath: expected_exit_code_failure_message=( f"Could not configure {str(cwd)} with meson using args {args}" ), + update_envs=update_envs, ) return cwd.joinpath(build_dir) diff --git a/lisa/tools/pkgconfig.py b/lisa/tools/pkgconfig.py index bc2bdd75ea..23b97736bd 100644 --- a/lisa/tools/pkgconfig.py +++ b/lisa/tools/pkgconfig.py @@ -1,6 +1,8 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +from typing import Optional + from assertpy import assert_that from semver import VersionInfo @@ -23,16 +25,34 @@ def install(self) -> bool: self.node.os.install_packages("pkg-config") return True - def package_info_exists(self, package_name: str) -> bool: - package_info_result = self.run(f"--modversion {package_name}", force_run=True) + def package_info_exists( + self, package_name: str, pkg_config_path: Optional[str] = None + ) -> bool: + if pkg_config_path: + update_env = {"PKG_CONFIG_PATH": f"{pkg_config_path}"} + else: + update_env = None + package_info_result = self.run( + f"--modversion {package_name}", + force_run=True, + shell=True, + update_envs=update_env, + ) return package_info_result.exit_code == 0 def get_package_info( self, package_name: str, update_cached: bool = False, + pkg_config_path: Optional[str] = None, ) -> str: - info_exists = self.package_info_exists(package_name=package_name) + info_exists = self.package_info_exists( + package_name=package_name, pkg_config_path=pkg_config_path + ) + if pkg_config_path: + update_env = {"PKG_CONFIG_PATH": f"{pkg_config_path}"} + else: + update_env = None assert_that(info_exists).described_as( ( f"pkg-config information was not available for {package_name}. " @@ -40,10 +60,17 @@ def get_package_info( f"ensure .pc file is available for {package_name} on this OS." ) ).is_true() - return self.run(f"--modversion {package_name}").stdout + return self.run( + f"--modversion {package_name}", shell=True, update_envs=update_env + ).stdout def get_package_version( - self, package_name: str, update_cached: bool = False + self, + package_name: str, + update_cached: bool = False, + pkg_config_path: Optional[str] = None, ) -> VersionInfo: - version_info = self.get_package_info(package_name, update_cached=update_cached) + version_info = self.get_package_info( + package_name, update_cached=update_cached, pkg_config_path=pkg_config_path + ) return parse_version(version_info) diff --git a/microsoft/testsuites/dpdk/common.py b/microsoft/testsuites/dpdk/common.py index b7c54bded1..94a6846ee9 100644 --- a/microsoft/testsuites/dpdk/common.py +++ b/microsoft/testsuites/dpdk/common.py @@ -11,15 +11,26 @@ from lisa import Node from lisa.executable import Tool from lisa.operating_system import Debian, Fedora, Oracle, Posix, Redhat, Suse, Ubuntu -from lisa.tools import Git, Tar, Wget -from lisa.util import UnsupportedDistroException +from lisa.tools import Git, Lscpu, Tar, Wget +from lisa.tools.lscpu import CpuArchitecture +from lisa.util import UnsupportedCpuArchitectureException, UnsupportedDistroException DPDK_STABLE_GIT_REPO = "https://dpdk.org/git/dpdk-stable" - +# 32bit test relies on newer versions of DPDK. +# Release candidates are not in stable, so use the github mirror. +DPDK_32BIT_DEFAULT_BRANCH = "v24.11-rc1" +# TODO: update these when v24.11 is released to stable. +DPDK_32BIT_DEFAULT_SOURCE = "https://github.com/DPDK/dpdk.git" # azure routing table magic subnet prefix # signals 'route all traffic on this subnet' AZ_ROUTE_ALL_TRAFFIC = "0.0.0.0/0" +ARCH_COMPATIBILITY_MATRIX = { + CpuArchitecture.X64: [CpuArchitecture.X64], + CpuArchitecture.I386: [CpuArchitecture.I386, CpuArchitecture.X64], + CpuArchitecture.ARM64: [CpuArchitecture.ARM64], +} + # Attempt to clean up the DPDK package dependency mess # Make a Installer class that implements the common steps @@ -34,7 +45,7 @@ class OsPackageDependencies: # the packages to install on that OS. def __init__( self, - matcher: Callable[[Posix], bool], + matcher: Callable[[Posix, Optional[CpuArchitecture]], bool], packages: Optional[Sequence[Union[str, Tool, Type[Tool]]]] = None, stop_on_match: bool = False, ) -> None: @@ -45,14 +56,21 @@ def __init__( class DependencyInstaller: # provide a list of OsPackageDependencies for a project - def __init__(self, requirements: List[OsPackageDependencies]) -> None: + def __init__( + self, + requirements: List[OsPackageDependencies], + arch: Optional[CpuArchitecture] = None, + ) -> None: self.requirements = requirements + self._arch = arch # evaluate the list of package dependencies, def install_required_packages( - self, node: Node, extra_args: Union[List[str], None] + self, + os: Posix, + extra_args: Union[List[str], None], + arch: Optional[CpuArchitecture] = None, ) -> None: - os = node.os assert isinstance(os, Posix), ( "DependencyInstaller is not compatible with this OS: " f"{os.information.vendor} {os.information.release}" @@ -61,13 +79,16 @@ def install_required_packages( # stop on list end or if exclusive_match parameter is true. packages: List[Union[str, Tool, Type[Tool]]] = [] for requirement in self.requirements: - if requirement.matcher(os) and requirement.packages: - packages += requirement.packages + if requirement.matcher(os, arch): + if requirement.packages is not None and len(requirement.packages) > 0: + packages += requirement.packages if requirement.stop_on_match: break + os.install_packages(packages=packages, extra_args=extra_args) # NOTE: It is up to the caller to raise an exception on an invalid OS + # see unsupported_os_thrower as a catch-all 'list end' function class Downloader: @@ -200,23 +221,43 @@ def _uninstall(self) -> None: def _install_dependencies(self) -> None: if self._os_dependencies is not None: self._os_dependencies.install_required_packages( - self._node, extra_args=self._package_manager_extra_args + self._os, extra_args=self._package_manager_extra_args, arch=self._arch ) # define how to check the installed version def get_installed_version(self) -> VersionInfo: raise NotImplementedError(f"get_installed_version {self._err_msg}") - def _should_install(self, required_version: Optional[VersionInfo] = None) -> bool: - return (not self._check_if_installed()) or ( - required_version is not None - and required_version > self.get_installed_version() + def _should_install( + self, + required_version: Optional[VersionInfo] = None, + required_arch: Optional[CpuArchitecture] = None, + ) -> bool: + return ( + (not self._check_if_installed()) + # NOTE: Taking advantage of longer-than-expected lifetimes here. + # If the tool still exists we ~should~ be able to check the old version + # from a previous test environment. + # At the moment, we use create() to force re-initialization. + # If we ever fix things so that we use .get, + # we will need this check. So add it now.q + or (self._arch and required_arch != self._arch) + or ( + required_version is not None + and required_version > self.get_installed_version() + ) ) # run the defined setup and installation steps. - def do_installation(self, required_version: Optional[VersionInfo] = None) -> None: + def do_installation( + self, + required_version: Optional[VersionInfo] = None, + required_arch: Optional[CpuArchitecture] = None, + ) -> None: self._setup_node() - if self._should_install(): + if self._should_install( + required_version=required_version, required_arch=required_arch + ): self._uninstall() self._install_dependencies() self._install() @@ -226,6 +267,7 @@ def __init__( node: Node, os_dependencies: Optional[DependencyInstaller] = None, downloader: Optional[Downloader] = None, + arch: Optional[CpuArchitecture] = None, ) -> None: self._node = node if not isinstance(self._node.os, Posix): @@ -236,11 +278,22 @@ def __init__( self._package_manager_extra_args: List[str] = [] self._os_dependencies = os_dependencies self._downloader = downloader + self._arch = arch + if self._arch: + # avoid building/running arm64 on i386, etc + system_arch = self._node.tools[Lscpu].get_architecture() + if system_arch not in ARCH_COMPATIBILITY_MATRIX[self._arch]: + raise UnsupportedCpuArchitectureException(system_arch) # Base class for package manager installation class PackageManagerInstall(Installer): - def __init__(self, node: Node, os_dependencies: DependencyInstaller) -> None: + def __init__( + self, + node: Node, + os_dependencies: DependencyInstaller, + arch: Optional[CpuArchitecture], + ) -> None: super().__init__(node, os_dependencies) # uninstall from the package manager @@ -249,7 +302,10 @@ def _uninstall(self) -> None: return if self._os_dependencies is not None: for os_package_check in self._os_dependencies.requirements: - if os_package_check.matcher(self._os) and os_package_check.packages: + if ( + os_package_check.matcher(self._os, self._arch) + and os_package_check.packages + ): self._os.uninstall_packages(os_package_check.packages) if os_package_check.stop_on_match: break @@ -262,7 +318,10 @@ def _check_if_installed(self) -> bool: # This will take too long if it's more than a few packages. if self._os_dependencies is not None: for os_package_check in self._os_dependencies.requirements: - if os_package_check.matcher(self._os) and os_package_check.packages: + if ( + os_package_check.matcher(self._os, self._arch) + and os_package_check.packages + ): for pkg in os_package_check.packages: if not self._os.package_exists(pkg): return False @@ -271,7 +330,19 @@ def _check_if_installed(self) -> bool: return True -def force_dpdk_default_source(variables: Dict[str, Any]) -> None: +def force_dpdk_default_source( + variables: Dict[str, Any], build_arch: Optional[CpuArchitecture] = None +) -> None: + if build_arch: + variables["build_arch"] = build_arch + + if build_arch == CpuArchitecture.I386: + if not variables.get("dpdk_branch", None): + # assign a default branch with needed MANA commits for 32bit test + variables["dpdk_branch"] = DPDK_32BIT_DEFAULT_BRANCH + if not variables.get("dpdk_source", None): + variables["dpdk_source"] = DPDK_32BIT_DEFAULT_SOURCE + if not variables.get("dpdk_source", None): variables["dpdk_source"] = DPDK_STABLE_GIT_REPO @@ -368,13 +439,27 @@ def is_url_for_git_repo(url: str) -> bool: return scheme == "git" or check_for_git_https -def unsupported_os_thrower(os: Posix) -> bool: +# utility matcher to throw an OS error type for a match. +def unsupported_os_thrower(os: Posix, arch: Optional[CpuArchitecture]) -> bool: + if arch: + message_suffix = f"OS and Architecture ({arch})" + else: + message_suffix = "OS" raise UnsupportedDistroException( os, - message=("Installer did not define dependencies for this os."), + message=f"Installer did not define dependencies for this {message_suffix}", ) +# utility matcher to throw an OS error when i386 is not supported +def i386_not_implemented_thrower(os: Posix, arch: Optional[CpuArchitecture]) -> bool: + if arch and arch == CpuArchitecture.I386: + raise NotImplementedError( + "i386 is not implemented for this (installer,OS) combo." + ) + return False + + def get_debian_backport_repo_args(os: Debian) -> List[str]: # ex: 'bionic-backports' or 'buster-backports' # these backport repos are available for the older OS's diff --git a/microsoft/testsuites/dpdk/dpdksuite.py b/microsoft/testsuites/dpdk/dpdksuite.py index 2b66047a14..5781a8b1cf 100644 --- a/microsoft/testsuites/dpdk/dpdksuite.py +++ b/microsoft/testsuites/dpdk/dpdksuite.py @@ -25,6 +25,7 @@ from lisa.testsuite import TestResult, simple_requirement from lisa.tools import Echo, Git, Hugepages, Ip, Kill, Lsmod, Make, Modprobe from lisa.tools.hugepages import HugePageSize +from lisa.tools.lscpu import CpuArchitecture from lisa.util.constants import SIGINT from microsoft.testsuites.dpdk.common import ( DPDK_STABLE_GIT_REPO, @@ -43,6 +44,7 @@ init_nodes_concurrent, initialize_node_resources, run_testpmd_concurrent, + skip_32bit_test_on_unsupported_distros, verify_dpdk_build, verify_dpdk_l3fwd_ntttcp_tcp, verify_dpdk_send_receive, @@ -101,6 +103,70 @@ def verify_dpdk_build_netvsc( node, log, variables, "netvsc", HugePageSize.HUGE_2MB, result=result ) + @TestCaseMetadata( + description=""" + netvsc pmd version. + This test case checks DPDK can be built and installed correctly. + Prerequisites, accelerated networking must be enabled. + The VM should have at least two network interfaces, + with one interface for management. + More details refer https://docs.microsoft.com/en-us/azure/virtual-network/setup-dpdk#prerequisites # noqa: E501 + """, + priority=2, + requirement=simple_requirement( + min_core_count=8, + min_nic_count=2, + network_interface=Sriov(), + unsupported_features=[Gpu, Infiniband], + ), + ) + def verify_dpdk_build_netvsc_32bit( + self, node: Node, log: Logger, variables: Dict[str, Any], result: TestResult + ) -> None: + skip_32bit_test_on_unsupported_distros(node.os) + force_dpdk_default_source(variables, build_arch=CpuArchitecture.I386) + verify_dpdk_build( + node, log, variables, "netvsc", HugePageSize.HUGE_2MB, result=result + ) + + @TestCaseMetadata( + description=""" + netvsc pmd version. + This test case checks DPDK can be built and installed correctly. + Prerequisites, accelerated networking must be enabled. + The VM should have at least two network interfaces, + with one interface for management. + More details refer https://docs.microsoft.com/en-us/azure/virtual-network/setup-dpdk#prerequisites # noqa: E501 + """, + priority=2, + requirement=simple_requirement( + min_count=2, + min_core_count=8, + min_nic_count=2, + network_interface=Sriov(), + unsupported_features=[Gpu, Infiniband], + ), + ) + def verify_dpdk_send_receive_netvsc_32bit( + self, + environment: Environment, + log: Logger, + variables: Dict[str, Any], + result: TestResult, + ) -> None: + node = environment.default_node + skip_32bit_test_on_unsupported_distros(node.os) + force_dpdk_default_source(variables, build_arch=CpuArchitecture.I386) + verify_dpdk_send_receive( + environment, + log, + variables, + "netvsc", + HugePageSize.HUGE_2MB, + multiple_queues=True, + result=result, + ) + @TestCaseMetadata( description=""" netvsc pmd version with 1GiB hugepages diff --git a/microsoft/testsuites/dpdk/dpdktestpmd.py b/microsoft/testsuites/dpdk/dpdktestpmd.py index 14aa38e936..1554a81b56 100644 --- a/microsoft/testsuites/dpdk/dpdktestpmd.py +++ b/microsoft/testsuites/dpdk/dpdktestpmd.py @@ -3,7 +3,7 @@ import re from pathlib import PurePath, PurePosixPath -from typing import Any, List, Tuple, Type +from typing import Any, Dict, List, Optional, Tuple, Type from assertpy import assert_that, fail from semver import VersionInfo @@ -28,6 +28,7 @@ Timeout, Wget, ) +from lisa.tools.lscpu import CpuArchitecture from lisa.util import ( LisaException, SkippedException, @@ -44,6 +45,7 @@ PackageManagerInstall, TarDownloader, get_debian_backport_repo_args, + i386_not_implemented_thrower, is_url_for_git_repo, is_url_for_tarball, unsupported_os_thrower, @@ -53,38 +55,42 @@ # declare package dependencies for package manager DPDK installation -DPDK_PACKAGE_MANAGER_PACKAGES = DependencyInstaller( +DPDK_PACKAGE_MANAGER_DEPENDENCIES = DependencyInstaller( requirements=[ + # raise exception early if package manager install is used for i386 test. + OsPackageDependencies(matcher=i386_not_implemented_thrower), # install linux-modules-extra-azure if it's available for mana_ib # older debian kernels won't have mana_ib packaged, # so skip the check on those kernels. OsPackageDependencies( - matcher=lambda x: isinstance(x, Debian) - and bool(x.get_kernel_information().version >= "5.15.0") - and x.is_package_in_repo("linux-modules-extra-azure"), + matcher=lambda os, arch=None: isinstance(os, Debian) # type: ignore + and bool(os.get_kernel_information().version >= "5.15.0") + and os.is_package_in_repo("linux-modules-extra-azure"), packages=["linux-modules-extra-azure"], ), OsPackageDependencies( - matcher=lambda x: isinstance(x, Debian), + matcher=lambda os, arch=None: isinstance(os, Debian), # type: ignore packages=["dpdk", "dpdk-dev"], stop_on_match=True, ), OsPackageDependencies( - matcher=lambda x: isinstance(x, Suse) - and bool(parse_version(x.information.release) == "15.5.0"), + matcher=lambda os, arch=None: isinstance(os, Suse) # type: ignore + and bool(parse_version(os.information.release) == "15.5.0"), packages=["dpdk22", "dpdk22-devel"], stop_on_match=True, ), OsPackageDependencies( # alma/rocky have started # including testpmd by default in 'dpdk' - matcher=lambda x: isinstance(x, Fedora) - and not x.is_package_in_repo("dpdk-devel"), + matcher=lambda os, arch=None: isinstance(os, Fedora) # type: ignore + and not os.is_package_in_repo("dpdk-devel"), packages=["dpdk"], stop_on_match=True, ), OsPackageDependencies( - matcher=lambda x: isinstance(x, (Fedora, Suse)), + matcher=lambda os, arch=None: isinstance( # type: ignore + os, (Fedora, Suse) + ), packages=["dpdk", "dpdk-devel"], stop_on_match=True, ), @@ -92,11 +98,11 @@ ] ) # declare package/tool dependencies for DPDK source installation -DPDK_SOURCE_INSTALL_PACKAGES = DependencyInstaller( +DPDK_SOURCE_DEPENDENCIES = DependencyInstaller( requirements=[ OsPackageDependencies( - matcher=lambda x: isinstance(x, Ubuntu) - and x.information.codename == "bionic", + matcher=lambda os, arch=None: isinstance(os, Ubuntu) # type: ignore + and os.information.codename == "bionic", packages=[ "build-essential", "libmnl-dev", @@ -116,13 +122,29 @@ # older debian kernels won't have mana_ib packaged, # so skip the check on those kernels. OsPackageDependencies( - matcher=lambda x: isinstance(x, Debian) - and bool(x.get_kernel_information().version >= "5.15.0") - and x.is_package_in_repo("linux-modules-extra-azure"), + matcher=lambda os, arch=None: isinstance(os, Debian) # type: ignore + and bool(os.get_kernel_information().version >= "5.15.0") + and os.is_package_in_repo("linux-modules-extra-azure"), packages=["linux-modules-extra-azure"], ), + # Install 32-bit dependencies if we're building for i386 OsPackageDependencies( - matcher=lambda x: isinstance(x, Debian), + matcher=lambda os, arch=None: isinstance(os, Debian) # type: ignore + and arch == CpuArchitecture.I386, + packages=[ + "python3-pyelftools", + "libelf-dev:i386", + "libnuma-dev:i386", + "pkg-config", + "python3-pip", + "cmake", + "libnl-3-dev:i386", + "meson", + "gcc-i686-linux-gnu", + ], + ), + OsPackageDependencies( + matcher=lambda os, arch=None: isinstance(os, Debian), # type: ignore packages=[ "build-essential", "libnuma-dev", @@ -134,7 +156,7 @@ stop_on_match=True, ), OsPackageDependencies( - matcher=lambda x: isinstance(x, Suse), + matcher=lambda os, arch=None: isinstance(os, Suse), # type: ignore packages=[ "psmisc", "libnuma-devel", @@ -145,7 +167,7 @@ stop_on_match=True, ), OsPackageDependencies( - matcher=lambda x: isinstance(x, (Fedora)), + matcher=lambda os, arch=None: isinstance(os, (Fedora)), # type: ignore packages=[ "psmisc", "numactl-devel", @@ -194,12 +216,58 @@ def _check_if_installed(self) -> bool: # implement SourceInstall for DPDK class DpdkSourceInstall(Installer): + def _get_pkgconfig_path(self) -> str: + if self._arch == CpuArchitecture.I386: + return "${PKG_CONFIG_PATH}:/usr/local/lib/i386-linux-gnu/pkgconfig" + else: + return "${PKG_CONFIG_PATH}:/usr/local/lib/pkgconfig" + + def _get_bashrc_defines(self) -> List[str]: + if self._arch == CpuArchitecture.I386: + arch_folder = "i386-linux-gnu" + else: + arch_folder = "lib64" + return [ + f"export PKG_CONFIG_PATH={self._get_pkgconfig_path()}", + "export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib/" + + f"{arch_folder}", + ] + _sample_applications = [ "l3fwd", "multi_process/client_server_mp/mp_server", "multi_process/client_server_mp/mp_client", ] + def _get_meson_defines(self) -> Optional[Dict[str, str]]: + if self._arch == CpuArchitecture.I386: + return { + "CC": "/usr/bin/i686-linux-gnu-gcc", + "LDFLAGS": "-m32", + "PKG_CONFIG_LIBDIR": "/usr/local/lib/i386-linux-gnu/pkgconfig", + } + else: + return {"PATH": "$PATH:/usr/local/bin:/home/$USER/.local/bin"} + + def _get_c_arguments(self) -> str: + if self._arch == CpuArchitecture.I386: + return "-Dc_link_args=-m32" + else: + return "" + + _build_dir: str = "build" + _enable_drivers: List[str] = [ + "*/mlx*", + "net/*netvsc", + "net/ring", + "net/virtio", + "net/bonding", + "bus/*", + "common/*", + "mempool/*", + ] + _enable_apps: List[str] = ["app/test-pmd"] + def _check_if_installed(self) -> bool: try: package_manager_install = self._os.package_exists("dpdk") @@ -218,6 +286,15 @@ def _check_if_installed(self) -> bool: def _setup_node(self) -> None: super()._setup_node() if isinstance(self._os, Debian): + if self._arch == CpuArchitecture.I386: + self._node.execute( + "dpkg --add-architecture i386", + sudo=True, + expected_exit_code=0, + expected_exit_code_failure_message=( + "Could not enable i386 packages." + ), + ) self._package_manager_extra_args = get_debian_backport_repo_args(self._os) if isinstance(self._os, Ubuntu) and self._os.information.version < "22.4.0": self._os.update_packages("linux-azure") @@ -252,21 +329,41 @@ def _uninstall(self) -> None: def get_installed_version(self) -> VersionInfo: return self._node.tools[Pkgconfig].get_package_version( - "libdpdk", update_cached=True + "libdpdk", update_cached=True, pkg_config_path=self._get_pkgconfig_path() ) + def _get_meson_parameters(self) -> str: + # enable any apps we need (namely, testpmd) + enable_apps = "-Denable_apps=" + ",".join(self._enable_apps) + # add net/mana to the pmd list if we need it + if self._node.nics.is_mana_device_present(): + self._enable_drivers += ["net/mana"] + # build the driver enable arg + enable_drivers = "-Denable_drivers=" + ",".join(self._enable_drivers) + # add any needed -Dc_flags or -Dc_link_args arguments + c_args = self._get_c_arguments() + return " ".join([c_args, enable_apps, enable_drivers]) + def _install(self) -> None: super()._install() if self._sample_applications: - sample_apps = f"-Dexamples={','.join(self._sample_applications)}" - else: - sample_apps = "" + self._meson_arguments = f"-Dexamples={','.join(self._sample_applications)}" + node = self._node # save the pythonpath for later python_path = node.tools[Python].get_python_path() + + # handle arch special cases for env vars + update_envs = self._get_meson_defines() + # run [DEFINE='...'] meson setup ... $build_dir + # configures the installation self.dpdk_build_path = node.tools[Meson].setup( - args=sample_apps, build_dir="build", cwd=self.asset_path + args=self._get_meson_parameters(), + cwd=self.asset_path, + build_dir=self._build_dir, + update_envs=update_envs, ) + # run the build node.tools[Ninja].run( cwd=self.dpdk_build_path, shell=True, @@ -300,12 +397,9 @@ def _install(self) -> None: expected_exit_code=0, expected_exit_code_failure_message="ldconfig failed, check for error spew.", ) - library_bashrc_lines = [ - "export PKG_CONFIG_PATH=${PKG_CONFIG_PATH}:/usr/local/lib64/pkgconfig/", - "export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib64/", - ] + # add the define lines we need node.tools[Echo].write_to_file( - ";".join(library_bashrc_lines), + ";".join(self._get_bashrc_defines()), node.get_pure_path("$HOME/.bashrc"), append=True, ) @@ -702,13 +796,14 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: self._dpdk_source = kwargs.pop("dpdk_source", PACKAGE_MANAGER_SOURCE) self._dpdk_branch = kwargs.pop("dpdk_branch", "main") self._sample_apps_to_build = kwargs.pop("sample_apps", []) + self._build_arch: Optional[CpuArchitecture] = kwargs.pop("build_arch", None) self._dpdk_version_info = VersionInfo(0, 0) self._testpmd_install_path: str = "" self._expected_install_path = "" self._determine_network_hardware() if self.use_package_manager_install(): self.installer: Installer = DpdkPackageManagerInstall( - self.node, DPDK_PACKAGE_MANAGER_PACKAGES + self.node, DPDK_PACKAGE_MANAGER_DEPENDENCIES, arch=self._build_arch ) # if not package manager, choose source installation else: @@ -736,10 +831,12 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: " Expected https://___/___.git or /path/to/tar.tar[.gz] or " "https://__/__.tar[.gz]" ) + self.installer = DpdkSourceInstall( node=self.node, - os_dependencies=DPDK_SOURCE_INSTALL_PACKAGES, + os_dependencies=DPDK_SOURCE_DEPENDENCIES, downloader=downloader, + arch=self._build_arch, ) # if dpdk is already installed, find the binary and check the version if self.find_testpmd_binary(assert_on_fail=False): @@ -801,7 +898,7 @@ def _install(self) -> bool: ): raise SkippedException("MANA DPDK test is not supported on this OS") - self.installer.do_installation() + self.installer.do_installation(required_arch=self._build_arch) self._dpdk_version_info = self.installer.get_installed_version() self._load_drivers_for_dpdk() self.find_testpmd_binary(check_path=self._expected_install_path) diff --git a/microsoft/testsuites/dpdk/dpdkutil.py b/microsoft/testsuites/dpdk/dpdkutil.py index 71753a730b..ce06dbf168 100644 --- a/microsoft/testsuites/dpdk/dpdkutil.py +++ b/microsoft/testsuites/dpdk/dpdkutil.py @@ -45,6 +45,7 @@ Timeout, ) from lisa.tools.hugepages import HugePageSize +from lisa.tools.lscpu import CpuArchitecture from lisa.util.constants import DEVICE_TYPE_SRIOV, SIGINT from lisa.util.parallel import TaskManager, run_in_parallel, run_in_parallel_async from microsoft.testsuites.dpdk.common import ( @@ -61,8 +62,9 @@ ) from microsoft.testsuites.dpdk.dpdktestpmd import PACKAGE_MANAGER_SOURCE, DpdkTestpmd from microsoft.testsuites.dpdk.rdmacore import ( + RDMA_CORE_I386_MANA_DEFAULT_SOURCE, RDMA_CORE_MANA_DEFAULT_SOURCE, - RDMA_CORE_PACKAGE_MANAGER_DEPENDENCIES, + RDMA_CORE_PACKAGE_DEPENDENCIES, RDMA_CORE_SOURCE_DEPENDENCIES, RdmaCorePackageManagerInstall, RdmaCoreSourceInstaller, @@ -128,14 +130,20 @@ def _set_forced_source_by_distro(node: Node, variables: Dict[str, Any]) -> None: def get_rdma_core_installer( - node: Node, dpdk_source: str, dpdk_branch: str, rdma_source: str, rdma_branch: str + node: Node, + rdma_source: str, + rdma_branch: str, + build_arch: Optional[CpuArchitecture] = None, ) -> Installer: # set rdma-core installer type. + if not build_arch: + build_arch = node.tools[Lscpu].get_architecture() + if rdma_source: if is_url_for_git_repo(rdma_source): # else, if we have a user provided rdma-core source, use it downloader: Downloader = GitDownloader(node, rdma_source, rdma_branch) - elif is_url_for_tarball(rdma_branch): + elif is_url_for_tarball(rdma_source): downloader = TarDownloader(node, rdma_source) else: # throw on unrecognized rdma core source type @@ -144,15 +152,22 @@ def get_rdma_core_installer( ) # handle MANA special case, build a default rdma-core with mana provider elif not rdma_source and node.nics.is_mana_device_present(): - downloader = TarDownloader(node, RDMA_CORE_MANA_DEFAULT_SOURCE) + if build_arch == CpuArchitecture.I386: + downloader = TarDownloader(node, RDMA_CORE_I386_MANA_DEFAULT_SOURCE) + else: + downloader = TarDownloader(node, RDMA_CORE_MANA_DEFAULT_SOURCE) else: # no rdma_source and not mana, just use the package manager return RdmaCorePackageManagerInstall( - node, os_dependencies=RDMA_CORE_PACKAGE_MANAGER_DEPENDENCIES + node, os_dependencies=RDMA_CORE_PACKAGE_DEPENDENCIES, arch=build_arch ) + # return the installer with the downloader we've picked return RdmaCoreSourceInstaller( - node, os_dependencies=RDMA_CORE_SOURCE_DEPENDENCIES, downloader=downloader + node, + os_dependencies=RDMA_CORE_SOURCE_DEPENDENCIES, + downloader=downloader, + arch=build_arch, ) @@ -300,6 +315,7 @@ def initialize_node_resources( "Dpdk initialize_node_resources running" f"found dpdk_source '{dpdk_source}' and dpdk_branch '{dpdk_branch}'" ) + build_arch = variables.get("build_arch", None) network_interface_feature = node.features[NetworkInterface] sriov_is_enabled = network_interface_feature.is_enabled_sriov() if not sriov_is_enabled: @@ -325,7 +341,7 @@ def initialize_node_resources( node.nics.check_pci_enabled(pci_enabled=True) update_kernel_from_repo(node) rdma_core = get_rdma_core_installer( - node, dpdk_source, dpdk_branch, rdma_source, rdma_branch + node, rdma_source, rdma_branch, build_arch=build_arch ) rdma_core.do_installation() # create tool, initialize testpmd tool (installs dpdk) @@ -337,6 +353,7 @@ def initialize_node_resources( dpdk_branch=dpdk_branch, sample_apps=sample_apps, force_net_failsafe_pmd=force_net_failsafe_pmd, + build_arch=build_arch, ) # Tools will skip installation if the binary is present, so # force invoke install. Installer will skip if the correct @@ -1110,7 +1127,7 @@ class NicType(Enum): # Short name for nic types -NIC_SHORT_NAMES = { +NIC_SHORT_NAMES: Dict[NicType, str] = { NicType.CX3: "cx3", NicType.CX4: "cx4", NicType.CX5: "cx5", @@ -1133,13 +1150,12 @@ def get_node_nic_short_name(node: Node) -> str: short_names = map(lambda x: x.value, non_mana_nics) known_nic_types = ",".join(short_names) found_nic_types = ",".join(map(str, [x.device_id for x in devices])) - node.log.debug( + # assert, this will be caught in annotate_dpdk_test_result + # and logged, rather than failing the test. + raise AssertionError( "Unknown NIC hardware was detected during DPDK test case. " f"Expected one of: {known_nic_types}. Found {found_nic_types}. " ) - # this is just a function for annotating a result, so don't assert - # if there's - return found_nic_types def _format_version_str(version: VersionInfo) -> str: @@ -1161,16 +1177,29 @@ def annotate_dpdk_test_result( test_result.information["dpdk_version"] = _format_version_str(dpdk_version) log.debug(f"Adding dpdk version: {dpdk_version}") except AssertionError as err: - test_kit.node.log.debug(f"Could not fetch DPDK version info: {str(err)}") + log.debug(f"Could not fetch DPDK version info: {str(err)}") try: rdma_version = test_kit.rdma_core.get_installed_version() test_result.information["rdma_version"] = _format_version_str(rdma_version) log.debug(f"Adding rdma version: {rdma_version}") except AssertionError as err: - test_kit.node.log.debug(f"Could not fetch RDMA version info: {str(err)}") + log.debug(f"Could not fetch RDMA version info: {str(err)}") try: nic_hw = get_node_nic_short_name(test_kit.node) test_result.information["nic_hw"] = nic_hw log.debug(f"Adding nic version: {nic_hw}") except AssertionError as err: - test_kit.node.log.debug(f"Could not fetch NIC short name: {str(err)}") + log.debug(f"Could not fetch NIC short name: {str(err)}") + + +def skip_32bit_test_on_unsupported_distros(os: OperatingSystem) -> None: + if not ( + isinstance(os, Ubuntu) + and os.information.version >= "22.4.0" + and os.information.version < "22.5.0" + ): + raise SkippedException( + UnsupportedDistroException( + os, "32bit test is only implemented for Ubuntu 22.04" + ) + ) diff --git a/microsoft/testsuites/dpdk/rdmacore.py b/microsoft/testsuites/dpdk/rdmacore.py index 0a0d650e1b..5c99ab1308 100644 --- a/microsoft/testsuites/dpdk/rdmacore.py +++ b/microsoft/testsuites/dpdk/rdmacore.py @@ -1,8 +1,10 @@ from assertpy import assert_that from semver import VersionInfo +from lisa import UnsupportedCpuArchitectureException from lisa.operating_system import Debian, Fedora, Suse from lisa.tools import Make, Pkgconfig +from lisa.tools.lscpu import CpuArchitecture from microsoft.testsuites.dpdk.common import ( DependencyInstaller, Installer, @@ -16,19 +18,55 @@ "https://github.com/linux-rdma/rdma-core/" "releases/download/v50.1/rdma-core-50.1.tar.gz" ) +RDMA_CORE_I386_MANA_DEFAULT_SOURCE = ( + "https://github.com/linux-rdma/rdma-core/" + "releases/download/v53.1/rdma-core-53.1.tar.gz" +) RDMA_CORE_SOURCE_DEPENDENCIES = DependencyInstaller( [ OsPackageDependencies( - matcher=lambda x: isinstance(x, Debian) + matcher=lambda os, _arch=None: isinstance(os, Debian) # type: ignore # install linux-modules-extra-azure if it's available for mana_ib # older debian kernels won't have mana_ib packaged, # so skip the check on those kernels. - and bool(x.get_kernel_information().version >= "5.15.0") - and x.is_package_in_repo("linux-modules-extra-azure"), + and bool(os.get_kernel_information().version >= "5.15.0") + and os.is_package_in_repo("linux-modules-extra-azure"), packages=["linux-modules-extra-azure"], ), OsPackageDependencies( - matcher=lambda x: isinstance(x, Debian), + matcher=lambda os, arch=None: isinstance(os, (Debian)) # type: ignore + and arch == CpuArchitecture.I386, + packages=[ + "python3-pyelftools", + "libelf-dev:i386", + "libnuma-dev:i386", + "pkg-config", + "python3-pip", + "cmake", + "libnl-3-dev:i386", + "libnl-route-3-dev:i386", + "meson", + "gcc-i686-linux-gnu", + "python3-dev:i386", + "libudev-dev:i386", + "libudev-dev", + "libnl-3-dev", + "libnl-route-3-dev", + "libssl-dev", + "libelf-dev", + ], + ), + OsPackageDependencies( + matcher=lambda os, arch=None: isinstance(os, (Debian)) # type: ignore + and arch == CpuArchitecture.I386, + # Weirdly, I've run into errors trying to + packages=[ + "cython3:i386", + ], + stop_on_match=True, + ), + OsPackageDependencies( + matcher=lambda os, _arch=None: isinstance(os, Debian), # type: ignore packages=[ "cmake", "libudev-dev", @@ -49,7 +87,7 @@ stop_on_match=True, ), OsPackageDependencies( - matcher=lambda x: isinstance(x, Fedora), + matcher=lambda os, _arch=None: isinstance(os, Fedora), # type: ignore packages=[ "cmake", "libudev-devel", @@ -82,31 +120,34 @@ ] ) -RDMA_CORE_PACKAGE_MANAGER_DEPENDENCIES = DependencyInstaller( + +RDMA_CORE_PACKAGE_DEPENDENCIES = DependencyInstaller( [ OsPackageDependencies( - matcher=lambda x: isinstance(x, Debian) + matcher=lambda os, _=None: isinstance(os, Debian) # type: ignore # install linux-modules-extra-azure if it's available for mana_ib # older debian kernels won't have mana_ib packaged, # so skip the check on those kernels. - and bool(x.get_kernel_information().version >= "5.15.0") - and x.is_package_in_repo("linux-modules-extra-azure"), + and bool(os.get_kernel_information().version >= "5.15.0") + and os.is_package_in_repo("linux-modules-extra-azure"), packages=["linux-modules-extra-azure"], ), OsPackageDependencies( - matcher=lambda x: isinstance(x, Debian), + matcher=lambda os, _=None: isinstance(os, Debian), # type: ignore packages=["ibverbs-providers", "libibverbs-dev"], ), OsPackageDependencies( - matcher=lambda x: isinstance(x, Suse), + matcher=lambda os, _=None: isinstance(os, Suse), # type: ignore packages=["rdma-core-devel", "librdmacm1"], ), OsPackageDependencies( - matcher=lambda x: isinstance(x, Fedora), + matcher=lambda os, _=None: isinstance(os, Fedora), # type: ignore packages=["librdmacm-devel"], ), OsPackageDependencies( - matcher=lambda x: isinstance(x, (Fedora, Debian, Suse)), + matcher=lambda os, _=None: isinstance( # type: ignore + os, (Fedora, Debian, Suse) + ), packages=["rdma-core"], stop_on_match=True, ), @@ -115,7 +156,12 @@ ) -class RdmaCorePackageManagerInstall(PackageManagerInstall): +# Common parent for isinstance matching +class RdmaCoreInstaller(Installer): + ... + + +class RdmaCorePackageManagerInstall(RdmaCoreInstaller, PackageManagerInstall): def _setup_node(self) -> None: if isinstance(self._os, Fedora): self._os.install_epel() @@ -131,7 +177,7 @@ def _check_if_installed(self) -> bool: # implement SourceInstall for DPDK -class RdmaCoreSourceInstaller(Installer): +class RdmaCoreSourceInstaller(RdmaCoreInstaller): def _check_if_installed(self) -> bool: try: package_manager_install = self._os.package_exists("rdma-core") @@ -148,10 +194,42 @@ def _check_if_installed(self) -> bool: return False def _setup_node(self) -> None: + self._pkg_config_path = None if isinstance(self._os, (Debian, Fedora, Suse)): self._os.uninstall_packages("rdma-core") if isinstance(self._os, Fedora): self._os.group_install_packages("Development Tools") + if not self._arch or self._arch in [CpuArchitecture.ARM64, CpuArchitecture.X64]: + self._cmake_command = ( + "cmake -DIN_PLACE=0 -DNO_MAN_PAGES=1 -DCMAKE_INSTALL_PREFIX=/usr" + ) + # Only support this 32bit build on one distro family. + elif isinstance(self._os, Debian) and self._arch == CpuArchitecture.I386: + self._pkg_config_path = "/usr/local/lib/i386-linux-gnu/pkgconfig" + # enable 32bit packages, needed for dependencies + self._node.execute( + "dpkg --add-architecture i386", + sudo=True, + expected_exit_code=0, + expected_exit_code_failure_message="Could not enable i386 packages.", + ) + self._node.execute( + "apt update", + sudo=True, + expected_exit_code=0, + expected_exit_code_failure_message=( + "Apt update after enabling i386 failed" + ), + ) + self._cmake_command = ( + "cmake " + "-DIN_PLACE=0 -DNO_MAN_PAGES=1 -DCMAKE_INSTALL_PREFIX=/usr " + "-DCMAKE_C_COMPILER=/usr/bin/i686-linux-gnu-gcc -DCMAKE_C_FLAGS=-m32" + ) + else: + # Will hit this when adding a new CPU Architecture, + # so hello to whoever is adding RISC-V :) + raise UnsupportedCpuArchitectureException(arch=self._arch) super()._setup_node() def _uninstall(self) -> None: @@ -178,7 +256,7 @@ def _uninstall(self) -> None: def get_installed_version(self) -> VersionInfo: return self._node.tools[Pkgconfig].get_package_version( - "libibverbs", update_cached=True + "libibverbs", update_cached=True, pkg_config_path=self._pkg_config_path ) def _install(self) -> None: @@ -186,7 +264,7 @@ def _install(self) -> None: node = self._node make = node.tools[Make] node.execute( - "cmake -DIN_PLACE=0 -DNO_MAN_PAGES=1 -DCMAKE_INSTALL_PREFIX=/usr", + self._cmake_command, shell=True, cwd=self.asset_path, sudo=True,