DPDK: Add 32bit test

lisa/tools/meson.py

@@ -2,7 +2,7 @@
 # Licensed under the MIT license.
 
 from pathlib import PurePath
-from typing import cast
+from typing import Dict, Optional, cast
 
 from semver import VersionInfo
 
@@ -50,15 +50,22 @@ def _install(self) -> bool:
             )
         return self._check_exists()
 
-    def setup(self, args: str, cwd: PurePath, build_dir: str = "build") -> PurePath:
+    def setup(
+        self,
+        args: str,
+        cwd: PurePath,
+        build_dir: str = "build",
+        update_envs: Optional[Dict[str, str]] = None,
+    ) -> PurePath:
         self.run(
-            f"{args} {build_dir}",
+            parameters=f"{args} {build_dir}",
             force_run=True,
             shell=True,
             cwd=cwd,
             expected_exit_code=0,
             expected_exit_code_failure_message=(
                 f"Could not configure {str(cwd)} with meson using args {args}"
             ),
+            update_envs=update_envs,
         )
         return cwd.joinpath(build_dir)

lisa/tools/pkgconfig.py

@@ -1,6 +1,8 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT license.
 
+from typing import Optional
+
 from assertpy import assert_that
 from semver import VersionInfo
 
@@ -23,27 +25,52 @@ def install(self) -> bool:
         self.node.os.install_packages("pkg-config")
         return True
 
-    def package_info_exists(self, package_name: str) -> bool:
-        package_info_result = self.run(f"--modversion {package_name}", force_run=True)
+    def package_info_exists(
+        self, package_name: str, pkg_config_path: Optional[str] = None
+    ) -> bool:
+        if pkg_config_path:
+            update_env = {"PKG_CONFIG_PATH": f"{pkg_config_path}"}
+        else:
+            update_env = None
+        package_info_result = self.run(
+            f"--modversion {package_name}",
+            force_run=True,
+            shell=True,
+            update_envs=update_env,
+        )
         return package_info_result.exit_code == 0
 
     def get_package_info(
         self,
         package_name: str,
         update_cached: bool = False,
+        pkg_config_path: Optional[str] = None,
     ) -> str:
-        info_exists = self.package_info_exists(package_name=package_name)
+        info_exists = self.package_info_exists(
+            package_name=package_name, pkg_config_path=pkg_config_path
+        )
+        if pkg_config_path:
+            update_env = {"PKG_CONFIG_PATH": f"{pkg_config_path}"}
+        else:
+            update_env = None
         assert_that(info_exists).described_as(
             (
                 f"pkg-config information was not available for {package_name}. "
                 "This indicates an installation or package detection bug. "
                 f"ensure .pc file is available for {package_name} on this OS."
             )
         ).is_true()
-        return self.run(f"--modversion {package_name}").stdout
+        return self.run(
+            f"--modversion {package_name}", shell=True, update_envs=update_env
+        ).stdout
 
     def get_package_version(
-        self, package_name: str, update_cached: bool = False
+        self,
+        package_name: str,
+        update_cached: bool = False,
+        pkg_config_path: Optional[str] = None,
     ) -> VersionInfo:
-        version_info = self.get_package_info(package_name, update_cached=update_cached)
+        version_info = self.get_package_info(
+            package_name, update_cached=update_cached, pkg_config_path=pkg_config_path
+        )
         return parse_version(version_info)

microsoft/testsuites/dpdk/common.py

@@ -11,15 +11,26 @@
 from lisa import Node
 from lisa.executable import Tool
 from lisa.operating_system import Debian, Fedora, Oracle, Posix, Redhat, Suse, Ubuntu
-from lisa.tools import Git, Tar, Wget
-from lisa.util import UnsupportedDistroException
+from lisa.tools import Git, Lscpu, Tar, Wget
+from lisa.tools.lscpu import CpuArchitecture
+from lisa.util import UnsupportedCpuArchitectureException, UnsupportedDistroException
 
 DPDK_STABLE_GIT_REPO = "https://dpdk.org/git/dpdk-stable"
-
+# 32bit test relies on newer versions of DPDK.
+# Release candidates are not in stable, so use the github mirror.
+DPDK_32BIT_DEFAULT_BRANCH = "v24.11-rc1"
+# TODO: update these when v24.11 is released to stable.
+DPDK_32BIT_DEFAULT_SOURCE = "https://github.com/DPDK/dpdk.git"
 # azure routing table magic subnet prefix
 # signals 'route all traffic on this subnet'
 AZ_ROUTE_ALL_TRAFFIC = "0.0.0.0/0"
 
+ARCH_COMPATIBILITY_MATRIX = {
+    CpuArchitecture.X64: [CpuArchitecture.X64],
+    CpuArchitecture.I386: [CpuArchitecture.I386, CpuArchitecture.X64],
+    CpuArchitecture.ARM64: [CpuArchitecture.ARM64],
+}
+
 
 # Attempt to clean up the DPDK package dependency mess
 # Make a Installer class that implements the common steps
@@ -34,7 +45,7 @@ class OsPackageDependencies:
     # the packages to install on that OS.
     def __init__(
         self,
-        matcher: Callable[[Posix], bool],
+        matcher: Callable[[Posix, Optional[CpuArchitecture]], bool],
         packages: Optional[Sequence[Union[str, Tool, Type[Tool]]]] = None,
         stop_on_match: bool = False,
     ) -> None:
@@ -45,14 +56,21 @@ def __init__(
 
 class DependencyInstaller:
     # provide a list of OsPackageDependencies for a project
-    def __init__(self, requirements: List[OsPackageDependencies]) -> None:
+    def __init__(
+        self,
+        requirements: List[OsPackageDependencies],
+        arch: Optional[CpuArchitecture] = None,
+    ) -> None:
         self.requirements = requirements
+        self._arch = arch
 
     # evaluate the list of package dependencies,
     def install_required_packages(
-        self, node: Node, extra_args: Union[List[str], None]
+        self,
+        os: Posix,
+        extra_args: Union[List[str], None],
+        arch: Optional[CpuArchitecture] = None,
     ) -> None:
-        os = node.os
         assert isinstance(os, Posix), (
             "DependencyInstaller is not compatible with this OS: "
             f"{os.information.vendor} {os.information.release}"
@@ -61,13 +79,16 @@ def install_required_packages(
         # stop on list end or if exclusive_match parameter is true.
         packages: List[Union[str, Tool, Type[Tool]]] = []
         for requirement in self.requirements:
-            if requirement.matcher(os) and requirement.packages:
-                packages += requirement.packages
+            if requirement.matcher(os, arch):
+                if requirement.packages is not None and len(requirement.packages) > 0:
+                    packages += requirement.packages
                 if requirement.stop_on_match:
                     break
+
         os.install_packages(packages=packages, extra_args=extra_args)
 
         # NOTE: It is up to the caller to raise an exception on an invalid OS
+        # see unsupported_os_thrower as a catch-all 'list end' function
 
 
 class Downloader:
@@ -200,23 +221,43 @@ def _uninstall(self) -> None:
     def _install_dependencies(self) -> None:
         if self._os_dependencies is not None:
             self._os_dependencies.install_required_packages(
-                self._node, extra_args=self._package_manager_extra_args
+                self._os, extra_args=self._package_manager_extra_args, arch=self._arch
             )
 
     # define how to check the installed version
     def get_installed_version(self) -> VersionInfo:
         raise NotImplementedError(f"get_installed_version {self._err_msg}")
 
-    def _should_install(self, required_version: Optional[VersionInfo] = None) -> bool:
-        return (not self._check_if_installed()) or (
-            required_version is not None
-            and required_version > self.get_installed_version()
+    def _should_install(
+        self,
+        required_version: Optional[VersionInfo] = None,
+        required_arch: Optional[CpuArchitecture] = None,
+    ) -> bool:
+        return (
+            (not self._check_if_installed())
+            # NOTE: Taking advantage of longer-than-expected lifetimes here.
+            # If the tool still exists we ~should~ be able to check the old version
+            # from a previous test environment.
+            # At the moment, we use create() to force re-initialization.
+            # If we ever fix things so that we use .get,
+            # we will need this check. So add it now.q
+            or (self._arch and required_arch != self._arch)
+            or (
+                required_version is not None
+                and required_version > self.get_installed_version()
+            )
         )
 
     # run the defined setup and installation steps.
-    def do_installation(self, required_version: Optional[VersionInfo] = None) -> None:
+    def do_installation(
+        self,
+        required_version: Optional[VersionInfo] = None,
+        required_arch: Optional[CpuArchitecture] = None,
+    ) -> None:
         self._setup_node()
-        if self._should_install():
+        if self._should_install(
+            required_version=required_version, required_arch=required_arch
+        ):
             self._uninstall()
             self._install_dependencies()
             self._install()
@@ -226,6 +267,7 @@ def __init__(
         node: Node,
         os_dependencies: Optional[DependencyInstaller] = None,
         downloader: Optional[Downloader] = None,
+        arch: Optional[CpuArchitecture] = None,
     ) -> None:
         self._node = node
         if not isinstance(self._node.os, Posix):
@@ -236,11 +278,22 @@ def __init__(
         self._package_manager_extra_args: List[str] = []
         self._os_dependencies = os_dependencies
         self._downloader = downloader
+        self._arch = arch
+        if self._arch:
+            # avoid building/running arm64 on i386, etc
+            system_arch = self._node.tools[Lscpu].get_architecture()
+            if system_arch not in ARCH_COMPATIBILITY_MATRIX[self._arch]:
+                raise UnsupportedCpuArchitectureException(system_arch)
 
 
 # Base class for package manager installation
 class PackageManagerInstall(Installer):
-    def __init__(self, node: Node, os_dependencies: DependencyInstaller) -> None:
+    def __init__(
+        self,
+        node: Node,
+        os_dependencies: DependencyInstaller,
+        arch: Optional[CpuArchitecture],
+    ) -> None:
         super().__init__(node, os_dependencies)
 
     # uninstall from the package manager
@@ -249,7 +302,10 @@ def _uninstall(self) -> None:
             return
         if self._os_dependencies is not None:
             for os_package_check in self._os_dependencies.requirements:
-                if os_package_check.matcher(self._os) and os_package_check.packages:
+                if (
+                    os_package_check.matcher(self._os, self._arch)
+                    and os_package_check.packages
+                ):
                     self._os.uninstall_packages(os_package_check.packages)
                     if os_package_check.stop_on_match:
                         break
@@ -262,7 +318,10 @@ def _check_if_installed(self) -> bool:
         # This will take too long if it's more than a few packages.
         if self._os_dependencies is not None:
             for os_package_check in self._os_dependencies.requirements:
-                if os_package_check.matcher(self._os) and os_package_check.packages:
+                if (
+                    os_package_check.matcher(self._os, self._arch)
+                    and os_package_check.packages
+                ):
                     for pkg in os_package_check.packages:
                         if not self._os.package_exists(pkg):
                             return False
@@ -271,7 +330,19 @@ def _check_if_installed(self) -> bool:
         return True
 
 
-def force_dpdk_default_source(variables: Dict[str, Any]) -> None:
+def force_dpdk_default_source(
+    variables: Dict[str, Any], build_arch: Optional[CpuArchitecture] = None
+) -> None:
+    if build_arch:
+        variables["build_arch"] = build_arch
+
+    if build_arch == CpuArchitecture.I386:
+        if not variables.get("dpdk_branch", None):
+            # assign a default branch with needed MANA commits for 32bit test
+            variables["dpdk_branch"] = DPDK_32BIT_DEFAULT_BRANCH
+        if not variables.get("dpdk_source", None):
+            variables["dpdk_source"] = DPDK_32BIT_DEFAULT_SOURCE
+
     if not variables.get("dpdk_source", None):
         variables["dpdk_source"] = DPDK_STABLE_GIT_REPO
 
@@ -368,13 +439,27 @@ def is_url_for_git_repo(url: str) -> bool:
     return scheme == "git" or check_for_git_https
 
 
-def unsupported_os_thrower(os: Posix) -> bool:
+# utility matcher to throw an OS error type for a match.
+def unsupported_os_thrower(os: Posix, arch: Optional[CpuArchitecture]) -> bool:
+    if arch:
+        message_suffix = f"OS and Architecture ({arch})"
+    else:
+        message_suffix = "OS"
     raise UnsupportedDistroException(
         os,
-        message=("Installer did not define dependencies for this os."),
+        message=f"Installer did not define dependencies for this {message_suffix}",
     )
 
 
+# utility matcher to throw an OS error when i386 is not supported
+def i386_not_implemented_thrower(os: Posix, arch: Optional[CpuArchitecture]) -> bool:
+    if arch and arch == CpuArchitecture.I386:
+        raise NotImplementedError(
+            "i386 is not implemented for this (installer,OS) combo."
+        )
+    return False
+
+
 def get_debian_backport_repo_args(os: Debian) -> List[str]:
     # ex: 'bionic-backports' or 'buster-backports'
     # these backport repos are available for the older OS's