tests.yaml

---
name: Tests
"on":
  push:
    branches:
      - main
  pull_request:
  schedule:
    # Run tests every Monday at 9:17 to catch regressions.
    - cron: "17 9 * * 1"

# XXX Concurrency detection sucks and jobs gets killed randomly.
# concurrency:
#   # Group workflow jobs so new commits cancels in-progress execution triggered by previous commits.
#   # Source: https://mail.python.org/archives/list/pypa-committers@python.org/thread/PCBCQMJF64JGRBOX7E2EE4YLKHT4DI55/
#   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
#   cancel-in-progress: true

jobs:

  test-matrix:
    # There is no way to selective flags collections of elements in a matrix, without having to flag all combinations.
    # This will became unmaintainable and tedious so we use this job to pre-compute which jobs is going to get our
    # "stable" flag.
    name: "OS/Python/stable matrix pre-compute"
    runs-on: ubuntu-24.04
    outputs:
      test_matrix: ${{ steps.create_matrix.outputs.matrix }}
    steps:
      - name: Create JSON matrix
        id: create_matrix
        shell: python
        run: |
          import json
          import os
          from itertools import product
          from pathlib import Path

          # To speed up the tests, we only test the latest available OS for each platform. For Python, we only test
          # against the oldest and newest major supported version and skip intermediates. We also test on the upcoming
          # development version to anticipate issues.
          variants: dict[str, set[str]] = {
              # Available OS: https://github.com/actions/runner-images#available-images
              "os": {
                  "ubuntu-24.04",
                  "macos-15",
                  "windows-2022",
              },
              # Available Python: https://github.com/actions/python-versions/blob/main/versions-manifest.json
              "python-version": {
                  "3.10",
                  "3.13",
                  "3.14-dev",
              }
          }
          # TODO: List of additional variants to include in the matrix.
          include: list[dict[str, str]] = []

          # List of variants to exclude from the matrix.
          exclude: list[dict[str, str]] = []

          # List of unstable criterions.
          unstable: list[dict[str, str]] = []

          # Build the job matrix.
          jobs: list[dict[str, str]] = []
          for variants in product(*[{(key, value) for value in values} for key, values in variants.items()]):
              job = dict(variants)

              # Match the job against the exclude criterions.
              exclude_job = False
              for criterion in exclude:
                  if set(criterion.items()).issubset(job.items()):
                      exclude_job = True
                      break
              if exclude_job:
                  continue

              # Match the job against the unstable criterions.
              job["state"] = "stable"
              for criterion in unstable:
                  if set(criterion.items()).issubset(job.items()):
                      job["state"] = "unstable"
                      break

              jobs.append(job)

          matrix = json.dumps({"include": jobs})

          env_file = Path(os.getenv("GITHUB_OUTPUT"))
          env_file.write_text(f"matrix={matrix}")

      - name: Print JSON matrix
        run: |
          echo '${{ steps.create_matrix.outputs.matrix }}'
          jq -aR <<< echo '${{ steps.create_matrix.outputs.matrix }}'

  tests:
    needs:
      - test-matrix
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.test-matrix.outputs.test_matrix) }}
    runs-on: ${{ matrix.os }}
    # We keep going when a job flagged as not stable fails.
    continue-on-error: ${{ matrix.state == 'unstable' }}
    env:
      # XXX Workaround for Windows runners redirecting the output of commands to files. See:
      # https://github.com/databrickslabs/dbx/issues/455#issuecomment-1312770919
      # https://github.com/pallets/click/issues/2121#issuecomment-1312773882
      # https://gist.github.com/NodeJSmith/e7e37f2d3f162456869f015f842bcf15
      PYTHONIOENCODING: "utf8"
      # Deactivate Homebrew verbose output and auto-update.
      HOMEBREW_NO_ENV_HINTS: "1"
      HOMEBREW_NO_AUTO_UPDATE: "1"
      # Do not let brew use its live API to fetch the latest version.
      # This variable forces brew to use the local repository instead (which we need in destructive tests to checkout
      # formulas from the past): https://github.com/Homebrew/brew/blob/10845a1/Library/Homebrew/env_config.rb#L314-L317
      # See: meta_package_manager/test_manager_homebrew.py::TestCask test.
      HOMEBREW_NO_INSTALL_FROM_API: "1"
    steps:
      - uses: actions/checkout@v4.2.2

      - name: Python ${{ matrix.python-version }} - Install
        uses: actions/setup-python@v5.2.0
        with:
          python-version: ${{ matrix.python-version }}

      - name: Linux - Fix runner cache folder permissions
        if: runner.os == 'Linux'
        run: |
          mkdir -p /home/runner/.cache
          sudo chown -R runner /home/runner/.cache

      ### Install or upgrade package managers on each platform, and made them
      ### ready to use for mpm.

      # Pip
      - name: Pip & Setuptools - Upgrade
        run: |
          python -m pip install --upgrade setuptools pip

      # Pipx
      - name: Pipx - Install
        run: |
          python -m pip install --upgrade pipx

      # Homebrew
      - name: Homebrew - macOS upgrade
        if: runner.os == 'macOS'
        run: |
          brew upgrade
          brew update
      - name: Homebrew - macOS cask repository copy
        if: runner.os == 'macOS'
        # Clone the full copy of the cask repository, so tests on macOS can locally checkout past versions of casks.
        # This is performed by the destructive meta_package_manager/test_manager_homebrew.py::TestCask test.
        run: |
          brew tap homebrew/cask
      - name: Homebrew - Linux install
        if: runner.os == 'Linux'
        run: |
          /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
          echo "/home/linuxbrew/.linuxbrew/bin/" >> "$GITHUB_PATH"
      - name: Homebrew - Doctor and tap repair
        if: runner.os != 'Windows'
        run: |
          brew doctor || true
          brew tap --repair

      # Apt
      - name: APT - Linux upgrade
        if: runner.os == 'Linux'
        run: |
          sudo apt --quiet --yes update

      # Chocolatey
      - name: Chocolatey - Windows self-upgrade
        if: runner.os == 'Windows'
        run: |
          choco upgrade chocolatey --no-progress --yes
      - name: Chocolatey - Pin un-upgradeable packages
        if: runner.os == 'Windows'
        # XXX The upgrade all step below fails for strawberryperl package on windows-2022 runners v2.313.0:
        #
        #   You have nginx v1.25.3 installed. Version 1.25.4 is available based on your source(s).
        #
        #   nginx v1.25.4 [Approved]
        #   nginx package files upgrade completed. Performing other installation steps.
        #   Port '80' is in use by 'System'...
        #   ERROR: Please specify a different port number...
        #   Environment Vars (like PATH) have changed. Close/reopen your shell to
        #    see the changes (or in powershell/cmd.exe just type `refreshenv`).
        #   The upgrade of nginx was NOT successful.
        #   Error while running 'C:\ProgramData\chocolatey\lib\nginx\tools\chocolateyInstall.ps1'.
        #    See log for details.
        #
        #   You have strawberryperl v5.32.1.1 installed. Version 5.38.2.2 is available based on your source(s).
        #
        #   strawberryperl v5.38.2.2 [Approved]
        #   strawberryperl package files upgrade completed. Performing other installation steps.
        #   Downloading strawberryperl 64 bit
        #     from 'https://github.com/StrawberryPerl/Perl-Dist-Strawberry/strawberry-perl-5.38.2.2-64bit.msi'
        #
        #   Download of strawberry-perl-5.38.2.2-64bit.msi (171.74 MB) completed.
        #   Hashes match.
        #   Installing strawberryperl...
        #   WARNING: Generic MSI Error. This is a local environment error, not an issue with a package or the MSI
        #   itself - it could mean a pending reboot is necessary prior to install or something else (like the same
        #   version is already installed). Please see MSI log if available. If not, try again adding
        #   '--install-arguments="'/l*v c:\StrawberryPerl_msi_install.log'"'. Then search the MSI Log for
        #   "Return Value 3" and look above that for the error.
        #   ERROR: Running ["C:\Windows\System32\msiexec.exe" /i "C:\Users\runneradmin\AppData\Local\Temp\chocolatey
        #   \StrawberryPerl\5.38.2.2\strawberry-perl-5.38.2.2-64bit.msi" /qn /norestart ] was not successful. Exit
        #   code was '1603'. Exit code indicates the following: Generic MSI Error. This is a local environment error,
        #   not an issue with a package or the MSI itself - it could mean a pending reboot is necessary prior to
        #   install or something else (like the same version is already installed). Please see MSI log if available.
        #   If not, try again adding '--install-arguments="'/l*v c:\StrawberryPerl_msi_install.log'"'. Then search
        #   the MSI Log for "Return Value 3" and look above that for the error..
        #   The upgrade of strawberryperl was NOT successful.
        run: |
          choco pin add --name="nginx"
          choco pin add --name="strawberryperl"
      - name: Chocolatey - Windows full upgrade
        if: runner.os == 'Windows'
        # We upgrade all packages preemptively as the test suite will attempt it but fail,
        # because of mpm's enforced timeout.
        run: |
          choco upgrade all --no-progress --yes --ignore-package-exit-codes --ignore-detected-reboot

      # NPM
      - name: NPM - Linux install
        if: runner.os == 'Linux'
        run: |
          sudo apt --quiet --yes install npm
      - name: NPM - macOS install
        if: runner.os == 'macOS'
        run: |
          brew install npm
      - name: NPM - Windows upgrade
        if: runner.os == 'Windows'
        run: |
          npm install --global npm

      # Cargo
      - name: Cargo - macOS install
        if: runner.os == 'macOS'
        run: |
          brew install rust
      - name: Cargo - Linux install
        if: runner.os == 'Linux'
        run: |
          sudo apt --quiet --yes install cargo
      - name: Cargo - Windows install
        if: runner.os == 'Windows'
        run: |
          choco install rust --no-progress --yes

      # Yarn
      - name: Yarn - macOS install
        if: runner.os == 'macOS'
        run: |
          brew install yarn

      # XXX Upgrading Yarn on Linux via mpm is not working. Like in Windows below.
      #
      #  ► sudo apt --quiet --yes install yarn
      #  The following NEW packages will be installed:
      #    yarn
      #  (...)
      #  Setting up yarn (1.22.19-1) ...
      #  (...)
      #  _____________ TestUpgrade.test_manager_selection[single_exclusion] _____________
      #  (...)
      #  ----------------------------- Captured stdout call -----------------------------
      #  (...)
      #  ► mpm --exclude pip upgrade --all
      #    Upgrade all outdated packages from apt...
      #    (...)
      #    Upgrade all outdated packages from yarn...
      #  Return code: 1
      #  Traceback (most recent call last):
      #    File ".../site-packages/click/testing.py", line 408, in invoke
      #      return_value = cli.main(args=args or (), prog_name=prog_name, **extra)
      #    File ".../site-packages/click_extra/commands.py", line 159, in main
      #      return super().main(*args, **kwargs)
      #    File ".../site-packages/click/core.py", line 1055, in main
      #      rv = self.invoke(ctx)
      #    File ".../site-packages/click_extra/commands.py", line 213, in invoke
      #      return super().invoke(ctx)
      #    File ".../site-packages/click/core.py", line 1657, in invoke
      #      return _process_result(sub_ctx.command.invoke(sub_ctx))
      #    File ".../site-packages/click/core.py", line 1404, in invoke
      #      return ctx.invoke(self.callback, **ctx.params)
      #    File ".../site-packages/click/core.py", line 760, in invoke
      #      return __callback(*args, **kwargs)
      #    File ".../site-packages/click/decorators.py", line 26, in new_func
      #      return f(get_current_context(), *args, **kwargs)
      #    File ".../meta_package_manager/cli.py", line 892, in upgrade
      #      output = manager.upgrade()
      #    File ".../meta_package_manager/base.py", line 974, in upgrade
      #      return self.run(cli, extra_env=self.extra_env)
      #    File ".../meta_package_manager/base.py", line 643, in run
      #      code, output, error = run_cmd(
      #    File ".../site-packages/click_extra/run.py", line 110, in run_cmd
      #      process = subprocess.run(
      #    File ".../python3.10/subprocess.py", line 503, in run
      #      with Popen(*popenargs, **kwargs) as process:
      #    File ".../python3.10/subprocess.py", line 971, in __init__
      #      self._execute_child(args, executable, preexec_fn, close_fds,
      #    File ".../python3.10/subprocess.py", line 1847, in _execute_child
      #      raise child_exception_type(errno_num, err_msg, err_filename)
      #  FileNotFoundError: [Errno 2] No such file or directory: '/usr/local/bin/yarn'
      #
      # - name: Yarn - Linux install
      #   if: runner.os == 'Linux'
      #   run: |
      #     curl -sSL https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add -
      #     echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list
      #     sudo apt --quiet --yes update
      #     sudo apt --quiet --yes install yarn

      # XXX Upgrading Yarn on Windows via mpm is not working, as it seems to requires PATH re-evaluation, which cannot
      # be achieve unless we re-open a new shell. A thing we cannot do in a GitHub Action. So we simply do not install
      # it on Windows for now.
      #
      #   ► choco install yarn --no-progress --yes
      #   Chocolatey v1.3.0
      #   Download of yarn-1.22.19.msi (1.6 MB) completed.
      #   yarn has been installed.
      #   (...)
      #   Environment Vars (like PATH) have changed. Close/reopen your shell to
      #    see the changes (or in powershell/cmd.exe just type `refreshenv`).
      #    The install of yarn was successful.
      #     Software installed as 'msi', install location is likely default.
      #   (...)
      #   ___________ TestUpgrade.test_manager_selection[multiple_exclusions] ___________
      #   (...)
      #   ---------------------------- Captured stdout call -----------------------------
      #   ► mpm --exclude pip --exclude gem upgrade --all
      #     mwarning: cargo does not implement Operations.upgrade_all.
      #     Upgrade all outdated packages from choco...
      #     (...)
      #     warning: Skip unavailable scoop manager.
      #     warning: steamcmd does not implement Operations.upgrade_all.
      #     warning: vscode does not implement Operations.upgrade_all.
      #     Upgrade all outdated packages from yarn...
      #   Return code: 1
      #   Traceback (most recent call last):
      #     File "...\site-packages\click\testing.py", line 408, in invoke
      #       return_value = cli.main(args=args or (), prog_name=prog_name, **extra)
      #     File "...\site-packages\click_extra\commands.py", line 159, in main
      #       return super().main(*args, **kwargs)
      #     File "...\site-packages\click\core.py", line 1055, in main
      #       rv = self.invoke(ctx)
      #     File "...\site-packages\click_extra\commands.py", line 213, in invoke
      #       return super().invoke(ctx)
      #     File "...\site-packages\click\core.py", line 1657, in invoke
      #       return _process_result(sub_ctx.command.invoke(sub_ctx))
      #     File "...\site-packages\click\core.py", line 1404, in invoke
      #       return ctx.invoke(self.callback, **ctx.params)
      #     File "...\site-packages\click\core.py", line 760, in invoke
      #       return __callback(*args, **kwargs)
      #     File "...\site-packages\click\decorators.py", line 26, in new_func
      #       return f(get_current_context(), *args, **kwargs)
      #     File "...\meta_package_manager\cli.py", line 892, in upgrade
      #       output = manager.upgrade()
      #     File "...\meta_package_manager\base.py", line 974, in upgrade
      #       return self.run(cli, extra_env=self.extra_env)
      #     File "...\meta_package_manager\base.py", line 643, in run
      #       code, output, error = run_cmd(
      #     File "...\site-packages\click_extra\run.py", line 110, in run_cmd
      #       process = subprocess.run(
      #     File "...\Python\3.11.2\x64\Lib\subprocess.py", line 548, in run
      #       with Popen(*popenargs, **kwargs) as process:
      #     File "...\Python\3.11.2\x64\Lib\subprocess.py", line 1024, in __init__
      #       self._execute_child(args, executable, preexec_fn, close_fds,
      #     File "...\Python\3.11.2\x64\Lib\subprocess.py", line 1493, in _execute_child
      #       hp, ht, pid, tid = _winapi.CreateProcess(executable, args,
      #   FileNotFoundError: [WinError 2] The system cannot find the file specified
      #
      # - name: Yarn - Windows install
      #   if: runner.os == 'Windows'
      #   run: |
      #     choco install yarn --no-progress --yes

      # Composer
      - name: Composer - macOS install
        if: runner.os == 'macOS'
        run: |
          brew install composer
      - name: Composer - Windows upgrade
        if: runner.os == 'Windows'
        run: |
          choco upgrade composer --no-progress --yes

      # MAS
      # XXX Deactivate MAS on macOS as it always timeout:
      #    subprocess.TimeoutExpired: Command '('/opt/homebrew/bin/mas', 'install', '747648890')'
      #    timed out after 500 seconds
      # Probably because the MAS CLI is not registered against an Apple account.
      # - name: MAS - macOS install
      #   if: runner.os == 'macOS'
      #   run: |
      #     brew install mas

      # Flatpak
      - name: Flatpak - Linux install
        if: runner.os == 'Linux'
        run: |
          sudo apt --quiet --yes install flatpak

      # Scoop
      - name: Scoop - Windows install
        if: runner.os == 'Windows'
        shell: pwsh
        run: |
          iwr get.scoop.sh -outfile 'install.ps1'
          .\install.ps1 -RunAsAdmin

      # WinGet
      # XXX Rely on a hacked action as WinGet is not installable on GitHub action runners as-is:
      # https://github.com/microsoft/winget-cli/issues/3872
      - name: WinGet - Windows install
        if: runner.os == 'Windows'
        uses: Cyberboss/install-winget@v1.0.6
        # run: |
        #   winget upgrade winget --accept-package-agreements --accept-source-agreements --disable-interactivity

      # DNF
      - name: DNF - Linux install
        if: runner.os == 'Linux'
        run: |
          sudo apt --quiet --yes install dnf

      # Zypper
      - name: Zypper - Linux install
        if: runner.os == 'Linux'
        run: |
          sudo apt --quiet --yes install zypper

      # Snap
      - name: Snap - Linux upgrade
        if: runner.os == 'Linux'
        run: |
          sudo apt --quiet --yes install snapd

      # VSCode
      - name: VSCode - macOS install
        if: runner.os == 'macOS'
        # XXX Tweak permission to fix this issue:
        # Hint: https://github.com/Homebrew/discussions/discussions/633#discussioncomment-1400084
        # > Run brew install --cask visual-studio-code
        # ==>Downloading https://update.code.visualstudio.com/1.68.1/darwin/stable
        # ==>Downloading from https://az764295.vo.msecnd.net/stable/35f373630/VSCode-darwin.zip
        # Warning: macOS's Gatekeeper has been disabled for this Cask
        # ==>Installing Cask visual-studio-code
        # ==>Purging files for version 1.68.1 of Cask visual-studio-code
        # Error: Directory not empty @ dir_s_rmdir - /private/tmp/d20220706-3993-qik29s
        # Error: Process completed with exit code 1.
        run: |
          sudo chown -R "$USER" /private/tmp/
          brew install --cask visual-studio-code
      - name: VSCode - Linux install
        if: runner.os == 'Linux'
        run: |
          sudo snap install --classic code
      - name: VSCode - Windows install
        if: runner.os == 'Windows'
        run: |
          choco install vscode --no-progress --yes

      # Gem
      - name: Ruby & Gem - Windows upgrade
        if: runner.os == 'Windows'
        # The 'refreshenv' call forces the new Ruby version installed by 'choco' to be registered in the 'PATH'.
        # The Import-Module workaround is inspired by:
        # https://github.com/actions/runner-images/discussions/6065
        # https://docs.chocolatey.org/en-us/troubleshooting#why-does-choco-tab-not-work-for-me
        run: |
          choco upgrade ruby --no-progress --yes
          Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
          refreshenv
          gem update --quiet --system
      - name: Ruby & Gem - macOS
        if: runner.os == 'macOS'
        run: |
          sudo gem update --quiet --system
      - name: Ruby & Gem - Ubuntu
        # As of Ubuntu 21.04, RubyGems is exclusively managed by APT. A call to "sudo gem update --system"
        # ends up with the following error:
        #   ERROR:  Your RubyGems was installed trough APT, and upgrading it through RubyGems
        #   itself is unsupported. If you really need the latest version of RubyGems (tip:
        #   you usually don't), then you need to install RubyGems (and Ruby) manually,
        #   maybe using tools like ruby-install, rvm, etc.
        if: runner.os == 'Linux'
        run: |
          sudo apt --quiet --yes install ruby-rubygems

      # SteamCMD
      - name: SteamCMD - macOS install
        if: runner.os == 'macOS'
        run: |
          brew install --cask steamcmd
      - name: SteamCMD - Linux install
        if: runner.os == 'Linux'
        # Some notes on the Steam situation on Linux:
        # https://popey.com/blog/2023/08/i386-in-ubuntu-wont-die/
        run: |
          sudo add-apt-repository --yes multiverse
          sudo dpkg --add-architecture i386
          sudo apt --quiet --yes update
          # Accept Steam license.
          # Source: https://askubuntu.com/a/1017487
          echo steam steam/question select "I AGREE" | sudo debconf-set-selections
          echo steam steam/license note '' | sudo debconf-set-selections
          sudo debconf-show steam
          sudo apt --quiet --yes install libgcc-s1:i386 libstdc++6:i386 libatomic1:i386
          sudo apt --quiet --yes install steam
          sudo apt --quiet --yes install steamcmd
      - name: SteamCMD - Windows install
        if: runner.os == 'Windows'
        run: |
          choco install steamcmd --no-progress --yes

      - name: Install uv
        run: |
          python -m pip install -r https://raw.githubusercontent.com/kdeldycke/workflows/v4.6.0/requirements/uv.txt
      - name: Install project
        run: |
          uv --no-progress venv
          uv --no-progress sync --extra test

      # CLI tests.
      - name: mpm --help
        run: |
          uv --no-progress run --frozen -- mpm
      - name: mpm --version
        run: |
          uv --no-progress run --frozen -- mpm --version
      - name: mpm managers
        run: |
          uv --no-progress run --frozen -- mpm managers
      - name: mpm --all-managers managers
        run: |
          uv --no-progress run --frozen -- mpm --all-managers managers
      - name: mpm installed
        run: |
          uv --no-progress run --frozen -- mpm installed
      - name: mpm outdated
        run: |
          uv --no-progress run --frozen -- mpm outdated
      - name: mpm which mpm
        run: |
          uv --no-progress run --frozen -- mpm which mpm
      - name: mpm backup
        run: |
          uv --no-progress run --frozen -- mpm backup
      - name: mpm sbom --spdx
        run: |
          uv --no-progress run --frozen -- mpm sbom --spdx
      - name: mpm sbom --cyclonedx
        run: |
          uv --no-progress run --frozen -- mpm sbom --cyclonedx

      - name: Parallel non-destructive tests
        run: >
          uv --no-progress run --frozen --
          pytest --exitfirst
          --numprocesses=auto --run-non-destructive --skip-destructive
          --ignore=./tests/test_bar_plugin.py

      - name: Sequential random destructive tests
        run: >
          uv --no-progress run --frozen --
          pytest --exitfirst
          --numprocesses=0 --skip-non-destructive --run-destructive
          --ignore=./tests/test_bar_plugin.py

      # Bar plugin test runs.
      - name: bar_plugin.py --search-mpm
        if: runner.os == 'macOS'
        run: |
          ./meta_package_manager/bar_plugin.py --search-mpm
      - name: uv run bar_plugin.py --search-mpm
        if: runner.os == 'macOS'
        run: |
          uv --no-progress run --frozen -- ./meta_package_manager/bar_plugin.py --search-mpm
      - name: bar_plugin.py
        if: runner.os == 'macOS'
        run: |
          ./meta_package_manager/bar_plugin.py
      - name: uv run bar_plugin.py
        if: runner.os == 'macOS'
        run: |
          uv --no-progress run --frozen -- ./meta_package_manager/bar_plugin.py

      - name: Sequential, bar plugin tests - macOS
        if: runner.os == 'macOS'
        # XXX bar plugin calls way too many external commands to be run in parallel.
        run: >
          uv --no-progress run --frozen --
          pytest --exitfirst
          --numprocesses=0 --run-non-destructive --run-destructive
          ./tests/test_bar_plugin.py

      - name: Coverage - Upload Codecov
        uses: codecov/codecov-action@v4.6.0
        with:
          token: ${{ secrets.CODECOV_TOKEN }}