feat: Install subworkflows with modules from different remotes #7692
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Python tests | |
| # This workflow is triggered on pushes and PRs to the repository. | |
| # Only run if we changed a Python file | |
| on: | |
| push: | |
| branches: | |
| - dev | |
| paths-ignore: | |
| - "docs/**" | |
| - "CHANGELOG.md" | |
| pull_request: | |
| paths-ignore: | |
| - "docs/**" | |
| - "CHANGELOG.md" | |
| # ignore github workflows except for the current one | |
| - ".github/**" | |
| - "!.github/workflows/pytest.yml" | |
| release: | |
| types: [published] | |
| workflow_dispatch: | |
| inputs: | |
| runners: | |
| description: "Runners to test on" | |
| type: choice | |
| options: | |
| - "ubuntu-latest" | |
| - "self-hosted" | |
| default: "self-hosted" | |
| # Cancel if a newer run with the same workflow name is queued | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| jobs: | |
| setup: | |
| runs-on: "ubuntu-latest" | |
| strategy: | |
| matrix: | |
| python-version: ["3.8", "3.12"] | |
| runner: ["ubuntu-latest"] | |
| include: | |
| - python-version: "3.8" | |
| runner: "ubuntu-20.04" | |
| steps: | |
| - name: Check conditions | |
| id: conditions | |
| run: echo "run-tests=${{ github.ref == 'refs/heads/master' || (matrix.runner == 'ubuntu-20.04' && matrix.python-version == '3.8') }}" >> "$GITHUB_OUTPUT" | |
| outputs: | |
| python-version: ${{ matrix.python-version }} | |
| runner: ${{ matrix.runner }} | |
| run-tests: ${{ steps.conditions.outputs.run-tests }} | |
| # create a test matrix based on all python files in /tests | |
| list_tests: | |
| name: Get test file matrix | |
| runs-on: "ubuntu-latest" | |
| steps: | |
| - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 | |
| name: Check out source-code repository | |
| - name: List tests | |
| id: list_tests | |
| run: | | |
| echo "tests=$(find tests -type f -name "test_*.py" | tac | sed 's/tests\///g' | jq -R -s -c '{test: (split("\n")[:-1])}')" >> $GITHUB_OUTPUT | |
| outputs: | |
| tests: ${{ steps.list_tests.outputs.tests }} | |
| test: | |
| name: Run ${{matrix.test}} with Python ${{ needs.setup.outputs.python-version }} on ${{ needs.setup.outputs.runner }} | |
| needs: [setup, list_tests] | |
| if: ${{ needs.setup.outputs.run-tests }} | |
| # run on self-hosted runners for test_components.py (because of the gitlab branch), based on the input if it is dispatched manually, on github if it is a rerun or on self-hosted by default | |
| runs-on: ${{ matrix.test == 'test_components.py' && 'self-hosted' || (github.event.inputs.runners || github.run_number > 1 && 'ubuntu-latest' || 'self-hosted') }} | |
| strategy: | |
| matrix: ${{ fromJson(needs.list_tests.outputs.tests) }} | |
| fail-fast: false # run all tests even if one fails | |
| steps: | |
| - name: go to subdirectory and change nextflow workdir | |
| run: | | |
| mkdir -p pytest | |
| cd pytest | |
| export NXF_WORK=$(pwd) | |
| - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 | |
| name: Check out source-code repository | |
| - name: Set up Python ${{ needs.setup.outputs.python-version }} | |
| uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 | |
| with: | |
| python-version: ${{ needs.setup.outputs.python-version }} | |
| cache: "pip" | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip -r requirements-dev.txt | |
| pip install -e . | |
| - name: Downgrade git to the Ubuntu official repository's version | |
| if: ${{ needs.setup.outputs.runner == 'ubuntu-20.04' && needs.setup.outputs.python-version == '3.8' }} | |
| run: | | |
| sudo apt update | |
| sudo apt remove -y git git-man | |
| sudo add-apt-repository --remove ppa:git-core/ppa | |
| sudo apt install -y git | |
| - name: Set up Singularity | |
| if: ${{ matrix.test == 'test_download.py'}} | |
| uses: eWaterCycle/setup-singularity@931d4e31109e875b13309ae1d07c70ca8fbc8537 # v7 | |
| with: | |
| singularity-version: 3.8.3 | |
| - name: Get current date | |
| id: date | |
| run: echo "date=$(date +'%Y-%m')" >> $GITHUB_ENV | |
| - name: Install Nextflow | |
| uses: nf-core/setup-nextflow@v2 | |
| - name: Install nf-test | |
| uses: nf-core/setup-nf-test@v1 | |
| - name: move coveragerc file up | |
| run: | | |
| mv .github/.coveragerc . | |
| - name: Test with pytest | |
| run: | | |
| python3 -m pytest tests/${{matrix.test}} --color=yes --cov --durations=0 && exit_code=0|| exit_code=$? | |
| # don't fail if no tests were collected, e.g. for test_licence.py | |
| if [ "${exit_code}" -eq 5 ]; then | |
| echo "No tests were collected" | |
| exit 0 | |
| elif [ "${exit_code}" -ne 0 ]; then | |
| echo "Tests failed with exit code ${exit_code}" | |
| exit 1 | |
| fi | |
| - name: Store snapshot report | |
| uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4 | |
| if: always() | |
| with: | |
| name: Snapshot Report ${{ matrix.test }} | |
| path: ./snapshot_report.html | |
| - name: remove slashes from test name | |
| run: | | |
| test=$(echo ${{ matrix.test }} | sed 's/\//__/g') | |
| echo "test=${test}" >> $GITHUB_ENV | |
| - name: Upload coverage | |
| uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4 | |
| with: | |
| name: coverage_${{ env.test }} | |
| path: .coverage | |
| coverage: | |
| needs: test | |
| # use the runner given by the input if it is dispatched manually, run on github if it is a rerun or on self-hosted by default | |
| runs-on: ${{ github.event.inputs.runners || github.run_number > 1 && 'ubuntu-latest' || 'self-hosted' }} | |
| steps: | |
| - name: go to subdirectory | |
| run: | | |
| mkdir -p pytest | |
| cd pytest | |
| - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 | |
| - name: Set up Python 3.12 | |
| uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 | |
| env: | |
| AGENT_TOOLSDIRECTORY: /opt/actions-runner/_work/tools/tools/ | |
| with: | |
| python-version: "3.12" | |
| cache: "pip" | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip -r requirements-dev.txt | |
| pip install -e . | |
| - name: move coveragerc file up | |
| run: | | |
| mv .github/.coveragerc . | |
| - name: Download all artifacts | |
| uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4 | |
| - name: Run coverage | |
| run: | | |
| coverage combine --keep coverage*/.coverage* | |
| coverage report | |
| coverage xml | |
| - uses: codecov/codecov-action@5ecb98a3c6b747ed38dc09f787459979aebb39be # v4 | |
| with: | |
| files: coverage.xml | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} |