qa.yml

# Runs the entire testsuite
name: QA
on:
  pull_request:
  schedule:
    - cron:  '0 18 * * *'
  workflow_dispatch:
    inputs:
      pcp_qa_args:
        description: '(optional) overwrite ./check args (e.g. "-g pmproxy -g pmseries")'

jobs:
  run_qa:
    name: ${{ matrix.platform }}
    # do not run this workflow on schedule for forks of the main repository
    if: github.event_name != 'schedule' || github.repository == 'performancecopilot/pcp'
    runs-on: ${{ matrix.os }}
    # do not mark the workflow as failed if an experimental distro (e.g. rawhide) fails
    continue-on-error: ${{ matrix.experimental }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - {platform: debian10-container,        os: ubuntu-20.04, experimental: false}
          - {platform: debian11-container,        os: ubuntu-20.04, experimental: false}
          - {platform: debian12-container,        os: ubuntu-20.04, experimental: false}
          - {platform: ubuntu1804-container,      os: ubuntu-20.04, experimental: false}
          - {platform: ubuntu1804-i386-container, os: ubuntu-20.04, experimental: false}
          - {platform: ubuntu2004-container,      os: ubuntu-20.04, experimental: false}
          - {platform: ubuntu2004,                os: ubuntu-20.04, experimental: false}
          - {platform: ubuntu2204-container,      os: ubuntu-20.04, experimental: false}
          - {platform: ubuntu2204,                os: ubuntu-22.04, experimental: false}
          - {platform: ubuntu2404-container,      os: ubuntu-20.04, experimental: false}
          - {platform: ubuntu2404,                os: ubuntu-22.04, experimental: false}
          - {platform: fedora38-container,        os: ubuntu-20.04, experimental: false}
          - {platform: fedora39-container,        os: ubuntu-20.04, experimental: false}
          - {platform: fedora40-container,        os: ubuntu-20.04, experimental: false}
          - {platform: fedora-rawhide-container,  os: ubuntu-20.04, experimental: true }
          - {platform: centos6-container,         os: ubuntu-20.04, experimental: false}
          - {platform: centos7-container,         os: ubuntu-20.04, experimental: false}
          - {platform: centos-stream8-container,  os: ubuntu-20.04, experimental: false}
          - {platform: centos-stream9-container,  os: ubuntu-20.04, experimental: false}
    steps:
      - name: Checkout sources
        uses: actions/checkout@v4

      - name: Fix local hostname DNS lookup
        run: echo 127.0.1.1 $(hostname --fqdn) $(hostname) | sudo tee -a /etc/hosts

      - name: Update version information
        run: |
          . VERSION.pcp
          PACKAGE_BUILD="0.$(date +'%Y%m%d').$(git rev-parse --short HEAD)"
          PCP_VERSION=${PACKAGE_MAJOR}.${PACKAGE_MINOR}.${PACKAGE_REVISION}
          PCP_BUILD_VERSION=${PCP_VERSION}-${PACKAGE_BUILD}
          sed -i "s/PACKAGE_BUILD=.*/PACKAGE_BUILD=${PACKAGE_BUILD}/" VERSION.pcp
          sed -i "1 s/(.*)/(${PCP_BUILD_VERSION})/" debian/changelog

      - name: Setup
        run: |
          python3 -c 'import yaml' || pip3 install pyyaml
          mkdir -p artifacts/build artifacts/test
          touch artifacts/build/.keep
          # crun >= 1.9.1 is required on Ubuntu 20.04.6
          # this workaround came from ...
          # https://github.com/kubernetes-sigs/kind/pull/3527
          curl -sLo /tmp/crun https://github.com/containers/crun/releases/download/1.14.3/crun-1.14.3-linux-amd64
          chmod +x /tmp/crun
          sudo cp /tmp/crun /usr/bin/crun

          build/ci/ci-run.py ${{ matrix.platform }} setup

      - name: Build
        run: build/ci/ci-run.py ${{ matrix.platform }} task build

      - name: Copy build artifacts
        run: build/ci/ci-run.py ${{ matrix.platform }} artifacts build --path artifacts/build

      - name: Upload build artifacts
        # always() is required here to run this step even if the build fails
        # otherwise the platform will be skipped in the test report (it should be flagged as broken)
        if: always()
        uses: actions/upload-artifact@v3
        with:
          name: build-${{ matrix.platform }}
          path: artifacts/build

      - name: Install
        run: build/ci/ci-run.py ${{ matrix.platform }} task install

      - name: Initialize QA
        id: init_qa
        run: build/ci/ci-run.py ${{ matrix.platform }} task init_qa

      - name: QA
        run: build/ci/ci-run.py ${{ matrix.platform }} task qa || true
        env:
          PCP_QA_ARGS: ${{ github.event.inputs.pcp_qa_args }}

      - name: Copy test results
        # always() is required here to run this step even if the QA step fails or times out
        if: always() && steps.init_qa.outcome == 'success'
        run: build/ci/ci-run.py ${{ matrix.platform }} artifacts test --path artifacts/test

      - name: Upload test results
        if: always() && steps.init_qa.outcome == 'success'
        uses: actions/upload-artifact@v3
        with:
          name: test-${{ matrix.platform }}
          path: artifacts/test

  create_report:
    name: Create Test Report
    # always() is required here to run this job even if any of the QA jobs fails
    if: always() && (github.event_name != 'schedule' || github.repository == 'performancecopilot/pcp')
    needs: run_qa
    runs-on: ubuntu-latest
    outputs:
      report_id: ${{ steps.parse_test_results.outputs.report_id }}
      summary_url: ${{ steps.publish_test_summary.outputs.summary_url }}
    steps:
      - name: Checkout sources
        uses: actions/checkout@v4

      - name: Checkout performancecopilot/qa-reports
        uses: actions/checkout@v4
        with:
          repository: performancecopilot/qa-reports
          path: qa-reports

      - name: Download artifacts
        uses: actions/download-artifact@v3
        with:
          path: artifacts

      - name: Setup Allure2
        run: |
          wget -q -O allure-commandline.zip https://repo1.maven.org/maven2/io/qameta/allure/allure-commandline/2.19.0/allure-commandline-2.19.0.zip
          unzip -q -d /opt allure-commandline.zip

      - name: Parse test results
        id: parse_test_results
        run: |
          REPORT_ID=$(date +'%Y%m%d_%H%M%S')-${GITHUB_SHA::8}
          echo "report_id=${REPORT_ID}" >> $GITHUB_OUTPUT

          # generate Allure inputs
          build/ci/test-report.py \
            --artifacts artifacts \
            --allure-results allure-results \
            --summary summary.txt \
            --source-url "https://github.com/${{ github.repository }}/tree/${GITHUB_SHA}" \
            --build-name "GitHub Actions Run #${{ github.run_id }}" \
            --build-url  "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" \
            --report-url "https://${{ github.repository_owner }}.github.io/qa-reports/reports/${REPORT_ID}/"

          cat summary.txt

      - name: Upload test summary
        uses: actions/upload-artifact@v3
        with:
          name: qasummary
          path: summary.txt

      - name: Publish test summary
        id: publish_test_summary
        if: github.event_name != 'pull_request'
        uses: actions/github-script@v7
        continue-on-error: true
        with:
          script: |
            const fs = require('fs');
            const head_sha = context.payload.pull_request ? context.payload.pull_request.head.sha : context.sha;
            const response = await github.rest.checks.create({
              owner: context.repo.owner,
              repo: context.repo.repo,
              head_sha,
              name: 'Test Summary',
              conclusion: 'success',
              output: {
                title: 'Test Summary',
                summary: '```\n' + fs.readFileSync('summary.txt', 'utf8') + '\n```',
              }
            });
            core.setOutput('summary_url', response.data.html_url);
            console.log(`::notice::Test summary is available at ${response.data.html_url}`);

      - name: Create interactive test report
        run: |
          cp build/ci/allure/categories.json allure-results
          if [[ -f qa-reports/reports/latest/history/history.json.gz ]]; then
            gunzip qa-reports/reports/latest/history/history.json.gz
            cp -r qa-reports/reports/latest/history allure-results
          fi
          /opt/allure-*/bin/allure generate

      - name: Create qareport.zip
        run: |
          cp build/ci/allure/README.md allure-report
          cp build/ci/allure/open_report.sh allure-report
          gzip allure-report/history/history.json
          # zip report before uploading, because GitHub uploads files in individual HTTP calls
          cd allure-report && zip -q -r ../qareport.zip . && cd ..

      - name: Upload test report
        uses: actions/upload-artifact@v3
        with:
          name: qareport
          path: qareport.zip

      - name: Check for regressions
        run: build/ci/check-trend.py --history-trend allure-report/history/history-trend.json
        continue-on-error: true

  publish_report:
    name: Publish Test Report
    # always() is required here to run this job even if any of the QA jobs fails
    if: always() && (github.repository == 'performancecopilot/pcp' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'))
    needs: create_report
    runs-on: ubuntu-latest
    steps:
      - name: Checkout sources
        uses: actions/checkout@v4

      - name: Checkout ${{ github.repository_owner }}/qa-reports
        uses: actions/checkout@v4
        with:
          repository: ${{ github.repository_owner }}/qa-reports
          path: qa-reports
          ssh-key: ${{ secrets.QA_REPORT_DEPLOY_KEY }}

      - name: Download artifacts
        uses: actions/download-artifact@v3
        with:
          path: artifacts

      - name: Prepare test report on GitHub Pages
        run: |
          rm -rf qa-reports/test-*
          cp -r artifacts/test-* qa-reports/
          unzip -q -d qa-reports/reports/${REPORT_ID} artifacts/qareport/qareport.zip

          pushd qa-reports/reports
          rm latest
          rm README.md
          ls -1r | tail -n +16 | xargs --no-run-if-empty rm -r # keep last 15 reports
          ln -s ${REPORT_ID} latest
          ls -1r -I README.md | awk 'BEGIN {print "# Recent Reports"} { print "## [" $1 "](" $1 "/)" }' > README.md
          popd
        env:
          REPORT_ID: ${{ needs.create_report.outputs.report_id }}

      - name: Publish test report on GitHub Pages
        run: |
          pushd qa-reports
          git config --local user.name "GitHub Actions"
          git config --local user.email "action@github.com"
          git add .
          git commit --amend -m "Added QA report of ${{ github.repository }}@${GITHUB_SHA}"
          git push -f
          popd

          echo "::notice::Test report is available at https://${{ github.repository_owner }}.github.io/qa-reports/reports/${REPORT_ID}"
        env:
          REPORT_ID: ${{ needs.create_report.outputs.report_id }}

      - name: Send test results to Slack channel
        run: |
          build/ci/test-report.py \
            --artifacts artifacts \
            --slack-channel "${SLACK_CHANNEL}" \
            --build-url  "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" \
            --report-url "https://${{ github.repository_owner }}.github.io/qa-reports/reports/${REPORT_ID}/" \
            --summary-url "${SUMMARY_URL}"
        env:
          SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
          REPORT_ID: ${{ needs.create_report.outputs.report_id }}
          SUMMARY_URL: ${{ needs.create_report.outputs.summary_url }}