benchmark.yaml

# Run resolving benchmark and store results. Fail if any resolved runtimes
# change from previous result
#
name: benchmark
on:
  release:
    types: [released]
  pull_request:
    types: [opened, synchronize, reopened, labeled]

jobs:
  run_benchmark:
    name: run_benchmark
    runs-on: ubuntu-latest

    if: ${{ github.event_name == 'release' || contains(github.event.pull_request.labels.*.name, 'run-benchmarks') }}

    strategy:
      matrix:
        python-version:
          - '3.7'

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Setup python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install Rez
        run: |
          mkdir ./installdir

          python ./install.py ./installdir

      - name: Run Benchmark
        run: |
          ./installdir/bin/rez/rez-benchmark --out ./out

          # remove benchmarking suite package repo
          rm -rf ./out/packages

      - name: Validate Result
        run: |
          python ./.github/scripts/validate_benchmark.py

      - uses: actions/upload-artifact@v4
        with:
          name: "benchmark-result-${{ matrix.python-version }}"
          path: ./out

  store_benchmark_result:
    name: store_benchmark_result
    runs-on: ubuntu-latest
    needs: run_benchmark

    strategy:
      matrix:
        python-version:
          - '3.7'

      # so we don't have jobs trying to push to git at the same time
      max-parallel: 1

    steps:
      - uses: actions/download-artifact@v4
        with:
          name: "benchmark-result-${{ matrix.python-version }}"
          path: .

      - name: Checkout (release)
        uses: actions/checkout@v4
        if: ${{ github.event_name =='release' }}
        with:
          ref: main
          path: src

          # This is a PAT from an Admin user. We do this in order to be able to write to a
          # protected branch (main) from this workflow.
          # See https://github.community/t/how-to-push-to-protected-branches-in-a-github-action/16101/14
          #
          # Disable for now until we find a better solution.
          # token: "${{ secrets.GH_ACTION_TOKEN }}"

      - name: Checkout (pr)
        uses: actions/checkout@v4
        if: ${{ github.event_name !='release' }}
        with:
          path: src

      - name: Setup python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      # Note failing due to
      # https://github.com/actions/virtual-environments/issues/675
      #
      # We'll just skip on failure, this just means the gnuplot doesn't render
      #
      - name: Install gnuplot
        run: |
          sudo apt-get update
          sudo apt-get install -y gnuplot || /bin/true

      - name: Store Benchmark Result
        run: |
          python ./.github/scripts/store_benchmark.py
        working-directory: src

      - name: Create summary
        run: |
          echo '<details>' >> $GITHUB_STEP_SUMMARY
          echo '<summary>Results</summary>' >> $GITHUB_STEP_SUMMARY
          cat metrics/benchmarking/RESULTS.md >> $GITHUB_STEP_SUMMARY
          echo '</details>' >> $GITHUB_STEP_SUMMARY
        working-directory: src

      # - name: Setup git config
      #   if: ${{ github.event_name == 'release' }}
      #   run: |
      #     git config user.name 'github-actions[bot]'
      #     git config user.email 'github-actions[bot]@users.noreply.github.com'
      #   working-directory: src

      # - name: Git commit and push
      #   if: ${{ github.event_name == 'release' }}
      #   run: |
      #     if [[ "$(git status --porcelain)" == "" ]]; then
      #       echo "Nothing new to commit"
      #     else
      #       git add --all
      #       git commit -m "Generated from GitHub "${{ github.workflow }}" Workflow"
      #       git push origin main
      #     fi
      #   working-directory: src