benchmark.yml

name: Benchmark

on:
  push:
    branches: [main]
  workflow_dispatch:
    inputs:
      num_threads:
        description: "Thread count to pin (clamped to runner CPUs)"
        type: string
        default: "2"

# Allow only one benchmark run at a time so results are stored sequentially.
concurrency:
  group: benchmark
  cancel-in-progress: false

permissions:
  id-token: write # required for requesting the JWT (for OIDC)
  contents: write
  deployments: write

env:
  AWS_REGION: ${{ vars.AWS_REGION }}
  ECR_REPOSITORY: hls-science-container
  BUILD_PLATFORM: linux/amd64

jobs:
  benchmark:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0

      - uses: ./.github/actions/free-disk-space

      - name: Configure AWS credentials (OIDC)
        uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708 # v5.1.1
        with:
          role-to-assume: ${{ vars.AWS_ROLE_TO_ASSUME_ARN }}
          role-session-name: gha-${{ github.run_id }}
          aws-region: ${{ env.AWS_REGION }}

      - name: Login to Amazon ECR
        id: login-ecr
        uses: aws-actions/amazon-ecr-login@03f1aad4c6c7ffd436567f42f9384779290529bd # v2.1.7

      - name: Compute pixi.lock hash
        run: echo "LOCK_HASH=$(sha256sum pixi.lock | awk '{print $1}')" >> $GITHUB_ENV

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0

      - name: Build benchmark image
        uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5.4.0
        with:
          context: .
          load: true
          provenance: false
          sbom: false
          target: benchmark
          build-args: |
            PLATFORM=${{ env.BUILD_PLATFORM }}
            LOCK_HASH=${{ env.LOCK_HASH }}
            GIT_SHA=${{ github.sha }}
          secrets: |
            aws_access_key_id=${{ env.AWS_ACCESS_KEY_ID }}
            aws_secret_access_key=${{ env.AWS_SECRET_ACCESS_KEY }}
            aws_session_token=${{ env.AWS_SESSION_TOKEN }}
          tags: ${{ env.ECR_REPOSITORY }}:benchmark
          cache-from: type=registry,ref=${{ steps.login-ecr.outputs.registry }}/${{ env.ECR_REPOSITORY }}:cache
          cache-to: type=registry,ref=${{ steps.login-ecr.outputs.registry }}/${{ env.ECR_REPOSITORY }}:cache,mode=max,image-manifest=true,oci-mediatypes=true

      - name: Run benchmarks
        env:
          IMAGE: ${{ env.ECR_REPOSITORY }}:benchmark
          BENCHMARK_S2_GRANULE_IDS: ${{ vars.BENCHMARK_S2_GRANULE_IDS }}
          BENCHMARK_LS_GRANULE_IDS: ${{ vars.BENCHMARK_LS_GRANULE_IDS }}
          BENCHMARK_INPUT_BUCKET: ${{ vars.BENCHMARK_INPUT_BUCKET }}
          BENCHMARK_INPUT_PREFIX: ${{ vars.BENCHMARK_INPUT_PREFIX }}
          BENCHMARK_AUX_BUCKET: ${{ vars.BENCHMARK_AUX_BUCKET }}
          BENCHMARK_AUX_PREFIX: ${{ vars.BENCHMARK_AUX_PREFIX }}
          BENCHMARK_NUM_THREADS: ${{ github.event.inputs.num_threads || '2' }}
        run: scripts/benchmark

      # One chart group covering runtime + peak memory + CPU. The pytest tool
      # only charts runtime, so all three metrics are emitted to resources.json
      # (customSmallerIsBetter) and published in a single step.
      - name: Publish benchmarks to GitHub Pages
        uses: benchmark-action/github-action-benchmark@4322e5726e6334590d251fc4f92bec0efafc45dc # v1.22.2
        with:
          name: HLS pipeline benchmarks
          tool: customSmallerIsBetter
          output-file-path: benchmark-output/resources.json
          github-token: ${{ secrets.GITHUB_TOKEN }}
          auto-push: true
          comment-on-alert: true
          alert-threshold: "150%"
          fail-on-alert: false