benchmarks.yml

name: Benchmarks

# Detect performance regressions.
# - Push to main: run benchmarks, save Criterion baseline, upload as artifact.
# - PRs: find latest main baseline artifact, download, compare, report.
# Regressions are warning-only — this workflow never fails on regression.

permissions:
  contents: read
  actions: read

on:
  push:
    branches:
      - main
    paths:
      - "src/**"
      - "benches/**"
      - "tests/exact_bench_config.rs"
      - "tests/vs_linalg_inputs.rs"
      - ".config/nextest.toml"
      - "Cargo.toml"
      - "Cargo.lock"
      - "justfile"
      - "rust-toolchain.toml"
      - ".github/actions/setup-just/action.yml"
      - ".github/workflows/benchmarks.yml"
  pull_request:
    branches:
      - main
    paths:
      - "src/**"
      - "benches/**"
      - "tests/exact_bench_config.rs"
      - "tests/vs_linalg_inputs.rs"
      - ".config/nextest.toml"
      - "Cargo.toml"
      - "Cargo.lock"
      - "justfile"
      - "rust-toolchain.toml"
      - ".github/actions/setup-just/action.yml"
      - ".github/workflows/benchmarks.yml"
  workflow_dispatch:

concurrency:
  group: >
    bench-${{ github.workflow }}-${{
      github.event_name == 'pull_request' &&
      github.event.pull_request.number ||
      github.ref
    }}
  cancel-in-progress: true

env:
  CARGO_TERM_COLOR: always
  RUST_BACKTRACE: 1

jobs:
  bench:
    runs-on: ubuntu-latest
    timeout-minutes: 30

    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false

      - name: Install Rust toolchain
        uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0
        with:
          cache: true
          cache-bin: false

      - name: Set up just
        uses: ./.github/actions/setup-just

      - name: Resolve cargo-nextest version
        id: cargo_nextest_version
        shell: bash
        run: |
          set -euo pipefail

          version="$(just --evaluate cargo_nextest_version)"
          if [[ -z "$version" ]]; then
            echo "::error::Could not resolve cargo_nextest_version from justfile"
            exit 1
          fi

          echo "version=$version" >> "$GITHUB_OUTPUT"

      - name: Install cargo-nextest
        uses: taiki-e/cache-cargo-install-action@9ee83daaa7b96a6fab930949ecf1122bba04a389 # v3.0.8
        with:
          tool: cargo-nextest@${{ steps.cargo_nextest_version.outputs.version }}

      - name: Validate benchmark inputs
        run: just test-bench-inputs

      # ── PR: find and download the latest main baseline ──────────────
      - name: Find latest main baseline
        if: github.event_name == 'pull_request'
        id: find-baseline
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        with:
          script: |
            // Find the most recent successful run of this workflow on main.
            const runs = await github.rest.actions.listWorkflowRuns({
              owner: context.repo.owner,
              repo: context.repo.repo,
              workflow_id: 'benchmarks.yml',
              branch: 'main',
              status: 'completed',
              conclusion: 'success',
              per_page: 5,
            });

            for (const run of runs.data.workflow_runs) {
              const artifacts = await github.rest.actions.listWorkflowRunArtifacts({
                owner: context.repo.owner,
                repo: context.repo.repo,
                run_id: run.id,
              });

              const baseline = artifacts.data.artifacts.find(
                a => a.name === 'bench-baseline-main' && !a.expired
              );

              if (baseline) {
                console.log(`Found baseline from run ${run.id} (${run.created_at})`);
                core.setOutput('found', 'true');
                core.setOutput('run_id', run.id.toString());
                return;
              }
            }

            console.log('No baseline artifact found');
            core.setOutput('found', 'false');

      - name: Download baseline artifact
        if: >
          github.event_name == 'pull_request' &&
          steps.find-baseline.outputs.found == 'true'
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: bench-baseline-main
          path: target/criterion
          run-id: ${{ steps.find-baseline.outputs.run_id }}
          github-token: ${{ secrets.GITHUB_TOKEN }}

      # ── PR: run benchmarks and compare ──────────────────────────────
      - name: Run benchmarks (compare against main)
        if: github.event_name == 'pull_request'
        id: bench-compare
        run: |
          set -euo pipefail

          if [ -d target/criterion/exact_d2/det/main ]; then
            echo "::notice::Baseline found — comparing against main"
            echo "comparison_available=true" >> "$GITHUB_OUTPUT"
            # --baseline-lenient rather than --baseline: benches added on the
            # PR branch that don't yet exist in the main baseline get a
            # "no baseline data" notice instead of aborting the whole run.
            cargo bench --locked --features bench,exact --bench exact \
              -- --baseline-lenient main 2>&1 | tee bench-output.txt
          else
            echo "::notice::No baseline found — running without comparison"
            echo "comparison_available=false" >> "$GITHUB_OUTPUT"
            cargo bench --locked --features bench,exact --bench exact \
              2>&1 | tee bench-output.txt
          fi

          if grep -q "Performance has regressed" bench-output.txt; then
            echo "regression=true" >> "$GITHUB_OUTPUT"
          else
            echo "regression=false" >> "$GITHUB_OUTPUT"
          fi

      # ── Main push: run benchmarks and save baseline ─────────────────
      - name: Run benchmarks (save baseline)
        if: >
          (github.event_name == 'push' || github.event_name == 'workflow_dispatch') &&
          github.ref == 'refs/heads/main'
        run: >
          cargo bench --locked --features bench,exact --bench exact
          -- --save-baseline main

      - name: Run benchmarks (manual ref)
        if: github.event_name == 'workflow_dispatch' && github.ref != 'refs/heads/main'
        run: cargo bench --locked --features bench,exact --bench exact

      - name: Upload baseline artifact
        if: >
          (github.event_name == 'push' || github.event_name == 'workflow_dispatch') &&
          github.ref == 'refs/heads/main'
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: bench-baseline-main
          path: target/criterion
          retention-days: 90
          if-no-files-found: error

      # ── PR: report results ──────────────────────────────────────────
      - name: Benchmark summary
        if: github.event_name == 'pull_request' && always()
        run: |
          set -euo pipefail

          comparison_available="${BENCH_COMPARISON_AVAILABLE:-}"
          regression="${BENCH_REGRESSION:-}"

          if [ "$comparison_available" != "true" ] || [ -z "$regression" ]; then
            {
              echo "### ❓ Benchmark Comparison Unavailable"
              echo ""
              echo "No usable comparison against the main baseline was produced."
              echo "The benchmark still ran, but no regression claim can be made."
            } >> "$GITHUB_STEP_SUMMARY"
            echo "::warning::Benchmark comparison unavailable"
          elif [ "$regression" = "true" ]; then
            {
              echo "### ⚠️ Performance Regression Detected"
              echo ""
              echo "Exact-arithmetic benchmarks show regression vs main baseline."
              echo "This is a **warning only** — the workflow will not fail."
              echo ""
              echo '```'
              grep -B1 "Performance has regressed" bench-output.txt || true
              echo '```'
            } >> "$GITHUB_STEP_SUMMARY"
            echo "::warning::Performance regression detected in exact-arithmetic benchmarks"
          else
            {
              echo "### ✅ No Performance Regression"
              echo ""
              echo "Exact-arithmetic benchmarks are within expected range vs main."
            } >> "$GITHUB_STEP_SUMMARY"
          fi
        env:
          BENCH_COMPARISON_AVAILABLE: ${{ steps.bench-compare.outputs.comparison_available }}
          BENCH_REGRESSION: ${{ steps.bench-compare.outputs.regression }}