Merge pull request #213 from acgetchell/release/v0.4.5 #138
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmarks | |
| # Detect performance regressions. | |
| # - Push to main: run benchmarks, save Criterion baseline, upload as artifact. | |
| # - PRs: find latest main baseline artifact, download, compare, report. | |
| # Regressions are warning-only — this workflow never fails on regression. | |
| permissions: | |
| contents: read | |
| actions: read | |
| on: | |
| push: | |
| branches: | |
| - main | |
| paths: | |
| - "src/**" | |
| - "benches/**" | |
| - "tests/exact_bench_config.rs" | |
| - "tests/vs_linalg_inputs.rs" | |
| - ".config/nextest.toml" | |
| - "Cargo.toml" | |
| - "Cargo.lock" | |
| - "justfile" | |
| - "rust-toolchain.toml" | |
| - ".github/actions/setup-just/action.yml" | |
| - ".github/workflows/benchmarks.yml" | |
| pull_request: | |
| branches: | |
| - main | |
| paths: | |
| - "src/**" | |
| - "benches/**" | |
| - "tests/exact_bench_config.rs" | |
| - "tests/vs_linalg_inputs.rs" | |
| - ".config/nextest.toml" | |
| - "Cargo.toml" | |
| - "Cargo.lock" | |
| - "justfile" | |
| - "rust-toolchain.toml" | |
| - ".github/actions/setup-just/action.yml" | |
| - ".github/workflows/benchmarks.yml" | |
| workflow_dispatch: | |
| concurrency: | |
| group: > | |
| bench-${{ github.workflow }}-${{ | |
| github.event_name == 'pull_request' && | |
| github.event.pull_request.number || | |
| github.ref | |
| }} | |
| cancel-in-progress: true | |
| env: | |
| CARGO_TERM_COLOR: always | |
| RUST_BACKTRACE: 1 | |
| jobs: | |
| bench: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| - name: Install Rust toolchain | |
| uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 | |
| with: | |
| cache: true | |
| cache-bin: false | |
| - name: Set up just | |
| uses: ./.github/actions/setup-just | |
| - name: Resolve cargo-nextest version | |
| id: cargo_nextest_version | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| version="$(just --evaluate cargo_nextest_version)" | |
| if [[ -z "$version" ]]; then | |
| echo "::error::Could not resolve cargo_nextest_version from justfile" | |
| exit 1 | |
| fi | |
| echo "version=$version" >> "$GITHUB_OUTPUT" | |
| - name: Install cargo-nextest | |
| uses: taiki-e/cache-cargo-install-action@9ee83daaa7b96a6fab930949ecf1122bba04a389 # v3.0.8 | |
| with: | |
| tool: cargo-nextest@${{ steps.cargo_nextest_version.outputs.version }} | |
| - name: Validate benchmark inputs | |
| run: just test-bench-inputs | |
| # ── PR: find and download the latest main baseline ────────────── | |
| - name: Find latest main baseline | |
| if: github.event_name == 'pull_request' | |
| id: find-baseline | |
| uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 | |
| with: | |
| script: | | |
| // Find the most recent successful run of this workflow on main. | |
| const runs = await github.rest.actions.listWorkflowRuns({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| workflow_id: 'benchmarks.yml', | |
| branch: 'main', | |
| status: 'completed', | |
| conclusion: 'success', | |
| per_page: 5, | |
| }); | |
| for (const run of runs.data.workflow_runs) { | |
| const artifacts = await github.rest.actions.listWorkflowRunArtifacts({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| run_id: run.id, | |
| }); | |
| const baseline = artifacts.data.artifacts.find( | |
| a => a.name === 'bench-baseline-main' && !a.expired | |
| ); | |
| if (baseline) { | |
| console.log(`Found baseline from run ${run.id} (${run.created_at})`); | |
| core.setOutput('found', 'true'); | |
| core.setOutput('run_id', run.id.toString()); | |
| return; | |
| } | |
| } | |
| console.log('No baseline artifact found'); | |
| core.setOutput('found', 'false'); | |
| - name: Download baseline artifact | |
| if: > | |
| github.event_name == 'pull_request' && | |
| steps.find-baseline.outputs.found == 'true' | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: bench-baseline-main | |
| path: target/criterion | |
| run-id: ${{ steps.find-baseline.outputs.run_id }} | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| # ── PR: run benchmarks and compare ────────────────────────────── | |
| - name: Run benchmarks (compare against main) | |
| if: github.event_name == 'pull_request' | |
| id: bench-compare | |
| run: | | |
| set -euo pipefail | |
| if [ -d target/criterion/exact_d2/det/main ]; then | |
| echo "::notice::Baseline found — comparing against main" | |
| echo "comparison_available=true" >> "$GITHUB_OUTPUT" | |
| # --baseline-lenient rather than --baseline: benches added on the | |
| # PR branch that don't yet exist in the main baseline get a | |
| # "no baseline data" notice instead of aborting the whole run. | |
| cargo bench --locked --features bench,exact --bench exact \ | |
| -- --baseline-lenient main 2>&1 | tee bench-output.txt | |
| else | |
| echo "::notice::No baseline found — running without comparison" | |
| echo "comparison_available=false" >> "$GITHUB_OUTPUT" | |
| cargo bench --locked --features bench,exact --bench exact \ | |
| 2>&1 | tee bench-output.txt | |
| fi | |
| if grep -q "Performance has regressed" bench-output.txt; then | |
| echo "regression=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "regression=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| # ── Main push: run benchmarks and save baseline ───────────────── | |
| - name: Run benchmarks (save baseline) | |
| if: > | |
| (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && | |
| github.ref == 'refs/heads/main' | |
| run: > | |
| cargo bench --locked --features bench,exact --bench exact | |
| -- --save-baseline main | |
| - name: Run benchmarks (manual ref) | |
| if: github.event_name == 'workflow_dispatch' && github.ref != 'refs/heads/main' | |
| run: cargo bench --locked --features bench,exact --bench exact | |
| - name: Upload baseline artifact | |
| if: > | |
| (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && | |
| github.ref == 'refs/heads/main' | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: bench-baseline-main | |
| path: target/criterion | |
| retention-days: 90 | |
| if-no-files-found: error | |
| # ── PR: report results ────────────────────────────────────────── | |
| - name: Benchmark summary | |
| if: github.event_name == 'pull_request' && always() | |
| run: | | |
| set -euo pipefail | |
| comparison_available="${BENCH_COMPARISON_AVAILABLE:-}" | |
| regression="${BENCH_REGRESSION:-}" | |
| if [ "$comparison_available" != "true" ] || [ -z "$regression" ]; then | |
| { | |
| echo "### ❓ Benchmark Comparison Unavailable" | |
| echo "" | |
| echo "No usable comparison against the main baseline was produced." | |
| echo "The benchmark still ran, but no regression claim can be made." | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| echo "::warning::Benchmark comparison unavailable" | |
| elif [ "$regression" = "true" ]; then | |
| { | |
| echo "### ⚠️ Performance Regression Detected" | |
| echo "" | |
| echo "Exact-arithmetic benchmarks show regression vs main baseline." | |
| echo "This is a **warning only** — the workflow will not fail." | |
| echo "" | |
| echo '```' | |
| grep -B1 "Performance has regressed" bench-output.txt || true | |
| echo '```' | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| echo "::warning::Performance regression detected in exact-arithmetic benchmarks" | |
| else | |
| { | |
| echo "### ✅ No Performance Regression" | |
| echo "" | |
| echo "Exact-arithmetic benchmarks are within expected range vs main." | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| fi | |
| env: | |
| BENCH_COMPARISON_AVAILABLE: ${{ steps.bench-compare.outputs.comparison_available }} | |
| BENCH_REGRESSION: ${{ steps.bench-compare.outputs.regression }} |