Benchmark PR #3194 #4172
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark | |
| on: | |
| pull_request: | |
| branches: [main] | |
| types: [opened, synchronize, reopened, ready_for_review] | |
| # GitHub records a workflow_dispatch run's head_sha from the branch or tag | |
| # passed via --ref, not from a later checkout. Dispatch with a ref whose tip | |
| # is exactly head_sha; the first step rejects mismatches so release evidence | |
| # cannot be attributed to the wrong commit. | |
| workflow_dispatch: | |
| inputs: | |
| base_sha: | |
| description: Full lowercase 40-character base commit SHA | |
| required: true | |
| type: string | |
| head_sha: | |
| description: Full lowercase 40-character head SHA; --ref must be a branch or tag at this commit | |
| required: true | |
| type: string | |
| run-name: "Benchmark ${{ github.event_name == 'workflow_dispatch' && format('{0}...{1}', inputs.base_sha, inputs.head_sha) || format('PR #{0}', github.event.pull_request.number) }}" | |
| concurrency: | |
| group: benchmark-${{ github.event_name == 'workflow_dispatch' && format('dispatch-{0}-{1}', inputs.base_sha, inputs.head_sha) || github.event.pull_request.number }} | |
| cancel-in-progress: true | |
| env: | |
| CARGO_TERM_COLOR: always | |
| FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true | |
| BENCHMARK_BASE_SHA: ${{ github.event_name == 'workflow_dispatch' && inputs.base_sha || github.event.pull_request.base.sha }} | |
| BENCHMARK_HEAD_SHA: ${{ github.event_name == 'workflow_dispatch' && inputs.head_sha || github.event.pull_request.head.sha }} | |
| # 1000 files keeps the fastest lane (compile, ~200us/file single-threaded) | |
| # above ~200ms so runner thermal drift between interleaved runs stays well | |
| # under the 5% threshold; at 300 files the lane ran ~65ms and perf-neutral | |
| # PRs tripped the budget on drift alone (#1411: +5.27%). | |
| VIZE_BENCH_FILE_COUNT: 1000 | |
| # Optimized (ci-opt) binaries run fast enough that 5 runs / 1 warmup left the | |
| # gate's noise floor above the 5% threshold: an identical-source PR (#1395) | |
| # tripped the budget at +5.12% on the compile lane. More samples tighten the | |
| # median instead of weakening the threshold. | |
| VIZE_BENCH_RUNS: 10 | |
| VIZE_BENCH_WARMUPS: 2 | |
| VIZE_BENCH_REGRESSION_THRESHOLD_PERCENT: 5 | |
| jobs: | |
| pr-benchmark: | |
| name: pr-benchmark | |
| runs-on: blacksmith-32vcpu-ubuntu-2404 | |
| timeout-minutes: 30 | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Validate dispatch SHAs | |
| if: ${{ github.event_name == 'workflow_dispatch' }} | |
| env: | |
| RUN_HEAD_SHA: ${{ github.sha }} | |
| run: | | |
| full_sha='^[0-9a-f]{40}$' | |
| if [[ ! "$BENCHMARK_BASE_SHA" =~ $full_sha ]]; then | |
| echo "::error title=Invalid base_sha::base_sha must be a full lowercase 40-character commit SHA" | |
| exit 1 | |
| fi | |
| if [[ ! "$BENCHMARK_HEAD_SHA" =~ $full_sha ]]; then | |
| echo "::error title=Invalid head_sha::head_sha must be a full lowercase 40-character commit SHA" | |
| exit 1 | |
| fi | |
| if [[ "$BENCHMARK_BASE_SHA" == "$BENCHMARK_HEAD_SHA" ]]; then | |
| echo "::error title=Invalid benchmark range::base_sha must differ from head_sha" | |
| exit 1 | |
| fi | |
| if [[ "$RUN_HEAD_SHA" != "$BENCHMARK_HEAD_SHA" ]]; then | |
| echo "::error title=Dispatch ref mismatch::run head_sha comes from --ref; dispatch with a branch or tag at $BENCHMARK_HEAD_SHA (got $RUN_HEAD_SHA)" | |
| exit 1 | |
| fi | |
| - name: Checkout head | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| with: | |
| path: head | |
| ref: ${{ env.BENCHMARK_HEAD_SHA }} | |
| fetch-depth: ${{ github.event_name == 'workflow_dispatch' && '0' || '1' }} | |
| - name: Checkout base | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| with: | |
| path: base | |
| ref: ${{ env.BENCHMARK_BASE_SHA }} | |
| - name: Check base checkout | |
| id: base-integrity | |
| working-directory: base | |
| env: | |
| BASE_SHA: ${{ env.BENCHMARK_BASE_SHA }} | |
| HEAD_SHA: ${{ env.BENCHMARK_HEAD_SHA }} | |
| EVENT_NAME: ${{ github.event_name }} | |
| run: | | |
| if [[ "$EVENT_NAME" == "workflow_dispatch" ]] && \ | |
| ! git -C "$GITHUB_WORKSPACE/head" merge-base --is-ancestor "$BASE_SHA" "$HEAD_SHA"; then | |
| echo "::error title=Invalid benchmark range::base_sha must be an ancestor of head_sha" | |
| exit 1 | |
| fi | |
| if git grep -n -E '^(<<<<<<<|=======|>>>>>>>)' -- . ':(exclude)tests/_fixtures/_git/**'; then | |
| echo "skip=true" >> "$GITHUB_OUTPUT" | |
| { | |
| echo "## PR Benchmark" | |
| echo | |
| echo "Skipped because the base checkout contains unresolved merge conflict markers." | |
| echo | |
| echo "- Base: \`$BASE_SHA\`" | |
| echo "- Head: \`$HEAD_SHA\`" | |
| } > "$GITHUB_WORKSPACE/benchmark-summary.md" | |
| printf '{"skipped":true,"reason":"base_has_conflict_markers","base":"%s","head":"%s"}\n' "$BASE_SHA" "$HEAD_SHA" > "$GITHUB_WORKSPACE/benchmark-results.json" | |
| else | |
| echo "skip=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable | |
| - uses: wild-linker/action@0bbbfa5df4380cab8e63cb8505a1ce65e1d10203 # v0.9.0 | |
| with: | |
| wild-version: "0.9.0" | |
| - uses: ./head/.github/actions/setup-rust-sticky-cache | |
| with: | |
| key: benchmark-head | |
| cache-key-suffix: ${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('head/Cargo.lock') }} | |
| target-path: head/target | |
| secondary-key: benchmark-base | |
| secondary-target-path: base/target | |
| secondary-cache-key-suffix: ${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('base/Cargo.lock') }} | |
| - uses: voidzero-dev/setup-vp@ca1c46663915d6c1042ae23bd39ab85718bfb0fa # v1 | |
| with: | |
| node-version-file: "head/.node-version" | |
| cache: true | |
| run-install: false | |
| - name: Install JS dependencies | |
| working-directory: head | |
| run: vp install --frozen-lockfile --prefer-offline | |
| - name: Validate base metadata | |
| id: validate-base | |
| if: steps.base-integrity.outputs.skip != 'true' | |
| continue-on-error: true | |
| run: cargo metadata --manifest-path base/Cargo.toml --format-version 1 --no-deps > /dev/null | |
| - name: Skip benchmark when base metadata is invalid | |
| if: steps.base-integrity.outputs.skip != 'true' && steps.validate-base.outcome != 'success' | |
| run: | | |
| { | |
| echo "## PR Benchmark" | |
| echo | |
| echo "Base metadata could not be parsed, so the benchmark was skipped for this PR." | |
| echo | |
| echo "| status | reason |" | |
| echo "| --- | --- |" | |
| echo "| skipped | base checkout is not buildable |" | |
| } > benchmark-summary.md | |
| printf '{"skipped":true,"reason":"base_metadata_invalid","base":"%s","head":"%s"}\n' \ | |
| "$BENCHMARK_BASE_SHA" \ | |
| "$BENCHMARK_HEAD_SHA" \ | |
| > benchmark-results.json | |
| - name: Cache base CLI | |
| id: cache-base-cli | |
| if: steps.base-integrity.outputs.skip != 'true' && steps.validate-base.outcome == 'success' | |
| uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 | |
| with: | |
| path: base/target/ci-opt/vize | |
| key: ${{ runner.os }}-benchmark-base-cli-ci-opt-${{ env.BENCHMARK_BASE_SHA }} | |
| # The ci-opt profile is also injected via --config so the base checkout | |
| # builds even when it predates the profile's Cargo.toml definition, and | |
| # both sides are guaranteed to measure under identical settings. | |
| - name: Build base CLI | |
| id: build-base | |
| if: steps.base-integrity.outputs.skip != 'true' && steps.validate-base.outcome == 'success' && steps.cache-base-cli.outputs.cache-hit != 'true' | |
| continue-on-error: true | |
| run: >- | |
| cargo build --manifest-path base/Cargo.toml --profile ci-opt -p vize | |
| --config 'profile.ci-opt.inherits="release"' | |
| --config 'profile.ci-opt.lto="thin"' | |
| --config 'profile.ci-opt.codegen-units=16' | |
| # A base that does not compile (e.g. a hotfix PR for a broken main) cannot | |
| # be benchmarked; skip the comparison instead of hard-failing the gate. | |
| - name: Skip benchmark when base CLI cannot be built | |
| if: steps.base-integrity.outputs.skip != 'true' && steps.validate-base.outcome == 'success' && steps.build-base.outcome == 'failure' | |
| run: | | |
| { | |
| echo "## PR Benchmark" | |
| echo | |
| echo "Base CLI could not be built, so the benchmark was skipped for this PR." | |
| echo | |
| echo "| status | reason |" | |
| echo "| --- | --- |" | |
| echo "| skipped | base checkout does not compile |" | |
| } > benchmark-summary.md | |
| printf '{"skipped":true,"reason":"base_not_buildable","base":"%s","head":"%s"}\n' \ | |
| "$BENCHMARK_BASE_SHA" \ | |
| "$BENCHMARK_HEAD_SHA" \ | |
| > benchmark-results.json | |
| - name: Cache head CLI | |
| id: cache-head-cli | |
| if: steps.base-integrity.outputs.skip != 'true' && steps.validate-base.outcome == 'success' && steps.build-base.outcome != 'failure' | |
| uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 | |
| with: | |
| path: head/target/ci-opt/vize | |
| key: ${{ runner.os }}-benchmark-head-cli-ci-opt-${{ env.BENCHMARK_HEAD_SHA }} | |
| - name: Build head CLI | |
| if: steps.base-integrity.outputs.skip != 'true' && steps.validate-base.outcome == 'success' && steps.cache-head-cli.outputs.cache-hit != 'true' && steps.build-base.outcome != 'failure' | |
| run: >- | |
| cargo build --manifest-path head/Cargo.toml --profile ci-opt -p vize | |
| --config 'profile.ci-opt.inherits="release"' | |
| --config 'profile.ci-opt.lto="thin"' | |
| --config 'profile.ci-opt.codegen-units=16' | |
| - name: Generate benchmark input | |
| if: steps.base-integrity.outputs.skip != 'true' && steps.validate-base.outcome == 'success' && steps.build-base.outcome != 'failure' | |
| working-directory: head | |
| run: node bench/generate.mjs "$VIZE_BENCH_FILE_COUNT" | |
| - name: Compare base and head | |
| if: steps.base-integrity.outputs.skip != 'true' && steps.validate-base.outcome == 'success' && steps.build-base.outcome != 'failure' | |
| run: | | |
| node head/bench/compare-pr.mjs \ | |
| --input "$GITHUB_WORKSPACE/head/bench/__in__" \ | |
| --base-bin "$GITHUB_WORKSPACE/base/target/ci-opt/vize" \ | |
| --head-bin "$GITHUB_WORKSPACE/head/target/ci-opt/vize" \ | |
| --base-label "$BENCHMARK_BASE_SHA" \ | |
| --head-label "$BENCHMARK_HEAD_SHA" \ | |
| --runs "$VIZE_BENCH_RUNS" \ | |
| --warmups "$VIZE_BENCH_WARMUPS" \ | |
| --threshold "$VIZE_BENCH_REGRESSION_THRESHOLD_PERCENT" \ | |
| --out "$GITHUB_WORKSPACE/benchmark-summary.md" \ | |
| --json "$GITHUB_WORKSPACE/benchmark-results.json" | |
| - name: Write job summary | |
| run: cat benchmark-summary.md >> "$GITHUB_STEP_SUMMARY" | |
| - name: Upload benchmark results | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: pr-benchmark | |
| path: | | |
| benchmark-summary.md | |
| benchmark-results.json | |
| if-no-files-found: warn | |
| retention-days: 14 | |
| pr-benchmark-budget: | |
| name: pr-benchmark-budget | |
| runs-on: blacksmith-32vcpu-ubuntu-2404 | |
| timeout-minutes: 5 | |
| needs: | |
| - pr-benchmark | |
| permissions: | |
| actions: read | |
| contents: read | |
| issues: read | |
| steps: | |
| - name: Checkout head | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| with: | |
| path: head | |
| ref: ${{ env.BENCHMARK_HEAD_SHA }} | |
| - name: Download benchmark results | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: pr-benchmark | |
| - name: Read current PR labels | |
| id: pr-labels | |
| if: ${{ github.event_name == 'pull_request' }} | |
| env: | |
| GITHUB_TOKEN: ${{ github.token }} | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| REPOSITORY: ${{ github.repository }} | |
| run: | | |
| node <<'EOF' >> "$GITHUB_OUTPUT" | |
| async function main() { | |
| const apiUrl = process.env.GITHUB_API_URL ?? "https://api.github.com"; | |
| const url = `${apiUrl}/repos/${process.env.REPOSITORY}/issues/${process.env.PR_NUMBER}/labels?per_page=100`; | |
| const response = await fetch(url, { | |
| headers: { | |
| accept: "application/vnd.github+json", | |
| authorization: `Bearer ${process.env.GITHUB_TOKEN}`, | |
| }, | |
| }); | |
| if (!response.ok) { | |
| throw new Error(`Failed to read PR labels: ${response.status} ${response.statusText}`); | |
| } | |
| const labels = await response.json(); | |
| console.log("labels<<JSON"); | |
| console.log(JSON.stringify(labels.map((label) => label.name))); | |
| console.log("JSON"); | |
| } | |
| main().catch((error) => { | |
| console.error(error); | |
| process.exit(1); | |
| }); | |
| EOF | |
| - name: Enforce benchmark budget | |
| env: | |
| PR_LABELS_JSON: ${{ github.event_name == 'pull_request' && steps.pr-labels.outputs.labels || '[]' }} | |
| run: >- | |
| node head/bench/enforce-pr-budget.mjs | |
| --json benchmark-results.json | |
| --labels-json "$PR_LABELS_JSON" | |
| pr-benchmark-comment: | |
| name: pr-benchmark-comment | |
| runs-on: blacksmith-32vcpu-ubuntu-2404 | |
| timeout-minutes: 5 | |
| if: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }} | |
| needs: | |
| - pr-benchmark | |
| permissions: | |
| actions: read | |
| contents: read | |
| issues: write | |
| pull-requests: write | |
| steps: | |
| - name: Checkout trusted base | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| with: | |
| ref: ${{ github.event.pull_request.base.sha }} | |
| - name: Download benchmark results | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: pr-benchmark | |
| - name: Comment on PR | |
| continue-on-error: true | |
| env: | |
| GITHUB_TOKEN: ${{ github.token }} | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| BENCHMARK_COMMENT_KEY: ${{ github.event.pull_request.head.sha }} | |
| run: node bench/comment-pr.mjs --body benchmark-summary.md --comment-key "$BENCHMARK_COMMENT_KEY" |