feat: implement solver registry architecture core #249
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmarks | |
| on: | |
| pull_request: | |
| paths: | |
| - "crates/**" | |
| - "Cargo.toml" | |
| - "Cargo.lock" | |
| - "justfile" | |
| - "scripts/bench.py" | |
| - "scripts/bench_guard.py" | |
| - "benchmarks/**" | |
| - ".github/workflows/benchmarks.yaml" | |
| push: | |
| branches: | |
| - main | |
| paths: | |
| - "crates/**" | |
| - "Cargo.toml" | |
| - "Cargo.lock" | |
| - "justfile" | |
| - "scripts/bench.py" | |
| - "scripts/bench_guard.py" | |
| - "benchmarks/**" | |
| - ".github/workflows/benchmarks.yaml" | |
| workflow_dispatch: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref_name }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| # Keep in sync with ci.yaml | |
| env: | |
| UV_VERSION: "0.9.4" | |
| JUST_VERSION: "1.43.0" | |
| PYTHON_LATEST: "3.12" | |
| RUST_TOOLCHAIN_VERSION: "1.85" | |
| # Enable sparse index for faster crate resolution | |
| CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse | |
| # Faster cargo builds | |
| CARGO_INCREMENTAL: 1 | |
| jobs: | |
| benchmark: | |
| name: Run benchmarks | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| permissions: | |
| contents: read | |
| pull-requests: write # Required to post benchmark summaries/comments back to the PR. | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| with: | |
| persist-credentials: false | |
| # Cache 1: rmon binary (saves ~20-40s avoiding git install) | |
| - name: Cache rmon binary | |
| uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 | |
| id: rmon-cache | |
| with: | |
| path: ~/.local/bin/rmon | |
| key: ${{ runner.os }}-rmon-${{ hashFiles('.github/workflows/benchmarks.yaml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-rmon- | |
| - name: Set up build environment | |
| uses: ./.github/actions/setup-build-env | |
| with: | |
| python-version: ${{ env.PYTHON_LATEST }} | |
| uv-version: ${{ env.UV_VERSION }} | |
| just-version: ${{ env.JUST_VERSION }} | |
| rust-toolchain-version: ${{ env.RUST_TOOLCHAIN_VERSION }} | |
| sync-python-licenses: "false" | |
| save-rust-cache: "false" | |
| rust-cache-shared-key: rust-bench | |
| # Cache 3: Pre-built arco-cli binary (saves ~2-5min when Cargo.lock unchanged) | |
| - name: Cache arco-cli binary | |
| uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 | |
| id: arco-binary-cache | |
| with: | |
| path: target/release/arco | |
| key: ${{ runner.os }}-arco-cli-${{ hashFiles('Cargo.lock', 'crates/arco-cli/**', 'crates/arco_*/**') }} | |
| restore-keys: | | |
| ${{ runner.os }}-arco-cli- | |
| - name: Build arco CLI | |
| if: steps.arco-binary-cache.outputs.cache-hit != 'true' | |
| run: cargo build --release -p arco-cli | |
| - name: Ensure rmon is installed | |
| run: | | |
| if [ ! -x "$HOME/.local/bin/rmon" ]; then | |
| uv tool install --force git+https://github.com/NatLabRockies/resource_monitor | |
| fi | |
| "$HOME/.local/bin/rmon" --version | |
| # Ensure rmon is in PATH even if cached | |
| - name: Add local bin to PATH | |
| run: echo "$HOME/.local/bin" >> "$GITHUB_PATH" | |
| - name: Run CLI benchmarks | |
| timeout-minutes: 15 | |
| run: | | |
| uv run python scripts/bench.py \ | |
| --arco-binary target/release/arco \ | |
| --workflows validate,run \ | |
| --repetitions 10 \ | |
| --output artifacts/benchmark-results.json | |
| # Cache 4: Benchmark data for trend analysis (already exists, but optimized key) | |
| - name: Restore previous benchmark data | |
| uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 | |
| id: benchmark-data-restore | |
| with: | |
| path: ./cache | |
| key: ${{ runner.os }}-benchmark-${{ github.ref_name }}-${{ github.run_id }} | |
| restore-keys: | | |
| ${{ runner.os }}-benchmark-main- | |
| ${{ runner.os }}-benchmark- | |
| - name: Store benchmark results | |
| uses: benchmark-action/github-action-benchmark@a60cea5bc7b49e15c1f58f411161f99e0df48372 # v1.22.0 | |
| with: | |
| tool: customSmallerIsBetter | |
| output-file-path: artifacts/benchmark-results.json | |
| external-data-json-path: ./cache/benchmark-data.json | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| summary-always: true | |
| comment-on-alert: ${{ github.event_name == 'pull_request' }} | |
| alert-threshold: "125%" | |
| fail-on-alert: false | |
| - name: Benchmark regression guard | |
| run: | | |
| uv run python scripts/bench_guard.py \ | |
| --current artifacts/benchmark-results.json \ | |
| --baseline cache/benchmark-data.json \ | |
| --ratio-threshold 1.25 \ | |
| --absolute-delta-ms 1.0 \ | |
| --min-baseline-samples 10 | |
| - name: Save benchmark data | |
| if: github.ref == 'refs/heads/main' && steps.benchmark-data-restore.outputs.cache-hit != 'true' | |
| uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 | |
| with: | |
| path: ./cache | |
| key: ${{ runner.os }}-benchmark-${{ github.ref_name }}-${{ github.run_id }} |