From 25140e256e8a7795c5fb2181b1702a5a074d75eb Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Thu, 27 Nov 2025 16:39:27 +0400 Subject: [PATCH 01/40] feat(ci): add performance tracking workflow for AvalancheGo benchmarks Implements workflow to trigger C-Chain reexecution benchmarks in AvalancheGo and track Firewood performance over time. Supports task-based and custom parameter modes. Results stored in GitHub Pages via github-action-benchmark. --- .github/workflows/track-performance.yml | 222 ++++++++++++++++++++++++ 1 file changed, 222 insertions(+) create mode 100644 .github/workflows/track-performance.yml diff --git a/.github/workflows/track-performance.yml b/.github/workflows/track-performance.yml new file mode 100644 index 000000000..0c5a1a179 --- /dev/null +++ b/.github/workflows/track-performance.yml @@ -0,0 +1,222 @@ +name: Track Performance in AvalancheGo + +on: + workflow_dispatch: + inputs: + firewood-commit: + description: 'Firewood commit/branch/tag to test (leave empty for current HEAD)' + default: '' + avalanchego-ref: + description: 'AvalancheGo ref to test against' + default: 'master' + task: + description: 'Predefined task (leave empty to use custom parameters below)' + type: choice + options: + - c-chain-reexecution-firewood-101-250k + - c-chain-reexecution-firewood-33m-33m500k + - c-chain-reexecution-firewood-33m-40m + config: + description: 'VM config (e.g., firewood, hashdb)' + default: '' + start-block: + default: '' + end-block: + default: '' + block-dir-src: + description: 'Block directory source e.g., cchain-mainnet-blocks-1m-ldb (without S3 path)' + default: '' + current-state-dir-src: + description: 'Current state directory source e.g., cchain-mainnet-blocks-30m-40m-ldb (without S3 path)' + default: '' + runner: + description: 'Runner to use in AvalancheGo' + required: true + type: choice + options: + - avalanche-avalanchego-runner-2ti + - avago-runner-i4i-4xlarge-local-ssd + - avago-runner-m6i-4xlarge-ebs-fast + +jobs: + trigger-and-collect: + runs-on: ubuntu-latest + permissions: + contents: write # Required for github-action-benchmark to push to gh-pages + steps: + - name: Checkout Firewood + uses: actions/checkout@v4 + with: + fetch-depth: 0 # Needed for github-action-benchmark + + - name: Determine Firewood commit + id: commit + run: | + COMMIT_SHA="${{ inputs.firewood-commit || github.sha }}" + echo "sha=$COMMIT_SHA" >> "$GITHUB_OUTPUT" + + - name: Validate inputs + run: | + if [ -z "${{ inputs.task }}" ]; then + missing=() + [ -z "${{ inputs.config }}" ] && missing+=("config") + [ -z "${{ inputs.start-block }}" ] && missing+=("start-block") + [ -z "${{ inputs.end-block }}" ] && missing+=("end-block") + [ -z "${{ inputs.block-dir-src }}" ] && missing+=("block-dir-src") + [ -z "${{ inputs.current-state-dir-src }}" ] && missing+=("current-state-dir-src") + + if [ ${#missing[@]} -gt 0 ]; then + echo "Error: When using custom mode, these fields are required: ${missing[*]}" + exit 1 + fi + fi + + - name: Trigger AvalancheGo benchmark + id: trigger + run: | + if [ -n "${{ inputs.task }}" ]; then + gh workflow run "Firewood Reexecution Benchmark" \ + --repo ava-labs/avalanchego \ + --ref "${{ inputs.avalanchego-ref }}" \ + -f firewood-commit="${{ steps.commit.outputs.sha }}" \ + -f task="${{ inputs.task }}" \ + -f runner="${{ inputs.runner }}" + else + gh workflow run "Firewood Reexecution Benchmark" \ + --repo ava-labs/avalanchego \ + --ref "${{ inputs.avalanchego-ref }}" \ + -f firewood-commit="${{ steps.commit.outputs.sha }}" \ + -f config="${{ inputs.config }}" \ + -f start-block="${{ inputs.start-block }}" \ + -f end-block="${{ inputs.end-block }}" \ + -f block-dir-src="${{ inputs.block-dir-src }}" \ + -f current-state-dir-src="${{ inputs.current-state-dir-src }}" \ + -f runner="${{ inputs.runner }}" + fi + + sleep 10 + + RUN_ID=$(gh run list \ + --repo ava-labs/avalanchego \ + --workflow "Firewood Reexecution Benchmark" \ + --limit 5 \ + --json databaseId,createdAt \ + --jq '[.[] | select(.createdAt | fromdateiso8601 > (now - 60))] | .[0].databaseId') + + if [ -z "$RUN_ID" ] || [ "$RUN_ID" = "null" ]; then + echo "Could not find triggered workflow run" + exit 1 + fi + + echo "run_id=$RUN_ID" >> "$GITHUB_OUTPUT" + echo "run_url=https://github.com/ava-labs/avalanchego/actions/runs/$RUN_ID" >> "$GITHUB_OUTPUT" + env: + GH_TOKEN: ${{ secrets.FIREWOOD_GO_GITHUB_TOKEN }} + + - name: Wait for benchmark completion + id: wait + run: | + RUN_ID="${{ steps.trigger.outputs.run_id }}" + TIMEOUT=3600 + ELAPSED=0 + INTERVAL=30 + + while [ $ELAPSED -lt $TIMEOUT ]; do + RUN_INFO=$(gh run view "$RUN_ID" \ + --repo ava-labs/avalanchego \ + --json status,conclusion) + + STATUS=$(echo "$RUN_INFO" | jq -r '.status') + CONCLUSION=$(echo "$RUN_INFO" | jq -r '.conclusion') + + if [ "$STATUS" = "completed" ]; then + if [ "$CONCLUSION" = "success" ]; then + echo "conclusion=success" >> "$GITHUB_OUTPUT" + break + else + echo "Benchmark failed with conclusion: $CONCLUSION" + echo "conclusion=$CONCLUSION" >> "$GITHUB_OUTPUT" + exit 1 + fi + fi + + sleep $INTERVAL + ELAPSED=$((ELAPSED + INTERVAL)) + done + + if [ $ELAPSED -ge $TIMEOUT ]; then + echo "Timeout waiting for benchmark completion" + exit 1 + fi + env: + GH_TOKEN: ${{ secrets.FIREWOOD_GO_GITHUB_TOKEN }} + timeout-minutes: 60 + + - name: Download benchmark results + id: download + run: | + mkdir -p ./results + + gh run download "${{ steps.trigger.outputs.run_id }}" \ + --repo ava-labs/avalanchego \ + --name benchmark-output \ + --dir ./results + + cat ./results/benchmark-output.txt + + MGAS_PER_SEC=$(grep -oP '\d+\.\d+ mgas/s' ./results/benchmark-output.txt || echo "N/A") + echo "performance=$MGAS_PER_SEC" >> "$GITHUB_OUTPUT" + env: + GH_TOKEN: ${{ secrets.FIREWOOD_GO_GITHUB_TOKEN }} + + - name: Store benchmark results + uses: benchmark-action/github-action-benchmark@v1 + with: + name: C-Chain Reexecution Performance + tool: 'go' + output-file-path: ./results/benchmark-output.txt + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true + # Store benchmark data in gh-pages branch + gh-pages-branch: gh-pages + benchmark-data-dir-path: dev/bench + # Don't fail the workflow if there's an issue with benchmark storage + fail-on-alert: false + comment-on-alert: false + # Add metadata + alert-comment-cc-users: '' + + - name: Summary + if: always() + run: | + echo "## Firewood Performance Benchmark Results" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "**Configuration:**" >> $GITHUB_STEP_SUMMARY + + if [ -n "${{ inputs.task }}" ]; then + echo "- Mode: Task-based" >> $GITHUB_STEP_SUMMARY + echo "- Task: \`${{ inputs.task }}\`" >> $GITHUB_STEP_SUMMARY + else + echo "- Mode: Custom parameters" >> $GITHUB_STEP_SUMMARY + echo "- Config: \`${{ inputs.config }}\`" >> $GITHUB_STEP_SUMMARY + echo "- Blocks: \`${{ inputs.start-block }}\` → \`${{ inputs.end-block }}\`" >> $GITHUB_STEP_SUMMARY + echo "- Block source: \`${{ inputs.block-dir-src }}\`" >> $GITHUB_STEP_SUMMARY + echo "- State source: \`${{ inputs.current-state-dir-src }}\`" >> $GITHUB_STEP_SUMMARY + fi + + echo "- Firewood commit: \`${{ steps.commit.outputs.sha }}\`" >> $GITHUB_STEP_SUMMARY + echo "- AvalancheGo ref: \`${{ inputs.avalanchego-ref }}\`" >> $GITHUB_STEP_SUMMARY + echo "- Runner: \`${{ inputs.runner }}\`" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + if [ "${{ steps.wait.outputs.conclusion }}" = "success" ]; then + echo "**Performance:** ${{ steps.download.outputs.performance }}" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "**Links:**" >> $GITHUB_STEP_SUMMARY + echo "- [AvalancheGo Workflow](${{ steps.trigger.outputs.run_url }})" >> $GITHUB_STEP_SUMMARY + echo "- [Performance Trends](https://ava-labs.github.io/firewood/dev/bench/)" >> $GITHUB_STEP_SUMMARY + else + echo "**Status:** Failed" >> $GITHUB_STEP_SUMMARY + echo "Check [workflow logs](${{ steps.trigger.outputs.run_url }}) for details." >> $GITHUB_STEP_SUMMARY + fi + From 1ea99bb603c5e51467ae904d4f27dae655ac2d96 Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Thu, 27 Nov 2025 16:42:49 +0400 Subject: [PATCH 02/40] ci: track performance --- .github/workflows/track-performance.yml | 2 +- README.md | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/track-performance.yml b/.github/workflows/track-performance.yml index 0c5a1a179..66f8d4ca4 100644 --- a/.github/workflows/track-performance.yml +++ b/.github/workflows/track-performance.yml @@ -1,4 +1,4 @@ -name: Track Performance in AvalancheGo +name: Track Performance on: workflow_dispatch: diff --git a/README.md b/README.md index e06489714..e3b1befac 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,17 @@ as well as carefully managing the free list during the creation and expiration o Firewood provides comprehensive metrics for monitoring database performance, resource utilization, and operational characteristics. For detailed information about all available metrics, how to enable them, and how to interpret them, see [METRICS.md](METRICS.md). +## Performance Tracking + +Firewood tracks its performance over time by running C-Chain reexecution benchmarks in AvalancheGo. This allows us to: +- Monitor performance across commits and releases +- Catch performance regressions early +- Validate optimizations against real-world blockchain workloads + +Performance data is automatically collected and published to [GitHub Pages](https://ava-labs.github.io/firewood/dev/bench/). + +For information on running benchmarks and interpreting results, see [docs/PERFORMANCE_TRACKING.md](docs/PERFORMANCE_TRACKING.md). + ## Build In order to build firewood, the following dependencies must be installed: From 1ce14817dcbb6f290eeabd6b7d2e907d8e80f6b3 Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Thu, 27 Nov 2025 16:53:16 +0400 Subject: [PATCH 03/40] test(ci): add PR label trigger for testing --- .github/workflows/track-performance.yml | 40 +++++++++++++++++++++---- 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/.github/workflows/track-performance.yml b/.github/workflows/track-performance.yml index 66f8d4ca4..05b4a6d2e 100644 --- a/.github/workflows/track-performance.yml +++ b/.github/workflows/track-performance.yml @@ -3,6 +3,8 @@ name: Track Performance on: workflow_dispatch: inputs: + pull_request: + types: [labeled] firewood-commit: description: 'Firewood commit/branch/tag to test (leave empty for current HEAD)' default: '' @@ -49,13 +51,35 @@ jobs: with: fetch-depth: 0 # Needed for github-action-benchmark + - name: Check if triggered by label + if: github.event_name == 'pull_request' + run: | + if [ "${{ github.event.label.name }}" != "run-benchmark" ]; then + echo "Skipping - only runs on 'run-benchmark' label" + exit 0 + fi + - name: Determine Firewood commit id: commit run: | COMMIT_SHA="${{ inputs.firewood-commit || github.sha }}" echo "sha=$COMMIT_SHA" >> "$GITHUB_OUTPUT" + - name: Set defaults for PR trigger + id: defaults + run: | + if [ "${{ github.event_name }}" = "pull_request" ]; then + echo "task=c-chain-reexecution-firewood-101-250k" >> "$GITHUB_OUTPUT" + echo "runner=avalanche-avalanchego-runner-2ti" >> "$GITHUB_OUTPUT" + echo "avalanchego-ref=master" >> "$GITHUB_OUTPUT" + else + echo "task=${{ inputs.task }}" >> "$GITHUB_OUTPUT" + echo "runner=${{ inputs.runner }}" >> "$GITHUB_OUTPUT" + echo "avalanchego-ref=${{ inputs.avalanchego-ref }}" >> "$GITHUB_OUTPUT" + fi + - name: Validate inputs + if: github.event_name == 'workflow_dispatch' run: | if [ -z "${{ inputs.task }}" ]; then missing=() @@ -74,24 +98,28 @@ jobs: - name: Trigger AvalancheGo benchmark id: trigger run: | - if [ -n "${{ inputs.task }}" ]; then + TASK="${{ github.event_name == 'pull_request' && steps.defaults.outputs.task || inputs.task }}" + RUNNER="${{ github.event_name == 'pull_request' && steps.defaults.outputs.runner || inputs.runner }}" + AVAGO_REF="${{ github.event_name == 'pull_request' && steps.defaults.outputs.avalanchego-ref || inputs.avalanchego-ref }}" + + if [ -n "$TASK" ]; then gh workflow run "Firewood Reexecution Benchmark" \ --repo ava-labs/avalanchego \ - --ref "${{ inputs.avalanchego-ref }}" \ + --ref "$AVAGO_REF" \ -f firewood-commit="${{ steps.commit.outputs.sha }}" \ - -f task="${{ inputs.task }}" \ - -f runner="${{ inputs.runner }}" + -f task="$TASK" \ + -f runner="$RUNNER" else gh workflow run "Firewood Reexecution Benchmark" \ --repo ava-labs/avalanchego \ - --ref "${{ inputs.avalanchego-ref }}" \ + --ref "$AVAGO_REF" \ -f firewood-commit="${{ steps.commit.outputs.sha }}" \ -f config="${{ inputs.config }}" \ -f start-block="${{ inputs.start-block }}" \ -f end-block="${{ inputs.end-block }}" \ -f block-dir-src="${{ inputs.block-dir-src }}" \ -f current-state-dir-src="${{ inputs.current-state-dir-src }}" \ - -f runner="${{ inputs.runner }}" + -f runner="$RUNNER" fi sleep 10 From 879425013978968b61932b10319bc060fa319644 Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Thu, 27 Nov 2025 16:55:32 +0400 Subject: [PATCH 04/40] temp fix ci --- .github/workflows/track-performance.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/track-performance.yml b/.github/workflows/track-performance.yml index 05b4a6d2e..2093d62c4 100644 --- a/.github/workflows/track-performance.yml +++ b/.github/workflows/track-performance.yml @@ -3,8 +3,6 @@ name: Track Performance on: workflow_dispatch: inputs: - pull_request: - types: [labeled] firewood-commit: description: 'Firewood commit/branch/tag to test (leave empty for current HEAD)' default: '' @@ -39,6 +37,8 @@ on: - avalanche-avalanchego-runner-2ti - avago-runner-i4i-4xlarge-local-ssd - avago-runner-m6i-4xlarge-ebs-fast + pull_request: + types: [labeled] jobs: trigger-and-collect: From 4aa0e08e786703d759b3698c474f3cf9eff87a8c Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Tue, 2 Dec 2025 22:13:18 +0400 Subject: [PATCH 05/40] ci: use switch CI token --- .github/workflows/track-performance.yml | 106 +++++------------------- 1 file changed, 23 insertions(+), 83 deletions(-) diff --git a/.github/workflows/track-performance.yml b/.github/workflows/track-performance.yml index 2093d62c4..92fab2c59 100644 --- a/.github/workflows/track-performance.yml +++ b/.github/workflows/track-performance.yml @@ -37,8 +37,6 @@ on: - avalanche-avalanchego-runner-2ti - avago-runner-i4i-4xlarge-local-ssd - avago-runner-m6i-4xlarge-ebs-fast - pull_request: - types: [labeled] jobs: trigger-and-collect: @@ -51,35 +49,7 @@ jobs: with: fetch-depth: 0 # Needed for github-action-benchmark - - name: Check if triggered by label - if: github.event_name == 'pull_request' - run: | - if [ "${{ github.event.label.name }}" != "run-benchmark" ]; then - echo "Skipping - only runs on 'run-benchmark' label" - exit 0 - fi - - - name: Determine Firewood commit - id: commit - run: | - COMMIT_SHA="${{ inputs.firewood-commit || github.sha }}" - echo "sha=$COMMIT_SHA" >> "$GITHUB_OUTPUT" - - - name: Set defaults for PR trigger - id: defaults - run: | - if [ "${{ github.event_name }}" = "pull_request" ]; then - echo "task=c-chain-reexecution-firewood-101-250k" >> "$GITHUB_OUTPUT" - echo "runner=avalanche-avalanchego-runner-2ti" >> "$GITHUB_OUTPUT" - echo "avalanchego-ref=master" >> "$GITHUB_OUTPUT" - else - echo "task=${{ inputs.task }}" >> "$GITHUB_OUTPUT" - echo "runner=${{ inputs.runner }}" >> "$GITHUB_OUTPUT" - echo "avalanchego-ref=${{ inputs.avalanchego-ref }}" >> "$GITHUB_OUTPUT" - fi - - name: Validate inputs - if: github.event_name == 'workflow_dispatch' run: | if [ -z "${{ inputs.task }}" ]; then missing=() @@ -98,22 +68,25 @@ jobs: - name: Trigger AvalancheGo benchmark id: trigger run: | - TASK="${{ github.event_name == 'pull_request' && steps.defaults.outputs.task || inputs.task }}" - RUNNER="${{ github.event_name == 'pull_request' && steps.defaults.outputs.runner || inputs.runner }}" - AVAGO_REF="${{ github.event_name == 'pull_request' && steps.defaults.outputs.avalanchego-ref || inputs.avalanchego-ref }}" + TASK="${{ inputs.task }}" + RUNNER="${{ inputs.runner }}" + AVAGO_REF="${{ inputs.avalanchego-ref }}" + FIREWOOD_COMMIT="${{ inputs.firewood-commit || github.sha }}" + + echo "firewood_commit=$FIREWOOD_COMMIT" >> "$GITHUB_OUTPUT" if [ -n "$TASK" ]; then gh workflow run "Firewood Reexecution Benchmark" \ --repo ava-labs/avalanchego \ --ref "$AVAGO_REF" \ - -f firewood-commit="${{ steps.commit.outputs.sha }}" \ + -f firewood-commit="$FIREWOOD_COMMIT" \ -f task="$TASK" \ -f runner="$RUNNER" else gh workflow run "Firewood Reexecution Benchmark" \ --repo ava-labs/avalanchego \ --ref "$AVAGO_REF" \ - -f firewood-commit="${{ steps.commit.outputs.sha }}" \ + -f firewood-commit="$FIREWOOD_COMMIT" \ -f config="${{ inputs.config }}" \ -f start-block="${{ inputs.start-block }}" \ -f end-block="${{ inputs.end-block }}" \ @@ -139,45 +112,12 @@ jobs: echo "run_id=$RUN_ID" >> "$GITHUB_OUTPUT" echo "run_url=https://github.com/ava-labs/avalanchego/actions/runs/$RUN_ID" >> "$GITHUB_OUTPUT" env: - GH_TOKEN: ${{ secrets.FIREWOOD_GO_GITHUB_TOKEN }} + GH_TOKEN: ${{ secrets.FIREWOOD_AVALANCHEGO_GITHUB_TOKEN }} - name: Wait for benchmark completion - id: wait - run: | - RUN_ID="${{ steps.trigger.outputs.run_id }}" - TIMEOUT=3600 - ELAPSED=0 - INTERVAL=30 - - while [ $ELAPSED -lt $TIMEOUT ]; do - RUN_INFO=$(gh run view "$RUN_ID" \ - --repo ava-labs/avalanchego \ - --json status,conclusion) - - STATUS=$(echo "$RUN_INFO" | jq -r '.status') - CONCLUSION=$(echo "$RUN_INFO" | jq -r '.conclusion') - - if [ "$STATUS" = "completed" ]; then - if [ "$CONCLUSION" = "success" ]; then - echo "conclusion=success" >> "$GITHUB_OUTPUT" - break - else - echo "Benchmark failed with conclusion: $CONCLUSION" - echo "conclusion=$CONCLUSION" >> "$GITHUB_OUTPUT" - exit 1 - fi - fi - - sleep $INTERVAL - ELAPSED=$((ELAPSED + INTERVAL)) - done - - if [ $ELAPSED -ge $TIMEOUT ]; then - echo "Timeout waiting for benchmark completion" - exit 1 - fi + run: gh run watch "${{ steps.trigger.outputs.run_id }}" --repo ava-labs/avalanchego --exit-status env: - GH_TOKEN: ${{ secrets.FIREWOOD_GO_GITHUB_TOKEN }} + GH_TOKEN: ${{ secrets.FIREWOOD_AVALANCHEGO_GITHUB_TOKEN }} timeout-minutes: 60 - name: Download benchmark results @@ -192,10 +132,15 @@ jobs: cat ./results/benchmark-output.txt - MGAS_PER_SEC=$(grep -oP '\d+\.\d+ mgas/s' ./results/benchmark-output.txt || echo "N/A") - echo "performance=$MGAS_PER_SEC" >> "$GITHUB_OUTPUT" + # Determine target dashboard + if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then + echo "data-dir=bench" >> "$GITHUB_OUTPUT" + else + SAFE_NAME=$(echo "${{ github.ref_name }}" | tr '/' '-') + echo "data-dir=dev/bench/$SAFE_NAME" >> "$GITHUB_OUTPUT" + fi env: - GH_TOKEN: ${{ secrets.FIREWOOD_GO_GITHUB_TOKEN }} + GH_TOKEN: ${{ secrets.FIREWOOD_AVALANCHEGO_GITHUB_TOKEN }} - name: Store benchmark results uses: benchmark-action/github-action-benchmark@v1 @@ -205,14 +150,11 @@ jobs: output-file-path: ./results/benchmark-output.txt github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true - # Store benchmark data in gh-pages branch gh-pages-branch: gh-pages - benchmark-data-dir-path: dev/bench + benchmark-data-dir-path: ${{ steps.download.outputs.data-dir }} # Don't fail the workflow if there's an issue with benchmark storage fail-on-alert: false comment-on-alert: false - # Add metadata - alert-comment-cc-users: '' - name: Summary if: always() @@ -232,17 +174,15 @@ jobs: echo "- State source: \`${{ inputs.current-state-dir-src }}\`" >> $GITHUB_STEP_SUMMARY fi - echo "- Firewood commit: \`${{ steps.commit.outputs.sha }}\`" >> $GITHUB_STEP_SUMMARY + echo "- Firewood commit: \`${{ steps.trigger.outputs.firewood_commit }}\`" >> $GITHUB_STEP_SUMMARY echo "- AvalancheGo ref: \`${{ inputs.avalanchego-ref }}\`" >> $GITHUB_STEP_SUMMARY echo "- Runner: \`${{ inputs.runner }}\`" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - if [ "${{ steps.wait.outputs.conclusion }}" = "success" ]; then - echo "**Performance:** ${{ steps.download.outputs.performance }}" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY + if [ "${{ steps.download.outcome }}" = "success" ]; then echo "**Links:**" >> $GITHUB_STEP_SUMMARY echo "- [AvalancheGo Workflow](${{ steps.trigger.outputs.run_url }})" >> $GITHUB_STEP_SUMMARY - echo "- [Performance Trends](https://ava-labs.github.io/firewood/dev/bench/)" >> $GITHUB_STEP_SUMMARY + echo "- [Performance Trends](https://ava-labs.github.io/firewood/${{ steps.download.outputs.data-dir }}/)" >> $GITHUB_STEP_SUMMARY else echo "**Status:** Failed" >> $GITHUB_STEP_SUMMARY echo "Check [workflow logs](${{ steps.trigger.outputs.run_url }}) for details." >> $GITHUB_STEP_SUMMARY From 32fd06f1d86800c1270fe1e67e1ec298f0632a37 Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Tue, 2 Dec 2025 23:39:43 +0400 Subject: [PATCH 06/40] ci: lint --- .github/workflows/track-performance.yml | 38 ++++++++++++++++++------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/.github/workflows/track-performance.yml b/.github/workflows/track-performance.yml index 92fab2c59..34b72c760 100644 --- a/.github/workflows/track-performance.yml +++ b/.github/workflows/track-performance.yml @@ -3,11 +3,14 @@ name: Track Performance on: workflow_dispatch: inputs: - firewood-commit: + firewood: description: 'Firewood commit/branch/tag to test (leave empty for current HEAD)' default: '' - avalanchego-ref: - description: 'AvalancheGo ref to test against' + libevm: + description: 'libevm commit/branch/tag to test (leave empty to skip)' + default: '' + avalanchego: + description: 'AvalancheGo commit/branch/tag to test against' default: 'master' task: description: 'Predefined task (leave empty to use custom parameters below)' @@ -39,7 +42,7 @@ on: - avago-runner-m6i-4xlarge-ebs-fast jobs: - trigger-and-collect: + c-chain-benchmark: runs-on: ubuntu-latest permissions: contents: write # Required for github-action-benchmark to push to gh-pages @@ -70,10 +73,18 @@ jobs: run: | TASK="${{ inputs.task }}" RUNNER="${{ inputs.runner }}" - AVAGO_REF="${{ inputs.avalanchego-ref }}" - FIREWOOD_COMMIT="${{ inputs.firewood-commit || github.sha }}" + AVAGO_REF="${{ inputs.avalanchego }}" + FIREWOOD_COMMIT="${{ inputs.firewood || github.sha }}" + + echo "firewood=$FIREWOOD_COMMIT" >> "$GITHUB_OUTPUT" - echo "firewood_commit=$FIREWOOD_COMMIT" >> "$GITHUB_OUTPUT" + LIBEVM_COMMIT="${{ inputs.libevm }}" + + # Build optional libevm flag + LIBEVM_FLAG="" + if [ -n "$LIBEVM_COMMIT" ]; then + LIBEVM_FLAG="-f libevm-commit=$LIBEVM_COMMIT" + fi if [ -n "$TASK" ]; then gh workflow run "Firewood Reexecution Benchmark" \ @@ -81,7 +92,8 @@ jobs: --ref "$AVAGO_REF" \ -f firewood-commit="$FIREWOOD_COMMIT" \ -f task="$TASK" \ - -f runner="$RUNNER" + -f runner="$RUNNER" \ + $LIBEVM_FLAG else gh workflow run "Firewood Reexecution Benchmark" \ --repo ava-labs/avalanchego \ @@ -92,7 +104,8 @@ jobs: -f end-block="${{ inputs.end-block }}" \ -f block-dir-src="${{ inputs.block-dir-src }}" \ -f current-state-dir-src="${{ inputs.current-state-dir-src }}" \ - -f runner="$RUNNER" + -f runner="$RUNNER" \ + $LIBEVM_FLAG fi sleep 10 @@ -174,8 +187,11 @@ jobs: echo "- State source: \`${{ inputs.current-state-dir-src }}\`" >> $GITHUB_STEP_SUMMARY fi - echo "- Firewood commit: \`${{ steps.trigger.outputs.firewood_commit }}\`" >> $GITHUB_STEP_SUMMARY - echo "- AvalancheGo ref: \`${{ inputs.avalanchego-ref }}\`" >> $GITHUB_STEP_SUMMARY + echo "- Firewood: \`${{ steps.trigger.outputs.firewood }}\`" >> $GITHUB_STEP_SUMMARY + if [ -n "${{ inputs.libevm }}" ]; then + echo "- libevm: \`${{ inputs.libevm }}\`" >> $GITHUB_STEP_SUMMARY + fi + echo "- AvalancheGo: \`${{ inputs.avalanchego }}\`" >> $GITHUB_STEP_SUMMARY echo "- Runner: \`${{ inputs.runner }}\`" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY From 7a750215580d41360f3aaededa07d727041b8b14 Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Tue, 2 Dec 2025 23:43:19 +0400 Subject: [PATCH 07/40] fix --- .github/workflows/track-performance.yml | 30 ++++++++++--------------- 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/.github/workflows/track-performance.yml b/.github/workflows/track-performance.yml index 34b72c760..9071ac291 100644 --- a/.github/workflows/track-performance.yml +++ b/.github/workflows/track-performance.yml @@ -71,40 +71,34 @@ jobs: - name: Trigger AvalancheGo benchmark id: trigger run: | - TASK="${{ inputs.task }}" - RUNNER="${{ inputs.runner }}" - AVAGO_REF="${{ inputs.avalanchego }}" - FIREWOOD_COMMIT="${{ inputs.firewood || github.sha }}" - - echo "firewood=$FIREWOOD_COMMIT" >> "$GITHUB_OUTPUT" - - LIBEVM_COMMIT="${{ inputs.libevm }}" + FIREWOOD="${{ inputs.firewood || github.sha }}" + echo "firewood=$FIREWOOD" >> "$GITHUB_OUTPUT" # Build optional libevm flag LIBEVM_FLAG="" - if [ -n "$LIBEVM_COMMIT" ]; then - LIBEVM_FLAG="-f libevm-commit=$LIBEVM_COMMIT" + if [ -n "${{ inputs.libevm }}" ]; then + LIBEVM_FLAG="-f libevm=${{ inputs.libevm }}" fi - if [ -n "$TASK" ]; then + if [ -n "${{ inputs.task }}" ]; then gh workflow run "Firewood Reexecution Benchmark" \ --repo ava-labs/avalanchego \ - --ref "$AVAGO_REF" \ - -f firewood-commit="$FIREWOOD_COMMIT" \ - -f task="$TASK" \ - -f runner="$RUNNER" \ + --ref "${{ inputs.avalanchego }}" \ + -f firewood="$FIREWOOD" \ + -f task="${{ inputs.task }}" \ + -f runner="${{ inputs.runner }}" \ $LIBEVM_FLAG else gh workflow run "Firewood Reexecution Benchmark" \ --repo ava-labs/avalanchego \ - --ref "$AVAGO_REF" \ - -f firewood-commit="$FIREWOOD_COMMIT" \ + --ref "${{ inputs.avalanchego }}" \ + -f firewood="$FIREWOOD" \ -f config="${{ inputs.config }}" \ -f start-block="${{ inputs.start-block }}" \ -f end-block="${{ inputs.end-block }}" \ -f block-dir-src="${{ inputs.block-dir-src }}" \ -f current-state-dir-src="${{ inputs.current-state-dir-src }}" \ - -f runner="$RUNNER" \ + -f runner="${{ inputs.runner }}" \ $LIBEVM_FLAG fi From e35fa5d9dc32988bb491045052b0004b69e151ef Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Wed, 3 Dec 2025 00:31:54 +0400 Subject: [PATCH 08/40] docs --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e3b1befac..954e5e3f5 100644 --- a/README.md +++ b/README.md @@ -81,14 +81,13 @@ Firewood provides comprehensive metrics for monitoring database performance, res ## Performance Tracking Firewood tracks its performance over time by running C-Chain reexecution benchmarks in AvalancheGo. This allows us to: -- Monitor performance across commits and releases + +- Monitor performance across commits and releases (a/b comparison tests) - Catch performance regressions early - Validate optimizations against real-world blockchain workloads Performance data is automatically collected and published to [GitHub Pages](https://ava-labs.github.io/firewood/dev/bench/). -For information on running benchmarks and interpreting results, see [docs/PERFORMANCE_TRACKING.md](docs/PERFORMANCE_TRACKING.md). - ## Build In order to build firewood, the following dependencies must be installed: From 18f4035e5d81cf21e6c93d04f46416f881334a00 Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Thu, 4 Dec 2025 20:54:43 +0400 Subject: [PATCH 09/40] ci: push performance to benchmark data --- .github/workflows/gh-pages.yaml | 8 ++++++++ .github/workflows/track-performance.yml | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/gh-pages.yaml b/.github/workflows/gh-pages.yaml index b767da48d..efe0393f1 100644 --- a/.github/workflows/gh-pages.yaml +++ b/.github/workflows/gh-pages.yaml @@ -30,6 +30,14 @@ jobs: run: | cp -rv target/doc/* ./_site cp -rv docs/assets ./_site + - name: Include benchmark data + run: | + git fetch origin benchmark-data || true + if git rev-parse origin/benchmark-data >/dev/null 2>&1; then + git checkout origin/benchmark-data -- dev bench 2>/dev/null || true + cp -rv dev _site/ 2>/dev/null || true + cp -rv bench _site/ 2>/dev/null || true + fi - uses: actions/upload-artifact@v4 with: name: pages diff --git a/.github/workflows/track-performance.yml b/.github/workflows/track-performance.yml index 9071ac291..b62dd81bf 100644 --- a/.github/workflows/track-performance.yml +++ b/.github/workflows/track-performance.yml @@ -157,7 +157,7 @@ jobs: output-file-path: ./results/benchmark-output.txt github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true - gh-pages-branch: gh-pages + gh-pages-branch: benchmark-data benchmark-data-dir-path: ${{ steps.download.outputs.data-dir }} # Don't fail the workflow if there's an issue with benchmark storage fail-on-alert: false From 73fc78193b29ff8bb67d352678058e92cf02f314 Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Thu, 4 Dec 2025 21:15:38 +0400 Subject: [PATCH 10/40] ci(perf): add benchmark workflow with nix-based just commands --- .github/workflows/track-performance.yml | 68 +++++++++--------- ffi/flake.nix | 6 ++ justfile | 93 +++++++++++++++++++++++++ 3 files changed, 131 insertions(+), 36 deletions(-) diff --git a/.github/workflows/track-performance.yml b/.github/workflows/track-performance.yml index b62dd81bf..8050ddb0e 100644 --- a/.github/workflows/track-performance.yml +++ b/.github/workflows/track-performance.yml @@ -52,6 +52,11 @@ jobs: with: fetch-depth: 0 # Needed for github-action-benchmark + - name: Install Nix + uses: cachix/install-nix-action@02a151ada4993995686f9ed4f1be7cfbb229e56f # v31 + with: + github_access_token: ${{ secrets.GITHUB_TOKEN }} + - name: Validate inputs run: | if [ -z "${{ inputs.task }}" ]; then @@ -70,25 +75,21 @@ jobs: - name: Trigger AvalancheGo benchmark id: trigger + shell: nix develop ./ffi --command bash {0} run: | FIREWOOD="${{ inputs.firewood || github.sha }}" echo "firewood=$FIREWOOD" >> "$GITHUB_OUTPUT" - # Build optional libevm flag - LIBEVM_FLAG="" - if [ -n "${{ inputs.libevm }}" ]; then - LIBEVM_FLAG="-f libevm=${{ inputs.libevm }}" - fi - if [ -n "${{ inputs.task }}" ]; then - gh workflow run "Firewood Reexecution Benchmark" \ - --repo ava-labs/avalanchego \ - --ref "${{ inputs.avalanchego }}" \ - -f firewood="$FIREWOOD" \ - -f task="${{ inputs.task }}" \ - -f runner="${{ inputs.runner }}" \ - $LIBEVM_FLAG + # Task-based mode: use just command + RUN_ID=$(just benchmark-trigger "$FIREWOOD" "${{ inputs.avalanchego }}" "${{ inputs.task }}" "${{ inputs.runner }}" "${{ inputs.libevm }}") else + # Granular mode: use direct gh with custom params + LIBEVM_FLAG="" + if [ -n "${{ inputs.libevm }}" ]; then + LIBEVM_FLAG="-f libevm=${{ inputs.libevm }}" + fi + gh workflow run "Firewood Reexecution Benchmark" \ --repo ava-labs/avalanchego \ --ref "${{ inputs.avalanchego }}" \ @@ -100,20 +101,20 @@ jobs: -f current-state-dir-src="${{ inputs.current-state-dir-src }}" \ -f runner="${{ inputs.runner }}" \ $LIBEVM_FLAG - fi - - sleep 10 - - RUN_ID=$(gh run list \ - --repo ava-labs/avalanchego \ - --workflow "Firewood Reexecution Benchmark" \ - --limit 5 \ - --json databaseId,createdAt \ - --jq '[.[] | select(.createdAt | fromdateiso8601 > (now - 60))] | .[0].databaseId') - - if [ -z "$RUN_ID" ] || [ "$RUN_ID" = "null" ]; then - echo "Could not find triggered workflow run" - exit 1 + + sleep 10 + + RUN_ID=$(gh run list \ + --repo ava-labs/avalanchego \ + --workflow "Firewood Reexecution Benchmark" \ + --limit 5 \ + --json databaseId,createdAt \ + --jq '[.[] | select(.createdAt | fromdateiso8601 > (now - 60))] | .[0].databaseId') + + if [ -z "$RUN_ID" ] || [ "$RUN_ID" = "null" ]; then + echo "Could not find triggered workflow run" + exit 1 + fi fi echo "run_id=$RUN_ID" >> "$GITHUB_OUTPUT" @@ -122,22 +123,17 @@ jobs: GH_TOKEN: ${{ secrets.FIREWOOD_AVALANCHEGO_GITHUB_TOKEN }} - name: Wait for benchmark completion - run: gh run watch "${{ steps.trigger.outputs.run_id }}" --repo ava-labs/avalanchego --exit-status + shell: nix develop ./ffi --command bash {0} + run: just benchmark-wait "${{ steps.trigger.outputs.run_id }}" env: GH_TOKEN: ${{ secrets.FIREWOOD_AVALANCHEGO_GITHUB_TOKEN }} timeout-minutes: 60 - name: Download benchmark results id: download + shell: nix develop ./ffi --command bash {0} run: | - mkdir -p ./results - - gh run download "${{ steps.trigger.outputs.run_id }}" \ - --repo ava-labs/avalanchego \ - --name benchmark-output \ - --dir ./results - - cat ./results/benchmark-output.txt + just benchmark-download "${{ steps.trigger.outputs.run_id }}" # Determine target dashboard if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then diff --git a/ffi/flake.nix b/ffi/flake.nix index dab896af8..a8df692b7 100644 --- a/ffi/flake.nix +++ b/ffi/flake.nix @@ -121,11 +121,17 @@ program = "${pkgs.just}/bin/just"; }; + apps.gh = { + type = "app"; + program = "${pkgs.gh}/bin/gh"; + }; + devShells.default = craneLib.devShell { inputsFrom = [ firewood-ffi ]; packages = with pkgs; [ firewood-ffi + gh go jq just diff --git a/justfile b/justfile index d0885778f..18934da0c 100644 --- a/justfile +++ b/justfile @@ -138,3 +138,96 @@ update-ffi-flake: check-nix echo "checking for a consistent golang verion" ../scripts/run-just.sh check-golang-version + +# Trigger Reexecution Benchmark +# Usage: just benchmark-trigger [libevm] +benchmark-trigger firewood avalanchego task runner libevm="": check-nix + #!/usr/bin/env bash + set -euo pipefail + + GH="nix run ./ffi#gh --" + + LIBEVM_FLAG="" + if [ -n "{{ libevm }}" ]; then + LIBEVM_FLAG="-f libevm={{ libevm }}" + fi + + $GH workflow run "Firewood Reexecution Benchmark" \ + --repo ava-labs/avalanchego \ + --ref "{{ avalanchego }}" \ + -f firewood="{{ firewood }}" \ + -f task="{{ task }}" \ + -f runner="{{ runner }}" \ + $LIBEVM_FLAG + + sleep 10 + + RUN_ID=$($GH run list \ + --repo ava-labs/avalanchego \ + --workflow "Firewood Reexecution Benchmark" \ + --limit 5 \ + --json databaseId,createdAt \ + --jq '[.[] | select(.createdAt | fromdateiso8601 > (now - 60))] | .[0].databaseId') + + if [ -z "$RUN_ID" ] || [ "$RUN_ID" = "null" ]; then + echo "Error: Could not find triggered workflow run" >&2 + exit 1 + fi + + echo "$RUN_ID" + +# Wait for reexecution benchmark run to complete +# Usage: just benchmark-wait +benchmark-wait run_id: check-nix + #!/usr/bin/env bash + set -euo pipefail + nix run ./ffi#gh -- run watch "{{ run_id }}" --repo ava-labs/avalanchego --exit-status + +# Download benchmark results +# Usage: just benchmark-download +benchmark-download run_id: check-nix + #!/usr/bin/env bash + set -euo pipefail + mkdir -p ./results + nix run ./ffi#gh -- run download "{{ run_id }}" \ + --repo ava-labs/avalanchego \ + --name benchmark-output \ + --dir ./results + cat ./results/benchmark-output.txt + +# Run full benchmark: trigger, wait, download (composes the above) +# Usage: just benchmark [firewood] [avalanchego] [task] [runner] [libevm] +benchmark firewood="HEAD" avalanchego="master" task="c-chain-reexecution-firewood-101-250k" runner="avalanche-avalanchego-runner-2ti" libevm="": check-nix + #!/usr/bin/env bash + set -euo pipefail + + FIREWOOD="{{ firewood }}" + if [[ "$FIREWOOD" == "HEAD" ]]; then + FIREWOOD=$(git rev-parse HEAD) + fi + + echo "Firewood: $FIREWOOD" + echo "AvalancheGo: {{ avalanchego }}" + echo "Task: {{ task }}" + echo "Runner: {{ runner }}" + if [ -n "{{ libevm }}" ]; then + echo "LibEVM: {{ libevm }}" + fi + + echo "Triggering reexecution benchmark in AvalancheGo..." + RUN_ID=$(just benchmark-trigger "$FIREWOOD" "{{ avalanchego }}" "{{ task }}" "{{ runner }}" "{{ libevm }}") + echo " Run ID: $RUN_ID" + echo " URL: https://github.com/ava-labs/avalanchego/actions/runs/$RUN_ID \n" + + echo "Waiting for benchmark completion..." + just benchmark-wait "$RUN_ID \n" + + echo "Downloading results..." + just benchmark-download "$RUN_ID \n" + echo "Results saved to: ./results/benchmark-output.txt" + +# List recent AvalancheGo benchmark runs +benchmark-list: check-nix + #!/usr/bin/env bash + set -euo pipefail + nix run ./ffi#gh -- run list --repo ava-labs/avalanchego --workflow="Firewood Reexecution Benchmark" --limit 10 From 668cef309346d3e5e44276c26e43fd0b05bf74cd Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Thu, 4 Dec 2025 21:21:20 +0400 Subject: [PATCH 11/40] docs --- METRICS.md | 43 +++++++++++++++++++++++++++++++++++++++++++ README.md | 10 ---------- 2 files changed, 43 insertions(+), 10 deletions(-) diff --git a/METRICS.md b/METRICS.md index fcac0c31a..cedfb0285 100644 --- a/METRICS.md +++ b/METRICS.md @@ -284,3 +284,46 @@ rate(firewood_space_from_end[5m]) rate(firewood_proposal_commit{success="false"}[5m]) / rate(firewood_proposal_commit[5m]) ``` + +## Performance Tracking + +Firewood tracks its performance over time by running C-Chain reexecution benchmarks in AvalancheGo. This allows us to: + +- Monitor performance across commits and releases +- Catch performance regressions early +- Validate optimizations against real-world blockchain workloads + +Performance data is collected via the `Track Performance` workflow and published to GitHub Pages. + +### Running Benchmarks Locally + +Benchmarks can be triggered locally using just commands (requires nix): + +```bash +# Run full benchmark: trigger, wait, download results +just benchmark + +# With specific versions +just benchmark v0.0.15 master c-chain-reexecution-firewood-101-250k avalanche-avalanchego-runner-2ti + +# With libevm +just benchmark v0.0.15 master c-chain-reexecution-firewood-101-250k avalanche-avalanchego-runner-2ti v1.0.0 +``` + +### Composable Commands + +Individual steps can be run separately: + +```bash +# Trigger benchmark, returns run_id +just benchmark-trigger [libevm] + +# Wait for a specific run to complete +just benchmark-wait + +# Download results from a specific run +just benchmark-download + +# List recent benchmark runs +just benchmark-list +``` diff --git a/README.md b/README.md index 954e5e3f5..e06489714 100644 --- a/README.md +++ b/README.md @@ -78,16 +78,6 @@ as well as carefully managing the free list during the creation and expiration o Firewood provides comprehensive metrics for monitoring database performance, resource utilization, and operational characteristics. For detailed information about all available metrics, how to enable them, and how to interpret them, see [METRICS.md](METRICS.md). -## Performance Tracking - -Firewood tracks its performance over time by running C-Chain reexecution benchmarks in AvalancheGo. This allows us to: - -- Monitor performance across commits and releases (a/b comparison tests) -- Catch performance regressions early -- Validate optimizations against real-world blockchain workloads - -Performance data is automatically collected and published to [GitHub Pages](https://ava-labs.github.io/firewood/dev/bench/). - ## Build In order to build firewood, the following dependencies must be installed: From 6e65816adc2de118672a134520504e39868ed085 Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Thu, 4 Dec 2025 22:16:27 +0400 Subject: [PATCH 12/40] chore: remove "\n" that got URL encoded --- justfile | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/justfile b/justfile index 18934da0c..eec57ca9d 100644 --- a/justfile +++ b/justfile @@ -217,13 +217,16 @@ benchmark firewood="HEAD" avalanchego="master" task="c-chain-reexecution-firewoo echo "Triggering reexecution benchmark in AvalancheGo..." RUN_ID=$(just benchmark-trigger "$FIREWOOD" "{{ avalanchego }}" "{{ task }}" "{{ runner }}" "{{ libevm }}") echo " Run ID: $RUN_ID" - echo " URL: https://github.com/ava-labs/avalanchego/actions/runs/$RUN_ID \n" + echo " URL: https://github.com/ava-labs/avalanchego/actions/runs/$RUN_ID" + echo "" echo "Waiting for benchmark completion..." - just benchmark-wait "$RUN_ID \n" + just benchmark-wait "$RUN_ID" + echo "" echo "Downloading results..." - just benchmark-download "$RUN_ID \n" + just benchmark-download "$RUN_ID" + echo "" echo "Results saved to: ./results/benchmark-output.txt" # List recent AvalancheGo benchmark runs From 1e2bdd3d2699859fca3dbf4007b03236af56b631 Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Thu, 4 Dec 2025 22:19:02 +0400 Subject: [PATCH 13/40] docs --- METRICS.md | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/METRICS.md b/METRICS.md index cedfb0285..46bb001a0 100644 --- a/METRICS.md +++ b/METRICS.md @@ -300,29 +300,52 @@ Performance data is collected via the `Track Performance` workflow and published Benchmarks can be triggered locally using just commands (requires nix): ```bash -# Run full benchmark: trigger, wait, download results +# Run with defaults (current HEAD, master, default task/runner) just benchmark -# With specific versions -just benchmark v0.0.15 master c-chain-reexecution-firewood-101-250k avalanche-avalanchego-runner-2ti +# With named parameters +just benchmark \ + firewood=v0.0.15 \ + avalanchego=master \ + task=c-chain-reexecution-firewood-101-250k \ + runner=avalanche-avalanchego-runner-2ti # With libevm -just benchmark v0.0.15 master c-chain-reexecution-firewood-101-250k avalanche-avalanchego-runner-2ti v1.0.0 +just benchmark \ + firewood=v0.0.15 \ + avalanchego=master \ + task=c-chain-reexecution-firewood-101-250k \ + runner=avalanche-avalanchego-runner-2ti \ + libevm=v1.0.0 ``` +**Parameters:** + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `firewood` | Firewood commit/branch/tag | `HEAD` | +| `avalanchego` | AvalancheGo commit/branch/tag | `master` | +| `task` | Benchmark task name | `c-chain-reexecution-firewood-101-250k` | +| `runner` | GitHub Actions runner | `avalanche-avalanchego-runner-2ti` | +| `libevm` | libevm commit/branch/tag (optional) | `` | + ### Composable Commands Individual steps can be run separately: ```bash # Trigger benchmark, returns run_id -just benchmark-trigger [libevm] +just benchmark-trigger \ + firewood=v0.0.15 \ + avalanchego=master \ + task=c-chain-reexecution-firewood-101-250k \ + runner=avalanche-avalanchego-runner-2ti # Wait for a specific run to complete -just benchmark-wait +just benchmark-wait run_id=19938272417 # Download results from a specific run -just benchmark-download +just benchmark-download run_id=19938272417 # List recent benchmark runs just benchmark-list From 01ee24f97b501862c74552d2995485fc42fada15 Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Mon, 8 Dec 2025 18:56:19 +0400 Subject: [PATCH 14/40] ci(track-performance): remove `if: always` --- .github/workflows/track-performance.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/track-performance.yml b/.github/workflows/track-performance.yml index 8050ddb0e..f22234c39 100644 --- a/.github/workflows/track-performance.yml +++ b/.github/workflows/track-performance.yml @@ -160,7 +160,6 @@ jobs: comment-on-alert: false - name: Summary - if: always() run: | echo "## Firewood Performance Benchmark Results" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY From 19d921ee9a0d3b030f44087fed7aa5566f730d2c Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Mon, 8 Dec 2025 19:29:49 +0400 Subject: [PATCH 15/40] address PR --- .github/workflows/track-performance.yml | 14 ++-- METRICS.md | 64 +++++++++++++----- justfile | 87 +++++++++++++------------ 3 files changed, 101 insertions(+), 64 deletions(-) diff --git a/.github/workflows/track-performance.yml b/.github/workflows/track-performance.yml index f22234c39..4c2cbc11b 100644 --- a/.github/workflows/track-performance.yml +++ b/.github/workflows/track-performance.yml @@ -4,7 +4,7 @@ on: workflow_dispatch: inputs: firewood: - description: 'Firewood commit/branch/tag to test (leave empty for current HEAD)' + description: 'Firewood commit/branch/tag to test (leave empty to use the commit that triggered the workflow)' default: '' libevm: description: 'libevm commit/branch/tag to test (leave empty to skip)' @@ -82,7 +82,7 @@ jobs: if [ -n "${{ inputs.task }}" ]; then # Task-based mode: use just command - RUN_ID=$(just benchmark-trigger "$FIREWOOD" "${{ inputs.avalanchego }}" "${{ inputs.task }}" "${{ inputs.runner }}" "${{ inputs.libevm }}") + RUN_ID=$(just trigger-benchmark "$FIREWOOD" "${{ inputs.avalanchego }}" "${{ inputs.task }}" "${{ inputs.runner }}" "${{ inputs.libevm }}") else # Granular mode: use direct gh with custom params LIBEVM_FLAG="" @@ -124,7 +124,7 @@ jobs: - name: Wait for benchmark completion shell: nix develop ./ffi --command bash {0} - run: just benchmark-wait "${{ steps.trigger.outputs.run_id }}" + run: just wait-benchmark "${{ steps.trigger.outputs.run_id }}" env: GH_TOKEN: ${{ secrets.FIREWOOD_AVALANCHEGO_GITHUB_TOKEN }} timeout-minutes: 60 @@ -133,7 +133,7 @@ jobs: id: download shell: nix develop ./ffi --command bash {0} run: | - just benchmark-download "${{ steps.trigger.outputs.run_id }}" + just download-benchmark-results "${{ steps.trigger.outputs.run_id }}" # Determine target dashboard if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then @@ -146,6 +146,8 @@ jobs: GH_TOKEN: ${{ secrets.FIREWOOD_AVALANCHEGO_GITHUB_TOKEN }} - name: Store benchmark results + id: store + continue-on-error: true uses: benchmark-action/github-action-benchmark@v1 with: name: C-Chain Reexecution Performance @@ -155,12 +157,14 @@ jobs: auto-push: true gh-pages-branch: benchmark-data benchmark-data-dir-path: ${{ steps.download.outputs.data-dir }} - # Don't fail the workflow if there's an issue with benchmark storage fail-on-alert: false comment-on-alert: false - name: Summary run: | + if [ "${{ steps.store.outcome }}" == "failure" ]; then + echo "::warning::Benchmark storage failed - results were not saved to GitHub Pages" + fi echo "## Firewood Performance Benchmark Results" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "**Configuration:**" >> $GITHUB_STEP_SUMMARY diff --git a/METRICS.md b/METRICS.md index 46bb001a0..80f5911a6 100644 --- a/METRICS.md +++ b/METRICS.md @@ -287,7 +287,11 @@ rate(firewood_proposal_commit[5m]) ## Performance Tracking -Firewood tracks its performance over time by running C-Chain reexecution benchmarks in AvalancheGo. This allows us to: +Firewood tracks its performance over time by running [C-Chain reexecution benchmarks](https://github.com/ava-labs/avalanchego/blob/master/tests/reexecute/c/README.md) in AvalancheGo. These benchmarks re-execute historical mainnet C-Chain blocks against a state snapshot, measuring throughput in mgas/s (million gas per second). + +By default, the benchmark processes ~250,000 blocks (101 → 250k) and takes approximately 7 minutes on self-hosted runners. + +This allows us to: - Monitor performance across commits and releases - Catch performance regressions early @@ -297,7 +301,45 @@ Performance data is collected via the `Track Performance` workflow and published ### Running Benchmarks Locally -Benchmarks can be triggered locally using just commands (requires nix): +Benchmarks can be triggered locally using `just benchmark` (requires nix). + +**Parameters:** + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `firewood` | Firewood commit/branch/tag | `HEAD` (currently checked out commit) | +| `avalanchego` | AvalancheGo commit/branch/tag | `master` | +| `task` | Benchmark task name | `c-chain-reexecution-firewood-101-250k` | +| `runner` | GitHub Actions runner (list can be found [here](https://github.com/ava-labs/devops-argocd/blob/main/base/system/actions-runners/action-runner.yaml) -> ctrl+f "runnerScaleSetName") | `avalanche-avalanchego-runner-2ti` | +| `libevm` | libevm commit/branch/tag (optional) | `` | + +**Task Names:** + +Predefined tasks follow the pattern `c-chain-reexecution-{db}-{start}-{end}` where: + +- `{db}` - database backend (e.g., `firewood`, `hashdb`) +- `{start}` - start block number +- `{end}` - end block (e.g., `250k` for 250,000 or `1m` for 1,000,000) + +To find available tasks, run in the [AvalancheGo](https://github.com/ava-labs/avalanchego/) repo: + +```bash +./scripts/run_task.sh --list | grep "c-chain-reexecution" +``` + +See the [AvalancheGo C-Chain benchmark docs](https://github.com/ava-labs/avalanchego/blob/master/tests/reexecute/c/README.md) for the full list and details. + +**Granular Inputs:** + +If no predefined task fits your needs, leave `task` empty and use granular inputs via the CI workflow: + +- `config` - VM config (e.g., `archive`, `firewood`) +- `start-block` - first block to execute +- `end-block` - last block to execute +- `block-dir-src` - S3 source for block data +- `current-state-dir-src` - S3 source for state snapshot + +**Examples:** ```bash # Run with defaults (current HEAD, master, default task/runner) @@ -319,34 +361,24 @@ just benchmark \ libevm=v1.0.0 ``` -**Parameters:** - -| Parameter | Description | Default | -|-----------|-------------|---------| -| `firewood` | Firewood commit/branch/tag | `HEAD` | -| `avalanchego` | AvalancheGo commit/branch/tag | `master` | -| `task` | Benchmark task name | `c-chain-reexecution-firewood-101-250k` | -| `runner` | GitHub Actions runner | `avalanche-avalanchego-runner-2ti` | -| `libevm` | libevm commit/branch/tag (optional) | `` | - ### Composable Commands Individual steps can be run separately: ```bash # Trigger benchmark, returns run_id -just benchmark-trigger \ +just trigger-benchmark \ firewood=v0.0.15 \ avalanchego=master \ task=c-chain-reexecution-firewood-101-250k \ runner=avalanche-avalanchego-runner-2ti # Wait for a specific run to complete -just benchmark-wait run_id=19938272417 +just wait-benchmark run_id=19938272417 # Download results from a specific run -just benchmark-download run_id=19938272417 +just download-benchmark-results run_id=19938272417 # List recent benchmark runs -just benchmark-list +just list-benchmarks ``` diff --git a/justfile b/justfile index eec57ca9d..55aa64846 100644 --- a/justfile +++ b/justfile @@ -140,10 +140,9 @@ update-ffi-flake: check-nix ../scripts/run-just.sh check-golang-version # Trigger Reexecution Benchmark -# Usage: just benchmark-trigger [libevm] -benchmark-trigger firewood avalanchego task runner libevm="": check-nix - #!/usr/bin/env bash - set -euo pipefail +# Usage: just trigger-benchmark [libevm] +trigger-benchmark firewood avalanchego task runner libevm="": check-nix + #!/usr/bin/env -S bash -euo pipefail GH="nix run ./ffi#gh --" @@ -160,34 +159,35 @@ benchmark-trigger firewood avalanchego task runner libevm="": check-nix -f runner="{{ runner }}" \ $LIBEVM_FLAG - sleep 10 - - RUN_ID=$($GH run list \ - --repo ava-labs/avalanchego \ - --workflow "Firewood Reexecution Benchmark" \ - --limit 5 \ - --json databaseId,createdAt \ - --jq '[.[] | select(.createdAt | fromdateiso8601 > (now - 60))] | .[0].databaseId') - - if [ -z "$RUN_ID" ] || [ "$RUN_ID" = "null" ]; then - echo "Error: Could not find triggered workflow run" >&2 - exit 1 - fi + # Wait for GitHub to register the run (retry up to 3min) + for i in {1..18}; do + sleep 10 + RUN_ID=$($GH run list \ + --repo ava-labs/avalanchego \ + --workflow "Firewood Reexecution Benchmark" \ + --limit 5 \ + --json databaseId,createdAt \ + --jq '[.[] | select(.createdAt | fromdateiso8601 > (now - 180))] | .[0].databaseId') + + if [ -n "$RUN_ID" ] && [ "$RUN_ID" != "null" ]; then + echo "$RUN_ID" + exit 0 + fi + done - echo "$RUN_ID" + echo "Error: Could not find triggered workflow run after 3min" >&2 + exit 1 -# Wait for reexecution benchmark run to complete -# Usage: just benchmark-wait -benchmark-wait run_id: check-nix - #!/usr/bin/env bash - set -euo pipefail - nix run ./ffi#gh -- run watch "{{ run_id }}" --repo ava-labs/avalanchego --exit-status +# Wait for reexecution benchmark run to complete (2h timeout) +# Usage: just wait-benchmark +wait-benchmark run_id: check-nix + #!/usr/bin/env -S bash -euo pipefail + timeout -k 1m 2h nix run ./ffi#gh -- run watch "{{ run_id }}" --repo ava-labs/avalanchego --exit-status # Download benchmark results -# Usage: just benchmark-download -benchmark-download run_id: check-nix - #!/usr/bin/env bash - set -euo pipefail +# Usage: just download-benchmark-results +download-benchmark-results run_id: check-nix + #!/usr/bin/env -S bash -euo pipefail mkdir -p ./results nix run ./ffi#gh -- run download "{{ run_id }}" \ --repo ava-labs/avalanchego \ @@ -198,39 +198,40 @@ benchmark-download run_id: check-nix # Run full benchmark: trigger, wait, download (composes the above) # Usage: just benchmark [firewood] [avalanchego] [task] [runner] [libevm] benchmark firewood="HEAD" avalanchego="master" task="c-chain-reexecution-firewood-101-250k" runner="avalanche-avalanchego-runner-2ti" libevm="": check-nix - #!/usr/bin/env bash - set -euo pipefail + #!/usr/bin/env -S bash -euo pipefail FIREWOOD="{{ firewood }}" if [[ "$FIREWOOD" == "HEAD" ]]; then FIREWOOD=$(git rev-parse HEAD) fi - echo "Firewood: $FIREWOOD" - echo "AvalancheGo: {{ avalanchego }}" - echo "Task: {{ task }}" - echo "Runner: {{ runner }}" + echo "Triggering reexecution benchmark in AvalancheGo with parameters:" + echo " Firewood: $FIREWOOD" + echo " AvalancheGo: {{ avalanchego }}" + echo " Task: {{ task }}" + echo " Runner: {{ runner }}" if [ -n "{{ libevm }}" ]; then - echo "LibEVM: {{ libevm }}" + echo " LibEVM: {{ libevm }}" + else + echo " LibEVM: (not set)" fi + echo "" - echo "Triggering reexecution benchmark in AvalancheGo..." - RUN_ID=$(just benchmark-trigger "$FIREWOOD" "{{ avalanchego }}" "{{ task }}" "{{ runner }}" "{{ libevm }}") + RUN_ID=$(just trigger-benchmark "$FIREWOOD" "{{ avalanchego }}" "{{ task }}" "{{ runner }}" "{{ libevm }}") echo " Run ID: $RUN_ID" - echo " URL: https://github.com/ava-labs/avalanchego/actions/runs/$RUN_ID" + echo " Monitor progress here: https://github.com/ava-labs/avalanchego/actions/runs/$RUN_ID" echo "" echo "Waiting for benchmark completion..." - just benchmark-wait "$RUN_ID" + just wait-benchmark "$RUN_ID" echo "" echo "Downloading results..." - just benchmark-download "$RUN_ID" + just download-benchmark-results "$RUN_ID" echo "" echo "Results saved to: ./results/benchmark-output.txt" # List recent AvalancheGo benchmark runs -benchmark-list: check-nix - #!/usr/bin/env bash - set -euo pipefail +list-benchmarks: check-nix + #!/usr/bin/env -S bash -euo pipefail nix run ./ffi#gh -- run list --repo ava-labs/avalanchego --workflow="Firewood Reexecution Benchmark" --limit 10 From 1131a1d8a14a0fa19fbb96ddcfcb158f5ca6a8ad Mon Sep 17 00:00:00 2001 From: Elvis <43846394+Elvis339@users.noreply.github.com> Date: Mon, 8 Dec 2025 19:31:57 +0400 Subject: [PATCH 16/40] Update .github/workflows/track-performance.yml Co-authored-by: rodrigo <77309055+RodrigoVillar@users.noreply.github.com> Signed-off-by: Elvis <43846394+Elvis339@users.noreply.github.com> --- .github/workflows/track-performance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/track-performance.yml b/.github/workflows/track-performance.yml index 4c2cbc11b..d381f4e4d 100644 --- a/.github/workflows/track-performance.yml +++ b/.github/workflows/track-performance.yml @@ -27,7 +27,7 @@ on: end-block: default: '' block-dir-src: - description: 'Block directory source e.g., cchain-mainnet-blocks-1m-ldb (without S3 path)' + description: 'Block directory source (e.g., cchain-mainnet-blocks-1m-ldb [without S3 path])' default: '' current-state-dir-src: description: 'Current state directory source e.g., cchain-mainnet-blocks-30m-40m-ldb (without S3 path)' From 5e32e9da7e34cc56485d72d1bef0c45e614aaa01 Mon Sep 17 00:00:00 2001 From: Elvis <43846394+Elvis339@users.noreply.github.com> Date: Mon, 8 Dec 2025 19:32:11 +0400 Subject: [PATCH 17/40] Update .github/workflows/track-performance.yml Co-authored-by: rodrigo <77309055+RodrigoVillar@users.noreply.github.com> Signed-off-by: Elvis <43846394+Elvis339@users.noreply.github.com> --- .github/workflows/track-performance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/track-performance.yml b/.github/workflows/track-performance.yml index d381f4e4d..79a35a2f9 100644 --- a/.github/workflows/track-performance.yml +++ b/.github/workflows/track-performance.yml @@ -30,7 +30,7 @@ on: description: 'Block directory source (e.g., cchain-mainnet-blocks-1m-ldb [without S3 path])' default: '' current-state-dir-src: - description: 'Current state directory source e.g., cchain-mainnet-blocks-30m-40m-ldb (without S3 path)' + description: 'Current state directory source (e.g., cchain-mainnet-blocks-30m-40m-ldb [without S3 path])' default: '' runner: description: 'Runner to use in AvalancheGo' From 80082648a0c7d86b2dda9ee3049553a178721d0e Mon Sep 17 00:00:00 2001 From: Elvis <43846394+Elvis339@users.noreply.github.com> Date: Mon, 8 Dec 2025 19:44:46 +0400 Subject: [PATCH 18/40] Update justfile Co-authored-by: Ron Kuris Signed-off-by: Elvis <43846394+Elvis339@users.noreply.github.com> --- justfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/justfile b/justfile index 55aa64846..4cad4667f 100644 --- a/justfile +++ b/justfile @@ -214,6 +214,8 @@ benchmark firewood="HEAD" avalanchego="master" task="c-chain-reexecution-firewoo echo " LibEVM: {{ libevm }}" else echo " LibEVM: (not set)" + else + echo "LibEVM: default" fi echo "" From f2cf5024631365abc2423bec5a486479d9589863 Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Mon, 8 Dec 2025 19:51:15 +0400 Subject: [PATCH 19/40] fix(ci): replace sleep with retry loop, improve justfile commands --- .github/workflows/track-performance.yml | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/.github/workflows/track-performance.yml b/.github/workflows/track-performance.yml index 4c2cbc11b..87f8e4326 100644 --- a/.github/workflows/track-performance.yml +++ b/.github/workflows/track-performance.yml @@ -102,17 +102,23 @@ jobs: -f runner="${{ inputs.runner }}" \ $LIBEVM_FLAG - sleep 10 - - RUN_ID=$(gh run list \ - --repo ava-labs/avalanchego \ - --workflow "Firewood Reexecution Benchmark" \ - --limit 5 \ - --json databaseId,createdAt \ - --jq '[.[] | select(.createdAt | fromdateiso8601 > (now - 60))] | .[0].databaseId') + # Wait for GitHub API to register the workflow run (retry up to 3min) + for i in {1..18}; do + sleep 10 + RUN_ID=$(gh run list \ + --repo ava-labs/avalanchego \ + --workflow "Firewood Reexecution Benchmark" \ + --limit 5 \ + --json databaseId,createdAt \ + --jq '[.[] | select(.createdAt | fromdateiso8601 > (now - 180))] | .[0].databaseId') + + if [ -n "$RUN_ID" ] && [ "$RUN_ID" != "null" ]; then + break + fi + done if [ -z "$RUN_ID" ] || [ "$RUN_ID" = "null" ]; then - echo "Could not find triggered workflow run" + echo "Could not find triggered workflow run after 3min" exit 1 fi fi From ffcb33328351f255b7378e131dbe2ba994d5c191 Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Mon, 8 Dec 2025 20:00:18 +0400 Subject: [PATCH 20/40] lint: descriptive link text --- METRICS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/METRICS.md b/METRICS.md index 80f5911a6..ec78b59f7 100644 --- a/METRICS.md +++ b/METRICS.md @@ -310,7 +310,7 @@ Benchmarks can be triggered locally using `just benchmark` (requires nix). | `firewood` | Firewood commit/branch/tag | `HEAD` (currently checked out commit) | | `avalanchego` | AvalancheGo commit/branch/tag | `master` | | `task` | Benchmark task name | `c-chain-reexecution-firewood-101-250k` | -| `runner` | GitHub Actions runner (list can be found [here](https://github.com/ava-labs/devops-argocd/blob/main/base/system/actions-runners/action-runner.yaml) -> ctrl+f "runnerScaleSetName") | `avalanche-avalanchego-runner-2ti` | +| `runner` | GitHub Actions runner (see [runner config](https://github.com/ava-labs/devops-argocd/blob/main/base/system/actions-runners/action-runner.yaml), search for "runnerScaleSetName") | `avalanche-avalanchego-runner-2ti` | | `libevm` | libevm commit/branch/tag (optional) | `` | **Task Names:** From 02e7e94b79af1aa9f84db1c8b6b8087b94a7cc34 Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Mon, 8 Dec 2025 20:03:51 +0400 Subject: [PATCH 21/40] docs --- METRICS.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/METRICS.md b/METRICS.md index ec78b59f7..06eeb897e 100644 --- a/METRICS.md +++ b/METRICS.md @@ -299,6 +299,15 @@ This allows us to: Performance data is collected via the `Track Performance` workflow and published to GitHub Pages. +### Running Benchmarks from GitHub UI + +The easiest way to trigger a benchmark is via the GitHub Actions UI: + +1. Go to [Actions → Track Performance](https://github.com/ava-labs/firewood/actions/workflows/track-performance.yml) +2. Click "Run workflow" +3. Select parameters from the dropdowns (task, runner) or enter custom values +4. Click "Run workflow" + ### Running Benchmarks Locally Benchmarks can be triggered locally using `just benchmark` (requires nix). @@ -376,7 +385,7 @@ just trigger-benchmark \ # Wait for a specific run to complete just wait-benchmark run_id=19938272417 -# Download results from a specific run +# Download results from a specific run (saves to ./results/) just download-benchmark-results run_id=19938272417 # List recent benchmark runs From 625049e5ac4afd1449ed8135ca51c7c035b5545a Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Tue, 9 Dec 2025 20:05:08 +0400 Subject: [PATCH 22/40] chore: fix syntax error for benchmark command --- justfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/justfile b/justfile index 4cad4667f..55aa64846 100644 --- a/justfile +++ b/justfile @@ -214,8 +214,6 @@ benchmark firewood="HEAD" avalanchego="master" task="c-chain-reexecution-firewoo echo " LibEVM: {{ libevm }}" else echo " LibEVM: (not set)" - else - echo "LibEVM: default" fi echo "" From bb2c97ea1bcf37f52af6cc750096fa494d13b3b6 Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Tue, 30 Dec 2025 19:26:24 +0400 Subject: [PATCH 23/40] ci: update workflow for C-Chain reexecution benchmarks and improve justfile commands --- .github/workflows/track-performance.yml | 125 +++++++++------------- METRICS.md | 91 ++++------------ justfile | 131 +++++++++++------------- 3 files changed, 127 insertions(+), 220 deletions(-) diff --git a/.github/workflows/track-performance.yml b/.github/workflows/track-performance.yml index c97515175..f1f4b0392 100644 --- a/.github/workflows/track-performance.yml +++ b/.github/workflows/track-performance.yml @@ -1,4 +1,6 @@ -name: Track Performance +# Triggers AvalancheGo's C-Chain reexecution benchmark and publishes +# results to GitHub Pages for trend analysis. +name: C-Chain Performance Tracking on: workflow_dispatch: @@ -40,9 +42,12 @@ on: - avalanche-avalanchego-runner-2ti - avago-runner-i4i-4xlarge-local-ssd - avago-runner-m6i-4xlarge-ebs-fast + timeout-minutes: + description: 'Timeout in minutes for the benchmark job' + default: '120' jobs: - c-chain-benchmark: + record-benchmark-to-gh-pages: runs-on: ubuntu-latest permissions: contents: write # Required for github-action-benchmark to push to gh-pages @@ -73,7 +78,7 @@ jobs: fi fi - - name: Trigger AvalancheGo benchmark + - name: Trigger C-Chain reexecution in AvalancheGo id: trigger shell: nix develop ./ffi --command bash {0} run: | @@ -81,46 +86,9 @@ jobs: echo "firewood=$FIREWOOD" >> "$GITHUB_OUTPUT" if [ -n "${{ inputs.task }}" ]; then - # Task-based mode: use just command - RUN_ID=$(just trigger-benchmark "$FIREWOOD" "${{ inputs.avalanchego }}" "${{ inputs.task }}" "${{ inputs.runner }}" "${{ inputs.libevm }}") + RUN_ID=$(just trigger-reexecution "$FIREWOOD" "${{ inputs.avalanchego }}" "${{ inputs.task }}" "${{ inputs.runner }}" "${{ inputs.libevm }}" "${{ inputs.timeout-minutes }}") else - # Granular mode: use direct gh with custom params - LIBEVM_FLAG="" - if [ -n "${{ inputs.libevm }}" ]; then - LIBEVM_FLAG="-f libevm=${{ inputs.libevm }}" - fi - - gh workflow run "Firewood Reexecution Benchmark" \ - --repo ava-labs/avalanchego \ - --ref "${{ inputs.avalanchego }}" \ - -f firewood="$FIREWOOD" \ - -f config="${{ inputs.config }}" \ - -f start-block="${{ inputs.start-block }}" \ - -f end-block="${{ inputs.end-block }}" \ - -f block-dir-src="${{ inputs.block-dir-src }}" \ - -f current-state-dir-src="${{ inputs.current-state-dir-src }}" \ - -f runner="${{ inputs.runner }}" \ - $LIBEVM_FLAG - - # Wait for GitHub API to register the workflow run (retry up to 3min) - for i in {1..18}; do - sleep 10 - RUN_ID=$(gh run list \ - --repo ava-labs/avalanchego \ - --workflow "Firewood Reexecution Benchmark" \ - --limit 5 \ - --json databaseId,createdAt \ - --jq '[.[] | select(.createdAt | fromdateiso8601 > (now - 180))] | .[0].databaseId') - - if [ -n "$RUN_ID" ] && [ "$RUN_ID" != "null" ]; then - break - fi - done - - if [ -z "$RUN_ID" ] || [ "$RUN_ID" = "null" ]; then - echo "Could not find triggered workflow run after 3min" - exit 1 - fi + RUN_ID=$(just trigger-custom-reexecution "$FIREWOOD" "${{ inputs.avalanchego }}" "${{ inputs.config }}" "${{ inputs.start-block }}" "${{ inputs.end-block }}" "${{ inputs.block-dir-src }}" "${{ inputs.current-state-dir-src }}" "${{ inputs.runner }}" "${{ inputs.libevm }}" "${{ inputs.timeout-minutes }}") fi echo "run_id=$RUN_ID" >> "$GITHUB_OUTPUT" @@ -128,18 +96,17 @@ jobs: env: GH_TOKEN: ${{ secrets.FIREWOOD_AVALANCHEGO_GITHUB_TOKEN }} - - name: Wait for benchmark completion + - name: Wait for completion shell: nix develop ./ffi --command bash {0} - run: just wait-benchmark "${{ steps.trigger.outputs.run_id }}" + run: just wait-reexecution "${{ steps.trigger.outputs.run_id }}" env: GH_TOKEN: ${{ secrets.FIREWOOD_AVALANCHEGO_GITHUB_TOKEN }} - timeout-minutes: 60 - - name: Download benchmark results + - name: Download artifacts id: download shell: nix develop ./ffi --command bash {0} run: | - just download-benchmark-results "${{ steps.trigger.outputs.run_id }}" + just download-reexecution-results "${{ steps.trigger.outputs.run_id }}" # Determine target dashboard if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then @@ -171,35 +138,39 @@ jobs: if [ "${{ steps.store.outcome }}" == "failure" ]; then echo "::warning::Benchmark storage failed - results were not saved to GitHub Pages" fi - echo "## Firewood Performance Benchmark Results" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "**Configuration:**" >> $GITHUB_STEP_SUMMARY - - if [ -n "${{ inputs.task }}" ]; then - echo "- Mode: Task-based" >> $GITHUB_STEP_SUMMARY - echo "- Task: \`${{ inputs.task }}\`" >> $GITHUB_STEP_SUMMARY - else - echo "- Mode: Custom parameters" >> $GITHUB_STEP_SUMMARY - echo "- Config: \`${{ inputs.config }}\`" >> $GITHUB_STEP_SUMMARY - echo "- Blocks: \`${{ inputs.start-block }}\` → \`${{ inputs.end-block }}\`" >> $GITHUB_STEP_SUMMARY - echo "- Block source: \`${{ inputs.block-dir-src }}\`" >> $GITHUB_STEP_SUMMARY - echo "- State source: \`${{ inputs.current-state-dir-src }}\`" >> $GITHUB_STEP_SUMMARY - fi - echo "- Firewood: \`${{ steps.trigger.outputs.firewood }}\`" >> $GITHUB_STEP_SUMMARY - if [ -n "${{ inputs.libevm }}" ]; then - echo "- libevm: \`${{ inputs.libevm }}\`" >> $GITHUB_STEP_SUMMARY - fi - echo "- AvalancheGo: \`${{ inputs.avalanchego }}\`" >> $GITHUB_STEP_SUMMARY - echo "- Runner: \`${{ inputs.runner }}\`" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - - if [ "${{ steps.download.outcome }}" = "success" ]; then - echo "**Links:**" >> $GITHUB_STEP_SUMMARY - echo "- [AvalancheGo Workflow](${{ steps.trigger.outputs.run_url }})" >> $GITHUB_STEP_SUMMARY - echo "- [Performance Trends](https://ava-labs.github.io/firewood/${{ steps.download.outputs.data-dir }}/)" >> $GITHUB_STEP_SUMMARY - else - echo "**Status:** Failed" >> $GITHUB_STEP_SUMMARY - echo "Check [workflow logs](${{ steps.trigger.outputs.run_url }}) for details." >> $GITHUB_STEP_SUMMARY - fi + { + echo "## Firewood Performance Benchmark Results" + echo + echo "**Configuration:**" + + if [ -n "${{ inputs.task }}" ]; then + echo "- Mode: Task-based" + echo "- Task: \`${{ inputs.task }}\`" + else + echo "- Mode: Custom parameters" + echo "- Config: \`${{ inputs.config }}\`" + echo "- Blocks: \`${{ inputs.start-block }}\` → \`${{ inputs.end-block }}\`" + echo "- Block source: \`${{ inputs.block-dir-src }}\`" + echo "- State source: \`${{ inputs.current-state-dir-src }}\`" + fi + + echo "- Firewood: \`${{ steps.trigger.outputs.firewood }}\`" + if [ -n "${{ inputs.libevm }}" ]; then + echo "- libevm: \`${{ inputs.libevm }}\`" + fi + echo "- AvalancheGo: \`${{ inputs.avalanchego }}\`" + echo "- Runner: \`${{ inputs.runner }}\`" + echo "- Timeout: \`${{ inputs.timeout-minutes }}\` minutes" + echo + + if [ "${{ steps.download.outcome }}" = "success" ]; then + echo "**Links:**" + echo "- [AvalancheGo Workflow](${{ steps.trigger.outputs.run_url }})" + echo "- [Performance Trends](https://ava-labs.github.io/firewood/${{ steps.download.outputs.data-dir }}/)" + else + echo "**Status:** Failed" + echo "Check [workflow logs](${{ steps.trigger.outputs.run_url }}) for details." + fi + } >> $GITHUB_STEP_SUMMARY diff --git a/METRICS.md b/METRICS.md index 06eeb897e..cd0fc11df 100644 --- a/METRICS.md +++ b/METRICS.md @@ -308,86 +308,37 @@ The easiest way to trigger a benchmark is via the GitHub Actions UI: 3. Select parameters from the dropdowns (task, runner) or enter custom values 4. Click "Run workflow" -### Running Benchmarks Locally +### Running Reexecution Locally -Benchmarks can be triggered locally using `just benchmark` (requires nix). +Reexecution test can be triggered locally using `just` commands (requires nix). -**Parameters:** - -| Parameter | Description | Default | -|-----------|-------------|---------| -| `firewood` | Firewood commit/branch/tag | `HEAD` (currently checked out commit) | -| `avalanchego` | AvalancheGo commit/branch/tag | `master` | -| `task` | Benchmark task name | `c-chain-reexecution-firewood-101-250k` | -| `runner` | GitHub Actions runner (see [runner config](https://github.com/ava-labs/devops-argocd/blob/main/base/system/actions-runners/action-runner.yaml), search for "runnerScaleSetName") | `avalanche-avalanchego-runner-2ti` | -| `libevm` | libevm commit/branch/tag (optional) | `` | - -**Task Names:** - -Predefined tasks follow the pattern `c-chain-reexecution-{db}-{start}-{end}` where: - -- `{db}` - database backend (e.g., `firewood`, `hashdb`) -- `{start}` - start block number -- `{end}` - end block (e.g., `250k` for 250,000 or `1m` for 1,000,000) - -To find available tasks, run in the [AvalancheGo](https://github.com/ava-labs/avalanchego/) repo: +Example: Trigger a C-Chain reexecution in AvalancheGo, wait for completion, and download results: ```bash -./scripts/run_task.sh --list | grep "c-chain-reexecution" +RUN_ID=$(just trigger-reexecution firewood=v0.0.15 avalanchego=master task=c-chain-reexecution-firewood-101-250k runner=avalanche-avalanchego-runner-2ti) \ + && just wait-reexecution run_id=$RUN_ID \ + && just download-reexecution-results run_id=$RUN_ID ``` -See the [AvalancheGo C-Chain benchmark docs](https://github.com/ava-labs/avalanchego/blob/master/tests/reexecute/c/README.md) for the full list and details. - -**Granular Inputs:** - -If no predefined task fits your needs, leave `task` empty and use granular inputs via the CI workflow: - -- `config` - VM config (e.g., `archive`, `firewood`) -- `start-block` - first block to execute -- `end-block` - last block to execute -- `block-dir-src` - S3 source for block data -- `current-state-dir-src` - S3 source for state snapshot - -**Examples:** +**Custom example:** Trigger with specific block range and config: ```bash -# Run with defaults (current HEAD, master, default task/runner) -just benchmark - -# With named parameters -just benchmark \ - firewood=v0.0.15 \ - avalanchego=master \ - task=c-chain-reexecution-firewood-101-250k \ - runner=avalanche-avalanchego-runner-2ti - -# With libevm -just benchmark \ - firewood=v0.0.15 \ - avalanchego=master \ - task=c-chain-reexecution-firewood-101-250k \ - runner=avalanche-avalanchego-runner-2ti \ - libevm=v1.0.0 +RUN_ID=$(just trigger-custom-reexecution firewood=v0.0.15 avalanchego=master config=firewood start-block=101 end-block=250000 block-dir-src=cchain-mainnet-blocks-1m-ldb current-state-dir-src=cchain-mainnet-state-100-fw runner=avalanche-avalanchego-runner-2ti) \ + && just wait-reexecution run_id=$RUN_ID \ + && just download-reexecution-results run_id=$RUN_ID ``` -### Composable Commands - -Individual steps can be run separately: +**Available commands:** -```bash -# Trigger benchmark, returns run_id -just trigger-benchmark \ - firewood=v0.0.15 \ - avalanchego=master \ - task=c-chain-reexecution-firewood-101-250k \ - runner=avalanche-avalanchego-runner-2ti - -# Wait for a specific run to complete -just wait-benchmark run_id=19938272417 +| Command | Description | +|---------|-------------| +| `trigger-reexecution` | Trigger task-based reexecution, returns run_id | +| `trigger-custom-reexecution` | Trigger with custom block range/config | +| `wait-reexecution` | Wait for run to complete | +| `download-reexecution-results` | Download results to `./results/` | +| `list-reexecutions` | List recent reexecution runs | -# Download results from a specific run (saves to ./results/) -just download-benchmark-results run_id=19938272417 +**Tasks and runners** are defined in AvalancheGo: -# List recent benchmark runs -just list-benchmarks -``` +- [Available tasks](https://github.com/ava-labs/avalanchego/blob/master/.github/workflows/c-chain-reexecution-benchmark-container.json) +- [C-Chain benchmark docs](https://github.com/ava-labs/avalanchego/blob/master/tests/reexecute/c/README.md) diff --git a/justfile b/justfile index 55aa64846..3a32aea57 100644 --- a/justfile +++ b/justfile @@ -139,99 +139,84 @@ update-ffi-flake: check-nix echo "checking for a consistent golang verion" ../scripts/run-just.sh check-golang-version -# Trigger Reexecution Benchmark -# Usage: just trigger-benchmark [libevm] -trigger-benchmark firewood avalanchego task runner libevm="": check-nix +# Trigger task based reexecution in AvalancheGo +trigger-reexecution firewood avalanchego task runner libevm="" timeout-minutes="120": + just _trigger-workflow "{{ firewood }}" "{{ libevm }}" "{{ avalanchego }}" "{{ runner }}" "{{ timeout-minutes }}" \ + -f "task={{ task }}" + +# Trigger custom param reexecution in AvalancheGo +trigger-custom-reexecution firewood avalanchego config start-block end-block block-dir-src current-state-dir-src runner libevm="" timeout-minutes="120": + just _trigger-workflow "{{ firewood }}" "{{ libevm }}" "{{ avalanchego }}" "{{ runner }}" "{{ timeout-minutes }}" \ + -f "config={{ config }}" \ + -f "start-block={{ start-block }}" \ + -f "end-block={{ end-block }}" \ + -f "block-dir-src={{ block-dir-src }}" \ + -f "current-state-dir-src={{ current-state-dir-src }}" + +# Wait for reexecution run to complete +wait-reexecution run_id: check-nix + nix run ./ffi#gh -- run watch "{{ run_id }}" --repo ava-labs/avalanchego --exit-status + +# Download reexecution results +download-reexecution-results run_id: check-nix + #!/usr/bin/env -S bash -euo pipefail + mkdir -p ./results + nix run ./ffi#gh -- run download "{{ run_id }}" \ + --repo ava-labs/avalanchego \ + --name benchmark-output \ + --dir ./results + cat ./results/benchmark-output.txt + +# List recent reexecution runs in AvalancheGo +list-reexecutions: check-nix + nix run ./ffi#gh -- run list --repo ava-labs/avalanchego --workflow="C-Chain Re-Execution Benchmark w/ Container" --limit 10 + +# Internal helpers (not intended for direct use) + +_trigger-workflow firewood libevm avalanchego runner timeout-minutes +extra_flags: check-nix #!/usr/bin/env -S bash -euo pipefail - GH="nix run ./ffi#gh --" - LIBEVM_FLAG="" + # Validate refs contain only safe characters + if [[ ! "{{ firewood }}" =~ ^[a-zA-Z0-9/_.-]+$ ]]; then + echo "Error: firewood ref contains invalid characters: {{ firewood }}" >&2 + exit 1 + fi + if [[ -n "{{ libevm }}" && ! "{{ libevm }}" =~ ^[a-zA-Z0-9/_.-]+$ ]]; then + echo "Error: libevm ref contains invalid characters: {{ libevm }}" >&2 + exit 1 + fi + + WITH_VALUE="firewood={{ firewood }}" if [ -n "{{ libevm }}" ]; then - LIBEVM_FLAG="-f libevm={{ libevm }}" + WITH_VALUE="${WITH_VALUE},libevm={{ libevm }}" fi - $GH workflow run "Firewood Reexecution Benchmark" \ + $GH workflow run "C-Chain Re-Execution Benchmark w/ Container" \ --repo ava-labs/avalanchego \ --ref "{{ avalanchego }}" \ - -f firewood="{{ firewood }}" \ - -f task="{{ task }}" \ -f runner="{{ runner }}" \ - $LIBEVM_FLAG + -f timeout-minutes="{{ timeout-minutes }}" \ + -f with="$WITH_VALUE" \ + {{ extra_flags }} - # Wait for GitHub to register the run (retry up to 3min) + just _wait-for-workflow-run + +_wait-for-workflow-run: check-nix + #!/usr/bin/env -S bash -euo pipefail + GH="nix run ./ffi#gh --" for i in {1..18}; do sleep 10 RUN_ID=$($GH run list \ --repo ava-labs/avalanchego \ - --workflow "Firewood Reexecution Benchmark" \ + --workflow "C-Chain Re-Execution Benchmark w/ Container" \ --limit 5 \ --json databaseId,createdAt \ --jq '[.[] | select(.createdAt | fromdateiso8601 > (now - 180))] | .[0].databaseId') - if [ -n "$RUN_ID" ] && [ "$RUN_ID" != "null" ]; then echo "$RUN_ID" exit 0 fi done - - echo "Error: Could not find triggered workflow run after 3min" >&2 + echo "Error: Could not find workflow run after 3min. Check AvalancheGo logs." >&2 exit 1 - -# Wait for reexecution benchmark run to complete (2h timeout) -# Usage: just wait-benchmark -wait-benchmark run_id: check-nix - #!/usr/bin/env -S bash -euo pipefail - timeout -k 1m 2h nix run ./ffi#gh -- run watch "{{ run_id }}" --repo ava-labs/avalanchego --exit-status - -# Download benchmark results -# Usage: just download-benchmark-results -download-benchmark-results run_id: check-nix - #!/usr/bin/env -S bash -euo pipefail - mkdir -p ./results - nix run ./ffi#gh -- run download "{{ run_id }}" \ - --repo ava-labs/avalanchego \ - --name benchmark-output \ - --dir ./results - cat ./results/benchmark-output.txt - -# Run full benchmark: trigger, wait, download (composes the above) -# Usage: just benchmark [firewood] [avalanchego] [task] [runner] [libevm] -benchmark firewood="HEAD" avalanchego="master" task="c-chain-reexecution-firewood-101-250k" runner="avalanche-avalanchego-runner-2ti" libevm="": check-nix - #!/usr/bin/env -S bash -euo pipefail - - FIREWOOD="{{ firewood }}" - if [[ "$FIREWOOD" == "HEAD" ]]; then - FIREWOOD=$(git rev-parse HEAD) - fi - - echo "Triggering reexecution benchmark in AvalancheGo with parameters:" - echo " Firewood: $FIREWOOD" - echo " AvalancheGo: {{ avalanchego }}" - echo " Task: {{ task }}" - echo " Runner: {{ runner }}" - if [ -n "{{ libevm }}" ]; then - echo " LibEVM: {{ libevm }}" - else - echo " LibEVM: (not set)" - fi - echo "" - - RUN_ID=$(just trigger-benchmark "$FIREWOOD" "{{ avalanchego }}" "{{ task }}" "{{ runner }}" "{{ libevm }}") - echo " Run ID: $RUN_ID" - echo " Monitor progress here: https://github.com/ava-labs/avalanchego/actions/runs/$RUN_ID" - echo "" - - echo "Waiting for benchmark completion..." - just wait-benchmark "$RUN_ID" - echo "" - - echo "Downloading results..." - just download-benchmark-results "$RUN_ID" - echo "" - echo "Results saved to: ./results/benchmark-output.txt" - -# List recent AvalancheGo benchmark runs -list-benchmarks: check-nix - #!/usr/bin/env -S bash -euo pipefail - nix run ./ffi#gh -- run list --repo ava-labs/avalanchego --workflow="Firewood Reexecution Benchmark" --limit 10 From 37f1325e19885ec94c305042c3f990f46ebfcb62 Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Thu, 22 Jan 2026 18:46:54 +0100 Subject: [PATCH 24/40] ci(perf): add C-Chain reexecution benchmark workflow - Add bench-cchain-reexecution.sh script to trigger and monitor AvalancheGo benchmarks - Add just commands: bench, bench-status, bench-list, bench-help - Add track-performance.yml workflow to publish results to GitHub Pages - Support predefined tests and custom block ranges via env vars --- .github/workflows/track-performance.yml | 118 ++++------ justfile | 128 ++++------- scripts/bench-cchain-reexecution.sh | 287 ++++++++++++++++++++++++ 3 files changed, 372 insertions(+), 161 deletions(-) create mode 100755 scripts/bench-cchain-reexecution.sh diff --git a/.github/workflows/track-performance.yml b/.github/workflows/track-performance.yml index f1f4b0392..592841a62 100644 --- a/.github/workflows/track-performance.yml +++ b/.github/workflows/track-performance.yml @@ -14,15 +14,11 @@ on: avalanchego: description: 'AvalancheGo commit/branch/tag to test against' default: 'master' - task: - description: 'Predefined task (leave empty to use custom parameters below)' - type: choice - options: - - c-chain-reexecution-firewood-101-250k - - c-chain-reexecution-firewood-33m-33m500k - - c-chain-reexecution-firewood-33m-40m + test: + description: 'Predefined test (leave empty to use custom parameters below)' # https://github.com/ava-labs/avalanchego/blob/a85295d87193b30ff17c594680dadd6618022f5e/scripts/benchmark_cchain_range.sh#L63 + default: '' config: - description: 'VM config (e.g., firewood, hashdb)' + description: 'Config (e.g., firewood, hashdb)' default: '' start-block: default: '' @@ -43,8 +39,8 @@ on: - avago-runner-i4i-4xlarge-local-ssd - avago-runner-m6i-4xlarge-ebs-fast timeout-minutes: - description: 'Timeout in minutes for the benchmark job' - default: '120' + description: 'Timeout in minutes' + default: '' jobs: record-benchmark-to-gh-pages: @@ -57,80 +53,48 @@ jobs: with: fetch-depth: 0 # Needed for github-action-benchmark - - name: Install Nix - uses: cachix/install-nix-action@02a151ada4993995686f9ed4f1be7cfbb229e56f # v31 - with: - github_access_token: ${{ secrets.GITHUB_TOKEN }} - - - name: Validate inputs - run: | - if [ -z "${{ inputs.task }}" ]; then - missing=() - [ -z "${{ inputs.config }}" ] && missing+=("config") - [ -z "${{ inputs.start-block }}" ] && missing+=("start-block") - [ -z "${{ inputs.end-block }}" ] && missing+=("end-block") - [ -z "${{ inputs.block-dir-src }}" ] && missing+=("block-dir-src") - [ -z "${{ inputs.current-state-dir-src }}" ] && missing+=("current-state-dir-src") - - if [ ${#missing[@]} -gt 0 ]; then - echo "Error: When using custom mode, these fields are required: ${missing[*]}" - exit 1 - fi - fi - - - name: Trigger C-Chain reexecution in AvalancheGo - id: trigger - shell: nix develop ./ffi --command bash {0} - run: | - FIREWOOD="${{ inputs.firewood || github.sha }}" - echo "firewood=$FIREWOOD" >> "$GITHUB_OUTPUT" - - if [ -n "${{ inputs.task }}" ]; then - RUN_ID=$(just trigger-reexecution "$FIREWOOD" "${{ inputs.avalanchego }}" "${{ inputs.task }}" "${{ inputs.runner }}" "${{ inputs.libevm }}" "${{ inputs.timeout-minutes }}") - else - RUN_ID=$(just trigger-custom-reexecution "$FIREWOOD" "${{ inputs.avalanchego }}" "${{ inputs.config }}" "${{ inputs.start-block }}" "${{ inputs.end-block }}" "${{ inputs.block-dir-src }}" "${{ inputs.current-state-dir-src }}" "${{ inputs.runner }}" "${{ inputs.libevm }}" "${{ inputs.timeout-minutes }}") - fi - - echo "run_id=$RUN_ID" >> "$GITHUB_OUTPUT" - echo "run_url=https://github.com/ava-labs/avalanchego/actions/runs/$RUN_ID" >> "$GITHUB_OUTPUT" - env: - GH_TOKEN: ${{ secrets.FIREWOOD_AVALANCHEGO_GITHUB_TOKEN }} - - - name: Wait for completion - shell: nix develop ./ffi --command bash {0} - run: just wait-reexecution "${{ steps.trigger.outputs.run_id }}" + - name: Trigger C-Chain Reexecution Benchmark + run: ./scripts/bench-cchain-reexecution.sh trigger "${{ inputs.test }}" env: GH_TOKEN: ${{ secrets.FIREWOOD_AVALANCHEGO_GITHUB_TOKEN }} + # Custom mode (ignored when test is specified) + CONFIG: ${{ inputs.config }} + START_BLOCK: ${{ inputs.start-block }} + END_BLOCK: ${{ inputs.end-block }} + BLOCK_DIR_SRC: ${{ inputs.block-dir-src }} + CURRENT_STATE_DIR_SRC: ${{ inputs.current-state-dir-src }} + # Refs + FIREWOOD_REF: ${{ inputs.firewood || github.sha }} + AVALANCHEGO_REF: ${{ inputs.avalanchego }} + LIBEVM_REF: ${{ inputs.libevm }} + # Execution + RUNNER: ${{ inputs.runner }} + TIMEOUT_MINUTES: ${{ inputs.timeout-minutes }} - - name: Download artifacts - id: download - shell: nix develop ./ffi --command bash {0} + # Store main branch results in bench/, feature branches in dev/bench/{branch}/ + # to keep official benchmark history clean + - name: Determine results location + id: location run: | - just download-reexecution-results "${{ steps.trigger.outputs.run_id }}" - - # Determine target dashboard if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then echo "data-dir=bench" >> "$GITHUB_OUTPUT" else - SAFE_NAME=$(echo "${{ github.ref_name }}" | tr '/' '-') - echo "data-dir=dev/bench/$SAFE_NAME" >> "$GITHUB_OUTPUT" + echo "data-dir=dev/bench/$(echo '${{ github.ref_name }}' | tr '/' '-')" >> "$GITHUB_OUTPUT" fi - env: - GH_TOKEN: ${{ secrets.FIREWOOD_AVALANCHEGO_GITHUB_TOKEN }} - - name: Store benchmark results + - name: Publish benchmark results id: store continue-on-error: true uses: benchmark-action/github-action-benchmark@v1 with: - name: C-Chain Reexecution Performance - tool: 'go' - output-file-path: ./results/benchmark-output.txt + name: C-Chain Reexecution with Firewood + tool: 'customBiggerIsBetter' + output-file-path: ./results/benchmark-output.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true gh-pages-branch: benchmark-data - benchmark-data-dir-path: ${{ steps.download.outputs.data-dir }} - fail-on-alert: false + benchmark-data-dir-path: ${{ steps.location.outputs.data-dir }} + fail-on-alert: true comment-on-alert: false - name: Summary @@ -144,9 +108,9 @@ jobs: echo echo "**Configuration:**" - if [ -n "${{ inputs.task }}" ]; then - echo "- Mode: Task-based" - echo "- Task: \`${{ inputs.task }}\`" + if [ -n "${{ inputs.test }}" ]; then + echo "- Mode: Predefined test" + echo "- Test: \`${{ inputs.test }}\`" else echo "- Mode: Custom parameters" echo "- Config: \`${{ inputs.config }}\`" @@ -155,7 +119,7 @@ jobs: echo "- State source: \`${{ inputs.current-state-dir-src }}\`" fi - echo "- Firewood: \`${{ steps.trigger.outputs.firewood }}\`" + echo "- Firewood: \`${{ inputs.firewood || github.sha }}\`" if [ -n "${{ inputs.libevm }}" ]; then echo "- libevm: \`${{ inputs.libevm }}\`" fi @@ -164,13 +128,7 @@ jobs: echo "- Timeout: \`${{ inputs.timeout-minutes }}\` minutes" echo - if [ "${{ steps.download.outcome }}" = "success" ]; then - echo "**Links:**" - echo "- [AvalancheGo Workflow](${{ steps.trigger.outputs.run_url }})" - echo "- [Performance Trends](https://ava-labs.github.io/firewood/${{ steps.download.outputs.data-dir }}/)" - else - echo "**Status:** Failed" - echo "Check [workflow logs](${{ steps.trigger.outputs.run_url }}) for details." - fi + echo "**Links:**" + echo "- [Performance Trends](https://ava-labs.github.io/firewood/${{ steps.location.outputs.data-dir }}/)" } >> $GITHUB_STEP_SUMMARY diff --git a/justfile b/justfile index b1eba5657..c17cb890a 100644 --- a/justfile +++ b/justfile @@ -169,84 +169,50 @@ release-step-refresh-changelog tag: echo "Generating changelog..." git cliff -o CHANGELOG.md --tag "{{tag}}" -# Trigger task based reexecution in AvalancheGo -trigger-reexecution firewood avalanchego task runner libevm="" timeout-minutes="120": - just _trigger-workflow "{{ firewood }}" "{{ libevm }}" "{{ avalanchego }}" "{{ runner }}" "{{ timeout-minutes }}" \ - -f "task={{ task }}" - -# Trigger custom param reexecution in AvalancheGo -trigger-custom-reexecution firewood avalanchego config start-block end-block block-dir-src current-state-dir-src runner libevm="" timeout-minutes="120": - just _trigger-workflow "{{ firewood }}" "{{ libevm }}" "{{ avalanchego }}" "{{ runner }}" "{{ timeout-minutes }}" \ - -f "config={{ config }}" \ - -f "start-block={{ start-block }}" \ - -f "end-block={{ end-block }}" \ - -f "block-dir-src={{ block-dir-src }}" \ - -f "current-state-dir-src={{ current-state-dir-src }}" - -# Wait for reexecution run to complete -wait-reexecution run_id: check-nix - nix run ./ffi#gh -- run watch "{{ run_id }}" --repo ava-labs/avalanchego --exit-status - -# Download reexecution results -download-reexecution-results run_id: check-nix - #!/usr/bin/env -S bash -euo pipefail - mkdir -p ./results - nix run ./ffi#gh -- run download "{{ run_id }}" \ - --repo ava-labs/avalanchego \ - --name benchmark-output \ - --dir ./results - cat ./results/benchmark-output.txt - -# List recent reexecution runs in AvalancheGo -list-reexecutions: check-nix - nix run ./ffi#gh -- run list --repo ava-labs/avalanchego --workflow="C-Chain Re-Execution Benchmark w/ Container" --limit 10 - -# Internal helpers (not intended for direct use) - -_trigger-workflow firewood libevm avalanchego runner timeout-minutes +extra_flags: check-nix - #!/usr/bin/env -S bash -euo pipefail - GH="nix run ./ffi#gh --" - - # Validate refs contain only safe characters - if [[ ! "{{ firewood }}" =~ ^[a-zA-Z0-9/_.-]+$ ]]; then - echo "Error: firewood ref contains invalid characters: {{ firewood }}" >&2 - exit 1 - fi - if [[ -n "{{ libevm }}" && ! "{{ libevm }}" =~ ^[a-zA-Z0-9/_.-]+$ ]]; then - echo "Error: libevm ref contains invalid characters: {{ libevm }}" >&2 - exit 1 - fi - - WITH_VALUE="firewood={{ firewood }}" - if [ -n "{{ libevm }}" ]; then - WITH_VALUE="${WITH_VALUE},libevm={{ libevm }}" - fi - - $GH workflow run "C-Chain Re-Execution Benchmark w/ Container" \ - --repo ava-labs/avalanchego \ - --ref "{{ avalanchego }}" \ - -f runner="{{ runner }}" \ - -f timeout-minutes="{{ timeout-minutes }}" \ - -f with="$WITH_VALUE" \ - {{ extra_flags }} - - just _wait-for-workflow-run - -_wait-for-workflow-run: check-nix - #!/usr/bin/env -S bash -euo pipefail - GH="nix run ./ffi#gh --" - for i in {1..18}; do - sleep 10 - RUN_ID=$($GH run list \ - --repo ava-labs/avalanchego \ - --workflow "C-Chain Re-Execution Benchmark w/ Container" \ - --limit 5 \ - --json databaseId,createdAt \ - --jq '[.[] | select(.createdAt | fromdateiso8601 > (now - 180))] | .[0].databaseId') - if [ -n "$RUN_ID" ] && [ "$RUN_ID" != "null" ]; then - echo "$RUN_ID" - exit 0 - fi - done - echo "Error: Could not find workflow run after 3min. Check AvalancheGo logs." >&2 - exit 1 +# Run a C-Chain reexecution benchmark +# just bench - predefined test +# FIREWOOD_REF=v0.1.0 just bench firewood-101-250k - with specific ref +# Custom: START_BLOCK=101 END_BLOCK=250000 BLOCK_DIR_SRC= CURRENT_STATE_DIR_SRC= just bench +bench test="": + ./scripts/bench-cchain-reexecution.sh trigger "{{ test }}" + +# Check status of a benchmark run +bench-status run_id: + ./scripts/bench-cchain-reexecution.sh status "{{ run_id }}" + +# List recent benchmark runs +bench-list: + ./scripts/bench-cchain-reexecution.sh list + +# Show benchmark help +bench-help: + #!/usr/bin/env bash + echo "USAGE" + echo " just bench Predefined test" + echo " [CUSTOM_VARS] just bench Custom mode" + echo " just bench-status " + echo " just bench-list" + echo "" + echo "ENVIRONMENT VARIABLES" + echo " FIREWOOD_REF Firewood ref (default: HEAD)" + echo " AVALANCHEGO_REF AvalancheGo ref (default: master)" + echo " LIBEVM_REF libevm ref (optional)" + echo " RUNNER GitHub Actions runner (default: avalanche-avalanchego-runner-2ti)" + echo " CONFIG VM config (default: firewood)" + echo " START_BLOCK Start block for custom mode" + echo " END_BLOCK End block for custom mode" + echo " BLOCK_DIR_SRC S3 block directory" + echo " CURRENT_STATE_DIR_SRC S3 state directory" + echo "" + echo "EXAMPLES" + echo " # Predefined test" + echo " just bench firewood-101-250k" + echo " FIREWOOD_REF=v0.1.0 just bench firewood-33m-40m" + echo "" + echo " # Custom mode" + echo " START_BLOCK=101 END_BLOCK=250000 \\" + echo " BLOCK_DIR_SRC=cchain-mainnet-blocks-1m-ldb \\" + echo " CURRENT_STATE_DIR_SRC=cchain-current-state-mainnet-ldb \\" + echo " just bench" + echo "" + echo "Advanced: ./scripts/bench-cchain-reexecution.sh help" diff --git a/scripts/bench-cchain-reexecution.sh b/scripts/bench-cchain-reexecution.sh new file mode 100755 index 000000000..733cb7e24 --- /dev/null +++ b/scripts/bench-cchain-reexecution.sh @@ -0,0 +1,287 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Triggers and monitors C-Chain re-execution benchmarks in AvalancheGo. +# +# USAGE +# ./bench-cchain-reexecution.sh [args] +# +# COMMANDS +# trigger [test] Trigger benchmark, wait, download results +# download Download results for a run +# status Check run status +# list List recent runs +# tests Show available tests +# help Show this help +# +# ENVIRONMENT +# GH_TOKEN (string, required) GitHub token for API access +# TEST (string, optional) Predefined test name (alternative to arg) +# FIREWOOD_REF (string, HEAD) Firewood commit/tag/branch +# AVALANCHEGO_REF (string, master) AvalancheGo ref to test against +# RUNNER (string, avalanche-avalanchego-runner-2ti) GitHub Actions runner label +# LIBEVM_REF (string, optional) Optional libevm ref +# TIMEOUT_MINUTES (int, optional) Workflow timeout +# +# Custom mode (when no TEST/test arg specified): +# CONFIG (string, firewood) VM config (https://github.com/ava-labs/avalanchego/blob/3c645de551294b8db0f695563e386a2f38c1aded/tests/reexecute/c/vm_reexecute.go#L64) +# START_BLOCK (required) First block number +# END_BLOCK (required) Last block number +# BLOCK_DIR_SRC (required) S3 block directory (without S3:// prefix, e.g., cchain-mainnet-blocks-200-ldb) +# CURRENT_STATE_DIR_SRC (required) S3 state directory (without S3:// prefix, e.g., cchain-current-state-mainnet-100-firewood) +# +# +# EXAMPLES +# ./bench-cchain-reexecution.sh trigger firewood-101-250k +# +# TEST=firewood-33m-40m FIREWOOD_REF=v0.1.0 ./bench-cchain-reexecution.sh trigger +# +# START_BLOCK=101 END_BLOCK=250000 \ +# BLOCK_DIR_SRC=cchain-mainnet-blocks-1m-ldb \ +# CURRENT_STATE_DIR_SRC=cchain-current-state-mainnet-ldb \ +# ./bench-cchain-reexecution.sh trigger +# +# ./bench-cchain-reexecution.sh download 12345678 + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" + +AVALANCHEGO_REPO="ava-labs/avalanchego" +WORKFLOW_NAME="C-Chain Re-Execution Benchmark w/ Container" + +: "${FIREWOOD_REF:=HEAD}" +: "${AVALANCHEGO_REF:=master}" +: "${RUNNER:=avalanche-avalanchego-runner-2ti}" +: "${LIBEVM_REF:=}" +: "${TIMEOUT_MINUTES:=}" +: "${CONFIG:=firewood}" + +# Polling config for workflow registration. gh workflow run doesn't return +# the run ID, so we poll until the run appears. 180s accommodates busy runners. +POLL_INTERVAL=1 +POLL_TIMEOUT=180 + +# Subset of tests for discoverability. AvalancheGo is the source of truth: +# https://github.com/ava-labs/avalanchego/blob/master/scripts/benchmark_cchain_range.sh +declare -A TESTS=( + ["firewood-101-250k"]="Blocks 101-250k" + ["firewood-archive-101-250k"]="Blocks 101-250k (archive)" + ["firewood-33m-33m500k"]="Blocks 33m-33.5m" + ["firewood-archive-33m-33m500k"]="Blocks 33m-33.5m (archive)" + ["firewood-33m-40m"]="Blocks 33m-40m" + ["firewood-archive-33m-40m"]="Blocks 33m-40m (archive)" +) + +log() { echo "==> $1"; } + +# Use GitHub Actions error annotation format in CI for better visibility +err() { + if [[ "${GITHUB_ACTIONS:-}" == "true" ]]; then + echo "::error::$1" + else + echo "error: $1" >&2 + fi +} + +die() { err "$1"; exit 1; } + +require_gh() { + if ! command -v gh &>/dev/null; then + die "gh CLI not found. Run from nix shell: nix develop ./ffi" + fi + [[ -z "${GH_TOKEN:-}" ]] && die "GH_TOKEN is required" +} + +# Prevent command injection via malicious ref names passed to gh CLI +validate_ref() { + local ref="$1" name="$2" + [[ "$ref" =~ ^[a-zA-Z0-9/_.-]+$ ]] || die "$name contains invalid characters: $ref" +} + +poll_workflow_registration() { + local trigger_time="$1" + log "Waiting for workflow to register..." + + for ((i=0; i $t)] | .[0].databaseId' 2>/dev/null || echo "") + + if [[ -n "$run_id" && "$run_id" != "null" ]]; then + echo "$run_id" + return 0 + fi + done + + die "workflow not found after ${POLL_TIMEOUT}s" +} + +trigger_workflow() { + local test="${1:-}" + local firewood="$FIREWOOD_REF" + + # Resolve HEAD to commit SHA for reproducibility—"HEAD" means different things at different times + [[ "$firewood" == "HEAD" ]] && firewood=$(git -C "$REPO_ROOT" rev-parse HEAD) + validate_ref "$firewood" "FIREWOOD_REF" + [[ -n "$LIBEVM_REF" ]] && validate_ref "$LIBEVM_REF" "LIBEVM_REF" + + local args=(-f runner="$RUNNER") + [[ -n "$TIMEOUT_MINUTES" ]] && args+=(-f timeout-minutes="$TIMEOUT_MINUTES") + + if [[ -n "$test" ]]; then + args+=(-f test="$test") + log "Triggering: $test" + else + # Custom mode: block params required, config defaults to firewood + [[ -z "${START_BLOCK:-}${END_BLOCK:-}${BLOCK_DIR_SRC:-}${CURRENT_STATE_DIR_SRC:-}" ]] && \ + die "Provide a test name or set START_BLOCK, END_BLOCK, BLOCK_DIR_SRC, CURRENT_STATE_DIR_SRC" + : "${START_BLOCK:?START_BLOCK required}" + : "${END_BLOCK:?END_BLOCK required}" + : "${BLOCK_DIR_SRC:?BLOCK_DIR_SRC required}" + : "${CURRENT_STATE_DIR_SRC:?CURRENT_STATE_DIR_SRC required}" + args+=( + -f config="$CONFIG" + -f start-block="$START_BLOCK" + -f end-block="$END_BLOCK" + -f block-dir-src="$BLOCK_DIR_SRC" + -f current-state-dir-src="$CURRENT_STATE_DIR_SRC" + ) + log "Triggering: $CONFIG $START_BLOCK-$END_BLOCK" + fi + + log "firewood: $firewood" + log "avalanchego: $AVALANCHEGO_REF" + log "runner: $RUNNER" + + # Record time BEFORE triggering to avoid race condition: we only look for + # runs created after this timestamp, so concurrent triggers don't collide + local trigger_time + trigger_time=$(date -u +%Y-%m-%dT%H:%M:%SZ) + + gh workflow run "$WORKFLOW_NAME" \ + --repo "$AVALANCHEGO_REPO" \ + --ref "$AVALANCHEGO_REF" \ + "${args[@]}" + + poll_workflow_registration "$trigger_time" +} + +wait_for_completion() { + local run_id="$1" + log "Waiting for run $run_id" + log "https://github.com/${AVALANCHEGO_REPO}/actions/runs/${run_id}" + gh run watch "$run_id" --repo "$AVALANCHEGO_REPO" --exit-status +} + +download_artifact() { + local run_id="$1" + local tmp_dir results_dir + # Download to tmp to avoid permission issues only the final JSON goes to results + tmp_dir=$(mktemp -d) + results_dir="${REPO_ROOT}/results" + + gh run download "$run_id" \ + --repo "$AVALANCHEGO_REPO" \ + --pattern "benchmark-output-*" \ + --dir "$tmp_dir" + + mkdir -p "$results_dir" + find "$tmp_dir" -name "benchmark-output.json" -exec cp {} "$results_dir/" \; + rm -rf "$tmp_dir" + + [[ -f "$results_dir/benchmark-output.json" ]] || die "No benchmark results found" + log "Results: $results_dir/benchmark-output.json" +} + +check_status() { + local run_id="$1" + echo "https://github.com/${AVALANCHEGO_REPO}/actions/runs/${run_id}" + + local status + status=$(gh run view "$run_id" --repo "$AVALANCHEGO_REPO" --json status,conclusion \ + --jq '.status + " (" + (.conclusion // "in progress") + ")"') + echo "status: $status" +} + +list_runs() { + gh run list --repo "$AVALANCHEGO_REPO" --workflow "$WORKFLOW_NAME" --limit 10 +} + +list_tests() { + for test in "${!TESTS[@]}"; do + [[ "$test" == firewood* ]] && printf " %-25s %s\n" "$test" "${TESTS[$test]}" + done | sort +} + +cmd_trigger() { + require_gh + + local test="${1:-${TEST:-}}" + local run_id + run_id=$(trigger_workflow "$test") + log "run_id: $run_id" + + wait_for_completion "$run_id" + download_artifact "$run_id" +} + +cmd_download() { + [[ -z "${1:-}" ]] && die "usage: $0 download " + require_gh + download_artifact "$1" +} + +cmd_status() { + [[ -z "${1:-}" ]] && die "usage: $0 status " + require_gh + check_status "$1" +} + +cmd_list() { + require_gh + list_runs +} + +cmd_tests() { + list_tests +} + +cmd_help() { + cat < [args] + +COMMANDS + trigger [test] Trigger benchmark, wait, download results + download Download results for a run + status Check run status + list List recent runs + tests Show available tests + help Show this help + +TESTS +EOF + list_tests +} + +main() { + local cmd="${1:-help}" + shift || true + + case "$cmd" in + trigger) cmd_trigger "$@" ;; + download) cmd_download "$@" ;; + status) cmd_status "$@" ;; + list) cmd_list "$@" ;; + tests) cmd_tests "$@" ;; + help|-h|--help) cmd_help ;; + *) err "unknown command: $cmd"; cmd_help; exit 1 ;; + esac +} + +main "$@" From 33e69219054fd9a60adb30298f1c23794cac6b68 Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Sun, 25 Jan 2026 09:41:42 +0100 Subject: [PATCH 25/40] chore: revert changes from gh-pages --- .github/workflows/gh-pages.yaml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/gh-pages.yaml b/.github/workflows/gh-pages.yaml index efe0393f1..b767da48d 100644 --- a/.github/workflows/gh-pages.yaml +++ b/.github/workflows/gh-pages.yaml @@ -30,14 +30,6 @@ jobs: run: | cp -rv target/doc/* ./_site cp -rv docs/assets ./_site - - name: Include benchmark data - run: | - git fetch origin benchmark-data || true - if git rev-parse origin/benchmark-data >/dev/null 2>&1; then - git checkout origin/benchmark-data -- dev bench 2>/dev/null || true - cp -rv dev _site/ 2>/dev/null || true - cp -rv bench _site/ 2>/dev/null || true - fi - uses: actions/upload-artifact@v4 with: name: pages From 268c05fd08644b74fc76898c307d76c69364ef3a Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Sun, 25 Jan 2026 10:17:31 +0100 Subject: [PATCH 26/40] ci(track-performance): remove checkout fetch depth 0 --- .github/workflows/track-performance.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/track-performance.yml b/.github/workflows/track-performance.yml index 592841a62..42a65be74 100644 --- a/.github/workflows/track-performance.yml +++ b/.github/workflows/track-performance.yml @@ -1,6 +1,6 @@ # Triggers AvalancheGo's C-Chain reexecution benchmark and publishes # results to GitHub Pages for trend analysis. -name: C-Chain Performance Tracking +name: C-Chain Reexecution Performance Tracking on: workflow_dispatch: @@ -50,8 +50,6 @@ jobs: steps: - name: Checkout Firewood uses: actions/checkout@v4 - with: - fetch-depth: 0 # Needed for github-action-benchmark - name: Trigger C-Chain Reexecution Benchmark run: ./scripts/bench-cchain-reexecution.sh trigger "${{ inputs.test }}" From 7055e9c355e125878d12e7a198099507be6692b1 Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Sun, 25 Jan 2026 10:45:02 +0100 Subject: [PATCH 27/40] refactor(bench): simplify C-Chain benchmark workflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Rename justfile commands to {action}-cchain pattern for ergonomics - bench → bench-cchain - bench-status → status-cchain - bench-list → list-cchain - bench-help → help-cchain - Remove standalone `download` command (trigger is the golden path) - Add configurable DOWNLOAD_DIR env var (default: ./results) - Simplify artifact download logic (flatten in place) --- justfile | 34 ++++++++++++------------ scripts/bench-cchain-reexecution.sh | 41 +++++++++++------------------ 2 files changed, 33 insertions(+), 42 deletions(-) diff --git a/justfile b/justfile index c17cb890a..9a9224bcc 100644 --- a/justfile +++ b/justfile @@ -170,28 +170,28 @@ release-step-refresh-changelog tag: git cliff -o CHANGELOG.md --tag "{{tag}}" # Run a C-Chain reexecution benchmark -# just bench - predefined test -# FIREWOOD_REF=v0.1.0 just bench firewood-101-250k - with specific ref -# Custom: START_BLOCK=101 END_BLOCK=250000 BLOCK_DIR_SRC= CURRENT_STATE_DIR_SRC= just bench -bench test="": +# just bench-cchain - predefined test +# FIREWOOD_REF=v0.1.0 just bench-cchain firewood-101-250k - with specific ref +# Custom: START_BLOCK=101 END_BLOCK=250000 BLOCK_DIR_SRC= CURRENT_STATE_DIR_SRC= just bench-cchain +bench-cchain test="": ./scripts/bench-cchain-reexecution.sh trigger "{{ test }}" -# Check status of a benchmark run -bench-status run_id: +# Check status of a C-Chain benchmark run +status-cchain run_id: ./scripts/bench-cchain-reexecution.sh status "{{ run_id }}" -# List recent benchmark runs -bench-list: +# List recent C-Chain benchmark runs +list-cchain: ./scripts/bench-cchain-reexecution.sh list -# Show benchmark help -bench-help: +# Show C-Chain benchmark help +help-cchain: #!/usr/bin/env bash echo "USAGE" - echo " just bench Predefined test" - echo " [CUSTOM_VARS] just bench Custom mode" - echo " just bench-status " - echo " just bench-list" + echo " just bench-cchain Predefined test" + echo " [CUSTOM_VARS] just bench-cchain Custom mode" + echo " just status-cchain " + echo " just list-cchain" echo "" echo "ENVIRONMENT VARIABLES" echo " FIREWOOD_REF Firewood ref (default: HEAD)" @@ -206,13 +206,13 @@ bench-help: echo "" echo "EXAMPLES" echo " # Predefined test" - echo " just bench firewood-101-250k" - echo " FIREWOOD_REF=v0.1.0 just bench firewood-33m-40m" + echo " just bench-cchain firewood-101-250k" + echo " FIREWOOD_REF=v0.1.0 just bench-cchain firewood-33m-40m" echo "" echo " # Custom mode" echo " START_BLOCK=101 END_BLOCK=250000 \\" echo " BLOCK_DIR_SRC=cchain-mainnet-blocks-1m-ldb \\" echo " CURRENT_STATE_DIR_SRC=cchain-current-state-mainnet-ldb \\" - echo " just bench" + echo " just bench-cchain" echo "" echo "Advanced: ./scripts/bench-cchain-reexecution.sh help" diff --git a/scripts/bench-cchain-reexecution.sh b/scripts/bench-cchain-reexecution.sh index 733cb7e24..ba47a148a 100755 --- a/scripts/bench-cchain-reexecution.sh +++ b/scripts/bench-cchain-reexecution.sh @@ -8,7 +8,6 @@ set -euo pipefail # # COMMANDS # trigger [test] Trigger benchmark, wait, download results -# download Download results for a run # status Check run status # list List recent runs # tests Show available tests @@ -22,6 +21,7 @@ set -euo pipefail # RUNNER (string, avalanche-avalanchego-runner-2ti) GitHub Actions runner label # LIBEVM_REF (string, optional) Optional libevm ref # TIMEOUT_MINUTES (int, optional) Workflow timeout +# DOWNLOAD_DIR (string, ./results) Directory for downloaded artifacts # # Custom mode (when no TEST/test arg specified): # CONFIG (string, firewood) VM config (https://github.com/ava-labs/avalanchego/blob/3c645de551294b8db0f695563e386a2f38c1aded/tests/reexecute/c/vm_reexecute.go#L64) @@ -41,7 +41,7 @@ set -euo pipefail # CURRENT_STATE_DIR_SRC=cchain-current-state-mainnet-ldb \ # ./bench-cchain-reexecution.sh trigger # -# ./bench-cchain-reexecution.sh download 12345678 +# ./bench-cchain-reexecution.sh status 12345678 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" @@ -55,6 +55,7 @@ WORKFLOW_NAME="C-Chain Re-Execution Benchmark w/ Container" : "${LIBEVM_REF:=}" : "${TIMEOUT_MINUTES:=}" : "${CONFIG:=firewood}" +: "${DOWNLOAD_DIR:=${REPO_ROOT}/results}" # Polling config for workflow registration. gh workflow run doesn't return # the run ID, so we poll until the run appears. 180s accommodates busy runners. @@ -180,22 +181,20 @@ wait_for_completion() { download_artifact() { local run_id="$1" - local tmp_dir results_dir - # Download to tmp to avoid permission issues only the final JSON goes to results - tmp_dir=$(mktemp -d) - results_dir="${REPO_ROOT}/results" + local output_file="$DOWNLOAD_DIR/benchmark-output.json" + mkdir -p "$DOWNLOAD_DIR" gh run download "$run_id" \ --repo "$AVALANCHEGO_REPO" \ --pattern "benchmark-output-*" \ - --dir "$tmp_dir" + --dir "$DOWNLOAD_DIR" - mkdir -p "$results_dir" - find "$tmp_dir" -name "benchmark-output.json" -exec cp {} "$results_dir/" \; - rm -rf "$tmp_dir" + # Flatten: gh extracts to $DOWNLOAD_DIR//benchmark-output.json + find "$DOWNLOAD_DIR" -name "benchmark-output.json" -type f -exec mv {} "$output_file" \; + find "$DOWNLOAD_DIR" -mindepth 1 -type d -delete 2>/dev/null || true - [[ -f "$results_dir/benchmark-output.json" ]] || die "No benchmark results found" - log "Results: $results_dir/benchmark-output.json" + [[ -f "$output_file" ]] || die "No benchmark results found" + log "Results: $output_file" } check_status() { @@ -230,12 +229,6 @@ cmd_trigger() { download_artifact "$run_id" } -cmd_download() { - [[ -z "${1:-}" ]] && die "usage: $0 download " - require_gh - download_artifact "$1" -} - cmd_status() { [[ -z "${1:-}" ]] && die "usage: $0 status " require_gh @@ -257,12 +250,11 @@ USAGE ./bench-cchain-reexecution.sh [args] COMMANDS - trigger [test] Trigger benchmark, wait, download results - download Download results for a run - status Check run status - list List recent runs - tests Show available tests - help Show this help + trigger [test] Trigger benchmark, wait, download results + status Check run status + list List recent runs + tests Show available tests + help Show this help TESTS EOF @@ -275,7 +267,6 @@ main() { case "$cmd" in trigger) cmd_trigger "$@" ;; - download) cmd_download "$@" ;; status) cmd_status "$@" ;; list) cmd_list "$@" ;; tests) cmd_tests "$@" ;; From 8b92cc6fa3485f703ad1802f79c1f0b63fa3ea83 Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Sun, 25 Jan 2026 13:15:00 +0100 Subject: [PATCH 28/40] docs --- METRICS.md | 82 +++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 62 insertions(+), 20 deletions(-) diff --git a/METRICS.md b/METRICS.md index 6ed5efa3e..ecd1a32dc 100644 --- a/METRICS.md +++ b/METRICS.md @@ -322,37 +322,79 @@ The easiest way to trigger a benchmark is via the GitHub Actions UI: 3. Select parameters from the dropdowns (task, runner) or enter custom values 4. Click "Run workflow" -### Running Reexecution Locally +### Triggering Benchmarks via CLI -Reexecution test can be triggered locally using `just` commands (requires nix). +Benchmarks run on AvalancheGo's self-hosted runners, not locally. This enables end-to-end integration testing where: -Example: Trigger a C-Chain reexecution in AvalancheGo, wait for completion, and download results: +- Firewood team can benchmark changes against the full AvalancheGo stack +- AvalancheGo team can iterate on their Firewood integration -```bash -RUN_ID=$(just trigger-reexecution firewood=v0.0.15 avalanchego=master task=c-chain-reexecution-firewood-101-250k runner=avalanche-avalanchego-runner-2ti) \ - && just wait-reexecution run_id=$RUN_ID \ - && just download-reexecution-results run_id=$RUN_ID +```mermaid +sequenceDiagram + participant F as Firewood + participant A as AvalancheGo + participant G as GitHub Pages + + F->>A: 1. trigger workflow + A->>A: 2. run benchmark + A-->>F: 3. download results + F->>G: 4. publish ``` -**Custom example:** Trigger with specific block range and config: +The CLI commands trigger the remote workflow, wait for completion, and download the results. ```bash -RUN_ID=$(just trigger-custom-reexecution firewood=v0.0.15 avalanchego=master config=firewood start-block=101 end-block=250000 block-dir-src=cchain-mainnet-blocks-1m-ldb current-state-dir-src=cchain-mainnet-state-100-fw runner=avalanche-avalanchego-runner-2ti) \ - && just wait-reexecution run_id=$RUN_ID \ - && just download-reexecution-results run_id=$RUN_ID +# Predefined test +just bench-cchain firewood-101-250k + +# With specific Firewood version +FIREWOOD_REF=v0.1.0 just bench-cchain firewood-33m-40m + +# Custom block range +START_BLOCK=101 END_BLOCK=250000 \ + BLOCK_DIR_SRC=cchain-mainnet-blocks-1m-ldb \ + CURRENT_STATE_DIR_SRC=cchain-current-state-firewood-100 \ + just bench-cchain ``` -**Available commands:** +**Commands:** | Command | Description | |---------|-------------| -| `trigger-reexecution` | Trigger task-based reexecution, returns run_id | -| `trigger-custom-reexecution` | Trigger with custom block range/config | -| `wait-reexecution` | Wait for run to complete | -| `download-reexecution-results` | Download results to `./results/` | -| `list-reexecutions` | List recent reexecution runs | +| `bench-cchain [test]` | Trigger remote benchmark, wait, download results | +| `status-cchain ` | Check status of a run | +| `list-cchain` | List recent runs | +| `help-cchain` | Show help and available options | + +**Environment variables:** + +| Variable | Default | Description | +|----------|---------|-------------| +| `GH_TOKEN` | required | GitHub token for API access | +| `FIREWOOD_REF` | HEAD | Firewood commit/tag/branch | +| `AVALANCHEGO_REF` | master | AvalancheGo ref to test against | +| `LIBEVM_REF` | - | Optional libevm ref | +| `RUNNER` | avalanche-avalanchego-runner-2ti | GitHub Actions runner | +| `TIMEOUT_MINUTES` | - | Workflow timeout | +| `DOWNLOAD_DIR` | ./results | Directory for downloaded artifacts | + +**Custom mode variables** (when no test specified): + +| Variable | Default | Description | +|----------|---------|-------------| +| `CONFIG` | firewood | VM config (firewood, hashdb, etc.) | +| `START_BLOCK` | required | First block number | +| `END_BLOCK` | required | Last block number | +| `BLOCK_DIR_SRC` | required | S3 block directory | +| `CURRENT_STATE_DIR_SRC` | required | S3 state directory | + +**Tests and runners** are defined in AvalancheGo: + +- [Available tests](https://github.com/ava-labs/avalanchego/blob/master/scripts/benchmark_cchain_range.sh) +- [C-Chain benchmark docs](https://github.com/ava-labs/avalanchego/blob/master/tests/reexecute/c/README.md) -**Tasks and runners** are defined in AvalancheGo: +### Viewing Results -- [Available tasks](https://github.com/ava-labs/avalanchego/blob/master/.github/workflows/c-chain-reexecution-benchmark-container.json) -- [C-Chain benchmark docs](https://github.com/ava-labs/avalanchego/blob/master/tests/reexecute/c/README.md) +Results are published to GitHub Pages via [github-action-benchmark](https://github.com/benchmark-action/github-action-benchmark). View trends at: + +- [Performance Trends](https://ava-labs.github.io/firewood/bench/) From 98c9d0058a971666159c15981e997d1674c4094e Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Sun, 25 Jan 2026 14:24:55 +0100 Subject: [PATCH 29/40] ci(track-performance): empty `inputs.test` was causing CI workflow to fail --- .github/workflows/track-performance.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/track-performance.yml b/.github/workflows/track-performance.yml index 42a65be74..2856d7627 100644 --- a/.github/workflows/track-performance.yml +++ b/.github/workflows/track-performance.yml @@ -52,7 +52,12 @@ jobs: uses: actions/checkout@v4 - name: Trigger C-Chain Reexecution Benchmark - run: ./scripts/bench-cchain-reexecution.sh trigger "${{ inputs.test }}" + run: | + if [[ -n "${{ inputs.test }}" ]]; then + ./scripts/bench-cchain-reexecution.sh trigger "${{ inputs.test }}" + else + ./scripts/bench-cchain-reexecution.sh trigger + fi env: GH_TOKEN: ${{ secrets.FIREWOOD_AVALANCHEGO_GITHUB_TOKEN }} # Custom mode (ignored when test is specified) From 2b128a82ca25520696d5e21d1b7017691ab86cd9 Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Sun, 25 Jan 2026 14:27:32 +0100 Subject: [PATCH 30/40] debug --- .github/workflows/track-performance.yml | 3 ++ scripts/bench-cchain-reexecution.sh | 67 +++++++++++++++++-------- 2 files changed, 48 insertions(+), 22 deletions(-) diff --git a/.github/workflows/track-performance.yml b/.github/workflows/track-performance.yml index 2856d7627..92c61390b 100644 --- a/.github/workflows/track-performance.yml +++ b/.github/workflows/track-performance.yml @@ -53,6 +53,9 @@ jobs: - name: Trigger C-Chain Reexecution Benchmark run: | + echo "==> Starting benchmark trigger" + echo "==> Script exists: $(test -f ./scripts/bench-cchain-reexecution.sh && echo yes || echo no)" + echo "==> Test input: '${{ inputs.test }}'" if [[ -n "${{ inputs.test }}" ]]; then ./scripts/bench-cchain-reexecution.sh trigger "${{ inputs.test }}" else diff --git a/scripts/bench-cchain-reexecution.sh b/scripts/bench-cchain-reexecution.sh index ba47a148a..eda11baf5 100755 --- a/scripts/bench-cchain-reexecution.sh +++ b/scripts/bench-cchain-reexecution.sh @@ -16,7 +16,7 @@ set -euo pipefail # ENVIRONMENT # GH_TOKEN (string, required) GitHub token for API access # TEST (string, optional) Predefined test name (alternative to arg) -# FIREWOOD_REF (string, HEAD) Firewood commit/tag/branch +# FIREWOOD_REF (string, optional) Firewood commit/tag/branch (empty = AvalancheGo's go.mod default) # AVALANCHEGO_REF (string, master) AvalancheGo ref to test against # RUNNER (string, avalanche-avalanchego-runner-2ti) GitHub Actions runner label # LIBEVM_REF (string, optional) Optional libevm ref @@ -28,7 +28,7 @@ set -euo pipefail # START_BLOCK (required) First block number # END_BLOCK (required) Last block number # BLOCK_DIR_SRC (required) S3 block directory (without S3:// prefix, e.g., cchain-mainnet-blocks-200-ldb) -# CURRENT_STATE_DIR_SRC (required) S3 state directory (without S3:// prefix, e.g., cchain-current-state-mainnet-100-firewood) +# CURRENT_STATE_DIR_SRC (optional) S3 state directory (empty = genesis run) # # # EXAMPLES @@ -49,7 +49,7 @@ REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" AVALANCHEGO_REPO="ava-labs/avalanchego" WORKFLOW_NAME="C-Chain Re-Execution Benchmark w/ Container" -: "${FIREWOOD_REF:=HEAD}" +: "${FIREWOOD_REF:=}" : "${AVALANCHEGO_REF:=master}" : "${RUNNER:=avalanche-avalanchego-runner-2ti}" : "${LIBEVM_REF:=}" @@ -73,7 +73,7 @@ declare -A TESTS=( ["firewood-archive-33m-40m"]="Blocks 33m-40m (archive)" ) -log() { echo "==> $1"; } +log() { echo "==> $1" >&2; } # Use GitHub Actions error annotation format in CI for better visibility err() { @@ -90,7 +90,9 @@ require_gh() { if ! command -v gh &>/dev/null; then die "gh CLI not found. Run from nix shell: nix develop ./ffi" fi - [[ -z "${GH_TOKEN:-}" ]] && die "GH_TOKEN is required" + if [[ -z "${GH_TOKEN:-}" ]]; then + die "GH_TOKEN is required" + fi } # Prevent command injection via malicious ref names passed to gh CLI @@ -101,62 +103,81 @@ validate_ref() { poll_workflow_registration() { local trigger_time="$1" - log "Waiting for workflow to register..." + log "Waiting for workflow to register (looking for runs after $trigger_time)..." + + # Verify we can list runs (fail fast on permission issues) + if ! gh run list --repo "$AVALANCHEGO_REPO" --workflow "$WORKFLOW_NAME" --limit 1 &>/dev/null; then + die "Cannot list runs in $AVALANCHEGO_REPO. Check token permissions." + fi for ((i=0; i $t)] | .[0].databaseId' 2>/dev/null || echo "") + --json databaseId,createdAt 2>&1) || { + log "gh run list failed: $raw_output" + continue + } + + # Debug: show first result on first iteration + ((i == 0)) && log "Latest run: $(echo "$raw_output" | jq -c '.[0] // "none"')" + + run_id=$(echo "$raw_output" | jq -r --arg t "$trigger_time" '[.[] | select(.createdAt > $t)] | .[0].databaseId // empty') if [[ -n "$run_id" && "$run_id" != "null" ]]; then echo "$run_id" return 0 fi + + # Progress indicator every 10 seconds + ((i % 10 == 0)) && ((i > 0)) && log "Polling (${i}s)" done - die "workflow not found after ${POLL_TIMEOUT}s" + die "Workflow not found after ${POLL_TIMEOUT}s. The workflow may have been triggered but couldn't be detected. Check: https://github.com/$AVALANCHEGO_REPO/actions/workflows" } trigger_workflow() { local test="${1:-}" local firewood="$FIREWOOD_REF" - # Resolve HEAD to commit SHA for reproducibility—"HEAD" means different things at different times - [[ "$firewood" == "HEAD" ]] && firewood=$(git -C "$REPO_ROOT" rev-parse HEAD) - validate_ref "$firewood" "FIREWOOD_REF" + # Validate refs if provided + if [[ -n "$firewood" ]]; then + # Resolve HEAD to commit SHA for reproducibility + [[ "$firewood" == "HEAD" ]] && firewood=$(git -C "$REPO_ROOT" rev-parse HEAD) + validate_ref "$firewood" "FIREWOOD_REF" + fi [[ -n "$LIBEVM_REF" ]] && validate_ref "$LIBEVM_REF" "LIBEVM_REF" local args=(-f runner="$RUNNER") + [[ -n "$firewood" ]] && args+=(-f firewood-ref="$firewood") + [[ -n "$LIBEVM_REF" ]] && args+=(-f libevm-ref="$LIBEVM_REF") [[ -n "$TIMEOUT_MINUTES" ]] && args+=(-f timeout-minutes="$TIMEOUT_MINUTES") if [[ -n "$test" ]]; then args+=(-f test="$test") log "Triggering: $test" else - # Custom mode: block params required, config defaults to firewood - [[ -z "${START_BLOCK:-}${END_BLOCK:-}${BLOCK_DIR_SRC:-}${CURRENT_STATE_DIR_SRC:-}" ]] && \ - die "Provide a test name or set START_BLOCK, END_BLOCK, BLOCK_DIR_SRC, CURRENT_STATE_DIR_SRC" + # Custom mode: block params required, CURRENT_STATE_DIR_SRC optional (empty = genesis) + [[ -z "${START_BLOCK:-}${END_BLOCK:-}${BLOCK_DIR_SRC:-}" ]] && \ + die "Provide a test name or set START_BLOCK, END_BLOCK, BLOCK_DIR_SRC" : "${START_BLOCK:?START_BLOCK required}" : "${END_BLOCK:?END_BLOCK required}" : "${BLOCK_DIR_SRC:?BLOCK_DIR_SRC required}" - : "${CURRENT_STATE_DIR_SRC:?CURRENT_STATE_DIR_SRC required}" args+=( -f config="$CONFIG" -f start-block="$START_BLOCK" -f end-block="$END_BLOCK" -f block-dir-src="$BLOCK_DIR_SRC" - -f current-state-dir-src="$CURRENT_STATE_DIR_SRC" ) - log "Triggering: $CONFIG $START_BLOCK-$END_BLOCK" + [[ -n "${CURRENT_STATE_DIR_SRC:-}" ]] && args+=(-f current-state-dir-src="$CURRENT_STATE_DIR_SRC") + log "Triggering: $CONFIG $START_BLOCK-$END_BLOCK${CURRENT_STATE_DIR_SRC:+ (with state)}" fi - log "firewood: $firewood" log "avalanchego: $AVALANCHEGO_REF" + log "firewood: ${firewood:-}" log "runner: $RUNNER" # Record time BEFORE triggering to avoid race condition: we only look for @@ -183,6 +204,7 @@ download_artifact() { local run_id="$1" local output_file="$DOWNLOAD_DIR/benchmark-output.json" + log "Downloading artifact..." mkdir -p "$DOWNLOAD_DIR" gh run download "$run_id" \ --repo "$AVALANCHEGO_REPO" \ @@ -223,10 +245,11 @@ cmd_trigger() { local test="${1:-${TEST:-}}" local run_id run_id=$(trigger_workflow "$test") - log "run_id: $run_id" + log "Run ID: $run_id" wait_for_completion "$run_id" download_artifact "$run_id" + log "Done" } cmd_status() { From a3966e6dca291b990ca44a3dcd178b26e2062d96 Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Sun, 25 Jan 2026 14:33:02 +0100 Subject: [PATCH 31/40] refactor(benchmark): trigger C-Chain benchmarks via Firewood CI workflow - Refactor `bench-cchain` to trigger track-performance.yml instead of directly calling AvalancheGo's workflow - Add input validation and helpful error messages for test/custom params - Use commit SHA instead of branch name for reproducibility - Fix AvalancheGo workflow input: use `with-dependencies` format ("firewood=abc,libevm=xyz") instead of separate firewood-ref/libevm-ref - Remove status-cchain, list-cchain, help-cchain (use GitHub UI instead) - Remove debug logging from track-performance.yml --- .github/workflows/track-performance.yml | 3 - justfile | 109 ++++++++++++++---------- scripts/bench-cchain-reexecution.sh | 8 +- 3 files changed, 70 insertions(+), 50 deletions(-) diff --git a/.github/workflows/track-performance.yml b/.github/workflows/track-performance.yml index 92c61390b..2856d7627 100644 --- a/.github/workflows/track-performance.yml +++ b/.github/workflows/track-performance.yml @@ -53,9 +53,6 @@ jobs: - name: Trigger C-Chain Reexecution Benchmark run: | - echo "==> Starting benchmark trigger" - echo "==> Script exists: $(test -f ./scripts/bench-cchain-reexecution.sh && echo yes || echo no)" - echo "==> Test input: '${{ inputs.test }}'" if [[ -n "${{ inputs.test }}" ]]; then ./scripts/bench-cchain-reexecution.sh trigger "${{ inputs.test }}" else diff --git a/justfile b/justfile index 9a9224bcc..c7e96649d 100644 --- a/justfile +++ b/justfile @@ -170,49 +170,68 @@ release-step-refresh-changelog tag: git cliff -o CHANGELOG.md --tag "{{tag}}" # Run a C-Chain reexecution benchmark -# just bench-cchain - predefined test -# FIREWOOD_REF=v0.1.0 just bench-cchain firewood-101-250k - with specific ref -# Custom: START_BLOCK=101 END_BLOCK=250000 BLOCK_DIR_SRC= CURRENT_STATE_DIR_SRC= just bench-cchain -bench-cchain test="": - ./scripts/bench-cchain-reexecution.sh trigger "{{ test }}" - -# Check status of a C-Chain benchmark run -status-cchain run_id: - ./scripts/bench-cchain-reexecution.sh status "{{ run_id }}" - -# List recent C-Chain benchmark runs -list-cchain: - ./scripts/bench-cchain-reexecution.sh list - -# Show C-Chain benchmark help -help-cchain: - #!/usr/bin/env bash - echo "USAGE" - echo " just bench-cchain Predefined test" - echo " [CUSTOM_VARS] just bench-cchain Custom mode" - echo " just status-cchain " - echo " just list-cchain" - echo "" - echo "ENVIRONMENT VARIABLES" - echo " FIREWOOD_REF Firewood ref (default: HEAD)" - echo " AVALANCHEGO_REF AvalancheGo ref (default: master)" - echo " LIBEVM_REF libevm ref (optional)" - echo " RUNNER GitHub Actions runner (default: avalanche-avalanchego-runner-2ti)" - echo " CONFIG VM config (default: firewood)" - echo " START_BLOCK Start block for custom mode" - echo " END_BLOCK End block for custom mode" - echo " BLOCK_DIR_SRC S3 block directory" - echo " CURRENT_STATE_DIR_SRC S3 state directory" - echo "" - echo "EXAMPLES" - echo " # Predefined test" - echo " just bench-cchain firewood-101-250k" - echo " FIREWOOD_REF=v0.1.0 just bench-cchain firewood-33m-40m" - echo "" - echo " # Custom mode" - echo " START_BLOCK=101 END_BLOCK=250000 \\" - echo " BLOCK_DIR_SRC=cchain-mainnet-blocks-1m-ldb \\" - echo " CURRENT_STATE_DIR_SRC=cchain-current-state-mainnet-ldb \\" - echo " just bench-cchain" +# Triggers Firewood's track-performance.yml which then triggers AvalancheGo. +# This ensures results appear in Firewood's workflow summary and get published +# to GitHub Pages for the current branch. +# +# By default, uses the current HEAD commit to build Firewood. If you want to +# benchmark a specific version (e.g., a release tag), set FIREWOOD_REF explicitly: +# FIREWOOD_REF=v0.1.0 just bench-cchain firewood-101-250k +# +# Examples: +# just bench-cchain firewood-101-250k +# FIREWOOD_REF=v0.1.0 just bench-cchain firewood-101-250k +# START_BLOCK=1 END_BLOCK=100 BLOCK_DIR_SRC=cchain-mainnet-blocks-200-ldb just bench-cchain +bench-cchain test="" runner="avalanche-avalanchego-runner-2ti": + #!/usr/bin/env -S bash -euo pipefail + + # Resolve gh CLI + if command -v gh &>/dev/null; then + GH=gh + elif command -v nix &>/dev/null; then + GH="nix run ./ffi#gh --" + else + echo "error: 'gh' CLI not found. Install it or use 'nix develop ./ffi'" >&2 + exit 1 + fi + + test="{{ test }}" + + # Validate: need either test name OR custom block params + if [[ -z "$test" && -z "${START_BLOCK:-}" ]]; then + echo "error: Provide a test name or set START_BLOCK, END_BLOCK, BLOCK_DIR_SRC" >&2 + echo "" >&2 + echo "Predefined tests:" >&2 + echo " firewood-101-250k, firewood-33m-33m500k, firewood-33m-40m" >&2 + echo " firewood-archive-101-250k, firewood-archive-33m-33m500k, firewood-archive-33m-40m" >&2 + echo "" >&2 + echo "Custom mode example:" >&2 + echo " START_BLOCK=1 END_BLOCK=100 BLOCK_DIR_SRC=cchain-mainnet-blocks-200-ldb just bench-cchain" >&2 + exit 1 + fi + + # Build workflow args + args=(-f runner="{{ runner }}") + [[ -n "$test" ]] && args+=(-f test="$test") + [[ -n "${FIREWOOD_REF:-}" ]] && args+=(-f firewood="$FIREWOOD_REF") + [[ -n "${LIBEVM_REF:-}" ]] && args+=(-f libevm="$LIBEVM_REF") + [[ -n "${AVALANCHEGO_REF:-}" ]] && args+=(-f avalanchego="$AVALANCHEGO_REF") + [[ -n "${CONFIG:-}" ]] && args+=(-f config="$CONFIG") + [[ -n "${START_BLOCK:-}" ]] && args+=(-f start-block="$START_BLOCK") + [[ -n "${END_BLOCK:-}" ]] && args+=(-f end-block="$END_BLOCK") + [[ -n "${BLOCK_DIR_SRC:-}" ]] && args+=(-f block-dir-src="$BLOCK_DIR_SRC") + [[ -n "${CURRENT_STATE_DIR_SRC:-}" ]] && args+=(-f current-state-dir-src="$CURRENT_STATE_DIR_SRC") + [[ -n "${TIMEOUT_MINUTES:-}" ]] && args+=(-f timeout-minutes="$TIMEOUT_MINUTES") + + commit=$(git rev-parse HEAD) + + echo "==> Triggering benchmark on commit: $commit" + [[ -n "$test" ]] && echo "==> Test: $test" + [[ -n "${START_BLOCK:-}" ]] && echo "==> Custom: blocks $START_BLOCK-${END_BLOCK:-?}" + echo "==> Runner: {{ runner }}" + + $GH workflow run track-performance.yml --ref "$commit" "${args[@]}" echo "" - echo "Advanced: ./scripts/bench-cchain-reexecution.sh help" + echo "Workflow triggered. View at:" + echo " https://github.com/ava-labs/firewood/actions/workflows/track-performance.yml" + diff --git a/scripts/bench-cchain-reexecution.sh b/scripts/bench-cchain-reexecution.sh index eda11baf5..c30830446 100755 --- a/scripts/bench-cchain-reexecution.sh +++ b/scripts/bench-cchain-reexecution.sh @@ -152,10 +152,14 @@ trigger_workflow() { [[ -n "$LIBEVM_REF" ]] && validate_ref "$LIBEVM_REF" "LIBEVM_REF" local args=(-f runner="$RUNNER") - [[ -n "$firewood" ]] && args+=(-f firewood-ref="$firewood") - [[ -n "$LIBEVM_REF" ]] && args+=(-f libevm-ref="$LIBEVM_REF") [[ -n "$TIMEOUT_MINUTES" ]] && args+=(-f timeout-minutes="$TIMEOUT_MINUTES") + # Build with-dependencies string (format: "firewood=abc,libevm=xyz") + local deps="" + [[ -n "$firewood" ]] && deps="firewood=$firewood" + [[ -n "$LIBEVM_REF" ]] && deps="${deps:+$deps,}libevm=$LIBEVM_REF" + [[ -n "$deps" ]] && args+=(-f with-dependencies="$deps") + if [[ -n "$test" ]]; then args+=(-f test="$test") log "Triggering: $test" From 65224f8f0a84af9658c8bbd52d2448a6c92fde34 Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Sun, 25 Jan 2026 14:44:08 +0100 Subject: [PATCH 32/40] temp --- scripts/bench-cchain-reexecution.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/bench-cchain-reexecution.sh b/scripts/bench-cchain-reexecution.sh index c30830446..1955c01be 100755 --- a/scripts/bench-cchain-reexecution.sh +++ b/scripts/bench-cchain-reexecution.sh @@ -191,7 +191,7 @@ trigger_workflow() { gh workflow run "$WORKFLOW_NAME" \ --repo "$AVALANCHEGO_REPO" \ - --ref "$AVALANCHEGO_REF" \ + --ref es/enable-firewood-dev-workflow \ # @TODO: update to use AVALANCHEGO_REF once '#4560' in AvalancheGo is merged "${args[@]}" poll_workflow_registration "$trigger_time" From 25cde31155e0dc671caf2cff5aed5a76daeb2884 Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Sun, 25 Jan 2026 14:47:00 +0100 Subject: [PATCH 33/40] temp --- scripts/bench-cchain-reexecution.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/bench-cchain-reexecution.sh b/scripts/bench-cchain-reexecution.sh index 1955c01be..13dd6971e 100755 --- a/scripts/bench-cchain-reexecution.sh +++ b/scripts/bench-cchain-reexecution.sh @@ -189,9 +189,10 @@ trigger_workflow() { local trigger_time trigger_time=$(date -u +%Y-%m-%dT%H:%M:%SZ) + # TODO: use $AVALANCHEGO_REF once ava-labs/avalanchego#4560 is merged gh workflow run "$WORKFLOW_NAME" \ --repo "$AVALANCHEGO_REPO" \ - --ref es/enable-firewood-dev-workflow \ # @TODO: update to use AVALANCHEGO_REF once '#4560' in AvalancheGo is merged + --ref es/enable-firewood-dev-workflow \ "${args[@]}" poll_workflow_registration "$trigger_time" From e19e9510812ba2ac7df19b84296b4bc67783b7a8 Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Sun, 25 Jan 2026 15:06:30 +0100 Subject: [PATCH 34/40] fix(track-performance): improve concurrent trigger handling Add input verification to reliably identify workflow runs when multiple triggers occur simultaneously. Previously, the script picked the newest run after a timestamp which could select the wrong run in concurrent scenarios. Changes: - Add verify_run_inputs() to match runs by job name parameters (test name, config, start-block, runner) - Iterate candidate runs oldest-first and verify inputs match - Increase run list limit from 5 to 10 for better coverage - Document dependency on AvalancheGo's job naming convention This partially handles concurrent triggers: runs triggered seconds apart are now correctly identified. Same-second collisions remain ambiguous but are rare in practice. --- .github/workflows/track-performance.yml | 1 + justfile | 34 +++++++++++-- scripts/bench-cchain-reexecution.sh | 67 ++++++++++++++++++++++--- 3 files changed, 89 insertions(+), 13 deletions(-) diff --git a/.github/workflows/track-performance.yml b/.github/workflows/track-performance.yml index 2856d7627..b27d348c8 100644 --- a/.github/workflows/track-performance.yml +++ b/.github/workflows/track-performance.yml @@ -94,6 +94,7 @@ jobs: tool: 'customBiggerIsBetter' output-file-path: ./results/benchmark-output.json github-token: ${{ secrets.GITHUB_TOKEN }} + summary-always: true auto-push: true gh-pages-branch: benchmark-data benchmark-data-dir-path: ${{ steps.location.outputs.data-dir }} diff --git a/justfile b/justfile index c7e96649d..f4c881665 100644 --- a/justfile +++ b/justfile @@ -174,6 +174,8 @@ release-step-refresh-changelog tag: # This ensures results appear in Firewood's workflow summary and get published # to GitHub Pages for the current branch. # +# Note: Changes must be pushed to the remote branch for the workflow to use them. +# # By default, uses the current HEAD commit to build Firewood. If you want to # benchmark a specific version (e.g., a release tag), set FIREWOOD_REF explicitly: # FIREWOOD_REF=v0.1.0 just bench-cchain firewood-101-250k @@ -223,15 +225,37 @@ bench-cchain test="" runner="avalanche-avalanchego-runner-2ti": [[ -n "${CURRENT_STATE_DIR_SRC:-}" ]] && args+=(-f current-state-dir-src="$CURRENT_STATE_DIR_SRC") [[ -n "${TIMEOUT_MINUTES:-}" ]] && args+=(-f timeout-minutes="$TIMEOUT_MINUTES") - commit=$(git rev-parse HEAD) + branch=$(git rev-parse --abbrev-ref HEAD) - echo "==> Triggering benchmark on commit: $commit" [[ -n "$test" ]] && echo "==> Test: $test" [[ -n "${START_BLOCK:-}" ]] && echo "==> Custom: blocks $START_BLOCK-${END_BLOCK:-?}" echo "==> Runner: {{ runner }}" - $GH workflow run track-performance.yml --ref "$commit" "${args[@]}" + # Record time before triggering to find our run (avoid race conditions) + trigger_time=$(date -u +%Y-%m-%dT%H:%M:%SZ) + + $GH workflow run track-performance.yml --ref "$branch" "${args[@]}" + + # Poll for workflow registration (runs created after trigger_time) echo "" - echo "Workflow triggered. View at:" - echo " https://github.com/ava-labs/firewood/actions/workflows/track-performance.yml" + echo "Polling for workflow to register..." + for i in {1..30}; do + sleep 1 + run_id=$($GH run list --workflow=track-performance.yml --limit=10 --json databaseId,createdAt \ + --jq "[.[] | select(.createdAt > \"$trigger_time\")] | .[-1].databaseId // empty") + [[ -n "$run_id" ]] && break + done + + if [[ -z "$run_id" ]]; then + echo "warning: Could not find run ID. Check manually at:" + echo " https://github.com/ava-labs/firewood/actions/workflows/track-performance.yml" + exit 0 + fi + + echo "" + echo "Workflow triggered. Run ID: $run_id" + echo "https://github.com/ava-labs/firewood/actions/runs/$run_id" + echo "" + + $GH run watch "$run_id" diff --git a/scripts/bench-cchain-reexecution.sh b/scripts/bench-cchain-reexecution.sh index 13dd6971e..525ff622c 100755 --- a/scripts/bench-cchain-reexecution.sh +++ b/scripts/bench-cchain-reexecution.sh @@ -101,8 +101,52 @@ validate_ref() { [[ "$ref" =~ ^[a-zA-Z0-9/_.-]+$ ]] || die "$name contains invalid characters: $ref" } +# Verify a run's inputs match what we triggered with +# Returns 0 if match, 1 if no match +# +# NOTE: We rely on AvalancheGo's workflow job naming convention: +# "c-chain-reexecution (start, end, block-dir, config, runner...)" +# If AvalancheGo changes this format, this matching logic will need updating. +verify_run_inputs() { + local run_id="$1" expected_test="$2" expected_config="$3" expected_start="$4" expected_end="$5" expected_runner="$6" + + # Get the c-chain-reexecution job name (second job, after define-matrix) + local job_name + job_name=$(gh run view "$run_id" --repo "$AVALANCHEGO_REPO" --json jobs \ + --jq '.jobs[] | select(.name | startswith("c-chain-reexecution")) | .name // ""' 2>/dev/null) || return 1 + + # If job hasn't started yet (only define-matrix exists), accept for now + [[ -z "$job_name" ]] && return 0 + + # If we have a test name, check if it appears in job name + if [[ -n "$expected_test" ]]; then + [[ "$job_name" == *"$expected_test"* ]] && return 0 + return 1 + fi + + # For custom mode, check that our params appear in job name + # Job name: "c-chain-reexecution (1, 100, cchain-mainnet-blocks-200-ldb, firewood, runner...)" + local match=true + [[ -n "$expected_start" && "$job_name" != *"$expected_start"* ]] && match=false + [[ -n "$expected_config" && "$job_name" != *"$expected_config"* ]] && match=false + [[ -n "$expected_runner" && "$job_name" != *"$expected_runner"* ]] && match=false + + $match && return 0 + + # Fallback: if no identifiers to match, accept the run + [[ -z "$expected_test" && -z "$expected_start" && -z "$expected_runner" ]] && return 0 + + return 1 +} + poll_workflow_registration() { local trigger_time="$1" + local expected_test="${2:-}" + local expected_config="${3:-}" + local expected_start="${4:-}" + local expected_end="${5:-}" + local expected_runner="${6:-}" + log "Waiting for workflow to register (looking for runs after $trigger_time)..." # Verify we can list runs (fail fast on permission issues) @@ -112,11 +156,11 @@ poll_workflow_registration() { for ((i=0; i&1) || { log "gh run list failed: $raw_output" continue @@ -125,12 +169,18 @@ poll_workflow_registration() { # Debug: show first result on first iteration ((i == 0)) && log "Latest run: $(echo "$raw_output" | jq -c '.[0] // "none"')" - run_id=$(echo "$raw_output" | jq -r --arg t "$trigger_time" '[.[] | select(.createdAt > $t)] | .[0].databaseId // empty') + # Get candidate runs (created after trigger_time), oldest first + local candidates + candidates=$(echo "$raw_output" | jq -r --arg t "$trigger_time" \ + '[.[] | select(.createdAt > $t)] | reverse | .[].databaseId') - if [[ -n "$run_id" && "$run_id" != "null" ]]; then - echo "$run_id" - return 0 - fi + # Check each candidate's inputs to find our run + for run_id in $candidates; do + if verify_run_inputs "$run_id" "$expected_test" "$expected_config" "$expected_start" "$expected_end" "$expected_runner"; then + echo "$run_id" + return 0 + fi + done # Progress indicator every 10 seconds ((i % 10 == 0)) && ((i > 0)) && log "Polling (${i}s)" @@ -195,7 +245,8 @@ trigger_workflow() { --ref es/enable-firewood-dev-workflow \ "${args[@]}" - poll_workflow_registration "$trigger_time" + # Pass test/custom params to help identify our specific run among concurrent triggers + poll_workflow_registration "$trigger_time" "$test" "$CONFIG" "${START_BLOCK:-}" "${END_BLOCK:-}" "$RUNNER" } wait_for_completion() { From 2b144f029eb6ed7ad81de5a5669791908660640e Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Sun, 25 Jan 2026 15:30:55 +0100 Subject: [PATCH 35/40] docs --- METRICS.md | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/METRICS.md b/METRICS.md index ecd1a32dc..2f720f072 100644 --- a/METRICS.md +++ b/METRICS.md @@ -303,8 +303,6 @@ rate(firewood_proposal_commit[5m]) Firewood tracks its performance over time by running [C-Chain reexecution benchmarks](https://github.com/ava-labs/avalanchego/blob/master/tests/reexecute/c/README.md) in AvalancheGo. These benchmarks re-execute historical mainnet C-Chain blocks against a state snapshot, measuring throughput in mgas/s (million gas per second). -By default, the benchmark processes ~250,000 blocks (101 → 250k) and takes approximately 7 minutes on self-hosted runners. - This allows us to: - Monitor performance across commits and releases @@ -344,6 +342,9 @@ sequenceDiagram The CLI commands trigger the remote workflow, wait for completion, and download the results. ```bash +nix run ./ffi#gh -- gh auth login +export GH_TOKEN=$(gh auth token) + # Predefined test just bench-cchain firewood-101-250k @@ -357,26 +358,25 @@ START_BLOCK=101 END_BLOCK=250000 \ just bench-cchain ``` -**Commands:** +**Command:** + +```bash +just bench-cchain [test] +``` + +Triggers Firewood's `track-performance.yml` workflow, which orchestrates the AvalancheGo benchmark. The command polls for the workflow run and watches progress in terminal. -| Command | Description | -|---------|-------------| -| `bench-cchain [test]` | Trigger remote benchmark, wait, download results | -| `status-cchain ` | Check status of a run | -| `list-cchain` | List recent runs | -| `help-cchain` | Show help and available options | +> **Note:** Changes must be pushed to the remote branch for the workflow to use them. By default, the workflow builds Firewood from the current commit. To benchmark a specific version (e.g., a release tag), set `FIREWOOD_REF` explicitly. **Environment variables:** | Variable | Default | Description | |----------|---------|-------------| -| `GH_TOKEN` | required | GitHub token for API access | -| `FIREWOOD_REF` | HEAD | Firewood commit/tag/branch | +| `FIREWOOD_REF` | current commit | Firewood commit/tag/branch to build | | `AVALANCHEGO_REF` | master | AvalancheGo ref to test against | | `LIBEVM_REF` | - | Optional libevm ref | | `RUNNER` | avalanche-avalanchego-runner-2ti | GitHub Actions runner | | `TIMEOUT_MINUTES` | - | Workflow timeout | -| `DOWNLOAD_DIR` | ./results | Directory for downloaded artifacts | **Custom mode variables** (when no test specified): @@ -386,7 +386,7 @@ START_BLOCK=101 END_BLOCK=250000 \ | `START_BLOCK` | required | First block number | | `END_BLOCK` | required | Last block number | | `BLOCK_DIR_SRC` | required | S3 block directory | -| `CURRENT_STATE_DIR_SRC` | required | S3 state directory | +| `CURRENT_STATE_DIR_SRC` | - | S3 state directory (empty = genesis run) | **Tests and runners** are defined in AvalancheGo: From fb0a9cc8d1972d89327c49844e62fe4ae098bdd6 Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Sun, 25 Jan 2026 15:37:58 +0100 Subject: [PATCH 36/40] docs --- .github/workflows/track-performance.yml | 5 ++--- METRICS.md | 1 + 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/track-performance.yml b/.github/workflows/track-performance.yml index b27d348c8..cd72736ee 100644 --- a/.github/workflows/track-performance.yml +++ b/.github/workflows/track-performance.yml @@ -87,7 +87,6 @@ jobs: - name: Publish benchmark results id: store - continue-on-error: true uses: benchmark-action/github-action-benchmark@v1 with: name: C-Chain Reexecution with Firewood @@ -96,10 +95,10 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} summary-always: true auto-push: true - gh-pages-branch: benchmark-data - benchmark-data-dir-path: ${{ steps.location.outputs.data-dir }} fail-on-alert: true comment-on-alert: false + gh-pages-branch: benchmark-data + benchmark-data-dir-path: ${{ steps.location.outputs.data-dir }} - name: Summary run: | diff --git a/METRICS.md b/METRICS.md index 2f720f072..d70f26a03 100644 --- a/METRICS.md +++ b/METRICS.md @@ -377,6 +377,7 @@ Triggers Firewood's `track-performance.yml` workflow, which orchestrates the Ava | `LIBEVM_REF` | - | Optional libevm ref | | `RUNNER` | avalanche-avalanchego-runner-2ti | GitHub Actions runner | | `TIMEOUT_MINUTES` | - | Workflow timeout | +| `DOWNLOAD_DIR` | ./results | Directory for downloaded artifacts | **Custom mode variables** (when no test specified): From bfc7c8a4a8ac770d2a8d0aac088e2d89775dcd33 Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Sun, 25 Jan 2026 15:56:51 +0100 Subject: [PATCH 37/40] ci(gh-pages): preserve benchmark data when deploying docs GitHub Pages overwrites on each deploy, which would lose benchmark history stored on the benchmark-data branch. This adds a step to merge benchmark directories (bench/, dev/) into the Pages deployment so both docs and performance charts are served together. - Fetch benchmark-data branch during docs build - Extract bench/ (main) and dev/ (feature branches) directories - Copy to _site/ before deployment - Handle bootstrap case when no benchmarks exist yet --- .github/workflows/gh-pages.yaml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/.github/workflows/gh-pages.yaml b/.github/workflows/gh-pages.yaml index b767da48d..5e6253274 100644 --- a/.github/workflows/gh-pages.yaml +++ b/.github/workflows/gh-pages.yaml @@ -30,6 +30,30 @@ jobs: run: | cp -rv target/doc/* ./_site cp -rv docs/assets ./_site + # GitHub Pages deploys from a single source (overwrites, doesn't merge). + # Benchmark history lives on benchmark-data branch (for append-only storage). + # We merge both into _site/ so a single deployment serves docs + benchmarks. + # See track-performance.yml for how benchmark data is collected and stored. + # + # Structure on benchmark-data branch (see track-performance.yml for how this is populated): + # bench/ - Official benchmark history (main branch only) + # dev/bench/{branch}/ - Feature branch benchmarks (experimental) + - name: Include benchmark data + run: | + # Fetch benchmark-data branch (may not exist on first run) + if ! git fetch origin benchmark-data 2>/dev/null; then + echo "No benchmark-data branch yet, skipping" + exit 0 + fi + + # Extract benchmark directories from benchmark-data branch into current dir + # These may not exist if no benchmarks have been run yet for that category + git checkout origin/benchmark-data -- dev 2>/dev/null || echo "No dev/ directory (no feature branch benchmarks yet)" + git checkout origin/benchmark-data -- bench 2>/dev/null || echo "No bench/ directory (no main branch benchmarks yet)" + + # Copy to _site (directories may not exist if no benchmarks run yet) + [[ -d dev ]] && cp -rv dev _site/ + [[ -d bench ]] && cp -rv bench _site/ - uses: actions/upload-artifact@v4 with: name: pages From 7d12619a1d3b66f3b7955a59f16dab0006dd6a81 Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Sun, 25 Jan 2026 16:07:25 +0100 Subject: [PATCH 38/40] ci(gh-pages): temp. add workflow_dispatch to rebuild Pages --- .github/workflows/gh-pages.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/gh-pages.yaml b/.github/workflows/gh-pages.yaml index 5e6253274..8099e2102 100644 --- a/.github/workflows/gh-pages.yaml +++ b/.github/workflows/gh-pages.yaml @@ -1,6 +1,7 @@ name: gh-pages on: + workflow_dispatch: push: branches: - "main" From 09e9f469d782b96b3f5deae43e7585c3a4b96cb1 Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Sun, 25 Jan 2026 16:15:59 +0100 Subject: [PATCH 39/40] fix(gh-pages) --- .github/workflows/gh-pages.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/gh-pages.yaml b/.github/workflows/gh-pages.yaml index 8099e2102..57ed5151b 100644 --- a/.github/workflows/gh-pages.yaml +++ b/.github/workflows/gh-pages.yaml @@ -52,9 +52,11 @@ jobs: git checkout origin/benchmark-data -- dev 2>/dev/null || echo "No dev/ directory (no feature branch benchmarks yet)" git checkout origin/benchmark-data -- bench 2>/dev/null || echo "No bench/ directory (no main branch benchmarks yet)" - # Copy to _site (directories may not exist if no benchmarks run yet) + # Copy to _site - at least one must exist [[ -d dev ]] && cp -rv dev _site/ [[ -d bench ]] && cp -rv bench _site/ + + [[ -d dev || -d bench ]] || { echo "::error::No benchmark data (dev/ or bench/) found"; exit 1; } - uses: actions/upload-artifact@v4 with: name: pages From c7cad2297d081c55a79595a3369adca81da9fb98 Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Sun, 25 Jan 2026 16:20:30 +0100 Subject: [PATCH 40/40] ci(gh-pages): remove temp. set workflow_dispatch --- .github/workflows/gh-pages.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/gh-pages.yaml b/.github/workflows/gh-pages.yaml index 57ed5151b..fae2589e8 100644 --- a/.github/workflows/gh-pages.yaml +++ b/.github/workflows/gh-pages.yaml @@ -1,7 +1,6 @@ name: gh-pages on: - workflow_dispatch: push: branches: - "main"