[ci] Support running re-execution benchmark with arbitrary version of Firewood

.github/actions/c-chain-reexecution-benchmark/action.yml

@@ -56,6 +56,9 @@ inputs:
   push-github-action-benchmark:
     description: 'Whether to push the benchmark result to GitHub.'
     required: true
+  skip-benchmark-comparison:
+    description: 'Skip benchmark comparison step (required when custom deps modify go.mod).'
+    default: 'false'
   push-post-state:
     description: 'S3 destination to copy the current-state directory after completing re-execution. If empty, this will be skipped.'
     default: ''
@@ -152,7 +155,17 @@ runs:
         GH_RUN_NUMBER: ${{ inputs.run-number }}
         GH_RUN_ATTEMPT: ${{ inputs.run-attempt }}
         GH_JOB_ID: ${{ inputs.job }}
+        # Required for Nix builds on self-hosted runners
+        TMPDIR: /tmp
+        NIX_BUILD_TOP: /tmp
+    - name: Upload Benchmark Artifact
+      uses: actions/upload-artifact@v4
+      with:
+        name: benchmark-output-${{ inputs.test }}-${{ inputs.runner_type }}
+        path: ${{ env.BENCHMARK_OUTPUT_FILE }}
+        if-no-files-found: error
     - name: Compare Benchmark Results
+      if: inputs.skip-benchmark-comparison != 'true'
       uses: benchmark-action/github-action-benchmark@v1
       with:
         tool: 'customBiggerIsBetter'

.github/workflows/c-chain-reexecution-benchmark-container.yml

@@ -31,6 +31,9 @@ on:
       timeout-minutes:
         description: 'Timeout in minutes for the job.'
         default: 30
+      with-dependencies:
+        description: 'Dependencies to use (e.g., "firewood=abc123" or "firewood=abc,libevm=xyz")'
+        default: ''
 
   # Disabled because scheduled trigger is empty. To enable, uncomment and add at least one vector to the schedule
   # entry in the corresponding JSON file.
@@ -49,17 +52,22 @@ jobs:
         shell: bash -x {0}
         run: |
           if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
+            FIREWOOD_REF="" LIBEVM_REF=""
+            [[ "${{ github.event.inputs.with-dependencies }}" =~ firewood=([^,]+) ]] && FIREWOOD_REF="${BASH_REMATCH[1]}"
+            [[ "${{ github.event.inputs.with-dependencies }}" =~ libevm=([^,]+) ]] && LIBEVM_REF="${BASH_REMATCH[1]}"
             {
               echo "matrix<<EOF"
-              printf '{ "include": [{ "test": "%s", "start-block": "%s", "end-block": "%s", "block-dir-src": "%s", "current-state-dir-src": "%s", "config": "%s", "runner": "%s", "timeout-minutes": %s }] }\n' \
+              printf '{ "include": [{ "test": "%s", "start-block": "%s", "end-block": "%s", "block-dir-src": "%s", "current-state-dir-src": "%s", "config": "%s", "runner": "%s", "timeout-minutes": %s, "firewood-ref": "%s", "libevm-ref": "%s" }] }\n' \
                 "${{ github.event.inputs.test }}" \
                 "${{ github.event.inputs.start-block }}" \
                 "${{ github.event.inputs.end-block }}" \
                 "${{ github.event.inputs.block-dir-src }}" \
                 "${{ github.event.inputs.current-state-dir-src }}" \
                 "${{ github.event.inputs.config }}" \
                 "${{ github.event.inputs.runner }}" \
-                "${{ github.event.inputs.timeout-minutes }}"
+                "${{ github.event.inputs.timeout-minutes }}" \
+                "$FIREWOOD_REF" \
+                "$LIBEVM_REF"
               echo EOF
             } >> "$GITHUB_OUTPUT"
           else
@@ -82,8 +90,6 @@ jobs:
       id-token: write
       contents: write
     runs-on: ${{ matrix.runner }}
-    container:
-      image: ghcr.io/actions/actions-runner:2.325.0
     steps:
       - uses: actions/checkout@v4
       - name: Install ARC Dependencies
@@ -94,6 +100,20 @@ jobs:
             sudo apt-get update
             sudo apt-get install -y xz-utils
           fi
+      - name: Install Nix
+        if: matrix.libevm-ref != '' || matrix.firewood-ref != ''
+        uses: cachix/install-nix-action@02a151ada4993995686f9ed4f1be7cfbb229e56f #v31
+        with:
+          github_access_token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Configure dependency versions
+        if: matrix.libevm-ref != '' || matrix.firewood-ref != ''
+        shell: nix develop --command bash {0}
+        run: ./scripts/run_task.sh polyrepo -- ${{ matrix.firewood-ref != '' && format('sync firewood@{0}', matrix.firewood-ref) || '' }}
+        env:
+          LIBEVM_REF: ${{ matrix.libevm-ref }}
+          # Required for Nix builds on ARC runners
+          TMPDIR: /tmp
+          NIX_BUILD_TOP: /tmp
       - name: Run C-Chain Re-Execution Benchmark
         uses: ./.github/actions/c-chain-reexecution-benchmark
         with:
@@ -107,7 +127,8 @@ jobs:
           prometheus-push-url: ${{ secrets.PROMETHEUS_PUSH_URL || '' }}
           prometheus-username: ${{ secrets.PROMETHEUS_USERNAME || '' }}
           prometheus-password: ${{ secrets.PROMETHEUS_PASSWORD || '' }}
-          push-github-action-benchmark: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.repository == 'ava-labs/avalanchego' && github.ref_name == 'master') }}
+          push-github-action-benchmark: ${{ (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.repository == 'ava-labs/avalanchego' && github.ref_name == 'master')) && matrix.firewood-ref == '' }}
+          skip-benchmark-comparison: ${{ matrix.firewood-ref != '' || matrix.libevm-ref != '' }}
           aws-role: ${{ github.event.inputs.push-post-state != '' && secrets.AWS_S3_RW_ROLE || secrets.AWS_S3_READ_ONLY_ROLE }}
           aws-region: 'us-east-2'
           github-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/c-chain-reexecution-benchmark-gh-native.yml

@@ -31,6 +31,9 @@ on:
       timeout-minutes:
         description: 'Timeout in minutes for the job.'
         default: 30
+      with-dependencies:
+        description: 'Dependencies to use (e.g., "firewood=abc123" or "firewood=abc,libevm=xyz")'
+        default: ''
 
   # Disabled because scheduled trigger is empty. To enable, uncomment and add at least one vector to the schedule
   # entry in the corresponding JSON file.
@@ -49,17 +52,23 @@ jobs:
         shell: bash -x {0}
         run: |
           if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
+            # Parse 'with-dependencies' input for dependency refs
+            FIREWOOD_REF="" LIBEVM_REF=""
+            [[ "${{ github.event.inputs.with-dependencies }}" =~ firewood=([^,]+) ]] && FIREWOOD_REF="${BASH_REMATCH[1]}"
+            [[ "${{ github.event.inputs.with-dependencies }}" =~ libevm=([^,]+) ]] && LIBEVM_REF="${BASH_REMATCH[1]}"
             {
               echo "matrix<<EOF"
-              printf '{ "include": [{ "test": "%s", "start-block": "%s", "end-block": "%s", "block-dir-src": "%s", "current-state-dir-src": "%s", "config": "%s", "runner": "%s", "timeout-minutes": %s }] }\n' \
+              printf '{ "include": [{ "test": "%s", "start-block": "%s", "end-block": "%s", "block-dir-src": "%s", "current-state-dir-src": "%s", "config": "%s", "runner": "%s", "timeout-minutes": %s, "firewood-ref": "%s", "libevm-ref": "%s" }] }\n' \
                 "${{ github.event.inputs.test }}" \
                 "${{ github.event.inputs.start-block }}" \
                 "${{ github.event.inputs.end-block }}" \
                 "${{ github.event.inputs.block-dir-src }}" \
                 "${{ github.event.inputs.current-state-dir-src }}" \
                 "${{ github.event.inputs.config }}" \
                 "${{ github.event.inputs.runner }}" \
-                "${{ github.event.inputs.timeout-minutes }}"
+                "${{ github.event.inputs.timeout-minutes }}" \
+                "$FIREWOOD_REF" \
+                "$LIBEVM_REF"
               echo EOF
             } >> "$GITHUB_OUTPUT"
           else
@@ -84,6 +93,17 @@ jobs:
     runs-on: ${{ matrix.runner }}
     steps:
       - uses: actions/checkout@v4
+      - name: Install Nix
+        if: matrix.libevm-ref != '' || matrix.firewood-ref != ''
+        uses: cachix/install-nix-action@02a151ada4993995686f9ed4f1be7cfbb229e56f #v31
+        with:
+          github_access_token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Configure dependency versions
+        if: matrix.libevm-ref != '' || matrix.firewood-ref != ''
+        shell: nix develop --command bash {0}
+        run: ./scripts/run_task.sh polyrepo -- ${{ matrix.firewood-ref != '' && format('sync firewood@{0}', matrix.firewood-ref) || '' }}
+        env:
+          LIBEVM_REF: ${{ matrix.libevm-ref }}
       - name: Run C-Chain Re-Execution Benchmark
         uses: ./.github/actions/c-chain-reexecution-benchmark
         with:
@@ -97,7 +117,8 @@ jobs:
           prometheus-push-url: ${{ secrets.PROMETHEUS_PUSH_URL || '' }}
           prometheus-username: ${{ secrets.PROMETHEUS_USERNAME || '' }}
           prometheus-password: ${{ secrets.PROMETHEUS_PASSWORD || '' }}
-          push-github-action-benchmark: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.repository == 'ava-labs/avalanchego' && github.ref_name == 'master') }}
+          push-github-action-benchmark: ${{ (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.repository == 'ava-labs/avalanchego' && github.ref_name == 'master')) && matrix.firewood-ref == '' }}
+          skip-benchmark-comparison: ${{ matrix.firewood-ref != '' || matrix.libevm-ref != '' }}
           aws-role: ${{ github.event.inputs.push-post-state != '' && secrets.AWS_S3_RW_ROLE || secrets.AWS_S3_READ_ONLY_ROLE }}
           aws-region: 'us-east-2'
           github-token: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -66,5 +66,9 @@ vendor
 # debug files
 __debug_*
 
+# polyrepo
+polyrepo.log
+firewood
+
 # solc downloads from setup-solc GitHub Action
 setup-solc_downloads/

Taskfile.yml

@@ -174,6 +174,10 @@ tasks:
     desc: Runs shellcheck to check sanity of shell scripts
     cmd: ./scripts/shellcheck.sh
 
+  polyrepo:
+    desc: "Run polyrepo for orchestrating local dependencies. Example: LIBEVM_REF=v1.2.3 task polyrepo -- sync firewood@abc123"
+    cmd: ./scripts/run_polyrepo.sh {{.CLI_ARGS}}
+
   setup-chain-data:
     desc: Sets up C-Chain test data (blocks from S3, state from S3 or empty for genesis).
     cmd: ./scripts/setup_cchain_data.sh

scripts/run_polyrepo.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# Run polyrepo tool for managing local dependencies.
+#
+# Usage:
+#   ./scripts/run_polyrepo.sh [polyrepo args...]
+#
+# Environment variables (optional):
+#   LIBEVM_REF - Git ref for libevm (runs: go get && go mod tidy)
+#
+# Examples:
+#   ./scripts/run_polyrepo.sh sync firewood@abc123   # sync firewood
+#   ./scripts/run_polyrepo.sh status                 # check status
+#   LIBEVM_REF=v1.2.3 ./scripts/run_polyrepo.sh sync firewood@abc123  # both
+
+POLYREPO_REVISION=6239973c9b
+
+if [[ -n "${LIBEVM_REF:-}" ]]; then
+  echo "Updating libevm to ${LIBEVM_REF}..."
+  go get "github.com/ava-labs/libevm@${LIBEVM_REF}"
+  go mod tidy
+fi
+
+if [[ $# -gt 0 ]]; then
+  go run github.com/ava-labs/avalanchego/tests/fixture/polyrepo@"${POLYREPO_REVISION}" "${@}"
+fi

tests/reexecute/c/README.md

@@ -250,6 +250,48 @@ To execute a benchmark with any of these options, you must use a compatible `CUR
 
 The `CONFIG` parameter currently only supports pre-defined configs and not passing a full JSON blob in, so that we can define corresponding names for each config option. The config name is attached as a label to the exported metrics and included in the name of the sub-benchmark (used by GitHub Action Benchmark to separate historical results with different configs).
 
+## Testing with Custom Dependency Versions
+
+The benchmarks support testing with custom versions of `libevm` and `firewood` dependencies. This is useful for:
+- Testing unreleased versions before merging
+- Benchmarking performance changes in dependencies
+- Reproducing issues with specific dependency versions
+
+### Local Usage
+
+**Prerequisite**: You must be in a nix shell (`nix develop`).
+
+Use `run_polyrepo.sh` to set up custom dependencies:
+- `LIBEVM_REF=<ref>` env var updates libevm via `go get`
+- `sync firewood@<ref>` arg syncs firewood via polyrepo
+
+```bash
+# Both libevm and firewood
+LIBEVM_REF=v1.2.3 ./scripts/run_polyrepo.sh sync firewood@abc123def
+
+# Only firewood
+./scripts/run_polyrepo.sh sync firewood@abc123def
+
+# Only libevm
+LIBEVM_REF=v1.2.3 ./scripts/run_polyrepo.sh
+
+# Then run the benchmark
+./scripts/run_task.sh test-cchain-reexecution -- firewood-101-250k
+```
+
+### CI Usage
+
+Use `with-dependencies` to specify custom versions. Either or both can be provided:
+
+```bash
+gh workflow run "C-Chain Re-Execution Benchmark GH Native" \
+  -f test=firewood-101-250k \
+  -f with-dependencies="firewood=abc123,libevm=v1.2.3" \
+  -f runner=blacksmith-4vcpu-ubuntu-2404
+```
+
+See [Trigger Workflow Dispatch with GitHub CLI](#trigger-workflow-dispatch-with-github-cli) for more examples.
+
 ## Metrics
 
 The C-Chain benchmarks export VM metrics to the same Grafana instance as AvalancheGo CI: https://grafana-poc.avax-dev.network/.
@@ -323,6 +365,18 @@ gh workflow run "C-Chain Re-Execution Benchmark GH Native" \
   -f timeout-minutes=60
 ```
 
+### Using Custom Dependency Versions
+
+Use `with-dependencies` to specify `firewood=<ref>` and/or `libevm=<ref>`:
+
+```bash
+gh workflow run "C-Chain Re-Execution Benchmark GH Native" \
+  -f test=firewood-101-250k \
+  -f with-dependencies="firewood=abc123def,libevm=v1.2.3" \
+  -f runner=blacksmith-4vcpu-ubuntu-2404 \
+  -f timeout-minutes=60
+```
+
 ### Using Custom Parameters
 
 ```bash
@@ -335,3 +389,13 @@ gh workflow run "C-Chain Re-Execution Benchmark GH Native" \
   -f runner=ubuntu-latest \
   -f timeout-minutes=360
 ```
+
+### Pushing Post-Execution State to S3
+
+```bash
+gh workflow run "C-Chain Re-Execution Benchmark GH Native" \
+  -f test=hashdb-101-250k \
+  -f runner=blacksmith-4vcpu-ubuntu-2404 \
+  -f push-post-state=s3://avalanchego-bootstrap-testing/cchain-current-state-new/ \
+  -f timeout-minutes=60
+```