diff --git a/.github/actions/c-chain-reexecution-benchmark/action.yml b/.github/actions/c-chain-reexecution-benchmark/action.yml index a42e163921ad..f8eefaf787b4 100644 --- a/.github/actions/c-chain-reexecution-benchmark/action.yml +++ b/.github/actions/c-chain-reexecution-benchmark/action.yml @@ -56,6 +56,9 @@ inputs: push-github-action-benchmark: description: 'Whether to push the benchmark result to GitHub.' required: true + skip-benchmark-comparison: + description: 'Skip benchmark comparison step (required when custom deps modify go.mod).' + default: 'false' push-post-state: description: 'S3 destination to copy the current-state directory after completing re-execution. If empty, this will be skipped.' default: '' @@ -152,7 +155,17 @@ runs: GH_RUN_NUMBER: ${{ inputs.run-number }} GH_RUN_ATTEMPT: ${{ inputs.run-attempt }} GH_JOB_ID: ${{ inputs.job }} + # Required for Nix builds on self-hosted runners + TMPDIR: /tmp + NIX_BUILD_TOP: /tmp + - name: Upload Benchmark Artifact + uses: actions/upload-artifact@v4 + with: + name: benchmark-output-${{ inputs.test }}-${{ inputs.runner_type }} + path: ${{ env.BENCHMARK_OUTPUT_FILE }} + if-no-files-found: error - name: Compare Benchmark Results + if: inputs.skip-benchmark-comparison != 'true' uses: benchmark-action/github-action-benchmark@v1 with: tool: 'customBiggerIsBetter' diff --git a/.github/workflows/c-chain-reexecution-benchmark-container.yml b/.github/workflows/c-chain-reexecution-benchmark-container.yml index 483029c1f21f..0da6d7974315 100644 --- a/.github/workflows/c-chain-reexecution-benchmark-container.yml +++ b/.github/workflows/c-chain-reexecution-benchmark-container.yml @@ -31,6 +31,9 @@ on: timeout-minutes: description: 'Timeout in minutes for the job.' default: 30 + with-dependencies: + description: 'Dependencies to use (e.g., "firewood=abc123" or "firewood=abc,libevm=xyz")' + default: '' # Disabled because scheduled trigger is empty. To enable, uncomment and add at least one vector to the schedule # entry in the corresponding JSON file. @@ -49,9 +52,12 @@ jobs: shell: bash -x {0} run: | if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + FIREWOOD_REF="" LIBEVM_REF="" + [[ "${{ github.event.inputs.with-dependencies }}" =~ firewood=([^,]+) ]] && FIREWOOD_REF="${BASH_REMATCH[1]}" + [[ "${{ github.event.inputs.with-dependencies }}" =~ libevm=([^,]+) ]] && LIBEVM_REF="${BASH_REMATCH[1]}" { echo "matrix<> "$GITHUB_OUTPUT" else @@ -82,8 +90,6 @@ jobs: id-token: write contents: write runs-on: ${{ matrix.runner }} - container: - image: ghcr.io/actions/actions-runner:2.325.0 steps: - uses: actions/checkout@v4 - name: Install ARC Dependencies @@ -94,6 +100,20 @@ jobs: sudo apt-get update sudo apt-get install -y xz-utils fi + - name: Install Nix + if: matrix.libevm-ref != '' || matrix.firewood-ref != '' + uses: cachix/install-nix-action@02a151ada4993995686f9ed4f1be7cfbb229e56f #v31 + with: + github_access_token: ${{ secrets.GITHUB_TOKEN }} + - name: Configure dependency versions + if: matrix.libevm-ref != '' || matrix.firewood-ref != '' + shell: nix develop --command bash {0} + run: ./scripts/run_task.sh polyrepo -- ${{ matrix.firewood-ref != '' && format('sync firewood@{0}', matrix.firewood-ref) || '' }} + env: + LIBEVM_REF: ${{ matrix.libevm-ref }} + # Required for Nix builds on ARC runners + TMPDIR: /tmp + NIX_BUILD_TOP: /tmp - name: Run C-Chain Re-Execution Benchmark uses: ./.github/actions/c-chain-reexecution-benchmark with: @@ -107,7 +127,8 @@ jobs: prometheus-push-url: ${{ secrets.PROMETHEUS_PUSH_URL || '' }} prometheus-username: ${{ secrets.PROMETHEUS_USERNAME || '' }} prometheus-password: ${{ secrets.PROMETHEUS_PASSWORD || '' }} - push-github-action-benchmark: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.repository == 'ava-labs/avalanchego' && github.ref_name == 'master') }} + push-github-action-benchmark: ${{ (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.repository == 'ava-labs/avalanchego' && github.ref_name == 'master')) && matrix.firewood-ref == '' }} + skip-benchmark-comparison: ${{ matrix.firewood-ref != '' || matrix.libevm-ref != '' }} aws-role: ${{ github.event.inputs.push-post-state != '' && secrets.AWS_S3_RW_ROLE || secrets.AWS_S3_READ_ONLY_ROLE }} aws-region: 'us-east-2' github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/c-chain-reexecution-benchmark-gh-native.yml b/.github/workflows/c-chain-reexecution-benchmark-gh-native.yml index d5ed9a6433c5..0749aa62ac1b 100644 --- a/.github/workflows/c-chain-reexecution-benchmark-gh-native.yml +++ b/.github/workflows/c-chain-reexecution-benchmark-gh-native.yml @@ -31,6 +31,9 @@ on: timeout-minutes: description: 'Timeout in minutes for the job.' default: 30 + with-dependencies: + description: 'Dependencies to use (e.g., "firewood=abc123" or "firewood=abc,libevm=xyz")' + default: '' # Disabled because scheduled trigger is empty. To enable, uncomment and add at least one vector to the schedule # entry in the corresponding JSON file. @@ -49,9 +52,13 @@ jobs: shell: bash -x {0} run: | if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + # Parse 'with-dependencies' input for dependency refs + FIREWOOD_REF="" LIBEVM_REF="" + [[ "${{ github.event.inputs.with-dependencies }}" =~ firewood=([^,]+) ]] && FIREWOOD_REF="${BASH_REMATCH[1]}" + [[ "${{ github.event.inputs.with-dependencies }}" =~ libevm=([^,]+) ]] && LIBEVM_REF="${BASH_REMATCH[1]}" { echo "matrix<> "$GITHUB_OUTPUT" else @@ -84,6 +93,17 @@ jobs: runs-on: ${{ matrix.runner }} steps: - uses: actions/checkout@v4 + - name: Install Nix + if: matrix.libevm-ref != '' || matrix.firewood-ref != '' + uses: cachix/install-nix-action@02a151ada4993995686f9ed4f1be7cfbb229e56f #v31 + with: + github_access_token: ${{ secrets.GITHUB_TOKEN }} + - name: Configure dependency versions + if: matrix.libevm-ref != '' || matrix.firewood-ref != '' + shell: nix develop --command bash {0} + run: ./scripts/run_task.sh polyrepo -- ${{ matrix.firewood-ref != '' && format('sync firewood@{0}', matrix.firewood-ref) || '' }} + env: + LIBEVM_REF: ${{ matrix.libevm-ref }} - name: Run C-Chain Re-Execution Benchmark uses: ./.github/actions/c-chain-reexecution-benchmark with: @@ -97,7 +117,8 @@ jobs: prometheus-push-url: ${{ secrets.PROMETHEUS_PUSH_URL || '' }} prometheus-username: ${{ secrets.PROMETHEUS_USERNAME || '' }} prometheus-password: ${{ secrets.PROMETHEUS_PASSWORD || '' }} - push-github-action-benchmark: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.repository == 'ava-labs/avalanchego' && github.ref_name == 'master') }} + push-github-action-benchmark: ${{ (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.repository == 'ava-labs/avalanchego' && github.ref_name == 'master')) && matrix.firewood-ref == '' }} + skip-benchmark-comparison: ${{ matrix.firewood-ref != '' || matrix.libevm-ref != '' }} aws-role: ${{ github.event.inputs.push-post-state != '' && secrets.AWS_S3_RW_ROLE || secrets.AWS_S3_READ_ONLY_ROLE }} aws-region: 'us-east-2' github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index d2a6f73d49b8..1123f54a56e9 100644 --- a/.gitignore +++ b/.gitignore @@ -66,5 +66,9 @@ vendor # debug files __debug_* +# polyrepo +polyrepo.log +firewood + # solc downloads from setup-solc GitHub Action setup-solc_downloads/ diff --git a/Taskfile.yml b/Taskfile.yml index 681760acbff3..00f7b446506a 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -174,6 +174,10 @@ tasks: desc: Runs shellcheck to check sanity of shell scripts cmd: ./scripts/shellcheck.sh + polyrepo: + desc: "Run polyrepo for orchestrating local dependencies. Example: LIBEVM_REF=v1.2.3 task polyrepo -- sync firewood@abc123" + cmd: ./scripts/run_polyrepo.sh {{.CLI_ARGS}} + setup-chain-data: desc: Sets up C-Chain test data (blocks from S3, state from S3 or empty for genesis). cmd: ./scripts/setup_cchain_data.sh diff --git a/scripts/run_polyrepo.sh b/scripts/run_polyrepo.sh new file mode 100755 index 000000000000..277ca66db9ba --- /dev/null +++ b/scripts/run_polyrepo.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Run polyrepo tool for managing local dependencies. +# +# Usage: +# ./scripts/run_polyrepo.sh [polyrepo args...] +# +# Environment variables (optional): +# LIBEVM_REF - Git ref for libevm (runs: go get && go mod tidy) +# +# Examples: +# ./scripts/run_polyrepo.sh sync firewood@abc123 # sync firewood +# ./scripts/run_polyrepo.sh status # check status +# LIBEVM_REF=v1.2.3 ./scripts/run_polyrepo.sh sync firewood@abc123 # both + +POLYREPO_REVISION=6239973c9b + +if [[ -n "${LIBEVM_REF:-}" ]]; then + echo "Updating libevm to ${LIBEVM_REF}..." + go get "github.com/ava-labs/libevm@${LIBEVM_REF}" + go mod tidy +fi + +if [[ $# -gt 0 ]]; then + go run github.com/ava-labs/avalanchego/tests/fixture/polyrepo@"${POLYREPO_REVISION}" "${@}" +fi diff --git a/tests/reexecute/c/README.md b/tests/reexecute/c/README.md index 281eff22a346..355234c9749d 100644 --- a/tests/reexecute/c/README.md +++ b/tests/reexecute/c/README.md @@ -250,6 +250,48 @@ To execute a benchmark with any of these options, you must use a compatible `CUR The `CONFIG` parameter currently only supports pre-defined configs and not passing a full JSON blob in, so that we can define corresponding names for each config option. The config name is attached as a label to the exported metrics and included in the name of the sub-benchmark (used by GitHub Action Benchmark to separate historical results with different configs). +## Testing with Custom Dependency Versions + +The benchmarks support testing with custom versions of `libevm` and `firewood` dependencies. This is useful for: +- Testing unreleased versions before merging +- Benchmarking performance changes in dependencies +- Reproducing issues with specific dependency versions + +### Local Usage + +**Prerequisite**: You must be in a nix shell (`nix develop`). + +Use `run_polyrepo.sh` to set up custom dependencies: +- `LIBEVM_REF=` env var updates libevm via `go get` +- `sync firewood@` arg syncs firewood via polyrepo + +```bash +# Both libevm and firewood +LIBEVM_REF=v1.2.3 ./scripts/run_polyrepo.sh sync firewood@abc123def + +# Only firewood +./scripts/run_polyrepo.sh sync firewood@abc123def + +# Only libevm +LIBEVM_REF=v1.2.3 ./scripts/run_polyrepo.sh + +# Then run the benchmark +./scripts/run_task.sh test-cchain-reexecution -- firewood-101-250k +``` + +### CI Usage + +Use `with-dependencies` to specify custom versions. Either or both can be provided: + +```bash +gh workflow run "C-Chain Re-Execution Benchmark GH Native" \ + -f test=firewood-101-250k \ + -f with-dependencies="firewood=abc123,libevm=v1.2.3" \ + -f runner=blacksmith-4vcpu-ubuntu-2404 +``` + +See [Trigger Workflow Dispatch with GitHub CLI](#trigger-workflow-dispatch-with-github-cli) for more examples. + ## Metrics The C-Chain benchmarks export VM metrics to the same Grafana instance as AvalancheGo CI: https://grafana-poc.avax-dev.network/. @@ -323,6 +365,18 @@ gh workflow run "C-Chain Re-Execution Benchmark GH Native" \ -f timeout-minutes=60 ``` +### Using Custom Dependency Versions + +Use `with-dependencies` to specify `firewood=` and/or `libevm=`: + +```bash +gh workflow run "C-Chain Re-Execution Benchmark GH Native" \ + -f test=firewood-101-250k \ + -f with-dependencies="firewood=abc123def,libevm=v1.2.3" \ + -f runner=blacksmith-4vcpu-ubuntu-2404 \ + -f timeout-minutes=60 +``` + ### Using Custom Parameters ```bash @@ -335,3 +389,13 @@ gh workflow run "C-Chain Re-Execution Benchmark GH Native" \ -f runner=ubuntu-latest \ -f timeout-minutes=360 ``` + +### Pushing Post-Execution State to S3 + +```bash +gh workflow run "C-Chain Re-Execution Benchmark GH Native" \ + -f test=hashdb-101-250k \ + -f runner=blacksmith-4vcpu-ubuntu-2404 \ + -f push-post-state=s3://avalanchego-bootstrap-testing/cchain-current-state-new/ \ + -f timeout-minutes=60 +```