benchmark.yml

name: Benchmark

on:
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

env:
  CARGO_TERM_COLOR: always
  SCCACHE_GHA_ENABLED: "true"
  RUSTC_WRAPPER: "sccache"
  BENCHMARK_REPO: "base/benchmark"
  BENCHMARK_REF: "48f0e0405fc9df31bd8b3e59d686374695e06d64"

permissions:
  contents: read

jobs:
  # Build base-reth-node, base-builder, and base-load-test together, cached by source hash
  build-binaries:
    name: Build binaries
    runs-on: ubuntu-latest
    permissions:
      contents: read
      actions: write
    steps:
      - name: Harden the runner (Audit all outbound calls)
        uses: step-security/harden-runner@a90bcbc6539c36a85cdfeb73f7e2f433735f215b # v2.15.0
        with:
          egress-policy: audit

      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          fetch-depth: 1

      # Cache based on Cargo files and crates source, not the whole repo
      # This allows iterating on benchmark workflow without rebuilding binaries
      - name: Cache binaries by source hash
        uses: actions/cache@2f8e54208210a422b2efd51efaa6bd6d7ca8920f # v3.4.3
        id: cache-binaries
        with:
          path: ~/bin
          key: ${{ runner.os }}-binaries-${{ hashFiles('Cargo.toml', 'Cargo.lock', 'crates/**', 'bin/**') }}

      - name: Setup Rust toolchain, cache, and dependencies
        if: steps.cache-binaries.outputs.cache-hit != 'true'
        uses: ./.github/actions/setup
        with:
          native-deps: "true"
          mold: "true"
          foundry: "true"
          rust-cache-shared-key: "benchmark"
          rust-cache-save: "true"

      - name: Build base-reth-node
        if: steps.cache-binaries.outputs.cache-hit != 'true'
        run: |
          cargo build --locked --bin base-reth-node --profile maxperf
          mkdir -p ~/bin
          cp target/maxperf/base-reth-node ~/bin/

      - name: Build base-builder
        if: steps.cache-binaries.outputs.cache-hit != 'true'
        run: |
          cargo build --locked --bin base-builder --profile maxperf
          cp target/maxperf/base-builder ~/bin/base-builder

      - name: Build base-load-test
        if: steps.cache-binaries.outputs.cache-hit != 'true'
        run: |
          cargo build --locked -p base-load-tests --bin base-load-test --profile maxperf
          cp target/maxperf/base-load-test ~/bin/base-load-test

      - name: Verify binaries exist
        run: |
          echo "Binaries in ~/bin:"
          ls -la ~/bin/

      - name: Upload binaries artifact
        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
        with:
          name: binaries
          path: ~/bin/
          retention-days: 1

  # Run the benchmark
  benchmark:
    name: Run Benchmark
    runs-on: ubuntu-latest
    needs: [build-binaries]
    permissions:
      contents: read
      actions: write
    steps:
      - name: Harden the runner (Audit all outbound calls)
        uses: step-security/harden-runner@a90bcbc6539c36a85cdfeb73f7e2f433735f215b # v2.15.0
        with:
          egress-policy: audit

      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          path: node-reth
          fetch-depth: 1

      - name: Checkout benchmark repo
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          repository: ${{ env.BENCHMARK_REPO }}
          ref: ${{ env.BENCHMARK_REF }}
          path: benchmark
          submodules: true
          fetch-depth: 1

      - name: Set up Go
        uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6.1.0

      - name: Install Go dependencies
        working-directory: benchmark
        run: go mod download

      - name: Download binaries
        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
        with:
          name: binaries
          path: ${{ runner.temp }}/bin/

      - name: Make binaries executable
        run: |
          chmod +x ${{ runner.temp }}/bin/*
          echo "Downloaded binaries:"
          ls -la ${{ runner.temp }}/bin

      - name: Run Benchmark
        working-directory: benchmark
        run: |
          mkdir -p ${{ runner.temp }}/data-dir
          mkdir -p ${{ runner.temp }}/output

          go run benchmark/cmd/main.go \
            --log.level info \
            run \
            --config ../node-reth/.github/benchmark/load-test.yml \
            --root-dir ${{ runner.temp }}/data-dir \
            --output-dir ${{ runner.temp }}/output \
            --builder-bin ${{ runner.temp }}/bin/base-builder \
            --base-reth-node-bin ${{ runner.temp }}/bin/base-reth-node \
            --load-test-bin ${{ runner.temp }}/bin/base-load-test

      - name: Setup Node.js
        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
        with:
          node-version: "20"

      - name: Build Report
        working-directory: benchmark
        run: |
          cp -r ${{ runner.temp }}/output/ ./output/
          pushd report
          npm install
          npm run build
          popd

      - name: Upload Output
        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
        if: always()
        with:
          name: benchmark-output
          path: ${{ runner.temp }}/output/
          retention-days: 7

      - name: Upload Report
        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
        with:
          name: benchmark-report
          path: benchmark/report/dist/
          retention-days: 7