test(reexecute): add firewood chaos test [2/2]

.github/workflows/chaos-test.yml

@@ -0,0 +1,105 @@
+name: Firewood Chaos Test
+
+on:
+  workflow_dispatch:
+    inputs:
+      test:
+        description: 'Test name to run (e.g., 101-250k). Leave empty to use custom inputs below.'
+        default: ''
+      # Custom inputs (used when test is not provided)
+      start-block:
+        description: 'The start block for the benchmark.'
+        default: ''
+      end-block:
+        description: 'The end block for the benchmark.'
+        default: ''
+      block-dir-src:
+        description: 'The source block directory. Supports S3 directory/zip and local directories.'
+        default: ''
+      current-state-dir-src:
+        description: 'The current state directory. Supports S3 directory/zip and local directories.'
+        default: ''
+      # Chaos test specific inputs
+      min-wait-time:
+        description: 'Minimum wait time before killing the process (e.g., 120s, 2m).'
+        default: '120s'
+      max-wait-time:
+        description: 'Maximum wait time before killing the process (e.g., 150s, 3m).'
+        default: '150s'
+      runner:
+        description: 'Runner to execute the chaos test. Input to the runs-on field of the job.'
+        required: true
+      timeout-minutes:
+        description: 'Timeout in minutes for the job.'
+        default: '60'
+  # XXX: remove this before merging
+  pull_request:
+  schedule:
+    - cron: '0 9 * * *' # Runs every day at 09:00 UTC (04:00 EST)
+
+jobs:
+  define-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.define-matrix.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Define Matrix
+        id: define-matrix
+        shell: bash -x {0}
+        run: |
+          if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
+            {
+              echo "matrix<<EOF"
+              printf '{ "include": [{ "test": "%s", "start-block": "%s", "end-block": "%s", "block-dir-src": "%s", "current-state-dir-src": "%s", "min-wait-time": "%s", "max-wait-time": "%s", "runner": "%s", "timeout-minutes": %s }] }\n' \
+                "${{ github.event.inputs.test }}" \
+                "${{ github.event.inputs.start-block }}" \
+                "${{ github.event.inputs.end-block }}" \
+                "${{ github.event.inputs.block-dir-src }}" \
+                "${{ github.event.inputs.current-state-dir-src }}" \
+                "${{ github.event.inputs.min-wait-time }}" \
+                "${{ github.event.inputs.max-wait-time }}" \
+                "${{ github.event.inputs.runner }}" \
+                "${{ github.event.inputs.timeout-minutes }}"
+              echo EOF
+            } >> "$GITHUB_OUTPUT"
+          else
+            {
+              echo "matrix<<EOF"
+              echo '{ "include": [{ "test": "101-250k" }, { "test": "archive-101-250k" }] }'
+              echo EOF
+            } >> "$GITHUB_OUTPUT"
+          fi
+
+  firewood-chaos-test:
+    needs: define-matrix
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.define-matrix.outputs.matrix) }}
+    timeout-minutes: ${{ matrix.timeout-minutes || 60 }}
+    runs-on: ${{ matrix.runner || 'ubuntu-latest' }}
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - uses: cachix/install-nix-action@02a151ada4993995686f9ed4f1be7cfbb229e56f #v31
+        with:
+          github_access_token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.AWS_S3_READ_ONLY_ROLE }}
+          aws-region: 'us-east-2'
+          role-duration-seconds: '43200'
+      - uses: actions/checkout@v4
+      - name: Run chaos test with Firewood
+        shell: nix develop --impure --command bash -x {0}
+        run: ./scripts/run_task.sh test-firewood-chaos -- "${{ matrix.test || '' }}"
+        env:
+          START_BLOCK: ${{ matrix.start-block }}
+          END_BLOCK: ${{ matrix.end-block }}
+          BLOCK_DIR_SRC: ${{ matrix.block-dir-src }}
+          CURRENT_STATE_DIR_SRC: ${{ matrix.current-state-dir-src }}
+          MIN_WAIT_TIME: ${{ matrix.min-wait-time }}
+          MAX_WAIT_TIME: ${{ matrix.max-wait-time }}
+

Taskfile.yml

@@ -239,6 +239,10 @@ tasks:
     cmds:
       - cmd: bash -x ./scripts/tests.e2e.kube.sh --ginkgo.focus-file=xsvm.go {{.CLI_ARGS}}
 
+  test-firewood-chaos:
+    desc: Runs Firewood chaos test. Run with 'help' to see all available tests.
+    cmd: ./scripts/tests.firewood_chaos.sh {{.CLI_ARGS}}
+
   # To use a different fuzz time, run `task test-fuzz FUZZTIME=[value in seconds]`.
   # A value of `-1` will run until it encounters a failing output.
 

scripts/tests.firewood_chaos.sh

@@ -0,0 +1,138 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# Firewood Chaos Test
+#
+# Usage:
+#   ./tests.firewood_chaos.sh [test-name]
+#
+# To see available tests: use `help` as the test name or invoke
+# without a test name and without required env vars.
+#
+# Test names configure defaults for S3 sources and block ranges.
+# All defaults can be overridden via environment variables.
+#
+# Environment variables:
+#   Data sources (provide S3 sources OR local paths):
+#     BLOCK_DIR_SRC: S3 object key for blocks (triggers S3 import).
+#     CURRENT_STATE_DIR_SRC: S3 object key for state (triggers S3 import).
+#     BLOCK_DIR: Path to local block directory.
+#     CURRENT_STATE_DIR: Path to local current state directory.
+#
+#   Required:
+#     START_BLOCK: The starting block height (inclusive).
+#     END_BLOCK: The ending block height (inclusive).
+#     MIN_WAIT_TIME: The minimum amount of time to wait before crashing.
+#     MAX_WAIT_TIME: The maximum amount of time to wait before crashing.
+#
+#   Optional:
+#     CONFIG: VM config preset (firewood, firewood-archive).
+
+show_usage() {
+    cat <<EOF
+Usage: $0 [test-name]
+
+Available tests:
+    help                    - Show this help message
+    101-250k                - Blocks 101-250k with Firewood
+    archive-101-250k        - Blocks 101-250k with Firewood archive mode
+EOF
+}
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# CI-aware error function
+error() {
+    if [[ "${GITHUB_ACTIONS:-}" == "true" ]]; then
+        echo "::error::$1"
+    else
+        echo "Error: $1" >&2
+    fi
+    exit 1
+}
+
+# Set defaults based on test name (if provided)
+TEST_NAME="${1:-}"
+if [[ -n "$TEST_NAME" ]]; then
+    shift
+    case "$TEST_NAME" in
+        help)
+            show_usage
+            exit 0
+            ;;
+        101-250k)
+            BLOCK_DIR_SRC="${BLOCK_DIR_SRC:-cchain-mainnet-blocks-1m-ldb}"
+            CURRENT_STATE_DIR_SRC="${CURRENT_STATE_DIR_SRC:-cchain-current-state-firewood-100}"
+            START_BLOCK="${START_BLOCK:-101}"
+            END_BLOCK="${END_BLOCK:-250000}"
+            MIN_WAIT_TIME="${MIN_WAIT_TIME:-120s}"
+            MAX_WAIT_TIME="${MAX_WAIT_TIME:-150s}"
+            CONFIG="${CONFIG:-firewood}"
+            ;;
+        archive-101-250k)
+            BLOCK_DIR_SRC="${BLOCK_DIR_SRC:-cchain-mainnet-blocks-1m-ldb}"
+            CURRENT_STATE_DIR_SRC="${CURRENT_STATE_DIR_SRC:-cchain-current-state-firewood-archive-100}"
+            START_BLOCK="${START_BLOCK:-101}"
+            END_BLOCK="${END_BLOCK:-250000}"
+            MIN_WAIT_TIME="${MIN_WAIT_TIME:-120s}"
+            MAX_WAIT_TIME="${MAX_WAIT_TIME:-150s}"
+            CONFIG="${CONFIG:-firewood-archive}"
+            ;;
+        *)
+            error "Unknown test '$TEST_NAME'"
+            ;;
+    esac
+fi
+
+# Determine data source: S3 import or local paths
+if [[ -n "${BLOCK_DIR_SRC:-}" && -n "${CURRENT_STATE_DIR_SRC:-}" ]]; then
+    # S3 mode - import data
+    TIMESTAMP=$(date '+%Y%m%d-%H%M%S')
+    EXECUTION_DATA_DIR="${EXECUTION_DATA_DIR:-/tmp/chaos-test-${TEST_NAME:-custom}-${TIMESTAMP}}"
+
+    BLOCK_DIR_SRC="${BLOCK_DIR_SRC}" \
+    CURRENT_STATE_DIR_SRC="${CURRENT_STATE_DIR_SRC}" \
+    EXECUTION_DATA_DIR="${EXECUTION_DATA_DIR}" \
+    "${SCRIPT_DIR}/import_cchain_data.sh"
+
+    BLOCK_DIR="${EXECUTION_DATA_DIR}/blocks"
+    CURRENT_STATE_DIR="${EXECUTION_DATA_DIR}/current-state"
+elif [[ -n "${BLOCK_DIR_SRC:-}" || -n "${CURRENT_STATE_DIR_SRC:-}" ]]; then
+    error "Both BLOCK_DIR_SRC and CURRENT_STATE_DIR_SRC must be provided together"
+elif [[ -z "${BLOCK_DIR:-}" || -z "${CURRENT_STATE_DIR:-}" ]]; then
+    show_usage
+    echo ""
+    echo "Env vars status:"
+    echo "  S3 sources:"
+    [[ -n "${BLOCK_DIR_SRC:-}" ]] && echo "    BLOCK_DIR_SRC: ${BLOCK_DIR_SRC}" || echo "    BLOCK_DIR_SRC: (not set)"
+    [[ -n "${CURRENT_STATE_DIR_SRC:-}" ]] && echo "    CURRENT_STATE_DIR_SRC: ${CURRENT_STATE_DIR_SRC}" || echo "    CURRENT_STATE_DIR_SRC: (not set)"
+    echo "  Local paths:"
+    [[ -n "${BLOCK_DIR:-}" ]] && echo "    BLOCK_DIR: ${BLOCK_DIR}" || echo "    BLOCK_DIR: (not set)"
+    [[ -n "${CURRENT_STATE_DIR:-}" ]] && echo "    CURRENT_STATE_DIR: ${CURRENT_STATE_DIR}" || echo "    CURRENT_STATE_DIR: (not set)"
+    echo "  Block range:"
+    [[ -n "${START_BLOCK:-}" ]] && echo "    START_BLOCK: ${START_BLOCK}" || echo "    START_BLOCK: (not set)"
+    [[ -n "${END_BLOCK:-}" ]] && echo "    END_BLOCK: ${END_BLOCK}" || echo "    END_BLOCK: (not set)"
+    echo "  Timeouts:"
+    [[ -n "${MIN_WAIT_TIME:-}" ]] && echo "    MIN_WAIT_TIME: ${MIN_WAIT_TIME}" || echo "    MIN_WAIT_TIME: (not set)"
+    [[ -n "${MAX_WAIT_TIME:-}" ]] && echo "    MAX_WAIT_TIME: ${MAX_WAIT_TIME}" || echo "    MAX_WAIT_TIME: (not set)"
+    exit 1
+fi
+
+# Validate block range
+if [[ -z "${START_BLOCK:-}" || -z "${END_BLOCK:-}" ]]; then
+    error "START_BLOCK and END_BLOCK are required"
+fi
+
+echo "=== Firewood Chaos Test: ${TEST_NAME:-custom} ==="
+echo "Blocks: ${START_BLOCK} - ${END_BLOCK}"
+echo "Crashing between ${MIN_WAIT_TIME} and ${MAX_WAIT_TIME}"
+
+echo "=== Running Chaos Test ==="
+go run ./tests/reexecute/chaos \
+    --start-block="${START_BLOCK}" \
+    --end-block="${END_BLOCK}" \
+    --current-state-dir="${CURRENT_STATE_DIR}" \
+    --block-dir="${BLOCK_DIR}" \
+    --min-wait-time="${MIN_WAIT_TIME}" \
+    --max-wait-time="${MAX_WAIT_TIME}"