test(reexecute): add firewood chaos test [2/2]

.github/workflows/chaos-test.json

@@ -0,0 +1,30 @@
+{
+    "pull_request": {
+        "include": [
+            {
+                "start-block": "101",
+                "end-block": "200000",
+                "block-dir-src": "cchain-mainnet-blocks-1m-ldb",
+                "current-state-dir-src": "cchain-current-state-firewood-100",
+                "min-wait-time": "120s",
+                "max-wait-time": "150s",
+                "runner": "ubuntu-latest",
+                "timeout-minutes": 60
+            }
+        ]
+    },
+    "schedule": {
+        "include": [
+            {
+                "start-block": "101",
+                "end-block": "200000",
+                "block-dir-src": "cchain-mainnet-blocks-1m-ldb",
+                "current-state-dir-src": "cchain-current-state-firewood-100",
+                "min-wait-time": "120s",
+                "max-wait-time": "150s",
+                "runner": "ubuntu-latest",
+                "timeout-minutes": 60
+            }
+        ]
+    }
+}

.github/workflows/chaos-test.yml

@@ -0,0 +1,108 @@
+name: Firewood Chaos Test
+
+on:
+  workflow_dispatch:
+    inputs:
+      start-block:
+        description: 'The start block for the chaos test.'
+        default: ''
+      end-block:
+        description: 'The end block for the chaos test.'
+        default: ''
+      block-dir-src:
+        description: 'The source block directory. Supports S3 directory/zip and local directories.'
+        default: ''
+      current-state-dir-src:
+        description: 'The current state directory. Supports S3 directory/zip and local directories.'
+        default: ''
+      min-wait-time:
+        description: 'Minimum wait time before killing the process (e.g., 120s, 2m).'
+        default: '120s'
+      max-wait-time:
+        description: 'Maximum wait time before killing the process (e.g., 150s, 3m).'
+        default: '150s'
+      runner:
+        description: 'Runner to execute the chaos test. Input to the runs-on field of the job.'
+        required: true
+      timeout-minutes:
+        description: 'Timeout in minutes for the job.'
+        default: '60'
+  # XXX: remove this before merging
+  pull_request:
+  schedule:
+    - cron: '0 9 * * *' # Runs every day at 09:00 UTC (04:00 EST)
+
+jobs:
+  define-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.define-matrix.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Define Matrix
+        id: define-matrix
+        shell: bash -x {0}
+        run: |
+          if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
+            {
+              echo "matrix<<EOF"
+              printf '{ "include": [{ "start-block": "%s", "end-block": "%s", "block-dir-src": "%s", "current-state-dir-src": "%s", "min-wait-time": "%s", "max-wait-time": "%s", "runner": "%s", "timeout-minutes": %s }] }\n' \
+                "${{ github.event.inputs.start-block }}" \
+                "${{ github.event.inputs.end-block }}" \
+                "${{ github.event.inputs.block-dir-src }}" \
+                "${{ github.event.inputs.current-state-dir-src }}" \
+                "${{ github.event.inputs.min-wait-time }}" \
+                "${{ github.event.inputs.max-wait-time }}" \
+                "${{ github.event.inputs.runner }}" \
+                "${{ github.event.inputs.timeout-minutes }}"
+              echo EOF
+            } >> "$GITHUB_OUTPUT"
+          else
+            json_string=$(jq -r ".\"${{ github.event_name }}\"" .github/workflows/chaos-test.json)
+            {
+              echo "matrix<<EOF"
+              echo "$json_string"
+              echo EOF
+            } >> "$GITHUB_OUTPUT"
+          fi
+
+  firewood-chaos-test:
+    needs: define-matrix
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.define-matrix.outputs.matrix) }}
+    timeout-minutes: ${{ matrix.timeout-minutes }}
+    runs-on: ${{ matrix.runner }}
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - uses: cachix/install-nix-action@02a151ada4993995686f9ed4f1be7cfbb229e56f #v31
+        with:
+          github_access_token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.AWS_S3_READ_ONLY_ROLE }}
+          aws-region: 'us-east-2'
+          role-duration-seconds: '43200'
+      - uses: actions/checkout@v4
+      - name: Set task env
+        shell: bash
+        run: |
+          TIMESTAMP=$(date '+%Y%m%d-%H%M%S')
+          echo "EXECUTION_DATA_DIR=/tmp/reexecution-data-${TIMESTAMP}" >> "$GITHUB_ENV"
+      - name: Run chaos test with Firewood
+        shell: nix develop --impure --command bash -x {0}
+        run: |
+          TIMESTAMP=$(date +%s)
+          EXECUTION_DATA_DIR="/tmp/reexecution-data-${TIMESTAMP}"
+          ./scripts/run_task.sh firewood-chaos-test-with-copied-data \
+            START_BLOCK=${{ matrix.start-block }} \
+            END_BLOCK=${{ matrix.end-block }} \
+            BLOCK_DIR_SRC=${{ matrix.block-dir-src }} \
+            EXECUTION_DATA_DIR=$EXECUTION_DATA_DIR \
+            CURRENT_STATE_DIR_SRC=${{ matrix.current-state-dir-src }} \
+            MIN_WAIT_TIME=${{ matrix.min-wait-time }} \
+            MAX_WAIT_TIME=${{ matrix.max-wait-time }}
+

Taskfile.yml

@@ -117,6 +117,45 @@ tasks:
     cmds:
       - cmd: bash -x ./scripts/copy_dir.sh {{.SRC}} {{.DST}}
 
+  firewood-chaos-test:
+    desc: Chaos test during reexecution test with Firewood
+    vars:
+      START_BLOCK: '{{.START_BLOCK}}'
+      END_BLOCK: '{{.END_BLOCK}}'
+      CURRENT_STATE_DIR: '{{.CURRENT_STATE_DIR}}'
+      BLOCK_DIR: '{{.BLOCK_DIR}}'
+      MIN_WAIT_TIME: '{{.MIN_WAIT_TIME}}'
+      MAX_WAIT_TIME: '{{.MAX_WAIT_TIME}}'
+    cmd: go run ./tests/reexecute/chaos --start-block={{.START_BLOCK}} 
+      --end-block={{.END_BLOCK}} --current-state-dir={{.CURRENT_STATE_DIR}} 
+      --block-dir={{.BLOCK_DIR}} --min-wait-time={{.MIN_WAIT_TIME}} 
+      --max-wait-time={{.MAX_WAIT_TIME}}
+
+  firewood-chaos-test-with-copied-data:
+    desc: Combines import-cchain-reexecute-range and firewood-chaos-test
+    vars:
+      START_BLOCK: '{{.START_BLOCK}}'
+      END_BLOCK: '{{.END_BLOCK}}'
+      CURRENT_STATE_DIR_SRC: '{{.CURRENT_STATE_DIR_SRC}}'
+      BLOCK_DIR_SRC: '{{.BLOCK_DIR_SRC}}'
+      EXECUTION_DATA_DIR: '{{.EXECUTION_DATA_DIR}}'
+      MIN_WAIT_TIME: '{{.MIN_WAIT_TIME}}'
+      MAX_WAIT_TIME: '{{.MAX_WAIT_TIME}}'
+    cmds:
+      - task: import-cchain-reexecute-range
+        vars:
+          BLOCK_DIR_SRC: '{{.S3_BOOTSTRAP_BUCKET}}/{{.BLOCK_DIR_SRC}}/**'
+          CURRENT_STATE_DIR_SRC: '{{.S3_BOOTSTRAP_BUCKET}}/{{.CURRENT_STATE_DIR_SRC}}/**'
+          EXECUTION_DATA_DIR: '{{.EXECUTION_DATA_DIR}}'
+      - task: firewood-chaos-test
+        vars:
+          START_BLOCK: '{{.START_BLOCK}}'
+          END_BLOCK: '{{.END_BLOCK}}'
+          CURRENT_STATE_DIR: '{{.EXECUTION_DATA_DIR}}/current-state'
+          BLOCK_DIR: '{{.EXECUTION_DATA_DIR}}/blocks'
+          MIN_WAIT_TIME: '{{.MIN_WAIT_TIME}}'
+          MAX_WAIT_TIME: '{{.MAX_WAIT_TIME}}'
+
   generate-mocks:
     desc: Generates testing mocks
     cmds:

tests/reexecute/c/vm_reexecute.go

@@ -22,35 +22,15 @@ import (
 	"go.uber.org/zap"
 
 	"github.com/ava-labs/avalanchego/api/metrics"
-	"github.com/ava-labs/avalanchego/chains/atomic"
-	"github.com/ava-labs/avalanchego/database"
 	"github.com/ava-labs/avalanchego/database/leveldb"
-	"github.com/ava-labs/avalanchego/database/prefixdb"
-	"github.com/ava-labs/avalanchego/genesis"
 	"github.com/ava-labs/avalanchego/graft/coreth/plugin/evm"
-	"github.com/ava-labs/avalanchego/graft/coreth/plugin/factory"
-	"github.com/ava-labs/avalanchego/ids"
-	"github.com/ava-labs/avalanchego/snow"
-	"github.com/ava-labs/avalanchego/snow/engine/enginetest"
 	"github.com/ava-labs/avalanchego/snow/engine/snowman/block"
-	"github.com/ava-labs/avalanchego/snow/validators/validatorstest"
 	"github.com/ava-labs/avalanchego/tests"
 	"github.com/ava-labs/avalanchego/tests/fixture/tmpnet"
 	"github.com/ava-labs/avalanchego/tests/reexecute"
-	"github.com/ava-labs/avalanchego/upgrade"
-	"github.com/ava-labs/avalanchego/utils/constants"
-	"github.com/ava-labs/avalanchego/utils/crypto/bls/signer/localsigner"
 	"github.com/ava-labs/avalanchego/utils/logging"
 	"github.com/ava-labs/avalanchego/utils/perms"
 	"github.com/ava-labs/avalanchego/utils/timer"
-	"github.com/ava-labs/avalanchego/vms/metervm"
-	"github.com/ava-labs/avalanchego/vms/platformvm/warp"
-)
-
-var (
-	mainnetXChainID    = ids.FromStringOrPanic("2oYMBNV4eNHyqk2fjjV5nVQLDbtmNJzq5s3qs3Lo6ftnC6FByM")
-	mainnetCChainID    = ids.FromStringOrPanic("2q9e4r6Mu3U68nU1fYjgbR6JvwrRx36CohpAX5UQxse55x1Q5")
-	mainnetAvaxAssetID = ids.FromStringOrPanic("FvwEAhmxKfeiG8SnEvq42hc6whRyY3EFYAvebMqDNDGCgxN5Z")
 )
 
 var (
@@ -252,7 +232,7 @@ func benchmarkReexecuteRange(
 		r.NoError(db.Close())
 	}()
 
-	vm, err := newMainnetCChainVM(
+	vm, err := reexecute.NewMainnetCChainVM(
 		ctx,
 		db,
 		chainDataDir,
@@ -289,87 +269,6 @@ func benchmarkReexecuteRange(
 	}
 }
 
-func newMainnetCChainVM(
-	ctx context.Context,
-	vmAndSharedMemoryDB database.Database,
-	chainDataDir string,
-	configBytes []byte,
-	vmMultiGatherer metrics.MultiGatherer,
-	meterVMRegistry prometheus.Registerer,
-) (block.ChainVM, error) {
-	factory := factory.Factory{}
-	vmIntf, err := factory.New(logging.NoLog{})
-	if err != nil {
-		return nil, fmt.Errorf("failed to create VM from factory: %w", err)
-	}
-	vm := vmIntf.(block.ChainVM)
-
-	blsKey, err := localsigner.New()
-	if err != nil {
-		return nil, fmt.Errorf("failed to create BLS key: %w", err)
-	}
-
-	blsPublicKey := blsKey.PublicKey()
-	warpSigner := warp.NewSigner(blsKey, constants.MainnetID, mainnetCChainID)
-
-	genesisConfig := genesis.GetConfig(constants.MainnetID)
-
-	sharedMemoryDB := prefixdb.New([]byte("sharedmemory"), vmAndSharedMemoryDB)
-	atomicMemory := atomic.NewMemory(sharedMemoryDB)
-
-	chainIDToSubnetID := map[ids.ID]ids.ID{
-		mainnetXChainID: constants.PrimaryNetworkID,
-		mainnetCChainID: constants.PrimaryNetworkID,
-		ids.Empty:       constants.PrimaryNetworkID,
-	}
-
-	vm = metervm.NewBlockVM(vm, meterVMRegistry)
-
-	if err := vm.Initialize(
-		ctx,
-		&snow.Context{
-			NetworkID:       constants.MainnetID,
-			SubnetID:        constants.PrimaryNetworkID,
-			ChainID:         mainnetCChainID,
-			NodeID:          ids.GenerateTestNodeID(),
-			PublicKey:       blsPublicKey,
-			NetworkUpgrades: upgrade.Mainnet,
-
-			XChainID:    mainnetXChainID,
-			CChainID:    mainnetCChainID,
-			AVAXAssetID: mainnetAvaxAssetID,
-
-			Log:          tests.NewDefaultLogger("mainnet-vm-reexecution"),
-			SharedMemory: atomicMemory.NewSharedMemory(mainnetCChainID),
-			BCLookup:     ids.NewAliaser(),
-			Metrics:      vmMultiGatherer,
-
-			WarpSigner: warpSigner,
-
-			ValidatorState: &validatorstest.State{
-				GetSubnetIDF: func(_ context.Context, chainID ids.ID) (ids.ID, error) {
-					subnetID, ok := chainIDToSubnetID[chainID]
-					if ok {
-						return subnetID, nil
-					}
-					return ids.Empty, fmt.Errorf("unknown chainID: %s", chainID)
-				},
-			},
-			ChainDataDir: chainDataDir,
-		},
-		prefixdb.New([]byte("vm"), vmAndSharedMemoryDB),
-		[]byte(genesisConfig.CChainGenesis),
-		nil,
-		configBytes,
-		nil,
-		&enginetest.Sender{},
-	); err != nil {
-		return nil, fmt.Errorf("failed to initialize VM: %w", err)
-	}
-
-	return vm, nil
-}
-
 type vmExecutorConfig struct {
 	Log logging.Logger
 	// Registry is the registry to register the metrics with.