Extract EVM state

cmd/util/main.go

@@ -0,0 +1,71 @@
+package main
+
+import (
+	"flag"
+	"os"
+
+	"github.com/rs/zerolog/log"
+
+	"github.com/onflow/flow-evm-gateway/storage/pebble"
+	"github.com/onflow/flow-go/fvm/evm"
+	"github.com/onflow/flow-go/fvm/evm/emulator/state"
+	"github.com/onflow/flow-go/fvm/evm/offchain/storage"
+	flowGo "github.com/onflow/flow-go/model/flow"
+)
+
+func main() {
+	var (
+		height           uint64
+		outputDir        string
+		registerStoreDir string
+	)
+
+	flag.Uint64Var(&height, "evm-height", 0, "EVM Block height for EVM state export")
+	flag.StringVar(&outputDir, "output", "", "Output directory for exported EVM state")
+	flag.StringVar(&registerStoreDir, "register-store", "", "Directory of the register store")
+
+	flag.Parse()
+
+	if height == 0 || outputDir == "" || registerStoreDir == "" {
+		log.Error().Msg("All flags (height, output, register-store) must be provided")
+		flag.Usage()
+		os.Exit(1)
+	}
+
+	chainID := flowGo.Testnet
+
+	log.Info().Msgf("exporting EVM state for height %v from registerStoreDir %v, outputDir: %v", height, registerStoreDir, outputDir)
+	err := ExportEVMStateForHeight(height, outputDir, registerStoreDir, chainID)
+	if err != nil {
+		log.Fatal().Err(err).Msgf("fail to export")
+	}
+
+	log.Info().Msgf("successfully exported EVM state to %v", outputDir)
+}
+
+func ExportEVMStateForHeight(height uint64, outputDir string, registerStoreDir string, chainID flowGo.ChainID) error {
+	store, err := pebble.New(registerStoreDir, log.Logger)
+	if err != nil {
+		return err
+	}
+
+	storageAddress := evm.StorageAccountAddress(chainID)
+	registerStore := pebble.NewRegisterStorage(store, storageAddress)
+	snapshot, err := registerStore.GetSnapshotAt(height)
+	if err != nil {
+		return err
+	}
+
+	ledger := storage.NewReadOnlyStorage(snapshot)
+	exporter, err := state.NewExporter(ledger, storageAddress)
+	if err != nil {
+		return err
+	}
+
+	err = exporter.ExportGob(outputDir)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/goccy/go-json v0.10.2
 	github.com/onflow/atree v0.8.0
 	github.com/onflow/cadence v1.2.2
-	github.com/onflow/flow-go v0.38.0-preview.0.0.20241125190444-25a8af57bea1
+	github.com/onflow/flow-go v0.38.0-preview.0.0.10
 	github.com/onflow/flow-go-sdk v1.2.3
 	github.com/onflow/go-ethereum v1.14.7
 	github.com/prometheus/client_golang v1.18.0
@@ -83,6 +83,7 @@ require (
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/pprof v0.0.0-20231229205709-960ae82b1e42 // indirect
 	github.com/google/s2a-go v0.1.7 // indirect
 	github.com/google/uuid v1.6.0 // indirect
@@ -211,6 +212,7 @@ require (
 	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
+	gotest.tools v2.2.0+incompatible // indirect
 	lukechampine.com/blake3 v1.3.0 // indirect
 	rsc.io/tmplfunc v0.0.3 // indirect
 )

go.sum

@@ -535,8 +535,8 @@ github.com/onflow/flow-ft/lib/go/contracts v1.0.1 h1:Ts5ob+CoCY2EjEd0W6vdLJ7hLL3
 github.com/onflow/flow-ft/lib/go/contracts v1.0.1/go.mod h1:PwsL8fC81cjnUnTfmyL/HOIyHnyaw/JA474Wfj2tl6A=
 github.com/onflow/flow-ft/lib/go/templates v1.0.1 h1:FDYKAiGowABtoMNusLuRCILIZDtVqJ/5tYI4VkF5zfM=
 github.com/onflow/flow-ft/lib/go/templates v1.0.1/go.mod h1:uQ8XFqmMK2jxyBSVrmyuwdWjTEb+6zGjRYotfDJ5pAE=
-github.com/onflow/flow-go v0.38.0-preview.0.0.20241125190444-25a8af57bea1 h1:tE21Kgx2Aqll9ywbiRDfc2BVIz5g6zKdrIom9U9eTE4=
-github.com/onflow/flow-go v0.38.0-preview.0.0.20241125190444-25a8af57bea1/go.mod h1:c4ubAQ2WIMYY/TOaBvbajROEFWv2HwhKeGOsEdLPIM0=
+github.com/onflow/flow-go v0.38.0-preview.0.0.10 h1:0hUwj9VAH2O2UHSmmt2tJW0P9p1PRpgIPToH3YLyPzM=
+github.com/onflow/flow-go v0.38.0-preview.0.0.10/go.mod h1:c4ubAQ2WIMYY/TOaBvbajROEFWv2HwhKeGOsEdLPIM0=
 github.com/onflow/flow-go-sdk v1.2.3 h1:jb+0dIXBO12Zt8x3c2xDXYPv6k3sRTUvhe59M+EcXTI=
 github.com/onflow/flow-go-sdk v1.2.3/go.mod h1:jMaffBTlAIdutx+pBhRIigLZFIBYSDDST0Uax1rW2qo=
 github.com/onflow/flow-nft/lib/go/contracts v1.2.2 h1:XFERNVUDGbZ4ViZjt7P1cGD80mO1PzUJYPfdhXFsGbQ=

services/evm/extract.go

@@ -0,0 +1,36 @@
+package evm
+
+import (
+	"fmt"
+
+	"github.com/onflow/flow-evm-gateway/storage/pebble"
+	"github.com/onflow/flow-go/fvm/evm"
+	"github.com/onflow/flow-go/fvm/evm/emulator/state"
+	"github.com/onflow/flow-go/fvm/evm/offchain/storage"
+	flowGo "github.com/onflow/flow-go/model/flow"
+)
+
+func ExtractEVMState(
+	chainID flowGo.ChainID,
+	evmHeight uint64,
+	store *pebble.Storage,
+) (*state.EVMState, error) {
+	storageRoot := evm.StorageAccountAddress(chainID)
+	registerStore := pebble.NewRegisterStorage(store, storageRoot)
+	snapshot, err := registerStore.GetSnapshotAt(evmHeight)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get snapshot at evm height %d: %w", evmHeight, err)
+	}
+
+	ledger := storage.NewReadOnlyStorage(snapshot)
+	bv, err := state.NewBaseView(ledger, storageRoot)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create base view: %w", err)
+	}
+
+	evmState, err := state.Extract(storageRoot, bv)
+	if err != nil {
+		return nil, err
+	}
+	return evmState, nil
+}

services/evm/extract_test.go

@@ -0,0 +1,45 @@
+package evm_test
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/onflow/flow-evm-gateway/storage/pebble"
+	"github.com/onflow/flow-go/fvm/evm/emulator/state"
+	flowGo "github.com/onflow/flow-go/model/flow"
+	"github.com/rs/zerolog/log"
+	"github.com/stretchr/testify/require"
+
+	evmState "github.com/onflow/flow-evm-gateway/services/evm"
+)
+
+func StateDiff(t *testing.T) {
+	state1 := extractEVMState(t, flowGo.Testnet, "/var/flow52/evm/data/db", uint64(17724990))
+	state2 := evmStateFromCheckpointExtract(t, "/var/flow52/evm-state-from-checkpoint-228901661")
+
+	differences := state.Diff(state1, state2)
+
+	for i, diff := range differences {
+		fmt.Printf("Difference %d: %v\n", i, diff)
+	}
+
+	require.Len(t, differences, 0)
+}
+
+func extractEVMState(
+	t *testing.T, chainID flowGo.ChainID,
+	registerStoreDir string, evmHeight uint64) *state.EVMState {
+
+	store, err := pebble.New(registerStoreDir, log.Logger)
+	require.NoError(t, err)
+
+	evmState, err := evmState.ExtractEVMState(chainID, evmHeight, store)
+	require.NoError(t, err)
+	return evmState
+}
+
+func evmStateFromCheckpointExtract(t *testing.T, dir string) *state.EVMState {
+	enState, err := state.ImportEVMStateFromGob(dir)
+	require.NoError(t, err)
+	return enState
+}

services/ingestion/engine.go

@@ -1,4 +1,4 @@
 package ingestion
 
 import (
 	"context"
@@ -248,7 +248,7 @@
 	// Step 1.2: Replay all block transactions
 	// If `ReplayBlock` returns any error, we abort the EVM events processing
 	blockEvents := events.BlockEventPayload()
-	res, err := replayer.ReplayBlock(events.TxEventPayloads(), blockEvents)
+	res, _, err := replayer.ReplayBlock(events.TxEventPayloads(), blockEvents)
 	if err != nil {
 		return fmt.Errorf("failed to replay block on height: %d, with: %w", events.Block().Height, err)
 	}