Add verification tool for evm offchain replay

cmd/util/cmd/root.go

@@ -41,6 +41,7 @@ import (
 	"github.com/onflow/flow-go/cmd/util/cmd/snapshot"
 	system_addresses "github.com/onflow/flow-go/cmd/util/cmd/system-addresses"
 	truncate_database "github.com/onflow/flow-go/cmd/util/cmd/truncate-database"
+	verify_evm_offchain_replay "github.com/onflow/flow-go/cmd/util/cmd/verify-evm-offchain-replay"
 	"github.com/onflow/flow-go/cmd/util/cmd/version"
 	"github.com/onflow/flow-go/module/profiler"
 )
@@ -126,6 +127,7 @@ func addCommands() {
 	rootCmd.AddCommand(debug_script.Cmd)
 	rootCmd.AddCommand(generate_authorization_fixes.Cmd)
 	rootCmd.AddCommand(evm_state_exporter.Cmd)
+	rootCmd.AddCommand(verify_evm_offchain_replay.Cmd)
 }
 
 func initConfig() {

cmd/util/cmd/verify-evm-offchain-replay/main.go

@@ -0,0 +1,88 @@
+package verify
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/rs/zerolog/log"
+	"github.com/spf13/cobra"
+
+	"github.com/onflow/flow-go/model/flow"
+)
+
+var (
+	flagDatadir          string
+	flagExecutionDataDir string
+	flagEVMStateGobDir   string
+	flagChain            string
+	flagFromTo           string
+	flagSaveEveryNBlocks uint64
+)
+
+// usage example
+//
+//		./util verify-evm-offchain-replay --chain flow-testnet --from_to 211176670-211177000
+//	     --datadir /var/flow/data/protocol --execution_data_dir /var/flow/data/execution_data
+var Cmd = &cobra.Command{
+	Use:   "verify-evm-offchain-replay",
+	Short: "verify evm offchain replay with execution data",
+	Run:   run,
+}
+
+func init() {
+	Cmd.Flags().StringVar(&flagChain, "chain", "", "Chain name")
+	_ = Cmd.MarkFlagRequired("chain")
+
+	Cmd.Flags().StringVar(&flagDatadir, "datadir", "/var/flow/data/protocol",
+		"directory that stores the protocol state")
+
+	Cmd.Flags().StringVar(&flagExecutionDataDir, "execution_data_dir", "/var/flow/data/execution_data",
+		"directory that stores the execution state")
+
+	Cmd.Flags().StringVar(&flagFromTo, "from_to", "",
+		"the flow height range to verify blocks, i.e, 1-1000, 1000-2000, 2000-3000, etc.")
+
+	Cmd.Flags().StringVar(&flagEVMStateGobDir, "evm_state_gob_dir", "/var/flow/data/evm_state_gob",
+		"directory that stores the evm state gob files as checkpoint")
+
+	Cmd.Flags().Uint64Var(&flagSaveEveryNBlocks, "save_every", uint64(1_000_000),
+		"save the evm state gob files every N blocks")
+}
+
+func run(*cobra.Command, []string) {
+	chainID := flow.ChainID(flagChain)
+
+	from, to, err := parseFromTo(flagFromTo)
+	if err != nil {
+		log.Fatal().Err(err).Msg("could not parse from_to")
+	}
+
+	err = Verify(log.Logger, from, to, chainID, flagDatadir, flagExecutionDataDir, flagEVMStateGobDir, flagSaveEveryNBlocks)
+	if err != nil {
+		log.Fatal().Err(err).Msg("could not verify height")
+	}
+}
+
+func parseFromTo(fromTo string) (from, to uint64, err error) {
+	parts := strings.Split(fromTo, "-")
+	if len(parts) != 2 {
+		return 0, 0, fmt.Errorf("invalid format: expected 'from-to', got '%s'", fromTo)
+	}
+
+	from, err = strconv.ParseUint(strings.TrimSpace(parts[0]), 10, 64)
+	if err != nil {
+		return 0, 0, fmt.Errorf("invalid 'from' value: %w", err)
+	}
+
+	to, err = strconv.ParseUint(strings.TrimSpace(parts[1]), 10, 64)
+	if err != nil {
+		return 0, 0, fmt.Errorf("invalid 'to' value: %w", err)
+	}
+
+	if from > to {
+		return 0, 0, fmt.Errorf("'from' value (%d) must be less than or equal to 'to' value (%d)", from, to)
+	}
+
+	return from, to, nil
+}

cmd/util/cmd/verify-evm-offchain-replay/verify.go

@@ -0,0 +1,173 @@
+package verify
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+
+	"github.com/dgraph-io/badger/v2"
+	badgerds "github.com/ipfs/go-ds-badger2"
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
+
+	"github.com/onflow/flow-go/cmd/util/cmd/common"
+	"github.com/onflow/flow-go/fvm/evm/offchain/utils"
+	"github.com/onflow/flow-go/fvm/evm/testutils"
+	"github.com/onflow/flow-go/model/flow"
+	"github.com/onflow/flow-go/module/blobs"
+	"github.com/onflow/flow-go/module/executiondatasync/execution_data"
+	"github.com/onflow/flow-go/storage"
+)
+
+// Verify verifies the offchain replay of EVM blocks from the given height range
+// and updates the EVM state gob files with the latest state
+func Verify(
+	log zerolog.Logger,
+	from uint64,
+	to uint64,
+	chainID flow.ChainID,
+	dataDir string,
+	executionDataDir string,
+	evmStateGobDir string,
+	saveEveryNBlocks uint64,
+) error {
+	lg := log.With().
+		Uint64("from", from).Uint64("to", to).
+		Str("chain", chainID.String()).
+		Str("dataDir", dataDir).
+		Str("executionDataDir", executionDataDir).
+		Str("evmStateGobDir", evmStateGobDir).
+		Uint64("saveEveryNBlocks", saveEveryNBlocks).
+		Logger()
+
+	lg.Info().Msgf("verifying range from %d to %d", from, to)
+
+	db, storages, executionDataStore, dsStore, err := initStorages(dataDir, executionDataDir)
+	if err != nil {
+		return fmt.Errorf("could not initialize storages: %w", err)
+	}
+
+	defer db.Close()
+	defer dsStore.Close()
+
+	var store *testutils.TestValueStore
+
+	// root block require the account status registers to be saved
+	isRoot := utils.IsEVMRootHeight(chainID, from)
+	if isRoot {
+		store = testutils.GetSimpleValueStore()
+	} else {
+		prev := from - 1
+		store, err = loadState(prev, evmStateGobDir)
+		if err != nil {
+			return fmt.Errorf("could not load EVM state from previous height %d: %w", prev, err)
+		}
+	}
+
+	// save state every N blocks
+	onHeightReplayed := func(height uint64) error {
+		log.Info().Msgf("replayed height %d", height)
+		if height%saveEveryNBlocks == 0 {
+			err := saveState(store, height, evmStateGobDir)
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+
+	// replay blocks
+	err = utils.OffchainReplayBackwardCompatibilityTest(
+		log,
+		chainID,
+		from,
+		to,
+		storages.Headers,
+		storages.Results,
+		executionDataStore,
+		store,
+		onHeightReplayed,
+	)
+
+	if err != nil {
+		return err
+	}
+
+	err = saveState(store, to, evmStateGobDir)
+	if err != nil {
+		return err
+	}
+
+	lg.Info().Msgf("successfully verified range from %d to %d", from, to)
+
+	return nil
+}
+
+func saveState(store *testutils.TestValueStore, height uint64, gobDir string) error {
+	valueFileName, allocatorFileName := evmStateGobFileNamesByEndHeight(gobDir, height)
+	values, allocators := store.Dump()
+	err := testutils.SerializeState(valueFileName, values)
+	if err != nil {
+		return err
+	}
+	err = testutils.SerializeAllocator(allocatorFileName, allocators)
+	if err != nil {
+		return err
+	}
+
+	log.Info().Msgf("saved EVM state to %s and %s", valueFileName, allocatorFileName)
+
+	return nil
+}
+
+func loadState(height uint64, gobDir string) (*testutils.TestValueStore, error) {
+	valueFileName, allocatorFileName := evmStateGobFileNamesByEndHeight(gobDir, height)
+	values, err := testutils.DeserializeState(valueFileName)
+	if err != nil {
+		return nil, fmt.Errorf("could not deserialize state %v: %w", valueFileName, err)
+	}
+
+	allocators, err := testutils.DeserializeAllocator(allocatorFileName)
+	if err != nil {
+		return nil, fmt.Errorf("could not deserialize allocator %v: %w", allocatorFileName, err)
+	}
+	store := testutils.GetSimpleValueStorePopulated(values, allocators)
+
+	log.Info().Msgf("loaded EVM state for height %d from gob file %v", height, valueFileName)
+	return store, nil
+}
+
+func initStorages(dataDir string, executionDataDir string) (
+	*badger.DB,
+	*storage.All,
+	execution_data.ExecutionDataGetter,
+	io.Closer,
+	error,
+) {
+	db := common.InitStorage(dataDir)
+
+	storages := common.InitStorages(db)
+
+	datastoreDir := filepath.Join(executionDataDir, "blobstore")
+	err := os.MkdirAll(datastoreDir, 0700)
+	if err != nil {
+		return nil, nil, nil, nil, err
+	}
+	dsOpts := &badgerds.DefaultOptions
+	ds, err := badgerds.NewDatastore(datastoreDir, dsOpts)
+	if err != nil {
+		return nil, nil, nil, nil, err
+	}
+
+	executionDataBlobstore := blobs.NewBlobstore(ds)
+	executionDataStore := execution_data.NewExecutionDataStore(executionDataBlobstore, execution_data.DefaultSerializer)
+
+	return db, storages, executionDataStore, ds, nil
+}
+
+func evmStateGobFileNamesByEndHeight(evmStateGobDir string, endHeight uint64) (string, string) {
+	valueFileName := filepath.Join(evmStateGobDir, fmt.Sprintf("values-%d.gob", endHeight))
+	allocatorFileName := filepath.Join(evmStateGobDir, fmt.Sprintf("allocators-%d.gob", endHeight))
+	return valueFileName, allocatorFileName
+}

fvm/evm/handler/blockHashList.go

@@ -3,6 +3,7 @@ package handler
 import (
 	"encoding/binary"
 	"fmt"
+	"strings"
 
 	gethCommon "github.com/onflow/go-ethereum/common"
 
@@ -26,6 +27,14 @@ const (
 		heightEncodingSize
 )
 
+func IsBlockHashListBucketKeyFormat(id flow.RegisterID) bool {
+	return strings.HasPrefix(id.Key, "BlockHashListBucket")
+}
+
+func IsBlockHashListMetaKey(id flow.RegisterID) bool {
+	return id.Key == blockHashListMetaKey
+}
+
 // BlockHashList stores the last `capacity` number of block hashes
 //
 // Under the hood it breaks the list of hashes into

fvm/evm/offchain/blocks/block_proposal.go

@@ -0,0 +1,34 @@
+package blocks
+
+import (
+	"github.com/onflow/flow-go/fvm/evm/events"
+	"github.com/onflow/flow-go/fvm/evm/types"
+)
+
+func ReconstructProposal(
+	blockEvent *events.BlockEventPayload,
+	results []*types.Result,
+) *types.BlockProposal {
+	receipts := make([]types.LightReceipt, 0, len(results))
+	txHashes := make(types.TransactionHashes, 0, len(results))
+
+	for _, result := range results {
+		receipts = append(receipts, *result.LightReceipt())
+		txHashes = append(txHashes, result.TxHash)
+	}
+
+	return &types.BlockProposal{
+		Block: types.Block{
+			ParentBlockHash:     blockEvent.ParentBlockHash,
+			Height:              blockEvent.Height,
+			Timestamp:           blockEvent.Timestamp,
+			TotalSupply:         blockEvent.TotalSupply.Big(),
+			ReceiptRoot:         blockEvent.ReceiptRoot,
+			TransactionHashRoot: blockEvent.TransactionHashRoot,
+			TotalGasUsed:        blockEvent.TotalGasUsed,
+			PrevRandao:          blockEvent.PrevRandao,
+		},
+		Receipts: receipts,
+		TxHashes: txHashes,
+	}
+}