From 3ba74a3176dd12747ff34c8478b9a3fd33c9ab82 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Tue, 26 Nov 2024 14:21:00 -0800 Subject: [PATCH 01/16] add util to extract evm state --- cmd/util/main.go | 71 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 cmd/util/main.go diff --git a/cmd/util/main.go b/cmd/util/main.go new file mode 100644 index 00000000..f777e387 --- /dev/null +++ b/cmd/util/main.go @@ -0,0 +1,71 @@ +package main + +import ( + "flag" + "os" + + "github.com/rs/zerolog/log" + + "github.com/onflow/flow-evm-gateway/storage/pebble" + "github.com/onflow/flow-go/fvm/evm" + "github.com/onflow/flow-go/fvm/evm/emulator/state" + "github.com/onflow/flow-go/fvm/evm/offchain/storage" + flowGo "github.com/onflow/flow-go/model/flow" +) + +func main() { + var ( + height uint64 + outputDir string + registerStoreDir string + ) + + flag.Uint64Var(&height, "height", 0, "Block height for EVM state export") + flag.StringVar(&outputDir, "output", "", "Output directory for exported EVM state") + flag.StringVar(®isterStoreDir, "register-store", "", "Directory of the register store") + + flag.Parse() + + if height == 0 || outputDir == "" || registerStoreDir == "" { + log.Error().Msg("All flags (height, output, register-store) must be provided") + flag.Usage() + os.Exit(1) + } + + chainID := flowGo.Testnet + + log.Info().Msgf("exporting EVM state for height %v from registerStoreDir %v, outputDir: %v", height, registerStoreDir, outputDir) + err := ExportEVMStateForHeight(height, outputDir, registerStoreDir, chainID) + if err != nil { + log.Fatal().Err(err).Msgf("fail to export") + } + + log.Info().Msgf("successfully exported EVM state to %v", outputDir) +} + +func ExportEVMStateForHeight(height uint64, outputDir string, registerStoreDir string, chainID flowGo.ChainID) error { + store, err := pebble.New(registerStoreDir, log.Logger) + if err != nil { + return err + } + + storageAddress := evm.StorageAccountAddress(chainID) + registerStore := pebble.NewRegisterStorage(store, storageAddress) + snapshot, err := registerStore.GetSnapshotAt(height) + if err != nil { + return err + } + + ledger := storage.NewReadOnlyStorage(snapshot) + exporter, err := state.NewExporter(ledger, storageAddress) + if err != nil { + return err + } + + err = exporter.ExportGob(outputDir) + if err != nil { + return err + } + + return nil +} From 3cf7b49e1ee5cb71369a688b88df953b1158ce17 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Tue, 26 Nov 2024 14:25:18 -0800 Subject: [PATCH 02/16] updat flow-go --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d0317546..fc4cbaf2 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/goccy/go-json v0.10.2 github.com/onflow/atree v0.8.0 github.com/onflow/cadence v1.2.2 - github.com/onflow/flow-go v0.38.0-preview.0.0.20241125190444-25a8af57bea1 + github.com/onflow/flow-go v0.38.0-preview.0.0.4 github.com/onflow/flow-go-sdk v1.2.3 github.com/onflow/go-ethereum v1.14.7 github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index 50db1a3a..8f802fc4 100644 --- a/go.sum +++ b/go.sum @@ -535,8 +535,8 @@ github.com/onflow/flow-ft/lib/go/contracts v1.0.1 h1:Ts5ob+CoCY2EjEd0W6vdLJ7hLL3 github.com/onflow/flow-ft/lib/go/contracts v1.0.1/go.mod h1:PwsL8fC81cjnUnTfmyL/HOIyHnyaw/JA474Wfj2tl6A= github.com/onflow/flow-ft/lib/go/templates v1.0.1 h1:FDYKAiGowABtoMNusLuRCILIZDtVqJ/5tYI4VkF5zfM= github.com/onflow/flow-ft/lib/go/templates v1.0.1/go.mod h1:uQ8XFqmMK2jxyBSVrmyuwdWjTEb+6zGjRYotfDJ5pAE= -github.com/onflow/flow-go v0.38.0-preview.0.0.20241125190444-25a8af57bea1 h1:tE21Kgx2Aqll9ywbiRDfc2BVIz5g6zKdrIom9U9eTE4= -github.com/onflow/flow-go v0.38.0-preview.0.0.20241125190444-25a8af57bea1/go.mod h1:c4ubAQ2WIMYY/TOaBvbajROEFWv2HwhKeGOsEdLPIM0= +github.com/onflow/flow-go v0.38.0-preview.0.0.4 h1:oVCnex8RjD4vUsCgXWFpN5fILckQV7y1FJs+WEMEPLY= +github.com/onflow/flow-go v0.38.0-preview.0.0.4/go.mod h1:c4ubAQ2WIMYY/TOaBvbajROEFWv2HwhKeGOsEdLPIM0= github.com/onflow/flow-go-sdk v1.2.3 h1:jb+0dIXBO12Zt8x3c2xDXYPv6k3sRTUvhe59M+EcXTI= github.com/onflow/flow-go-sdk v1.2.3/go.mod h1:jMaffBTlAIdutx+pBhRIigLZFIBYSDDST0Uax1rW2qo= github.com/onflow/flow-nft/lib/go/contracts v1.2.2 h1:XFERNVUDGbZ4ViZjt7P1cGD80mO1PzUJYPfdhXFsGbQ= From 07f60d142ab8639f175a62fed988af8b63cf056f Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Tue, 26 Nov 2024 19:33:16 -0800 Subject: [PATCH 03/16] update go.mod --- go.mod | 4 +++- go.sum | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index fc4cbaf2..93c3104a 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/goccy/go-json v0.10.2 github.com/onflow/atree v0.8.0 github.com/onflow/cadence v1.2.2 - github.com/onflow/flow-go v0.38.0-preview.0.0.4 + github.com/onflow/flow-go v0.38.0-preview.0.0.5 github.com/onflow/flow-go-sdk v1.2.3 github.com/onflow/go-ethereum v1.14.7 github.com/prometheus/client_golang v1.18.0 @@ -83,6 +83,7 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect + github.com/google/go-cmp v0.6.0 // indirect github.com/google/pprof v0.0.0-20231229205709-960ae82b1e42 // indirect github.com/google/s2a-go v0.1.7 // indirect github.com/google/uuid v1.6.0 // indirect @@ -211,6 +212,7 @@ require ( google.golang.org/protobuf v1.33.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + gotest.tools v2.2.0+incompatible // indirect lukechampine.com/blake3 v1.3.0 // indirect rsc.io/tmplfunc v0.0.3 // indirect ) diff --git a/go.sum b/go.sum index 8f802fc4..dbefec49 100644 --- a/go.sum +++ b/go.sum @@ -535,8 +535,8 @@ github.com/onflow/flow-ft/lib/go/contracts v1.0.1 h1:Ts5ob+CoCY2EjEd0W6vdLJ7hLL3 github.com/onflow/flow-ft/lib/go/contracts v1.0.1/go.mod h1:PwsL8fC81cjnUnTfmyL/HOIyHnyaw/JA474Wfj2tl6A= github.com/onflow/flow-ft/lib/go/templates v1.0.1 h1:FDYKAiGowABtoMNusLuRCILIZDtVqJ/5tYI4VkF5zfM= github.com/onflow/flow-ft/lib/go/templates v1.0.1/go.mod h1:uQ8XFqmMK2jxyBSVrmyuwdWjTEb+6zGjRYotfDJ5pAE= -github.com/onflow/flow-go v0.38.0-preview.0.0.4 h1:oVCnex8RjD4vUsCgXWFpN5fILckQV7y1FJs+WEMEPLY= -github.com/onflow/flow-go v0.38.0-preview.0.0.4/go.mod h1:c4ubAQ2WIMYY/TOaBvbajROEFWv2HwhKeGOsEdLPIM0= +github.com/onflow/flow-go v0.38.0-preview.0.0.5 h1:rc9lIlR32bi1Z9ojKHyG4WbC2pl7drkva6hI5zVQNAQ= +github.com/onflow/flow-go v0.38.0-preview.0.0.5/go.mod h1:c4ubAQ2WIMYY/TOaBvbajROEFWv2HwhKeGOsEdLPIM0= github.com/onflow/flow-go-sdk v1.2.3 h1:jb+0dIXBO12Zt8x3c2xDXYPv6k3sRTUvhe59M+EcXTI= github.com/onflow/flow-go-sdk v1.2.3/go.mod h1:jMaffBTlAIdutx+pBhRIigLZFIBYSDDST0Uax1rW2qo= github.com/onflow/flow-nft/lib/go/contracts v1.2.2 h1:XFERNVUDGbZ4ViZjt7P1cGD80mO1PzUJYPfdhXFsGbQ= From e9166c9d53b90ddc6bd7e868d1a7d83a51eb43dc Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Tue, 26 Nov 2024 20:36:54 -0800 Subject: [PATCH 04/16] add extract EVM state tests --- cmd/util/main_test.go | 89 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 cmd/util/main_test.go diff --git a/cmd/util/main_test.go b/cmd/util/main_test.go new file mode 100644 index 00000000..b424131a --- /dev/null +++ b/cmd/util/main_test.go @@ -0,0 +1,89 @@ +package main_test + +import ( + "fmt" + "path/filepath" + "testing" + + "github.com/onflow/flow-evm-gateway/storage/pebble" + "github.com/onflow/flow-go-sdk" + + "github.com/onflow/flow-go/fvm/evm" + "github.com/onflow/flow-go/fvm/evm/emulator/state" + "github.com/onflow/flow-go/fvm/evm/offchain/storage" + "github.com/onflow/flow-go/fvm/evm/testutils" + flowGo "github.com/onflow/flow-go/model/flow" + "github.com/rs/zerolog/log" + "github.com/stretchr/testify/require" +) + +func TestStateDiff(t *testing.T) { + + state1 := ExtractEVMState(t, flow.Testnet, + "/var/flow/gw/data/db", uint64(218215348)) + state2 := state.EVMStateFromReplayGobDir(t, + "/var/flow2/evm-state-from-gobs-218215348/", uint64(218215348)) + // state2 := state.EVMStateFromCheckpointExtract(t, "/var/flow2/evm-state-from-checkpoint-218215348/") + + differences := state.Diff(state1, state2) + + for i, diff := range differences { + fmt.Printf("Difference %d: %v\n", i, diff) + } + + require.Len(t, differences, 0) +} + +func ExtractEVMState( + t *testing.T, chainID flowGo.ChainID, + registerStoreDir string, flowHeight uint64) *state.EVMState { + + store, err := pebble.New(registerStoreDir, log.Logger) + require.NoError(t, err) + + storageRoot := evm.StorageAccountAddress(chainID) + registerStore := pebble.NewRegisterStorage(store, storageRoot) + snapshot, err := registerStore.GetSnapshotAt(flowHeight) + require.NoError(t, err) + + ledger := storage.NewReadOnlyStorage(snapshot) + bv, err := state.NewBaseView(ledger, storageRoot) + require.NoError(t, err) + + evmState, err := state.Extract(storageRoot, bv) + require.NoError(t, err) + return evmState +} + +func EVMStateFromCheckpointExtract(t *testing.T, dir string) *state.EVMState { + enState, err := state.ImportEVMStateFromGob("/var/flow2/evm-state-from-gobs-218215348/") + require.NoError(t, err) + return enState +} + +func EVMStateFromReplayGobDir(t *testing.T, gobDir string, flowHeight uint64) *state.EVMState { + valueFileName, allocatorFileName := evmStateGobFileNamesByEndHeight(gobDir, flowHeight) + chainID := flow.Testnet + + allocatorGobs, err := testutils.DeserializeAllocator(allocatorFileName) + require.NoError(t, err) + + storageRoot := evm.StorageAccountAddress(chainID) + valuesGob, err := testutils.DeserializeState(valueFileName) + require.NoError(t, err) + + store := testutils.GetSimpleValueStorePopulated(valuesGob, allocatorGobs) + + bv, err := state.NewBaseView(store, storageRoot) + require.NoError(t, err) + + evmState, err := state.Extract(storageRoot, bv) + require.NoError(t, err) + return evmState +} + +func evmStateGobFileNamesByEndHeight(evmStateGobDir string, endHeight uint64) (string, string) { + valueFileName := filepath.Join(evmStateGobDir, fmt.Sprintf("values-%d.gob", endHeight)) + allocatorFileName := filepath.Join(evmStateGobDir, fmt.Sprintf("allocators-%d.gob", endHeight)) + return valueFileName, allocatorFileName +} From 27ecf37a8ce655a2536078ff7b391dbd125f8731 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Tue, 26 Nov 2024 20:38:22 -0800 Subject: [PATCH 05/16] fix replay --- services/ingestion/engine.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/ingestion/engine.go b/services/ingestion/engine.go index 2145e649..52249366 100644 --- a/services/ingestion/engine.go +++ b/services/ingestion/engine.go @@ -248,7 +248,7 @@ func (e *Engine) indexEvents(events *models.CadenceEvents, batch *pebbleDB.Batch // Step 1.2: Replay all block transactions // If `ReplayBlock` returns any error, we abort the EVM events processing blockEvents := events.BlockEventPayload() - res, err := replayer.ReplayBlock(events.TxEventPayloads(), blockEvents) + res, _, err := replayer.ReplayBlock(events.TxEventPayloads(), blockEvents) if err != nil { return fmt.Errorf("failed to replay block on height: %d, with: %w", events.Block().Height, err) } From e90f00ede23fff6d61fd6434c4ecd0b775afc495 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Tue, 26 Nov 2024 20:39:56 -0800 Subject: [PATCH 06/16] fix lint --- cmd/util/main_test.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cmd/util/main_test.go b/cmd/util/main_test.go index b424131a..6cc3d593 100644 --- a/cmd/util/main_test.go +++ b/cmd/util/main_test.go @@ -6,7 +6,6 @@ import ( "testing" "github.com/onflow/flow-evm-gateway/storage/pebble" - "github.com/onflow/flow-go-sdk" "github.com/onflow/flow-go/fvm/evm" "github.com/onflow/flow-go/fvm/evm/emulator/state" @@ -19,9 +18,9 @@ import ( func TestStateDiff(t *testing.T) { - state1 := ExtractEVMState(t, flow.Testnet, + state1 := ExtractEVMState(t, flowGo.Testnet, "/var/flow/gw/data/db", uint64(218215348)) - state2 := state.EVMStateFromReplayGobDir(t, + state2 := EVMStateFromReplayGobDir(t, "/var/flow2/evm-state-from-gobs-218215348/", uint64(218215348)) // state2 := state.EVMStateFromCheckpointExtract(t, "/var/flow2/evm-state-from-checkpoint-218215348/") @@ -63,7 +62,7 @@ func EVMStateFromCheckpointExtract(t *testing.T, dir string) *state.EVMState { func EVMStateFromReplayGobDir(t *testing.T, gobDir string, flowHeight uint64) *state.EVMState { valueFileName, allocatorFileName := evmStateGobFileNamesByEndHeight(gobDir, flowHeight) - chainID := flow.Testnet + chainID := flowGo.Testnet allocatorGobs, err := testutils.DeserializeAllocator(allocatorFileName) require.NoError(t, err) From 01e70cf5331a112cda9a743bf09c01e800df92c3 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Wed, 27 Nov 2024 06:58:04 -0800 Subject: [PATCH 07/16] update tests --- cmd/util/main_test.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/cmd/util/main_test.go b/cmd/util/main_test.go index 6cc3d593..ead92a30 100644 --- a/cmd/util/main_test.go +++ b/cmd/util/main_test.go @@ -86,3 +86,27 @@ func evmStateGobFileNamesByEndHeight(evmStateGobDir string, endHeight uint64) (s allocatorFileName := filepath.Join(evmStateGobDir, fmt.Sprintf("allocators-%d.gob", endHeight)) return valueFileName, allocatorFileName } + +func TestReadPebbleRootAccountStorageKeyID(t *testing.T) { + chainID := flowGo.Testnet + registerStoreDir := "/var/flow/gw/data/db" + flowHeight := uint64(218215348) + + store, err := pebble.New(registerStoreDir, log.Logger) + require.NoError(t, err) + + storageRoot := evm.StorageAccountAddress(chainID) + registerStore := pebble.NewRegisterStorage(store, storageRoot) + regID := flowGo.RegisterID{ + Owner: string(storageRoot.Bytes()), + Key: state.AccountsStorageIDKey, + } + regVal, err := registerStore.Get(regID, flowHeight) + require.NoError(t, err) + + require.NotNil(t, regVal) + fmt.Printf("Register value: %x\n", regVal) + + _, err = registerStore.GetSnapshotAt(flowHeight) + require.NoError(t, err) +} From d619ac80ffcd677cac1b852df0fe5ba6ac45df94 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Wed, 27 Nov 2024 09:39:14 -0800 Subject: [PATCH 08/16] update version --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 93c3104a..93d5f35b 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/goccy/go-json v0.10.2 github.com/onflow/atree v0.8.0 github.com/onflow/cadence v1.2.2 - github.com/onflow/flow-go v0.38.0-preview.0.0.5 + github.com/onflow/flow-go v0.38.0-preview.0.0.7 github.com/onflow/flow-go-sdk v1.2.3 github.com/onflow/go-ethereum v1.14.7 github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index dbefec49..0cd26354 100644 --- a/go.sum +++ b/go.sum @@ -535,8 +535,8 @@ github.com/onflow/flow-ft/lib/go/contracts v1.0.1 h1:Ts5ob+CoCY2EjEd0W6vdLJ7hLL3 github.com/onflow/flow-ft/lib/go/contracts v1.0.1/go.mod h1:PwsL8fC81cjnUnTfmyL/HOIyHnyaw/JA474Wfj2tl6A= github.com/onflow/flow-ft/lib/go/templates v1.0.1 h1:FDYKAiGowABtoMNusLuRCILIZDtVqJ/5tYI4VkF5zfM= github.com/onflow/flow-ft/lib/go/templates v1.0.1/go.mod h1:uQ8XFqmMK2jxyBSVrmyuwdWjTEb+6zGjRYotfDJ5pAE= -github.com/onflow/flow-go v0.38.0-preview.0.0.5 h1:rc9lIlR32bi1Z9ojKHyG4WbC2pl7drkva6hI5zVQNAQ= -github.com/onflow/flow-go v0.38.0-preview.0.0.5/go.mod h1:c4ubAQ2WIMYY/TOaBvbajROEFWv2HwhKeGOsEdLPIM0= +github.com/onflow/flow-go v0.38.0-preview.0.0.7 h1:T3jRN8cettIccxNhfMwZgPPp08446dW8fXXHEUMQKwg= +github.com/onflow/flow-go v0.38.0-preview.0.0.7/go.mod h1:c4ubAQ2WIMYY/TOaBvbajROEFWv2HwhKeGOsEdLPIM0= github.com/onflow/flow-go-sdk v1.2.3 h1:jb+0dIXBO12Zt8x3c2xDXYPv6k3sRTUvhe59M+EcXTI= github.com/onflow/flow-go-sdk v1.2.3/go.mod h1:jMaffBTlAIdutx+pBhRIigLZFIBYSDDST0Uax1rW2qo= github.com/onflow/flow-nft/lib/go/contracts v1.2.2 h1:XFERNVUDGbZ4ViZjt7P1cGD80mO1PzUJYPfdhXFsGbQ= From b63d49f237b1f8aececcc0e6669d8265591c2a7e Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Wed, 27 Nov 2024 09:42:44 -0800 Subject: [PATCH 09/16] update version --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 93d5f35b..bec3df70 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/goccy/go-json v0.10.2 github.com/onflow/atree v0.8.0 github.com/onflow/cadence v1.2.2 - github.com/onflow/flow-go v0.38.0-preview.0.0.7 + github.com/onflow/flow-go v0.38.0-preview.0.0.8 github.com/onflow/flow-go-sdk v1.2.3 github.com/onflow/go-ethereum v1.14.7 github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index 0cd26354..d1bac113 100644 --- a/go.sum +++ b/go.sum @@ -535,8 +535,8 @@ github.com/onflow/flow-ft/lib/go/contracts v1.0.1 h1:Ts5ob+CoCY2EjEd0W6vdLJ7hLL3 github.com/onflow/flow-ft/lib/go/contracts v1.0.1/go.mod h1:PwsL8fC81cjnUnTfmyL/HOIyHnyaw/JA474Wfj2tl6A= github.com/onflow/flow-ft/lib/go/templates v1.0.1 h1:FDYKAiGowABtoMNusLuRCILIZDtVqJ/5tYI4VkF5zfM= github.com/onflow/flow-ft/lib/go/templates v1.0.1/go.mod h1:uQ8XFqmMK2jxyBSVrmyuwdWjTEb+6zGjRYotfDJ5pAE= -github.com/onflow/flow-go v0.38.0-preview.0.0.7 h1:T3jRN8cettIccxNhfMwZgPPp08446dW8fXXHEUMQKwg= -github.com/onflow/flow-go v0.38.0-preview.0.0.7/go.mod h1:c4ubAQ2WIMYY/TOaBvbajROEFWv2HwhKeGOsEdLPIM0= +github.com/onflow/flow-go v0.38.0-preview.0.0.8 h1:LBlYsBYmf0lTSfQnbAicyfwDh6m5Uw0cwcHZbNq6UcE= +github.com/onflow/flow-go v0.38.0-preview.0.0.8/go.mod h1:c4ubAQ2WIMYY/TOaBvbajROEFWv2HwhKeGOsEdLPIM0= github.com/onflow/flow-go-sdk v1.2.3 h1:jb+0dIXBO12Zt8x3c2xDXYPv6k3sRTUvhe59M+EcXTI= github.com/onflow/flow-go-sdk v1.2.3/go.mod h1:jMaffBTlAIdutx+pBhRIigLZFIBYSDDST0Uax1rW2qo= github.com/onflow/flow-nft/lib/go/contracts v1.2.2 h1:XFERNVUDGbZ4ViZjt7P1cGD80mO1PzUJYPfdhXFsGbQ= From a3c8b9fe0bdba0a2586831b10fc89741f5f9ba81 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Wed, 27 Nov 2024 09:55:54 -0800 Subject: [PATCH 10/16] update version --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index bec3df70..df3b0094 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/goccy/go-json v0.10.2 github.com/onflow/atree v0.8.0 github.com/onflow/cadence v1.2.2 - github.com/onflow/flow-go v0.38.0-preview.0.0.8 + github.com/onflow/flow-go v0.38.0-preview.0.0.9 github.com/onflow/flow-go-sdk v1.2.3 github.com/onflow/go-ethereum v1.14.7 github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index d1bac113..b5a97c17 100644 --- a/go.sum +++ b/go.sum @@ -535,8 +535,8 @@ github.com/onflow/flow-ft/lib/go/contracts v1.0.1 h1:Ts5ob+CoCY2EjEd0W6vdLJ7hLL3 github.com/onflow/flow-ft/lib/go/contracts v1.0.1/go.mod h1:PwsL8fC81cjnUnTfmyL/HOIyHnyaw/JA474Wfj2tl6A= github.com/onflow/flow-ft/lib/go/templates v1.0.1 h1:FDYKAiGowABtoMNusLuRCILIZDtVqJ/5tYI4VkF5zfM= github.com/onflow/flow-ft/lib/go/templates v1.0.1/go.mod h1:uQ8XFqmMK2jxyBSVrmyuwdWjTEb+6zGjRYotfDJ5pAE= -github.com/onflow/flow-go v0.38.0-preview.0.0.8 h1:LBlYsBYmf0lTSfQnbAicyfwDh6m5Uw0cwcHZbNq6UcE= -github.com/onflow/flow-go v0.38.0-preview.0.0.8/go.mod h1:c4ubAQ2WIMYY/TOaBvbajROEFWv2HwhKeGOsEdLPIM0= +github.com/onflow/flow-go v0.38.0-preview.0.0.9 h1:k8l2cWVDcXZceyMEG2F+V9IvXXm85r6Q2IFDnhS0A00= +github.com/onflow/flow-go v0.38.0-preview.0.0.9/go.mod h1:c4ubAQ2WIMYY/TOaBvbajROEFWv2HwhKeGOsEdLPIM0= github.com/onflow/flow-go-sdk v1.2.3 h1:jb+0dIXBO12Zt8x3c2xDXYPv6k3sRTUvhe59M+EcXTI= github.com/onflow/flow-go-sdk v1.2.3/go.mod h1:jMaffBTlAIdutx+pBhRIigLZFIBYSDDST0Uax1rW2qo= github.com/onflow/flow-nft/lib/go/contracts v1.2.2 h1:XFERNVUDGbZ4ViZjt7P1cGD80mO1PzUJYPfdhXFsGbQ= From 90f7695c88f0bc69d5fae4d94eee81d49cf34a63 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Wed, 27 Nov 2024 10:01:08 -0800 Subject: [PATCH 11/16] update version --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index df3b0094..4a4b2211 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/goccy/go-json v0.10.2 github.com/onflow/atree v0.8.0 github.com/onflow/cadence v1.2.2 - github.com/onflow/flow-go v0.38.0-preview.0.0.9 + github.com/onflow/flow-go v0.38.0-preview.0.0.10 github.com/onflow/flow-go-sdk v1.2.3 github.com/onflow/go-ethereum v1.14.7 github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index b5a97c17..c2f8185b 100644 --- a/go.sum +++ b/go.sum @@ -535,8 +535,8 @@ github.com/onflow/flow-ft/lib/go/contracts v1.0.1 h1:Ts5ob+CoCY2EjEd0W6vdLJ7hLL3 github.com/onflow/flow-ft/lib/go/contracts v1.0.1/go.mod h1:PwsL8fC81cjnUnTfmyL/HOIyHnyaw/JA474Wfj2tl6A= github.com/onflow/flow-ft/lib/go/templates v1.0.1 h1:FDYKAiGowABtoMNusLuRCILIZDtVqJ/5tYI4VkF5zfM= github.com/onflow/flow-ft/lib/go/templates v1.0.1/go.mod h1:uQ8XFqmMK2jxyBSVrmyuwdWjTEb+6zGjRYotfDJ5pAE= -github.com/onflow/flow-go v0.38.0-preview.0.0.9 h1:k8l2cWVDcXZceyMEG2F+V9IvXXm85r6Q2IFDnhS0A00= -github.com/onflow/flow-go v0.38.0-preview.0.0.9/go.mod h1:c4ubAQ2WIMYY/TOaBvbajROEFWv2HwhKeGOsEdLPIM0= +github.com/onflow/flow-go v0.38.0-preview.0.0.10 h1:0hUwj9VAH2O2UHSmmt2tJW0P9p1PRpgIPToH3YLyPzM= +github.com/onflow/flow-go v0.38.0-preview.0.0.10/go.mod h1:c4ubAQ2WIMYY/TOaBvbajROEFWv2HwhKeGOsEdLPIM0= github.com/onflow/flow-go-sdk v1.2.3 h1:jb+0dIXBO12Zt8x3c2xDXYPv6k3sRTUvhe59M+EcXTI= github.com/onflow/flow-go-sdk v1.2.3/go.mod h1:jMaffBTlAIdutx+pBhRIigLZFIBYSDDST0Uax1rW2qo= github.com/onflow/flow-nft/lib/go/contracts v1.2.2 h1:XFERNVUDGbZ4ViZjt7P1cGD80mO1PzUJYPfdhXFsGbQ= From aa66a249092b6bca4bab5e14927663e6a09a8a03 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Wed, 27 Nov 2024 10:19:14 -0800 Subject: [PATCH 12/16] fix extraction --- cmd/util/main_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/util/main_test.go b/cmd/util/main_test.go index ead92a30..473240ec 100644 --- a/cmd/util/main_test.go +++ b/cmd/util/main_test.go @@ -45,7 +45,7 @@ func ExtractEVMState( snapshot, err := registerStore.GetSnapshotAt(flowHeight) require.NoError(t, err) - ledger := storage.NewReadOnlyStorage(snapshot) + ledger := storage.NewEphemeralStorage(storage.NewReadOnlyStorage(snapshot)) bv, err := state.NewBaseView(ledger, storageRoot) require.NoError(t, err) From c65b30a9faabbde35fb37f630e294f36dc5a4275 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Wed, 27 Nov 2024 13:34:17 -0800 Subject: [PATCH 13/16] get account balance --- cmd/util/main_test.go | 49 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 41 insertions(+), 8 deletions(-) diff --git a/cmd/util/main_test.go b/cmd/util/main_test.go index 473240ec..8897a79b 100644 --- a/cmd/util/main_test.go +++ b/cmd/util/main_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/onflow/flow-evm-gateway/storage/pebble" + gethCommon "github.com/onflow/go-ethereum/common" "github.com/onflow/flow-go/fvm/evm" "github.com/onflow/flow-go/fvm/evm/emulator/state" @@ -18,11 +19,16 @@ import ( func TestStateDiff(t *testing.T) { - state1 := ExtractEVMState(t, flowGo.Testnet, - "/var/flow/gw/data/db", uint64(218215348)) - state2 := EVMStateFromReplayGobDir(t, - "/var/flow2/evm-state-from-gobs-218215348/", uint64(218215348)) - // state2 := state.EVMStateFromCheckpointExtract(t, "/var/flow2/evm-state-from-checkpoint-218215348/") + height := uint64(218215348) + + // state1 := ExtractEVMState(t, flowGo.Testnet, "/var/flow52/evm/data/db", height) + + state1 := EVMStateFromReplayGobDir(t, "/var/flow/nov26_testnet_evm_state_gob", height) + state2 := EVMStateFromCheckpointExtract(t, "/var/flow52/evm-state-from-checkpoint-218215348-migrated/") + + // state1 := ExtractEVMState(t, flowGo.Testnet, + // "/var/flow52/evm/data/db", uint64(228901661)) + // state2 := EVMStateFromCheckpointExtract(t, "/var/flow52/evm-state-from-checkpoint-228901661") differences := state.Diff(state1, state2) @@ -55,7 +61,7 @@ func ExtractEVMState( } func EVMStateFromCheckpointExtract(t *testing.T, dir string) *state.EVMState { - enState, err := state.ImportEVMStateFromGob("/var/flow2/evm-state-from-gobs-218215348/") + enState, err := state.ImportEVMStateFromGob(dir) require.NoError(t, err) return enState } @@ -89,8 +95,8 @@ func evmStateGobFileNamesByEndHeight(evmStateGobDir string, endHeight uint64) (s func TestReadPebbleRootAccountStorageKeyID(t *testing.T) { chainID := flowGo.Testnet - registerStoreDir := "/var/flow/gw/data/db" - flowHeight := uint64(218215348) + registerStoreDir := "/var/flow52/evm/data/db" + flowHeight := uint64(228901661) store, err := pebble.New(registerStoreDir, log.Logger) require.NoError(t, err) @@ -110,3 +116,30 @@ func TestReadPebbleRootAccountStorageKeyID(t *testing.T) { _, err = registerStore.GetSnapshotAt(flowHeight) require.NoError(t, err) } + +func TestReadAccount(t *testing.T) { + + chainID := flowGo.Testnet + registerStoreDir := "/var/flow52/evm/data/db" + flowHeight := uint64(228901661) + accountStr := "17DDB8C5aA1382Fb04d057e3ec42B65a43A5fF49" + + // Convert string to common.Address + account := gethCommon.HexToAddress(accountStr) + + store, err := pebble.New(registerStoreDir, log.Logger) + require.NoError(t, err) + + storageRoot := evm.StorageAccountAddress(chainID) + registerStore := pebble.NewRegisterStorage(store, storageRoot) + + snapshot, err := registerStore.GetSnapshotAt(flowHeight) + require.NoError(t, err) + ledger := storage.NewEphemeralStorage(storage.NewReadOnlyStorage(snapshot)) + + stateDB, err := state.NewStateDB(ledger, storageRoot) + require.NoError(t, err) + + balance := stateDB.GetBalance(account) + fmt.Println("balance: ", balance) +} From c35e4d1fb9dbb67e39a1a50ced1a4f5b5665e78e Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Wed, 27 Nov 2024 13:47:12 -0800 Subject: [PATCH 14/16] print account balance from base view --- cmd/util/main_test.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/cmd/util/main_test.go b/cmd/util/main_test.go index 8897a79b..94933998 100644 --- a/cmd/util/main_test.go +++ b/cmd/util/main_test.go @@ -51,7 +51,7 @@ func ExtractEVMState( snapshot, err := registerStore.GetSnapshotAt(flowHeight) require.NoError(t, err) - ledger := storage.NewEphemeralStorage(storage.NewReadOnlyStorage(snapshot)) + ledger := storage.NewReadOnlyStorage(snapshot) bv, err := state.NewBaseView(ledger, storageRoot) require.NoError(t, err) @@ -135,11 +135,19 @@ func TestReadAccount(t *testing.T) { snapshot, err := registerStore.GetSnapshotAt(flowHeight) require.NoError(t, err) - ledger := storage.NewEphemeralStorage(storage.NewReadOnlyStorage(snapshot)) + ledger := storage.NewReadOnlyStorage(snapshot) + bv, err := state.NewBaseView(ledger, storageRoot) + require.NoError(t, err) stateDB, err := state.NewStateDB(ledger, storageRoot) require.NoError(t, err) balance := stateDB.GetBalance(account) - fmt.Println("balance: ", balance) + + fmt.Println("balance from stateDB: ", balance) + + balanceFromBaseView, err := bv.GetBalance(account) + require.NoError(t, err) + fmt.Println("balance from base view: ", balanceFromBaseView) + } From 512a96c98c215801a65397577f1ae28144e481f2 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Wed, 27 Nov 2024 16:03:38 -0800 Subject: [PATCH 15/16] update evm state extraction --- cmd/util/main.go | 2 +- cmd/util/main_test.go | 153 ----------------------------------- services/evm/extract.go | 36 +++++++++ services/evm/extract_test.go | 45 +++++++++++ 4 files changed, 82 insertions(+), 154 deletions(-) delete mode 100644 cmd/util/main_test.go create mode 100644 services/evm/extract.go create mode 100644 services/evm/extract_test.go diff --git a/cmd/util/main.go b/cmd/util/main.go index f777e387..a88022df 100644 --- a/cmd/util/main.go +++ b/cmd/util/main.go @@ -20,7 +20,7 @@ func main() { registerStoreDir string ) - flag.Uint64Var(&height, "height", 0, "Block height for EVM state export") + flag.Uint64Var(&height, "evm-height", 0, "EVM Block height for EVM state export") flag.StringVar(&outputDir, "output", "", "Output directory for exported EVM state") flag.StringVar(®isterStoreDir, "register-store", "", "Directory of the register store") diff --git a/cmd/util/main_test.go b/cmd/util/main_test.go deleted file mode 100644 index 94933998..00000000 --- a/cmd/util/main_test.go +++ /dev/null @@ -1,153 +0,0 @@ -package main_test - -import ( - "fmt" - "path/filepath" - "testing" - - "github.com/onflow/flow-evm-gateway/storage/pebble" - gethCommon "github.com/onflow/go-ethereum/common" - - "github.com/onflow/flow-go/fvm/evm" - "github.com/onflow/flow-go/fvm/evm/emulator/state" - "github.com/onflow/flow-go/fvm/evm/offchain/storage" - "github.com/onflow/flow-go/fvm/evm/testutils" - flowGo "github.com/onflow/flow-go/model/flow" - "github.com/rs/zerolog/log" - "github.com/stretchr/testify/require" -) - -func TestStateDiff(t *testing.T) { - - height := uint64(218215348) - - // state1 := ExtractEVMState(t, flowGo.Testnet, "/var/flow52/evm/data/db", height) - - state1 := EVMStateFromReplayGobDir(t, "/var/flow/nov26_testnet_evm_state_gob", height) - state2 := EVMStateFromCheckpointExtract(t, "/var/flow52/evm-state-from-checkpoint-218215348-migrated/") - - // state1 := ExtractEVMState(t, flowGo.Testnet, - // "/var/flow52/evm/data/db", uint64(228901661)) - // state2 := EVMStateFromCheckpointExtract(t, "/var/flow52/evm-state-from-checkpoint-228901661") - - differences := state.Diff(state1, state2) - - for i, diff := range differences { - fmt.Printf("Difference %d: %v\n", i, diff) - } - - require.Len(t, differences, 0) -} - -func ExtractEVMState( - t *testing.T, chainID flowGo.ChainID, - registerStoreDir string, flowHeight uint64) *state.EVMState { - - store, err := pebble.New(registerStoreDir, log.Logger) - require.NoError(t, err) - - storageRoot := evm.StorageAccountAddress(chainID) - registerStore := pebble.NewRegisterStorage(store, storageRoot) - snapshot, err := registerStore.GetSnapshotAt(flowHeight) - require.NoError(t, err) - - ledger := storage.NewReadOnlyStorage(snapshot) - bv, err := state.NewBaseView(ledger, storageRoot) - require.NoError(t, err) - - evmState, err := state.Extract(storageRoot, bv) - require.NoError(t, err) - return evmState -} - -func EVMStateFromCheckpointExtract(t *testing.T, dir string) *state.EVMState { - enState, err := state.ImportEVMStateFromGob(dir) - require.NoError(t, err) - return enState -} - -func EVMStateFromReplayGobDir(t *testing.T, gobDir string, flowHeight uint64) *state.EVMState { - valueFileName, allocatorFileName := evmStateGobFileNamesByEndHeight(gobDir, flowHeight) - chainID := flowGo.Testnet - - allocatorGobs, err := testutils.DeserializeAllocator(allocatorFileName) - require.NoError(t, err) - - storageRoot := evm.StorageAccountAddress(chainID) - valuesGob, err := testutils.DeserializeState(valueFileName) - require.NoError(t, err) - - store := testutils.GetSimpleValueStorePopulated(valuesGob, allocatorGobs) - - bv, err := state.NewBaseView(store, storageRoot) - require.NoError(t, err) - - evmState, err := state.Extract(storageRoot, bv) - require.NoError(t, err) - return evmState -} - -func evmStateGobFileNamesByEndHeight(evmStateGobDir string, endHeight uint64) (string, string) { - valueFileName := filepath.Join(evmStateGobDir, fmt.Sprintf("values-%d.gob", endHeight)) - allocatorFileName := filepath.Join(evmStateGobDir, fmt.Sprintf("allocators-%d.gob", endHeight)) - return valueFileName, allocatorFileName -} - -func TestReadPebbleRootAccountStorageKeyID(t *testing.T) { - chainID := flowGo.Testnet - registerStoreDir := "/var/flow52/evm/data/db" - flowHeight := uint64(228901661) - - store, err := pebble.New(registerStoreDir, log.Logger) - require.NoError(t, err) - - storageRoot := evm.StorageAccountAddress(chainID) - registerStore := pebble.NewRegisterStorage(store, storageRoot) - regID := flowGo.RegisterID{ - Owner: string(storageRoot.Bytes()), - Key: state.AccountsStorageIDKey, - } - regVal, err := registerStore.Get(regID, flowHeight) - require.NoError(t, err) - - require.NotNil(t, regVal) - fmt.Printf("Register value: %x\n", regVal) - - _, err = registerStore.GetSnapshotAt(flowHeight) - require.NoError(t, err) -} - -func TestReadAccount(t *testing.T) { - - chainID := flowGo.Testnet - registerStoreDir := "/var/flow52/evm/data/db" - flowHeight := uint64(228901661) - accountStr := "17DDB8C5aA1382Fb04d057e3ec42B65a43A5fF49" - - // Convert string to common.Address - account := gethCommon.HexToAddress(accountStr) - - store, err := pebble.New(registerStoreDir, log.Logger) - require.NoError(t, err) - - storageRoot := evm.StorageAccountAddress(chainID) - registerStore := pebble.NewRegisterStorage(store, storageRoot) - - snapshot, err := registerStore.GetSnapshotAt(flowHeight) - require.NoError(t, err) - ledger := storage.NewReadOnlyStorage(snapshot) - bv, err := state.NewBaseView(ledger, storageRoot) - require.NoError(t, err) - - stateDB, err := state.NewStateDB(ledger, storageRoot) - require.NoError(t, err) - - balance := stateDB.GetBalance(account) - - fmt.Println("balance from stateDB: ", balance) - - balanceFromBaseView, err := bv.GetBalance(account) - require.NoError(t, err) - fmt.Println("balance from base view: ", balanceFromBaseView) - -} diff --git a/services/evm/extract.go b/services/evm/extract.go new file mode 100644 index 00000000..e37cb1f8 --- /dev/null +++ b/services/evm/extract.go @@ -0,0 +1,36 @@ +package evm + +import ( + "fmt" + + "github.com/onflow/flow-evm-gateway/storage/pebble" + "github.com/onflow/flow-go/fvm/evm" + "github.com/onflow/flow-go/fvm/evm/emulator/state" + "github.com/onflow/flow-go/fvm/evm/offchain/storage" + flowGo "github.com/onflow/flow-go/model/flow" +) + +func ExtractEVMState( + chainID flowGo.ChainID, + evmHeight uint64, + store *pebble.Storage, +) (*state.EVMState, error) { + storageRoot := evm.StorageAccountAddress(chainID) + registerStore := pebble.NewRegisterStorage(store, storageRoot) + snapshot, err := registerStore.GetSnapshotAt(evmHeight) + if err != nil { + return nil, fmt.Errorf("failed to get snapshot at evm height %d: %w", evmHeight, err) + } + + ledger := storage.NewReadOnlyStorage(snapshot) + bv, err := state.NewBaseView(ledger, storageRoot) + if err != nil { + return nil, fmt.Errorf("failed to create base view: %w", err) + } + + evmState, err := state.Extract(storageRoot, bv) + if err != nil { + return nil, err + } + return evmState, nil +} diff --git a/services/evm/extract_test.go b/services/evm/extract_test.go new file mode 100644 index 00000000..dc7e08de --- /dev/null +++ b/services/evm/extract_test.go @@ -0,0 +1,45 @@ +package evm_test + +import ( + "fmt" + "testing" + + "github.com/onflow/flow-evm-gateway/storage/pebble" + "github.com/onflow/flow-go/fvm/evm/emulator/state" + flowGo "github.com/onflow/flow-go/model/flow" + "github.com/rs/zerolog/log" + "github.com/stretchr/testify/require" + + evmState "github.com/onflow/flow-evm-gateway/services/evm" +) + +func StateDiff(t *testing.T) { + state1 := extractEVMState(t, flowGo.Testnet, "/var/flow52/evm/data/db", uint64(17724990)) + state2 := evmStateFromCheckpointExtract(t, "/var/flow52/evm-state-from-checkpoint-228901661") + + differences := state.Diff(state1, state2) + + for i, diff := range differences { + fmt.Printf("Difference %d: %v\n", i, diff) + } + + require.Len(t, differences, 0) +} + +func extractEVMState( + t *testing.T, chainID flowGo.ChainID, + registerStoreDir string, evmHeight uint64) *state.EVMState { + + store, err := pebble.New(registerStoreDir, log.Logger) + require.NoError(t, err) + + evmState, err := evmState.ExtractEVMState(chainID, evmHeight, store) + require.NoError(t, err) + return evmState +} + +func evmStateFromCheckpointExtract(t *testing.T, dir string) *state.EVMState { + enState, err := state.ImportEVMStateFromGob(dir) + require.NoError(t, err) + return enState +} From 423c9928f6ee2e4cf2661f24ece69306f0661b82 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Mon, 2 Dec 2024 14:15:15 -0800 Subject: [PATCH 16/16] tidy --- go.mod | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.mod b/go.mod index 4a4b2211..9537563b 100644 --- a/go.mod +++ b/go.mod @@ -83,7 +83,6 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect - github.com/google/go-cmp v0.6.0 // indirect github.com/google/pprof v0.0.0-20231229205709-960ae82b1e42 // indirect github.com/google/s2a-go v0.1.7 // indirect github.com/google/uuid v1.6.0 // indirect @@ -212,7 +211,6 @@ require ( google.golang.org/protobuf v1.33.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - gotest.tools v2.2.0+incompatible // indirect lukechampine.com/blake3 v1.3.0 // indirect rsc.io/tmplfunc v0.0.3 // indirect )