diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 92da5a94e84..cad6b7ad3c6 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -82,6 +82,7 @@ jobs: docker-push: name: ${{ matrix.role }} images runs-on: ubuntu-latest + environment: Production Docker Registry needs: matrix_builder # setup jobs for each role diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index b708d0b973c..3b5620f43c2 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -1,10 +1,18 @@ name: CD on: - push: - tags: - - '*' - - "!daily-*" +# Workflow dispatch for now, while we're testing environments + # push: + # tags: + # - '*' + # - "!daily-*" + workflow_dispatch: + inputs: + tag: + description: 'Tag/commit' + required: true + type: string + env: GO_VERSION: "1.22" @@ -13,6 +21,7 @@ jobs: docker-push: name: Push to container registry runs-on: ubuntu-latest + environment: Production Docker Registry steps: - name: Setup Go uses: actions/setup-go@v4 @@ -21,6 +30,8 @@ jobs: go-version: ${{ env.GO_VERSION }} - name: Checkout repo uses: actions/checkout@v2 + with: + ref: ${{ inputs.tag }} # Provide Google Service Account credentials to Github Action, allowing interaction with the Google Container Registry # Logging in as github-actions@dl-flow.iam.gserviceaccount.com - id: auth diff --git a/cmd/access/node_builder/access_node_builder.go b/cmd/access/node_builder/access_node_builder.go index 595f797d600..1da04c85888 100644 --- a/cmd/access/node_builder/access_node_builder.go +++ b/cmd/access/node_builder/access_node_builder.go @@ -176,6 +176,7 @@ type AccessNodeConfig struct { checkPayerBalanceMode string versionControlEnabled bool stopControlEnabled bool + registerDBPruneThreshold uint64 } type PublicNetworkConfig struct { @@ -280,6 +281,7 @@ func DefaultAccessNodeConfig() *AccessNodeConfig { checkPayerBalanceMode: accessNode.Disabled.String(), versionControlEnabled: true, stopControlEnabled: false, + registerDBPruneThreshold: pruner.DefaultThreshold, } } @@ -905,7 +907,7 @@ func (builder *FlowAccessNodeBuilder) BuildExecutionSyncComponents() *FlowAccess } } - registers, err := pstorage.NewRegisters(pdb) + registers, err := pstorage.NewRegisters(pdb, builder.registerDBPruneThreshold) if err != nil { return nil, fmt.Errorf("could not create registers storage: %w", err) } @@ -1421,6 +1423,12 @@ func (builder *FlowAccessNodeBuilder) extraFlags() { "check-payer-balance-mode", defaultConfig.checkPayerBalanceMode, "flag for payer balance validation that specifies whether or not to enforce the balance check. one of [disabled(default), warn, enforce]") + + // Register DB Pruning + flags.Uint64Var(&builder.registerDBPruneThreshold, + "registerdb-pruning-threshold", + defaultConfig.registerDBPruneThreshold, + fmt.Sprintf("specifies the number of blocks below the latest stored block height to keep in register db. default: %d", defaultConfig.registerDBPruneThreshold)) }).ValidateFlags(func() error { if builder.supportsObserver && (builder.PublicNetworkConfig.BindAddress == cmd.NotSet || builder.PublicNetworkConfig.BindAddress == "") { return errors.New("public-network-address must be set if supports-observer is true") diff --git a/cmd/execution_builder.go b/cmd/execution_builder.go index 4c7f5f511e3..9add704760f 100644 --- a/cmd/execution_builder.go +++ b/cmd/execution_builder.go @@ -889,7 +889,7 @@ func (exeNode *ExecutionNode) LoadRegisterStore( return fmt.Errorf("could not import registers from checkpoint: %w", err) } } - diskStore, err := storagepebble.NewRegisters(pebbledb) + diskStore, err := storagepebble.NewRegisters(pebbledb, storagepebble.PruningDisabled) if err != nil { return fmt.Errorf("could not create registers storage: %w", err) } diff --git a/cmd/observer/node_builder/observer_builder.go b/cmd/observer/node_builder/observer_builder.go index 27cc55dc432..21d8211daa6 100644 --- a/cmd/observer/node_builder/observer_builder.go +++ b/cmd/observer/node_builder/observer_builder.go @@ -174,6 +174,7 @@ type ObserverServiceConfig struct { registerCacheType string registerCacheSize uint programCacheSize uint + registerDBPruneThreshold uint64 } // DefaultObserverServiceConfig defines all the default values for the ObserverServiceConfig @@ -252,11 +253,12 @@ func DefaultObserverServiceConfig() *ObserverServiceConfig { RetryDelay: edrequester.DefaultRetryDelay, MaxRetryDelay: edrequester.DefaultMaxRetryDelay, }, - scriptExecMinBlock: 0, - scriptExecMaxBlock: math.MaxUint64, - registerCacheType: pstorage.CacheTypeTwoQueue.String(), - registerCacheSize: 0, - programCacheSize: 0, + scriptExecMinBlock: 0, + scriptExecMaxBlock: math.MaxUint64, + registerCacheType: pstorage.CacheTypeTwoQueue.String(), + registerCacheSize: 0, + programCacheSize: 0, + registerDBPruneThreshold: pruner.DefaultThreshold, } } @@ -810,6 +812,12 @@ func (builder *ObserverServiceBuilder) extraFlags() { "program-cache-size", defaultConfig.programCacheSize, "[experimental] number of blocks to cache for cadence programs. use 0 to disable cache. default: 0. Note: this is an experimental feature and may cause nodes to become unstable under certain workloads. Use with caution.") + + // Register DB Pruning + flags.Uint64Var(&builder.registerDBPruneThreshold, + "registerdb-pruning-threshold", + defaultConfig.registerDBPruneThreshold, + fmt.Sprintf("specifies the number of blocks below the latest stored block height to keep in register db. default: %d", defaultConfig.registerDBPruneThreshold)) }).ValidateFlags(func() error { if builder.executionDataSyncEnabled { if builder.executionDataConfig.FetchTimeout <= 0 { @@ -1436,7 +1444,7 @@ func (builder *ObserverServiceBuilder) BuildExecutionSyncComponents() *ObserverS } } - registers, err := pstorage.NewRegisters(pdb) + registers, err := pstorage.NewRegisters(pdb, builder.registerDBPruneThreshold) if err != nil { return nil, fmt.Errorf("could not create registers storage: %w", err) } diff --git a/engine/access/rpc/backend/script_executor_test.go b/engine/access/rpc/backend/script_executor_test.go index 542d84e417d..70445b8ab8d 100644 --- a/engine/access/rpc/backend/script_executor_test.go +++ b/engine/access/rpc/backend/script_executor_test.go @@ -126,7 +126,7 @@ func (s *ScriptExecutorSuite) SetupTest() { s.dbDir = unittest.TempDir(s.T()) db := pebbleStorage.NewBootstrappedRegistersWithPathForTest(s.T(), s.dbDir, s.height, s.height) - pebbleRegisters, err := pebbleStorage.NewRegisters(db) + pebbleRegisters, err := pebbleStorage.NewRegisters(db, pebbleStorage.PruningDisabled) s.Require().NoError(err) s.registerIndex = pebbleRegisters diff --git a/engine/common/requester/engine.go b/engine/common/requester/engine.go index b1e35bcb642..3a9031e3b5a 100644 --- a/engine/common/requester/engine.go +++ b/engine/common/requester/engine.go @@ -421,15 +421,6 @@ func (e *Engine) dispatchRequest() (bool, error) { } e.requests[req.Nonce] = req - if e.log.Debug().Enabled() { - e.log.Debug(). - Hex("provider", logging.ID(providerID)). - Uint64("nonce", req.Nonce). - Strs("entities", logging.IDs(entityIDs)). - TimeDiff("duration", time.Now(), requestStart). - Msg("entity request sent") - } - // NOTE: we forget about requests after the expiry of the shortest retry time // from the entities in the list; this means that we purge requests aggressively. // However, most requests should be responded to on the first attempt and clearing @@ -443,11 +434,15 @@ func (e *Engine) dispatchRequest() (bool, error) { delete(e.requests, req.Nonce) }() + if e.log.Debug().Enabled() { + e.log.Debug(). + Hex("provider", logging.ID(providerID)). + Uint64("nonce", req.Nonce). + Strs("entities", logging.IDs(entityIDs)). + TimeDiff("duration", time.Now(), requestStart). + Msg("entity request sent") + } e.metrics.MessageSent(e.channel.String(), metrics.MessageEntityRequest) - e.log.Debug(). - Uint64("nonce", req.Nonce). - Strs("entity_ids", flow.IdentifierList(req.EntityIDs).Strings()). - Msg("entity request sent") return true, nil } diff --git a/engine/testutil/nodes.go b/engine/testutil/nodes.go index b6d1037b500..ee3b3b10ee7 100644 --- a/engine/testutil/nodes.go +++ b/engine/testutil/nodes.go @@ -616,7 +616,7 @@ func ExecutionNode(t *testing.T, hub *stub.Hub, identity bootstrap.NodeInfo, ide checkpointHeight := uint64(0) require.NoError(t, esbootstrap.ImportRegistersFromCheckpoint(node.Log, checkpointFile, checkpointHeight, matchTrie.RootHash(), pebbledb, 2)) - diskStore, err := storagepebble.NewRegisters(pebbledb) + diskStore, err := storagepebble.NewRegisters(pebbledb, storagepebble.PruningDisabled) require.NoError(t, err) reader := finalizedreader.NewFinalizedReader(headersStorage, checkpointHeight) diff --git a/fvm/evm/debug/tracer_test.go b/fvm/evm/debug/tracer_test.go index 89e62944e83..32e694b4ae7 100644 --- a/fvm/evm/debug/tracer_test.go +++ b/fvm/evm/debug/tracer_test.go @@ -1,4 +1,4 @@ -package debug +package debug_test import ( "encoding/json" @@ -11,6 +11,7 @@ import ( "github.com/rs/zerolog" "github.com/stretchr/testify/require" + "github.com/onflow/flow-go/fvm/evm/debug" "github.com/onflow/flow-go/fvm/evm/testutils" "github.com/onflow/flow-go/model/flow" ) @@ -23,13 +24,13 @@ func Test_CallTracer(t *testing.T) { mockUpload := &testutils.MockUploader{ UploadFunc: func(id string, message json.RawMessage) error { - require.Equal(t, TraceID(txID, blockID), id) + require.Equal(t, debug.TraceID(txID, blockID), id) require.Equal(t, res, message) return nil }, } - tracer, err := NewEVMCallTracer(mockUpload, zerolog.Nop()) + tracer, err := debug.NewEVMCallTracer(mockUpload, zerolog.Nop()) require.NoError(t, err) tracer.WithBlockID(blockID) @@ -61,13 +62,13 @@ func Test_CallTracer(t *testing.T) { mockUpload := &testutils.MockUploader{ UploadFunc: func(id string, message json.RawMessage) error { - require.Equal(t, TraceID(txID, blockID), id) + require.Equal(t, debug.TraceID(txID, blockID), id) require.Equal(t, res, message) return nil }, } - tracer, err := NewEVMCallTracer(mockUpload, zerolog.Nop()) + tracer, err := debug.NewEVMCallTracer(mockUpload, zerolog.Nop()) require.NoError(t, err) tracer.WithBlockID(blockID) @@ -107,7 +108,7 @@ func Test_CallTracer(t *testing.T) { }, } - tracer, err := NewEVMCallTracer(mockUpload, zerolog.Nop()) + tracer, err := debug.NewEVMCallTracer(mockUpload, zerolog.Nop()) require.NoError(t, err) tracer.WithBlockID(blockID) @@ -117,7 +118,7 @@ func Test_CallTracer(t *testing.T) { }) t.Run("nop tracer", func(t *testing.T) { - tracer := nopTracer{} + tracer := debug.NopTracer require.Nil(t, tracer.TxTracer()) }) } diff --git a/fvm/evm/debug/uploader_test.go b/fvm/evm/debug/uploader_test.go index c8052a95bc6..b531ad74d3b 100644 --- a/fvm/evm/debug/uploader_test.go +++ b/fvm/evm/debug/uploader_test.go @@ -1,4 +1,4 @@ -package debug +package debug_test import ( "context" @@ -16,6 +16,7 @@ import ( "github.com/rs/zerolog" "github.com/stretchr/testify/require" + "github.com/onflow/flow-go/fvm/evm/debug" "github.com/onflow/flow-go/model/flow" testutils "github.com/onflow/flow-go/utils/unittest" ) @@ -27,7 +28,7 @@ func Test_Uploader(t *testing.T) { testutils.SkipUnless(t, testutils.TEST_REQUIRES_GCP_ACCESS, "requires GCP Bucket setup") t.Run("successfuly upload traces", func(t *testing.T) { - uploader, err := NewGCPUploader(bucket) + uploader, err := debug.NewGCPUploader(bucket) require.NoError(t, err) const testID = "test_p" @@ -56,10 +57,10 @@ func Test_TracerUploaderIntegration(t *testing.T) { testutils.SkipUnless(t, testutils.TEST_REQUIRES_GCP_ACCESS, "requires GCP Bucket setup") t.Run("successfuly uploads traces", func(t *testing.T) { - uploader, err := NewGCPUploader(bucket) + uploader, err := debug.NewGCPUploader(bucket) require.NoError(t, err) - tracer, err := NewEVMCallTracer(uploader, zerolog.Nop()) + tracer, err := debug.NewEVMCallTracer(uploader, zerolog.Nop()) require.NoError(t, err) tr := tracer.TxTracer() diff --git a/fvm/evm/emulator/emulator_test.go b/fvm/evm/emulator/emulator_test.go index beea66aee31..79d20378be5 100644 --- a/fvm/evm/emulator/emulator_test.go +++ b/fvm/evm/emulator/emulator_test.go @@ -1257,7 +1257,7 @@ func TestTransactionTracing(t *testing.T) { testAccount.Address(), types.Address{0x01, 0x02}, testContract.ByteCode, - 1_000_000, + 2_000_000, big.NewInt(0), testAccount.Nonce(), ), diff --git a/fvm/evm/events/events.go b/fvm/evm/events/events.go index e9a50903124..584f6814104 100644 --- a/fvm/evm/events/events.go +++ b/fvm/evm/events/events.go @@ -6,7 +6,6 @@ import ( "github.com/onflow/cadence" "github.com/onflow/cadence/encoding/ccf" gethCommon "github.com/onflow/go-ethereum/common" - "github.com/onflow/go-ethereum/rlp" "github.com/onflow/flow-go/fvm/evm/stdlib" "github.com/onflow/flow-go/fvm/evm/types" @@ -56,51 +55,27 @@ func NewTransactionEvent( } func (p *transactionEvent) ToCadence(chainID flow.ChainID) (cadence.Event, error) { - var encodedLogs []byte - var err error - if len(p.Result.Logs) > 0 { - encodedLogs, err = rlp.EncodeToBytes(p.Result.Logs) - if err != nil { - return cadence.Event{}, err - } - } - - deployedAddress := cadence.String("") - if p.Result.DeployedContractAddress != nil { - deployedAddress = cadence.String(p.Result.DeployedContractAddress.String()) - } - - errorMsg := "" - if p.Result.VMError != nil { - errorMsg = p.Result.VMError.Error() - } - // both error would never happen at the same time - // but in case the priority is by validation error - if p.Result.ValidationError != nil { - errorMsg = p.Result.ValidationError.Error() + encodedLogs, err := p.Result.RLPEncodedLogs() + if err != nil { + return cadence.Event{}, err } eventType := stdlib.CadenceTypesForChain(chainID).TransactionExecuted - // the first 4 bytes of StateChangeCommitment is used as checksum - var checksum [ChecksumLength]byte - if len(p.Result.StateChangeCommitment) >= ChecksumLength { - copy(checksum[:ChecksumLength], p.Result.StateChangeCommitment[:ChecksumLength]) - } return cadence.NewEvent([]cadence.Value{ hashToCadenceArrayValue(p.Result.TxHash), cadence.NewUInt16(p.Result.Index), cadence.NewUInt8(p.Result.TxType), bytesToCadenceUInt8ArrayValue(p.Payload), cadence.NewUInt16(uint16(p.Result.ResultSummary().ErrorCode)), - cadence.String(errorMsg), + cadence.String(p.Result.ErrorMsg()), cadence.NewUInt64(p.Result.GasConsumed), - deployedAddress, + cadence.String(p.Result.DeployedContractAddressString()), bytesToCadenceUInt8ArrayValue(encodedLogs), cadence.NewUInt64(p.BlockHeight), bytesToCadenceUInt8ArrayValue(p.Result.ReturnedData), bytesToCadenceUInt8ArrayValue(p.Result.PrecompiledCalls), - checksumToCadenceArrayValue(checksum), + checksumToCadenceArrayValue(p.Result.StateChangeChecksum()), }).WithType(eventType), nil } @@ -194,19 +169,19 @@ func DecodeBlockEventPayload(event cadence.Event) (*BlockEventPayload, error) { } type TransactionEventPayload struct { - Hash gethCommon.Hash `cadence:"hash"` - Index uint16 `cadence:"index"` - TransactionType uint8 `cadence:"type"` - Payload []byte `cadence:"payload"` - ErrorCode uint16 `cadence:"errorCode"` - GasConsumed uint64 `cadence:"gasConsumed"` - ContractAddress string `cadence:"contractAddress"` - Logs []byte `cadence:"logs"` - BlockHeight uint64 `cadence:"blockHeight"` - ErrorMessage string `cadence:"errorMessage"` - ReturnedData []byte `cadence:"returnedData"` - PrecompiledCalls []byte `cadence:"precompiledCalls"` - StateUpdateChecksum [ChecksumLength]byte `cadence:"stateUpdateChecksum"` + Hash gethCommon.Hash `cadence:"hash"` + Index uint16 `cadence:"index"` + TransactionType uint8 `cadence:"type"` + Payload []byte `cadence:"payload"` + ErrorCode uint16 `cadence:"errorCode"` + GasConsumed uint64 `cadence:"gasConsumed"` + ContractAddress string `cadence:"contractAddress"` + Logs []byte `cadence:"logs"` + BlockHeight uint64 `cadence:"blockHeight"` + ErrorMessage string `cadence:"errorMessage"` + ReturnedData []byte `cadence:"returnedData"` + PrecompiledCalls []byte `cadence:"precompiledCalls"` + StateUpdateChecksum [types.ChecksumLength]byte `cadence:"stateUpdateChecksum"` } // transactionEventPayloadV0 legacy format of the transaction event without stateUpdateChecksum field diff --git a/fvm/evm/events/events_test.go b/fvm/evm/events/events_test.go index 08f9f5abb01..32544759a1f 100644 --- a/fvm/evm/events/events_test.go +++ b/fvm/evm/events/events_test.go @@ -131,7 +131,7 @@ func TestEVMTransactionExecutedEventCCFEncodingDecoding(t *testing.T) { assert.Equal(t, tep.GasConsumed, txResult.GasConsumed) assert.Equal(t, tep.ErrorMessage, txResult.VMError.Error()) assert.Equal(t, tep.ReturnedData, txResult.ReturnedData) - assert.Equal(t, tep.StateUpdateChecksum[:], stateUpdateCommit[:events.ChecksumLength]) + assert.Equal(t, tep.StateUpdateChecksum[:], stateUpdateCommit[:types.ChecksumLength]) assert.Equal( t, tep.ContractAddress, diff --git a/fvm/evm/events/utils.go b/fvm/evm/events/utils.go index 3e54a6f4f05..1fe164d6b54 100644 --- a/fvm/evm/events/utils.go +++ b/fvm/evm/events/utils.go @@ -3,6 +3,8 @@ package events import ( "github.com/onflow/cadence" gethCommon "github.com/onflow/go-ethereum/common" + + "github.com/onflow/flow-go/fvm/evm/types" ) // cadenceArrayTypeOfUInt8 is the Cadence type [UInt8] @@ -31,16 +33,13 @@ func hashToCadenceArrayValue(hash gethCommon.Hash) cadence.Array { WithType(cadenceHashType) } -// ChecksumLength captures number of bytes a checksum uses -const ChecksumLength = 4 - // checksumType is the Cadence type [UInt8;4] -var checksumType = cadence.NewConstantSizedArrayType(ChecksumLength, cadence.UInt8Type) +var checksumType = cadence.NewConstantSizedArrayType(types.ChecksumLength, cadence.UInt8Type) // checksumToCadenceArrayValue converts a checksum ([4]byte) into a Cadence array of type [UInt8;4] -func checksumToCadenceArrayValue(checksum [ChecksumLength]byte) cadence.Array { - values := make([]cadence.Value, ChecksumLength) - for i := 0; i < ChecksumLength; i++ { +func checksumToCadenceArrayValue(checksum [types.ChecksumLength]byte) cadence.Array { + values := make([]cadence.Value, types.ChecksumLength) + for i := 0; i < types.ChecksumLength; i++ { values[i] = cadence.NewUInt8(checksum[i]) } return cadence.NewArray(values). diff --git a/fvm/evm/handler/blockHashList.go b/fvm/evm/handler/blockHashList.go index cd5bda0caa2..86ea5738b5a 100644 --- a/fvm/evm/handler/blockHashList.go +++ b/fvm/evm/handler/blockHashList.go @@ -32,7 +32,7 @@ const ( // smaller fixed size buckets to minimize the // number of bytes read and written during set/get operations. type BlockHashList struct { - backend types.Backend + backend types.BackendStorage rootAddress flow.Address // cached meta data @@ -46,7 +46,7 @@ type BlockHashList struct { // It tries to load the metadata from the backend // and if not exist it creates one func NewBlockHashList( - backend types.Backend, + backend types.BackendStorage, rootAddress flow.Address, capacity int, ) (*BlockHashList, error) { diff --git a/fvm/evm/offchain/blocks/blocks.go b/fvm/evm/offchain/blocks/blocks.go new file mode 100644 index 00000000000..fce1bc004fe --- /dev/null +++ b/fvm/evm/offchain/blocks/blocks.go @@ -0,0 +1,122 @@ +package blocks + +import ( + "fmt" + + gethCommon "github.com/onflow/go-ethereum/common" + + "github.com/onflow/flow-go/fvm/evm/handler" + "github.com/onflow/flow-go/fvm/evm/types" + "github.com/onflow/flow-go/model/flow" +) + +const BlockStoreLatestBlockMetaKey = "LatestBlockMeta" + +// Blocks facilitates access to the recent block hash values +// and also the latest executed block meta data +type Blocks struct { + chainID flow.ChainID + storage types.BackendStorage + rootAddress flow.Address + bhl *handler.BlockHashList +} + +// NewBlocks constructs a new blocks type +func NewBlocks( + chainID flow.ChainID, + rootAddress flow.Address, + storage types.BackendStorage, +) (*Blocks, error) { + var err error + blocks := &Blocks{ + chainID: chainID, + storage: storage, + rootAddress: rootAddress, + } + blocks.bhl, err = handler.NewBlockHashList( + storage, + rootAddress, + handler.BlockHashListCapacity, + ) + if err != nil { + return nil, err + } + // if empty insert genesis block hash + if blocks.bhl.IsEmpty() { + genesis := types.GenesisBlock(chainID) + err = blocks.PushBlockMeta( + NewMeta( + genesis.Height, + genesis.Timestamp, + genesis.PrevRandao, + )) + if err != nil { + return nil, err + } + // push block hash + err = blocks.PushBlockHash( + genesis.Height, + types.GenesisBlockHash(chainID)) + if err != nil { + return nil, err + } + } + return blocks, nil +} + +// PushBlock pushes a new block into the storage +func (b *Blocks) PushBlockMeta( + meta *Meta, +) error { + // check height order + if meta.Height > 0 { + bm, err := b.LatestBlock() + if err != nil { + return err + } + if meta.Height != bm.Height+1 { + return fmt.Errorf("out of order block meta push! got: %d, expected %d ", meta.Height, bm.Height+1) + } + } + return b.storeBlockMetaData(meta) +} + +// PushBlockHash pushes a new block block hash into the storage +func (b *Blocks) PushBlockHash( + height uint64, + hash gethCommon.Hash, +) error { + return b.bhl.Push(height, hash) +} + +func (b *Blocks) LatestBlock() (*Meta, error) { + return b.loadBlockMetaData() +} + +// BlockHash returns the block hash for the given height +func (b *Blocks) BlockHash(height uint64) (gethCommon.Hash, error) { + _, hash, err := b.bhl.BlockHashByHeight(height) + return hash, err +} + +// storeBlockMetaData stores the block meta data into storage +func (b *Blocks) storeBlockMetaData(bm *Meta) error { + // store the encoded data into backend + return b.storage.SetValue( + b.rootAddress[:], + []byte(BlockStoreLatestBlockMetaKey), + bm.Encode(), + ) +} + +// loadBlockMetaData loads the block meta data from the storage +func (b *Blocks) loadBlockMetaData() (*Meta, error) { + data, err := b.storage.GetValue( + b.rootAddress[:], + []byte(BlockStoreLatestBlockMetaKey), + ) + if err != nil { + return nil, err + } + return MetaFromEncoded(data) +} diff --git a/fvm/evm/offchain/blocks/blocks_test.go b/fvm/evm/offchain/blocks/blocks_test.go new file mode 100644 index 00000000000..a5268ca66b6 --- /dev/null +++ b/fvm/evm/offchain/blocks/blocks_test.go @@ -0,0 +1,55 @@ +package blocks_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/fvm/evm/offchain/blocks" + "github.com/onflow/flow-go/fvm/evm/testutils" + "github.com/onflow/flow-go/fvm/evm/types" + "github.com/onflow/flow-go/model/flow" +) + +func TestBlocks(t *testing.T) { + storage := testutils.GetSimpleValueStore() + chainID := flow.Emulator.Chain().ChainID() + rootAddr := flow.Address{1, 2, 3, 4} + blks, err := blocks.NewBlocks(chainID, rootAddr, storage) + require.NoError(t, err) + + // no insertion - genesis block + bm, err := blks.LatestBlock() + require.NoError(t, err) + genesis := types.GenesisBlock(chainID) + require.Equal(t, genesis.Height, bm.Height) + require.Equal(t, genesis.Timestamp, bm.Timestamp) + require.Equal(t, genesis.PrevRandao, bm.Random) + + h, err := blks.BlockHash(0) + require.NoError(t, err) + expectedHash, err := genesis.Hash() + require.NoError(t, err) + require.Equal(t, expectedHash, h) + + // push next block + height := uint64(1) + timestamp := uint64(2) + random := testutils.RandomCommonHash(t) + hash := testutils.RandomCommonHash(t) + + err = blks.PushBlockMeta(blocks.NewMeta(height, timestamp, random)) + require.NoError(t, err) + err = blks.PushBlockHash(height, hash) + require.NoError(t, err) + + // check values + h, err = blks.BlockHash(1) + require.NoError(t, err) + require.Equal(t, hash, h) + bm, err = blks.LatestBlock() + require.NoError(t, err) + require.Equal(t, height, bm.Height) + require.Equal(t, timestamp, bm.Timestamp) + require.Equal(t, random, bm.Random) +} diff --git a/fvm/evm/offchain/blocks/meta.go b/fvm/evm/offchain/blocks/meta.go new file mode 100644 index 00000000000..c47ed71fdbc --- /dev/null +++ b/fvm/evm/offchain/blocks/meta.go @@ -0,0 +1,81 @@ +package blocks + +import ( + "encoding/binary" + "fmt" + + gethCommon "github.com/onflow/go-ethereum/common" +) + +const ( + heightEncodingSize = 8 + timestampEncodingSize = 8 + randomEncodingSize = 32 + metaEncodingSize = heightEncodingSize + + timestampEncodingSize + + randomEncodingSize +) + +// Meta holds meta data about a block +type Meta struct { + Height uint64 + Timestamp uint64 + Random gethCommon.Hash +} + +// NewBlockMeta constructs a new block meta +func NewMeta( + height uint64, + timestamp uint64, + random gethCommon.Hash, +) *Meta { + return &Meta{ + Height: height, + Timestamp: timestamp, + Random: random, + } +} + +// Encode encodes a meta +func (bm *Meta) Encode() []byte { + // encode meta data + buffer := make([]byte, metaEncodingSize) + pos := 0 + + // encode height + binary.BigEndian.PutUint64(buffer[pos:], uint64(bm.Height)) + pos += heightEncodingSize + + // encode timestamp + binary.BigEndian.PutUint64(buffer[pos:], uint64(bm.Timestamp)) + pos += timestampEncodingSize + + // encode random + copy(buffer[pos:pos+randomEncodingSize], bm.Random[:]) + + return buffer +} + +// MetaFromEncoded constructs a Meta from encoded data +func MetaFromEncoded(data []byte) (*Meta, error) { + // check the data size + if len(data) < metaEncodingSize { + return nil, fmt.Errorf("encoded input too short: %d < %d", len(data), metaEncodingSize) + } + + bm := &Meta{} + + pos := 0 + // decode height + bm.Height = binary.BigEndian.Uint64(data[pos:]) + pos += heightEncodingSize + + // decode timestamp + bm.Timestamp = binary.BigEndian.Uint64(data[pos:]) + pos += timestampEncodingSize + + // decode random + bm.Random = gethCommon.BytesToHash(data[pos:]) + + return bm, nil +} diff --git a/fvm/evm/offchain/blocks/meta_test.go b/fvm/evm/offchain/blocks/meta_test.go new file mode 100644 index 00000000000..a777200533d --- /dev/null +++ b/fvm/evm/offchain/blocks/meta_test.go @@ -0,0 +1,18 @@ +package blocks_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/fvm/evm/offchain/blocks" + "github.com/onflow/flow-go/fvm/evm/testutils" +) + +func TestBlockMetaEncodingDecoding(t *testing.T) { + bm := blocks.NewMeta(1, 2, testutils.RandomCommonHash(t)) + + ret, err := blocks.MetaFromEncoded(bm.Encode()) + require.NoError(t, err) + require.Equal(t, ret, bm) +} diff --git a/fvm/evm/testutils/contracts/test.sol b/fvm/evm/testutils/contracts/test.sol index 5036e239dcf..d5deadd9b37 100644 --- a/fvm/evm/testutils/contracts/test.sol +++ b/fvm/evm/testutils/contracts/test.sol @@ -18,6 +18,11 @@ contract Storage { number = num; } + function checkThenStore(uint256 prev, uint256 num)public { + require(number == prev, "stored value check failed"); + number = num; + } + function storeWithLog(uint256 num) public { emit NewStore(msg.sender, num); number = num; @@ -36,6 +41,10 @@ contract Storage { return block.number; } + function checkBlockNumber(uint expected) public view { + require(expected == block.number, "block number check failed"); + } + function blockTime() public view returns (uint) { return block.timestamp; } @@ -44,6 +53,14 @@ contract Storage { return blockhash(num); } + function checkBlockHash(uint num, bytes32 expected) public view { + require(expected == blockhash(num), "hash check failed"); + } + + function checkBalance(address addr, uint expected) public view{ + require(expected == addr.balance, "balance check failed"); + } + function random() public view returns (uint256) { return block.prevrandao; } @@ -93,4 +110,4 @@ contract Storage { require(expected == output, "output doesnt match the expected value"); return output; } -} +} \ No newline at end of file diff --git a/fvm/evm/testutils/contracts/test_abi.json b/fvm/evm/testutils/contracts/test_abi.json index e693dcc01ad..aff2b6afe16 100644 --- a/fvm/evm/testutils/contracts/test_abi.json +++ b/fvm/evm/testutils/contracts/test_abi.json @@ -117,6 +117,73 @@ "stateMutability": "view", "type": "function" }, + { + "inputs": [ + { + "internalType": "address", + "name": "addr", + "type": "address" + }, + { + "internalType": "uint256", + "name": "expected", + "type": "uint256" + } + ], + "name": "checkBalance", + "outputs": [], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "num", + "type": "uint256" + }, + { + "internalType": "bytes32", + "name": "expected", + "type": "bytes32" + } + ], + "name": "checkBlockHash", + "outputs": [], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "expected", + "type": "uint256" + } + ], + "name": "checkBlockNumber", + "outputs": [], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "prev", + "type": "uint256" + }, + { + "internalType": "uint256", + "name": "num", + "type": "uint256" + } + ], + "name": "checkThenStore", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, { "inputs": [], "name": "customError", diff --git a/fvm/evm/testutils/contracts/test_bytes.hex b/fvm/evm/testutils/contracts/test_bytes.hex index badff4c6411..b7a74895f9f 100644 --- a/fvm/evm/testutils/contracts/test_bytes.hex +++ b/fvm/evm/testutils/contracts/test_bytes.hex @@ -1 +1 @@ -60806040526112e1806100115f395ff3fe608060405234801561000f575f80fd5b5060043610610109575f3560e01c8063828dd048116100a0578063adc879e91161006f578063adc879e9146102a3578063b2821c8f146102c1578063cbaff5f9146102df578063d0d250bd146102e9578063dda3a7bd1461030757610109565b8063828dd0481461020957806383197ef01461023957806385df51fd14610243578063911007b41461027357610109565b806357e871e7116100dc57806357e871e7146101955780635ec01e4d146101b35780636057361d146101d15780636babb224146101ed57610109565b80632e64cec11461010d57806348b151661461012b5780634cbefa6a1461014957806352e2402414610165575b5f80fd5b610115610311565b6040516101229190610a6f565b60405180910390f35b610133610319565b6040516101409190610a6f565b60405180910390f35b610163600480360381019061015e9190610ac3565b610320565b005b61017f600480360381019061017a9190610b2b565b610329565b60405161018c9190610b65565b60405180910390f35b61019d6104d6565b6040516101aa9190610a6f565b60405180910390f35b6101bb6104dd565b6040516101c89190610a6f565b60405180910390f35b6101eb60048036038101906101e69190610ac3565b6104e4565b005b61020760048036038101906102029190610ac3565b6104ed565b005b610223600480360381019061021e9190610d7c565b61053a565b6040516102309190610e0b565b60405180910390f35b6102416106e9565b005b61025d60048036038101906102589190610ac3565b610702565b60405161026a9190610e33565b60405180910390f35b61028d60048036038101906102889190610b2b565b61070c565b60405161029a9190610e33565b60405180910390f35b6102ab61086e565b6040516102b89190610a6f565b60405180910390f35b6102c9610875565b6040516102d69190610b65565b60405180910390f35b6102e76109ca565b005b6102f1610a0c565b6040516102fe9190610e5b565b60405180910390f35b61030f610a19565b005b5f8054905090565b5f42905090565b805f8190555f80fd5b5f805f6801000000000000000173ffffffffffffffffffffffffffffffffffffffff166040516024016040516020818303038152906040527f53e87d66000000000000000000000000000000000000000000000000000000007bffffffffffffffffffffffffffffffffffffffffffffffffffffffff19166020820180517bffffffffffffffffffffffffffffffffffffffffffffffffffffffff83818316178352505050506040516103dc9190610ec6565b5f60405180830381855afa9150503d805f8114610414576040519150601f19603f3d011682016040523d82523d5f602084013e610419565b606091505b50915091508161045e576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040161045590610f36565b60405180910390fd5b5f818060200190518101906104739190610f68565b90508067ffffffffffffffff168567ffffffffffffffff16146104cb576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004016104c290611003565b60405180910390fd5b809350505050919050565b5f43905090565b5f44905090565b805f8190555050565b803373ffffffffffffffffffffffffffffffffffffffff167f043cc306157a91d747b36aba0e235bbbc5771d75aba162f6e5540767d22673c660405160405180910390a3805f8190555050565b5f805f6801000000000000000173ffffffffffffffffffffffffffffffffffffffff1686868660405160240161057293929190611069565b6040516020818303038152906040527f5ee837e7000000000000000000000000000000000000000000000000000000007bffffffffffffffffffffffffffffffffffffffffffffffffffffffff19166020820180517bffffffffffffffffffffffffffffffffffffffffffffffffffffffff83818316178352505050506040516105fc9190610ec6565b5f60405180830381855afa9150503d805f8114610634576040519150601f19603f3d011682016040523d82523d5f602084013e610639565b606091505b50915091508161067e576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401610675906110ef565b60405180910390fd5b5f818060200190518101906106939190611121565b9050801515881515146106db576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004016106d290611003565b60405180910390fd5b809350505050949350505050565b3373ffffffffffffffffffffffffffffffffffffffff16ff5b5f81409050919050565b5f805f6801000000000000000173ffffffffffffffffffffffffffffffffffffffff16846040516024016107409190610b65565b6040516020818303038152906040527f78a75fbe000000000000000000000000000000000000000000000000000000007bffffffffffffffffffffffffffffffffffffffffffffffffffffffff19166020820180517bffffffffffffffffffffffffffffffffffffffffffffffffffffffff83818316178352505050506040516107ca9190610ec6565b5f60405180830381855afa9150503d805f8114610802576040519150601f19603f3d011682016040523d82523d5f602084013e610807565b606091505b50915091508161084c576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040161084390610f36565b60405180910390fd5b5f818060200190518101906108619190611160565b9050809350505050919050565b5f46905090565b5f805f6801000000000000000173ffffffffffffffffffffffffffffffffffffffff166040516024016040516020818303038152906040527f705fab20000000000000000000000000000000000000000000000000000000007bffffffffffffffffffffffffffffffffffffffffffffffffffffffff19166020820180517bffffffffffffffffffffffffffffffffffffffffffffffffffffffff83818316178352505050506040516109289190610ec6565b5f60405180830381855afa9150503d805f8114610960576040519150601f19603f3d011682016040523d82523d5f602084013e610965565b606091505b5091509150816109aa576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004016109a1906110ef565b60405180910390fd5b5f818060200190518101906109bf9190610f68565b905080935050505090565b5f610a0a576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401610a01906111d5565b60405180910390fd5b565b6801000000000000000181565b60056040517f9195785a000000000000000000000000000000000000000000000000000000008152600401610a4e919061127f565b60405180910390fd5b5f819050919050565b610a6981610a57565b82525050565b5f602082019050610a825f830184610a60565b92915050565b5f604051905090565b5f80fd5b5f80fd5b610aa281610a57565b8114610aac575f80fd5b50565b5f81359050610abd81610a99565b92915050565b5f60208284031215610ad857610ad7610a91565b5b5f610ae584828501610aaf565b91505092915050565b5f67ffffffffffffffff82169050919050565b610b0a81610aee565b8114610b14575f80fd5b50565b5f81359050610b2581610b01565b92915050565b5f60208284031215610b4057610b3f610a91565b5b5f610b4d84828501610b17565b91505092915050565b610b5f81610aee565b82525050565b5f602082019050610b785f830184610b56565b92915050565b5f8115159050919050565b610b9281610b7e565b8114610b9c575f80fd5b50565b5f81359050610bad81610b89565b92915050565b5f73ffffffffffffffffffffffffffffffffffffffff82169050919050565b5f610bdc82610bb3565b9050919050565b610bec81610bd2565b8114610bf6575f80fd5b50565b5f81359050610c0781610be3565b92915050565b5f819050919050565b610c1f81610c0d565b8114610c29575f80fd5b50565b5f81359050610c3a81610c16565b92915050565b5f80fd5b5f80fd5b5f601f19601f8301169050919050565b7f4e487b71000000000000000000000000000000000000000000000000000000005f52604160045260245ffd5b610c8e82610c48565b810181811067ffffffffffffffff82111715610cad57610cac610c58565b5b80604052505050565b5f610cbf610a88565b9050610ccb8282610c85565b919050565b5f67ffffffffffffffff821115610cea57610ce9610c58565b5b610cf382610c48565b9050602081019050919050565b828183375f83830152505050565b5f610d20610d1b84610cd0565b610cb6565b905082815260208101848484011115610d3c57610d3b610c44565b5b610d47848285610d00565b509392505050565b5f82601f830112610d6357610d62610c40565b5b8135610d73848260208601610d0e565b91505092915050565b5f805f8060808587031215610d9457610d93610a91565b5b5f610da187828801610b9f565b9450506020610db287828801610bf9565b9350506040610dc387828801610c2c565b925050606085013567ffffffffffffffff811115610de457610de3610a95565b5b610df087828801610d4f565b91505092959194509250565b610e0581610b7e565b82525050565b5f602082019050610e1e5f830184610dfc565b92915050565b610e2d81610c0d565b82525050565b5f602082019050610e465f830184610e24565b92915050565b610e5581610bd2565b82525050565b5f602082019050610e6e5f830184610e4c565b92915050565b5f81519050919050565b5f81905092915050565b8281835e5f83830152505050565b5f610ea082610e74565b610eaa8185610e7e565b9350610eba818560208601610e88565b80840191505092915050565b5f610ed18284610e96565b915081905092915050565b5f82825260208201905092915050565b7f756e7375636365737366756c2063616c6c20746f2061726368200000000000005f82015250565b5f610f20601a83610edc565b9150610f2b82610eec565b602082019050919050565b5f6020820190508181035f830152610f4d81610f14565b9050919050565b5f81519050610f6281610b01565b92915050565b5f60208284031215610f7d57610f7c610a91565b5b5f610f8a84828501610f54565b91505092915050565b7f6f757470757420646f65736e74206d61746368207468652065787065637465645f8201527f2076616c75650000000000000000000000000000000000000000000000000000602082015250565b5f610fed602683610edc565b9150610ff882610f93565b604082019050919050565b5f6020820190508181035f83015261101a81610fe1565b9050919050565b5f82825260208201905092915050565b5f61103b82610e74565b6110458185611021565b9350611055818560208601610e88565b61105e81610c48565b840191505092915050565b5f60608201905061107c5f830186610e4c565b6110896020830185610e24565b818103604083015261109b8184611031565b9050949350505050565b7f756e7375636365737366756c2063616c6c20746f2061726368000000000000005f82015250565b5f6110d9601983610edc565b91506110e4826110a5565b602082019050919050565b5f6020820190508181035f830152611106816110cd565b9050919050565b5f8151905061111b81610b89565b92915050565b5f6020828403121561113657611135610a91565b5b5f6111438482850161110d565b91505092915050565b5f8151905061115a81610c16565b92915050565b5f6020828403121561117557611174610a91565b5b5f6111828482850161114c565b91505092915050565b7f417373657274204572726f72204d6573736167650000000000000000000000005f82015250565b5f6111bf601483610edc565b91506111ca8261118b565b602082019050919050565b5f6020820190508181035f8301526111ec816111b3565b9050919050565b5f819050919050565b5f819050919050565b5f61121f61121a611215846111f3565b6111fc565b610a57565b9050919050565b61122f81611205565b82525050565b7f56616c756520697320746f6f206c6f77000000000000000000000000000000005f82015250565b5f611269601083610edc565b915061127482611235565b602082019050919050565b5f6040820190506112925f830184611226565b81810360208301526112a38161125d565b90509291505056fea2646970667358221220c8f61ffdc0539e91e3d12903ea95b1869afd0ff3b8f25bcbdd2c3e2f7a3b55ca64736f6c634300081a0033 \ No newline at end of file +608060405261170d806100115f395ff3fe608060405234801561000f575f80fd5b5060043610610135575f3560e01c806383197ef0116100b6578063b2821c8f1161007a578063b2821c8f14610325578063cbaff5f914610343578063d0d250bd1461034d578063d462f09b1461036b578063d695cdca14610387578063dda3a7bd146103a357610135565b806383197ef01461028157806385df51fd1461028b578063911007b4146102bb578063a7b93d28146102eb578063adc879e91461030757610135565b806357e871e7116100fd57806357e871e7146101dd5780635ec01e4d146101fb5780636057361d146102195780636babb22414610235578063828dd0481461025157610135565b80632e64cec11461013957806348b15166146101575780634cbefa6a146101755780634d7b9bd51461019157806352e24024146101ad575b5f80fd5b6101416103ad565b60405161014e9190610c41565b60405180910390f35b61015f6103b5565b60405161016c9190610c41565b60405180910390f35b61018f600480360381019061018a9190610c95565b6103bc565b005b6101ab60048036038101906101a69190610d1a565b6103c5565b005b6101c760048036038101906101c29190610d95565b610422565b6040516101d49190610dcf565b60405180910390f35b6101e56105cf565b6040516101f29190610c41565b60405180910390f35b6102036105d6565b6040516102109190610c41565b60405180910390f35b610233600480360381019061022e9190610c95565b6105dd565b005b61024f600480360381019061024a9190610c95565b6105e6565b005b61026b60048036038101906102669190610f8c565b610633565b604051610278919061101b565b60405180910390f35b6102896107e2565b005b6102a560048036038101906102a09190610c95565b6107fb565b6040516102b29190611043565b60405180910390f35b6102d560048036038101906102d09190610d95565b610805565b6040516102e29190611043565b60405180910390f35b6103056004803603810190610300919061105c565b610967565b005b61030f6109b4565b60405161031c9190610c41565b60405180910390f35b61032d6109bb565b60405161033a9190610dcf565b60405180910390f35b61034b610b10565b005b610355610b52565b60405161036291906110a9565b60405180910390f35b610385600480360381019061038091906110c2565b610b5f565b005b6103a1600480360381019061039c9190610c95565b610ba6565b005b6103ab610beb565b005b5f8054905090565b5f42905090565b805f8190555f80fd5b8173ffffffffffffffffffffffffffffffffffffffff1631811461041e576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004016104159061115a565b60405180910390fd5b5050565b5f805f6801000000000000000173ffffffffffffffffffffffffffffffffffffffff166040516024016040516020818303038152906040527f53e87d66000000000000000000000000000000000000000000000000000000007bffffffffffffffffffffffffffffffffffffffffffffffffffffffff19166020820180517bffffffffffffffffffffffffffffffffffffffffffffffffffffffff83818316178352505050506040516104d591906111ca565b5f60405180830381855afa9150503d805f811461050d576040519150601f19603f3d011682016040523d82523d5f602084013e610512565b606091505b509150915081610557576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040161054e9061122a565b60405180910390fd5b5f8180602001905181019061056c919061125c565b90508067ffffffffffffffff168567ffffffffffffffff16146105c4576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004016105bb906112f7565b60405180910390fd5b809350505050919050565b5f43905090565b5f44905090565b805f8190555050565b803373ffffffffffffffffffffffffffffffffffffffff167f043cc306157a91d747b36aba0e235bbbc5771d75aba162f6e5540767d22673c660405160405180910390a3805f8190555050565b5f805f6801000000000000000173ffffffffffffffffffffffffffffffffffffffff1686868660405160240161066b9392919061135d565b6040516020818303038152906040527f5ee837e7000000000000000000000000000000000000000000000000000000007bffffffffffffffffffffffffffffffffffffffffffffffffffffffff19166020820180517bffffffffffffffffffffffffffffffffffffffffffffffffffffffff83818316178352505050506040516106f591906111ca565b5f60405180830381855afa9150503d805f811461072d576040519150601f19603f3d011682016040523d82523d5f602084013e610732565b606091505b509150915081610777576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040161076e906113e3565b60405180910390fd5b5f8180602001905181019061078c9190611415565b9050801515881515146107d4576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004016107cb906112f7565b60405180910390fd5b809350505050949350505050565b3373ffffffffffffffffffffffffffffffffffffffff16ff5b5f81409050919050565b5f805f6801000000000000000173ffffffffffffffffffffffffffffffffffffffff16846040516024016108399190610dcf565b6040516020818303038152906040527f78a75fbe000000000000000000000000000000000000000000000000000000007bffffffffffffffffffffffffffffffffffffffffffffffffffffffff19166020820180517bffffffffffffffffffffffffffffffffffffffffffffffffffffffff83818316178352505050506040516108c391906111ca565b5f60405180830381855afa9150503d805f81146108fb576040519150601f19603f3d011682016040523d82523d5f602084013e610900565b606091505b509150915081610945576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040161093c9061122a565b60405180910390fd5b5f8180602001905181019061095a9190611454565b9050809350505050919050565b815f54146109aa576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004016109a1906114c9565b60405180910390fd5b805f819055505050565b5f46905090565b5f805f6801000000000000000173ffffffffffffffffffffffffffffffffffffffff166040516024016040516020818303038152906040527f705fab20000000000000000000000000000000000000000000000000000000007bffffffffffffffffffffffffffffffffffffffffffffffffffffffff19166020820180517bffffffffffffffffffffffffffffffffffffffffffffffffffffffff8381831617835250505050604051610a6e91906111ca565b5f60405180830381855afa9150503d805f8114610aa6576040519150601f19603f3d011682016040523d82523d5f602084013e610aab565b606091505b509150915081610af0576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401610ae7906113e3565b60405180910390fd5b5f81806020019051810190610b05919061125c565b905080935050505090565b5f610b50576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401610b4790611531565b60405180910390fd5b565b6801000000000000000181565b81408114610ba2576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401610b9990611599565b60405180910390fd5b5050565b438114610be8576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401610bdf90611601565b60405180910390fd5b50565b60056040517f9195785a000000000000000000000000000000000000000000000000000000008152600401610c2091906116ab565b60405180910390fd5b5f819050919050565b610c3b81610c29565b82525050565b5f602082019050610c545f830184610c32565b92915050565b5f604051905090565b5f80fd5b5f80fd5b610c7481610c29565b8114610c7e575f80fd5b50565b5f81359050610c8f81610c6b565b92915050565b5f60208284031215610caa57610ca9610c63565b5b5f610cb784828501610c81565b91505092915050565b5f73ffffffffffffffffffffffffffffffffffffffff82169050919050565b5f610ce982610cc0565b9050919050565b610cf981610cdf565b8114610d03575f80fd5b50565b5f81359050610d1481610cf0565b92915050565b5f8060408385031215610d3057610d2f610c63565b5b5f610d3d85828601610d06565b9250506020610d4e85828601610c81565b9150509250929050565b5f67ffffffffffffffff82169050919050565b610d7481610d58565b8114610d7e575f80fd5b50565b5f81359050610d8f81610d6b565b92915050565b5f60208284031215610daa57610da9610c63565b5b5f610db784828501610d81565b91505092915050565b610dc981610d58565b82525050565b5f602082019050610de25f830184610dc0565b92915050565b5f8115159050919050565b610dfc81610de8565b8114610e06575f80fd5b50565b5f81359050610e1781610df3565b92915050565b5f819050919050565b610e2f81610e1d565b8114610e39575f80fd5b50565b5f81359050610e4a81610e26565b92915050565b5f80fd5b5f80fd5b5f601f19601f8301169050919050565b7f4e487b71000000000000000000000000000000000000000000000000000000005f52604160045260245ffd5b610e9e82610e58565b810181811067ffffffffffffffff82111715610ebd57610ebc610e68565b5b80604052505050565b5f610ecf610c5a565b9050610edb8282610e95565b919050565b5f67ffffffffffffffff821115610efa57610ef9610e68565b5b610f0382610e58565b9050602081019050919050565b828183375f83830152505050565b5f610f30610f2b84610ee0565b610ec6565b905082815260208101848484011115610f4c57610f4b610e54565b5b610f57848285610f10565b509392505050565b5f82601f830112610f7357610f72610e50565b5b8135610f83848260208601610f1e565b91505092915050565b5f805f8060808587031215610fa457610fa3610c63565b5b5f610fb187828801610e09565b9450506020610fc287828801610d06565b9350506040610fd387828801610e3c565b925050606085013567ffffffffffffffff811115610ff457610ff3610c67565b5b61100087828801610f5f565b91505092959194509250565b61101581610de8565b82525050565b5f60208201905061102e5f83018461100c565b92915050565b61103d81610e1d565b82525050565b5f6020820190506110565f830184611034565b92915050565b5f806040838503121561107257611071610c63565b5b5f61107f85828601610c81565b925050602061109085828601610c81565b9150509250929050565b6110a381610cdf565b82525050565b5f6020820190506110bc5f83018461109a565b92915050565b5f80604083850312156110d8576110d7610c63565b5b5f6110e585828601610c81565b92505060206110f685828601610e3c565b9150509250929050565b5f82825260208201905092915050565b7f62616c616e636520636865636b206661696c65640000000000000000000000005f82015250565b5f611144601483611100565b915061114f82611110565b602082019050919050565b5f6020820190508181035f83015261117181611138565b9050919050565b5f81519050919050565b5f81905092915050565b8281835e5f83830152505050565b5f6111a482611178565b6111ae8185611182565b93506111be81856020860161118c565b80840191505092915050565b5f6111d5828461119a565b915081905092915050565b7f756e7375636365737366756c2063616c6c20746f2061726368200000000000005f82015250565b5f611214601a83611100565b915061121f826111e0565b602082019050919050565b5f6020820190508181035f83015261124181611208565b9050919050565b5f8151905061125681610d6b565b92915050565b5f6020828403121561127157611270610c63565b5b5f61127e84828501611248565b91505092915050565b7f6f757470757420646f65736e74206d61746368207468652065787065637465645f8201527f2076616c75650000000000000000000000000000000000000000000000000000602082015250565b5f6112e1602683611100565b91506112ec82611287565b604082019050919050565b5f6020820190508181035f83015261130e816112d5565b9050919050565b5f82825260208201905092915050565b5f61132f82611178565b6113398185611315565b935061134981856020860161118c565b61135281610e58565b840191505092915050565b5f6060820190506113705f83018661109a565b61137d6020830185611034565b818103604083015261138f8184611325565b9050949350505050565b7f756e7375636365737366756c2063616c6c20746f2061726368000000000000005f82015250565b5f6113cd601983611100565b91506113d882611399565b602082019050919050565b5f6020820190508181035f8301526113fa816113c1565b9050919050565b5f8151905061140f81610df3565b92915050565b5f6020828403121561142a57611429610c63565b5b5f61143784828501611401565b91505092915050565b5f8151905061144e81610e26565b92915050565b5f6020828403121561146957611468610c63565b5b5f61147684828501611440565b91505092915050565b7f73746f7265642076616c756520636865636b206661696c6564000000000000005f82015250565b5f6114b3601983611100565b91506114be8261147f565b602082019050919050565b5f6020820190508181035f8301526114e0816114a7565b9050919050565b7f417373657274204572726f72204d6573736167650000000000000000000000005f82015250565b5f61151b601483611100565b9150611526826114e7565b602082019050919050565b5f6020820190508181035f8301526115488161150f565b9050919050565b7f6861736820636865636b206661696c65640000000000000000000000000000005f82015250565b5f611583601183611100565b915061158e8261154f565b602082019050919050565b5f6020820190508181035f8301526115b081611577565b9050919050565b7f626c6f636b206e756d62657220636865636b206661696c6564000000000000005f82015250565b5f6115eb601983611100565b91506115f6826115b7565b602082019050919050565b5f6020820190508181035f830152611618816115df565b9050919050565b5f819050919050565b5f819050919050565b5f61164b6116466116418461161f565b611628565b610c29565b9050919050565b61165b81611631565b82525050565b7f56616c756520697320746f6f206c6f77000000000000000000000000000000005f82015250565b5f611695601083611100565b91506116a082611661565b602082019050919050565b5f6040820190506116be5f830184611652565b81810360208301526116cf81611689565b90509291505056fea26469706673582212200ade6e0552548fb1b9355b0e891417786eff51bed669555dec499c0a4512075c64736f6c634300081a0033 \ No newline at end of file diff --git a/fvm/evm/types/backend.go b/fvm/evm/types/backend.go index 7f46cddecc8..985a8bc29e1 100644 --- a/fvm/evm/types/backend.go +++ b/fvm/evm/types/backend.go @@ -4,11 +4,16 @@ import ( "github.com/onflow/flow-go/fvm/environment" ) +// BackendStorage provides an interface for storage of registers +type BackendStorage interface { + environment.ValueStore +} + // Backend provides a subset of the FVM environment functionality // Any error returned by a Backend is expected to be a `FatalError` or // a `BackendError`. type Backend interface { - environment.ValueStore + BackendStorage environment.Meter environment.EventEmitter environment.BlockInfo diff --git a/fvm/evm/types/block.go b/fvm/evm/types/block.go index 458e2961bc3..f4123b71ddc 100644 --- a/fvm/evm/types/block.go +++ b/fvm/evm/types/block.go @@ -120,8 +120,23 @@ func GenesisBlock(chainID flow.ChainID) *Block { } } +// when testnet was launched the block structure +// didn't have the preRandao filed so the hash of the event +// was different from hashing the genesis block struct. +var TestNetGenesisHash = gethCommon.Hash{ + 60, 220, 118, 103, 27, 85, 73, 205, + 46, 2, 83, 105, 179, 240, 255, 14, + 55, 21, 42, 211, 55, 87, 177, 115, + 118, 144, 125, 37, 146, 116, 168, 229, +} + // GenesisBlockHash returns the genesis block hash in the EVM environment func GenesisBlockHash(chainID flow.ChainID) gethCommon.Hash { + // for the case of testnet, the block didn't initially + // had the preRandao and it was not part of the hash calculation. + if chainID == flow.Testnet { + return TestNetGenesisHash + } h, err := GenesisBlock(chainID).Hash() if err != nil { // this never happens panic(err) diff --git a/fvm/evm/types/block_test.go b/fvm/evm/types/block_test.go index 2ccb0ddcd65..a87f3eca008 100644 --- a/fvm/evm/types/block_test.go +++ b/fvm/evm/types/block_test.go @@ -17,14 +17,12 @@ func Test_GenesisBlock(t *testing.T) { testnetGenesis := GenesisBlock(flow.Testnet) require.Equal(t, testnetGenesis.Timestamp, GenesisTimestamp(flow.Testnet)) testnetGenesisHash := GenesisBlockHash(flow.Testnet) - h, err := testnetGenesis.Hash() - require.NoError(t, err) - require.Equal(t, h, testnetGenesisHash) + require.Equal(t, TestNetGenesisHash, testnetGenesisHash) mainnetGenesis := GenesisBlock(flow.Mainnet) require.Equal(t, mainnetGenesis.Timestamp, GenesisTimestamp(flow.Mainnet)) mainnetGenesisHash := GenesisBlockHash(flow.Mainnet) - h, err = mainnetGenesis.Hash() + h, err := mainnetGenesis.Hash() require.NoError(t, err) require.Equal(t, h, mainnetGenesisHash) diff --git a/fvm/evm/types/result.go b/fvm/evm/types/result.go index 31176529f7e..2ad16a0b5df 100644 --- a/fvm/evm/types/result.go +++ b/fvm/evm/types/result.go @@ -1,9 +1,9 @@ package types import ( - "github.com/onflow/go-ethereum/common" gethCommon "github.com/onflow/go-ethereum/common" gethTypes "github.com/onflow/go-ethereum/core/types" + "github.com/onflow/go-ethereum/rlp" ) // InvalidTransactionGasCost is a gas cost we charge when @@ -17,6 +17,9 @@ import ( // user for the validation fee. const InvalidTransactionGasCost = 1_000 +// ChecksumLength captures number of bytes a checksum uses +const ChecksumLength = 4 + // Status captures the status of an interaction to the emulator type Status uint8 @@ -127,6 +130,59 @@ func (res *Result) VMErrorString() string { return "" } +// ErrorMsg returns the error message, if any VM or Validation error +// both error would never happen at the same time +// but if it happens the priority is by validation error +func (res *Result) ErrorMsg() string { + errorMsg := "" + if res.VMError != nil { + errorMsg = res.VMError.Error() + } + if res.ValidationError != nil { + errorMsg = res.ValidationError.Error() + } + return errorMsg +} + +// RLPEncodedLogs returns the rlp encoding of the logs +func (res *Result) RLPEncodedLogs() ([]byte, error) { + var encodedLogs []byte + var err error + if len(res.Logs) > 0 { + encodedLogs, err = rlp.EncodeToBytes(res.Logs) + if err != nil { + return encodedLogs, err + } + } + return encodedLogs, nil +} + +// DeployedContractAddressString returns an string of the deployed address +// it returns an empty string if the deployed address is nil +func (res *Result) DeployedContractAddressString() string { + deployedAddress := "" + if res.DeployedContractAddress != nil { + deployedAddress = res.DeployedContractAddress.String() + } + return deployedAddress +} + +// StateChangeChecksum constructs a checksum +// based on the state change commitment on the result +func (res *Result) StateChangeChecksum() [ChecksumLength]byte { + return SliceToChecksumLength(res.StateChangeCommitment) +} + +// SliceToChecksumLength cuts the first 4 bytes of the input and convert it into checksum +func SliceToChecksumLength(input []byte) [ChecksumLength]byte { + // the first 4 bytes of StateChangeCommitment is used as checksum + var checksum [ChecksumLength]byte + if len(input) >= ChecksumLength { + copy(checksum[:ChecksumLength], input[:ChecksumLength]) + } + return checksum +} + // Receipt constructs an EVM-style receipt // can be used by json-rpc and other integration to be returned. // @@ -231,9 +287,9 @@ func (res *Result) ResultSummary() *ResultSummary { // used by the LightReceipt type LightLog struct { // address of the contract that generated the event - Address common.Address + Address gethCommon.Address // list of topics provided by the contract. - Topics []common.Hash + Topics []gethCommon.Hash // supplied by the contract, usually ABI-encoded Data []byte } diff --git a/go.mod b/go.mod index ce21004255f..dbed42d9182 100644 --- a/go.mod +++ b/go.mod @@ -48,7 +48,7 @@ require ( github.com/multiformats/go-multiaddr-dns v0.3.1 github.com/multiformats/go-multihash v0.2.3 github.com/onflow/atree v0.8.0-rc.6 - github.com/onflow/cadence v1.0.0 + github.com/onflow/cadence v1.0.1 github.com/onflow/crypto v0.25.2 github.com/onflow/flow v0.3.4 github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1 diff --git a/go.sum b/go.sum index 2f51ed1016e..8f4f3a4f317 100644 --- a/go.sum +++ b/go.sum @@ -2171,8 +2171,8 @@ github.com/onflow/atree v0.8.0-rc.6/go.mod h1:yccR+LR7xc1Jdic0mrjocbHvUD7lnVvg8/ github.com/onflow/boxo v0.0.0-20240201202436-f2477b92f483 h1:LpiQhTAfM9CAmNVEs0n//cBBgCg+vJSiIxTHYUklZ84= github.com/onflow/boxo v0.0.0-20240201202436-f2477b92f483/go.mod h1:pIZgTWdm3k3pLF9Uq6MB8JEcW07UDwNJjlXW1HELW80= github.com/onflow/cadence v1.0.0-M3/go.mod h1:odXGZZ/wGNA5mwT8bC9v8u8EXACHllB2ABSZK65TGL8= -github.com/onflow/cadence v1.0.0 h1:bvT75F2LZJvDCBmmajAv7QLISK6Qp30FAKcSwqNNH+o= -github.com/onflow/cadence v1.0.0/go.mod h1:7wvvecnAZtYOspLOS3Lh+FuAmMeSrXhAWiycC3kQ1UU= +github.com/onflow/cadence v1.0.1 h1:Vv/txHQsaT5BzS14SrsMRQgj164JdNVKafekXrZ5Fx4= +github.com/onflow/cadence v1.0.1/go.mod h1:7wvvecnAZtYOspLOS3Lh+FuAmMeSrXhAWiycC3kQ1UU= github.com/onflow/crypto v0.25.0/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= github.com/onflow/crypto v0.25.2 h1:GjHunqVt+vPcdqhxxhAXiMIF3YiLX7gTuTR5O+VG2ns= github.com/onflow/crypto v0.25.2/go.mod h1:fY7eLqUdMKV8EGOw301unP8h7PvLVy8/6gVR++/g0BY= diff --git a/insecure/go.mod b/insecure/go.mod index 060fa230a0f..5eedd7c0c17 100644 --- a/insecure/go.mod +++ b/insecure/go.mod @@ -203,7 +203,7 @@ require ( github.com/nxadm/tail v1.4.8 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/onflow/atree v0.8.0-rc.6 // indirect - github.com/onflow/cadence v1.0.0 // indirect + github.com/onflow/cadence v1.0.1 // indirect github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1 // indirect github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1 // indirect github.com/onflow/flow-ft/lib/go/contracts v1.0.0 // indirect diff --git a/insecure/go.sum b/insecure/go.sum index 8b88066e0ed..7910b609280 100644 --- a/insecure/go.sum +++ b/insecure/go.sum @@ -2159,8 +2159,8 @@ github.com/onflow/atree v0.6.1-0.20230711151834-86040b30171f/go.mod h1:xvP61FoOs github.com/onflow/atree v0.8.0-rc.6 h1:GWgaylK24b5ta2Hq+TvyOF7X5tZLiLzMMn7lEt59fsA= github.com/onflow/atree v0.8.0-rc.6/go.mod h1:yccR+LR7xc1Jdic0mrjocbHvUD7lnVvg8/Ct1AA5zBo= github.com/onflow/cadence v1.0.0-M3/go.mod h1:odXGZZ/wGNA5mwT8bC9v8u8EXACHllB2ABSZK65TGL8= -github.com/onflow/cadence v1.0.0 h1:bvT75F2LZJvDCBmmajAv7QLISK6Qp30FAKcSwqNNH+o= -github.com/onflow/cadence v1.0.0/go.mod h1:7wvvecnAZtYOspLOS3Lh+FuAmMeSrXhAWiycC3kQ1UU= +github.com/onflow/cadence v1.0.1 h1:Vv/txHQsaT5BzS14SrsMRQgj164JdNVKafekXrZ5Fx4= +github.com/onflow/cadence v1.0.1/go.mod h1:7wvvecnAZtYOspLOS3Lh+FuAmMeSrXhAWiycC3kQ1UU= github.com/onflow/crypto v0.25.0/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= github.com/onflow/crypto v0.25.2 h1:GjHunqVt+vPcdqhxxhAXiMIF3YiLX7gTuTR5O+VG2ns= github.com/onflow/crypto v0.25.2/go.mod h1:fY7eLqUdMKV8EGOw301unP8h7PvLVy8/6gVR++/g0BY= diff --git a/integration/go.mod b/integration/go.mod index 9bae86babca..2dcdefd2aee 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -20,7 +20,7 @@ require ( github.com/ipfs/go-ds-badger2 v0.1.3 github.com/ipfs/go-ds-pebble v0.3.1-0.20240828032824-d745b9d3200b github.com/libp2p/go-libp2p v0.32.2 - github.com/onflow/cadence v1.0.0 + github.com/onflow/cadence v1.0.1 github.com/onflow/crypto v0.25.2 github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1 github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1 diff --git a/integration/go.sum b/integration/go.sum index 4128721ef99..68675018855 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -2144,8 +2144,8 @@ github.com/onflow/atree v0.6.1-0.20230711151834-86040b30171f/go.mod h1:xvP61FoOs github.com/onflow/atree v0.8.0-rc.6 h1:GWgaylK24b5ta2Hq+TvyOF7X5tZLiLzMMn7lEt59fsA= github.com/onflow/atree v0.8.0-rc.6/go.mod h1:yccR+LR7xc1Jdic0mrjocbHvUD7lnVvg8/Ct1AA5zBo= github.com/onflow/cadence v1.0.0-M3/go.mod h1:odXGZZ/wGNA5mwT8bC9v8u8EXACHllB2ABSZK65TGL8= -github.com/onflow/cadence v1.0.0 h1:bvT75F2LZJvDCBmmajAv7QLISK6Qp30FAKcSwqNNH+o= -github.com/onflow/cadence v1.0.0/go.mod h1:7wvvecnAZtYOspLOS3Lh+FuAmMeSrXhAWiycC3kQ1UU= +github.com/onflow/cadence v1.0.1 h1:Vv/txHQsaT5BzS14SrsMRQgj164JdNVKafekXrZ5Fx4= +github.com/onflow/cadence v1.0.1/go.mod h1:7wvvecnAZtYOspLOS3Lh+FuAmMeSrXhAWiycC3kQ1UU= github.com/onflow/crypto v0.25.0/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= github.com/onflow/crypto v0.25.2 h1:GjHunqVt+vPcdqhxxhAXiMIF3YiLX7gTuTR5O+VG2ns= github.com/onflow/crypto v0.25.2/go.mod h1:fY7eLqUdMKV8EGOw301unP8h7PvLVy8/6gVR++/g0BY= diff --git a/module/execution/scripts_test.go b/module/execution/scripts_test.go index 28653047b45..42fe97519ae 100644 --- a/module/execution/scripts_test.go +++ b/module/execution/scripts_test.go @@ -178,7 +178,7 @@ func (s *scriptTestSuite) SetupTest() { s.dbDir = unittest.TempDir(s.T()) db := pebbleStorage.NewBootstrappedRegistersWithPathForTest(s.T(), s.dbDir, s.height, s.height) - pebbleRegisters, err := pebbleStorage.NewRegisters(db) + pebbleRegisters, err := pebbleStorage.NewRegisters(db, pebbleStorage.PruningDisabled) s.Require().NoError(err) s.registerIndex = pebbleRegisters diff --git a/storage/pebble/bootstrap_test.go b/storage/pebble/bootstrap_test.go index b0fccc8c269..c8a59d6fbe8 100644 --- a/storage/pebble/bootstrap_test.go +++ b/storage/pebble/bootstrap_test.go @@ -59,7 +59,7 @@ func TestRegisterBootstrap_IndexCheckpointFile_Happy(t *testing.T) { require.NoError(t, err) // create registers instance and check values - reg, err := NewRegisters(pb) + reg, err := NewRegisters(pb, PruningDisabled) require.NoError(t, err) require.Equal(t, reg.LatestHeight(), rootHeight) @@ -94,7 +94,7 @@ func TestRegisterBootstrap_IndexCheckpointFile_Empty(t *testing.T) { require.NoError(t, err) // create registers instance and check values - reg, err := NewRegisters(pb) + reg, err := NewRegisters(pb, PruningDisabled) require.NoError(t, err) require.Equal(t, reg.LatestHeight(), rootHeight) @@ -176,7 +176,7 @@ func TestRegisterBootstrap_IndexCheckpointFile_MultipleBatch(t *testing.T) { require.NoError(t, err) // create registers instance and check values - reg, err := NewRegisters(pb) + reg, err := NewRegisters(pb, PruningDisabled) require.NoError(t, err) require.Equal(t, reg.LatestHeight(), rootHeight) diff --git a/storage/pebble/open.go b/storage/pebble/open.go index 4e81f1155e1..224782fed4a 100644 --- a/storage/pebble/open.go +++ b/storage/pebble/open.go @@ -20,7 +20,7 @@ func NewBootstrappedRegistersWithPath(dir string) (*Registers, *pebble.DB, error if err != nil { return nil, nil, fmt.Errorf("failed to initialize pebble db: %w", err) } - registers, err := NewRegisters(db) + registers, err := NewRegisters(db, PruningDisabled) if err != nil { if errors.Is(err, storage.ErrNotBootstrapped) { // closing the db if not bootstrapped diff --git a/storage/pebble/open_test.go b/storage/pebble/open_test.go index 6a7ffacf592..bb5a1a32917 100644 --- a/storage/pebble/open_test.go +++ b/storage/pebble/open_test.go @@ -66,7 +66,7 @@ func TestNewBootstrappedRegistersWithPath(t *testing.T) { firstHeight := uint64(10) require.NoError(t, initHeights(db2, firstHeight)) - registers, err := NewRegisters(db2) + registers, err := NewRegisters(db2, PruningDisabled) require.NoError(t, err) require.Equal(t, firstHeight, registers.FirstHeight()) require.Equal(t, firstHeight, registers.LatestHeight()) diff --git a/storage/pebble/registers.go b/storage/pebble/registers.go index 2d889f8602f..2f5c78e8ff1 100644 --- a/storage/pebble/registers.go +++ b/storage/pebble/registers.go @@ -3,6 +3,7 @@ package pebble import ( "encoding/binary" "fmt" + "math" "github.com/cockroachdb/pebble" "github.com/pkg/errors" @@ -15,17 +16,21 @@ import ( // Registers library that implements pebble storage for registers // given a pebble instance with root block and root height populated type Registers struct { - db *pebble.DB - firstHeight uint64 - latestHeight *atomic.Uint64 + db *pebble.DB + firstHeight uint64 + latestHeight *atomic.Uint64 + pruneThreshold uint64 } +// PruningDisabled represents the absence of a pruning threshold. +const PruningDisabled = math.MaxUint64 + var _ storage.RegisterIndex = (*Registers)(nil) // NewRegisters takes a populated pebble instance with LatestHeight and FirstHeight set. // return storage.ErrNotBootstrapped if they those two keys are unavailable as it implies a uninitialized state // return other error if database is in a corrupted state -func NewRegisters(db *pebble.DB) (*Registers, error) { +func NewRegisters(db *pebble.DB, pruneThreshold uint64) (*Registers, error) { // check height keys and populate cache. These two variables will have been set firstHeight, latestHeight, err := ReadHeightsFromBootstrappedDB(db) if err != nil { @@ -33,11 +38,17 @@ func NewRegisters(db *pebble.DB) (*Registers, error) { return nil, fmt.Errorf("unable to initialize register storage, latest height unavailable in db: %w", err) } + // If no pruning threshold is provided, disable pruning. + if pruneThreshold == 0 { + pruneThreshold = PruningDisabled + } + // All registers between firstHeight and lastHeight have been indexed return &Registers{ - db: db, - firstHeight: firstHeight, - latestHeight: atomic.NewUint64(latestHeight), + db: db, + firstHeight: firstHeight, + latestHeight: atomic.NewUint64(latestHeight), + pruneThreshold: pruneThreshold, }, nil } @@ -53,12 +64,14 @@ func (s *Registers) Get( reg flow.RegisterID, height uint64, ) (flow.RegisterValue, error) { - latestHeight := s.latestHeight.Load() - if height > latestHeight || height < s.firstHeight { - return nil, errors.Wrap( - storage.ErrHeightNotIndexed, - fmt.Sprintf("height %d not indexed, indexed range is [%d-%d]", height, s.firstHeight, latestHeight), - ) + latestHeight := s.LatestHeight() + if height > latestHeight { + return nil, fmt.Errorf("height %d not indexed, latestHeight: %d, %w", height, latestHeight, storage.ErrHeightNotIndexed) + } + + firstHeight := s.calculateFirstHeight(latestHeight) + if height < firstHeight { + return nil, fmt.Errorf("height %d not indexed, indexed range: [%d-%d], %w", height, firstHeight, latestHeight, storage.ErrHeightNotIndexed) } key := newLookupKey(height, reg) return s.lookupRegister(key.Bytes()) @@ -132,6 +145,7 @@ func (s *Registers) Store( if err != nil { return fmt.Errorf("failed to commit batch: %w", err) } + s.latestHeight.Store(height) return nil @@ -144,7 +158,30 @@ func (s *Registers) LatestHeight() uint64 { // FirstHeight first indexed height found in the store, typically root block for the spork func (s *Registers) FirstHeight() uint64 { - return s.firstHeight + return s.calculateFirstHeight(s.LatestHeight()) +} + +// calculateFirstHeight calculates the first indexed height that is stored in the register index, based on the +// latest height and the configured pruning threshold. If the latest height is below the pruning threshold, the +// first indexed height will be the same as the initial height when the store was initialized. If the pruning +// threshold has been exceeded, the first indexed height is adjusted accordingly. +// +// Parameters: +// - latestHeight: the most recent height of complete registers available. +// +// Returns: +// - The first indexed height, either as the initialized height or adjusted for pruning. +func (s *Registers) calculateFirstHeight(latestHeight uint64) uint64 { + if latestHeight < s.pruneThreshold { + return s.firstHeight + } + + pruneHeight := latestHeight - s.pruneThreshold + if pruneHeight < s.firstHeight { + return s.firstHeight + } + + return pruneHeight } func firstStoredHeight(db *pebble.DB) (uint64, error) { diff --git a/storage/pebble/registers_test.go b/storage/pebble/registers_test.go index 34fd26ad602..7cf5329c3c7 100644 --- a/storage/pebble/registers_test.go +++ b/storage/pebble/registers_test.go @@ -25,7 +25,7 @@ func TestRegisters_Initialize(t *testing.T) { t.Parallel() p, dir := unittest.TempPebbleDBWithOpts(t, nil) // fail on blank database without FirstHeight and LastHeight set - _, err := NewRegisters(p) + _, err := NewRegisters(p, PruningDisabled) require.Error(t, err) // verify the error type require.True(t, errors.Is(err, storage.ErrNotBootstrapped)) @@ -296,7 +296,7 @@ func Benchmark_PayloadStorage(b *testing.B) { dbpath := path.Join(b.TempDir(), "benchmark1.db") db, err := pebble.Open(dbpath, opts) require.NoError(b, err) - s, err := NewRegisters(db) + s, err := NewRegisters(db, PruningDisabled) require.NoError(b, err) require.NotNil(b, s) diff --git a/storage/pebble/testutil.go b/storage/pebble/testutil.go index 64bd5a71a21..0d890fef85d 100644 --- a/storage/pebble/testutil.go +++ b/storage/pebble/testutil.go @@ -12,7 +12,7 @@ import ( func RunWithRegistersStorageAtInitialHeights(tb testing.TB, first uint64, latest uint64, f func(r *Registers)) { unittest.RunWithTempDir(tb, func(dir string) { db := NewBootstrappedRegistersWithPathForTest(tb, dir, first, latest) - r, err := NewRegisters(db) + r, err := NewRegisters(db, PruningDisabled) require.NoError(tb, err) f(r)