Merge branch 'master' into petera/allow-version-control-ignored-versions

.github/workflows/builds.yml

@@ -82,6 +82,7 @@ jobs:
   docker-push:
     name: ${{ matrix.role }} images
     runs-on: ubuntu-latest
+    environment: Production Docker Registry
     needs: matrix_builder
 
     # setup jobs for each role

.github/workflows/cd.yml

@@ -1,10 +1,18 @@
 name: CD
 
 on:
-  push:
-    tags:
-      - '*'
-      - "!daily-*"
+# Workflow dispatch for now, while we're testing environments
+  # push:
+  #   tags:
+  #     - '*'
+  #     - "!daily-*"
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: 'Tag/commit'
+        required: true
+        type: string
+
 
 env:
   GO_VERSION: "1.22"
@@ -13,6 +21,7 @@ jobs:
   docker-push:
     name: Push to container registry
     runs-on: ubuntu-latest
+    environment: Production Docker Registry
     steps:
     - name: Setup Go
       uses: actions/setup-go@v4
@@ -21,6 +30,8 @@ jobs:
         go-version: ${{ env.GO_VERSION }}
     - name: Checkout repo
       uses: actions/checkout@v2
+      with:
+        ref: ${{ inputs.tag }}
     # Provide Google Service Account credentials to Github Action, allowing interaction with the Google Container Registry
     # Logging in as [email protected]
     - id: auth

cmd/access/node_builder/access_node_builder.go

@@ -176,6 +176,7 @@ type AccessNodeConfig struct {
 	checkPayerBalanceMode                string
 	versionControlEnabled                bool
 	stopControlEnabled                   bool
+	registerDBPruneThreshold             uint64
 }
 
 type PublicNetworkConfig struct {
@@ -280,6 +281,7 @@ func DefaultAccessNodeConfig() *AccessNodeConfig {
 		checkPayerBalanceMode:                accessNode.Disabled.String(),
 		versionControlEnabled:                true,
 		stopControlEnabled:                   false,
+		registerDBPruneThreshold:             pruner.DefaultThreshold,
 	}
 }
 
@@ -905,7 +907,7 @@ func (builder *FlowAccessNodeBuilder) BuildExecutionSyncComponents() *FlowAccess
 					}
 				}
 
-				registers, err := pstorage.NewRegisters(pdb)
+				registers, err := pstorage.NewRegisters(pdb, builder.registerDBPruneThreshold)
 				if err != nil {
 					return nil, fmt.Errorf("could not create registers storage: %w", err)
 				}
@@ -1421,6 +1423,12 @@ func (builder *FlowAccessNodeBuilder) extraFlags() {
 			"check-payer-balance-mode",
 			defaultConfig.checkPayerBalanceMode,
 			"flag for payer balance validation that specifies whether or not to enforce the balance check. one of [disabled(default), warn, enforce]")
+
+		// Register DB Pruning
+		flags.Uint64Var(&builder.registerDBPruneThreshold,
+			"registerdb-pruning-threshold",
+			defaultConfig.registerDBPruneThreshold,
+			fmt.Sprintf("specifies the number of blocks below the latest stored block height to keep in register db. default: %d", defaultConfig.registerDBPruneThreshold))
 	}).ValidateFlags(func() error {
 		if builder.supportsObserver && (builder.PublicNetworkConfig.BindAddress == cmd.NotSet || builder.PublicNetworkConfig.BindAddress == "") {
 			return errors.New("public-network-address must be set if supports-observer is true")

cmd/execution_builder.go

@@ -889,7 +889,7 @@ func (exeNode *ExecutionNode) LoadRegisterStore(
 			return fmt.Errorf("could not import registers from checkpoint: %w", err)
 		}
 	}
-	diskStore, err := storagepebble.NewRegisters(pebbledb)
+	diskStore, err := storagepebble.NewRegisters(pebbledb, storagepebble.PruningDisabled)
 	if err != nil {
 		return fmt.Errorf("could not create registers storage: %w", err)
 	}

cmd/observer/node_builder/observer_builder.go

@@ -174,6 +174,7 @@ type ObserverServiceConfig struct {
 	registerCacheType                    string
 	registerCacheSize                    uint
 	programCacheSize                     uint
+	registerDBPruneThreshold             uint64
 }
 
 // DefaultObserverServiceConfig defines all the default values for the ObserverServiceConfig
@@ -252,11 +253,12 @@ func DefaultObserverServiceConfig() *ObserverServiceConfig {
 			RetryDelay:         edrequester.DefaultRetryDelay,
 			MaxRetryDelay:      edrequester.DefaultMaxRetryDelay,
 		},
-		scriptExecMinBlock: 0,
-		scriptExecMaxBlock: math.MaxUint64,
-		registerCacheType:  pstorage.CacheTypeTwoQueue.String(),
-		registerCacheSize:  0,
-		programCacheSize:   0,
+		scriptExecMinBlock:       0,
+		scriptExecMaxBlock:       math.MaxUint64,
+		registerCacheType:        pstorage.CacheTypeTwoQueue.String(),
+		registerCacheSize:        0,
+		programCacheSize:         0,
+		registerDBPruneThreshold: pruner.DefaultThreshold,
 	}
 }
 
@@ -810,6 +812,12 @@ func (builder *ObserverServiceBuilder) extraFlags() {
 			"program-cache-size",
 			defaultConfig.programCacheSize,
 			"[experimental] number of blocks to cache for cadence programs. use 0 to disable cache. default: 0. Note: this is an experimental feature and may cause nodes to become unstable under certain workloads. Use with caution.")
+
+		// Register DB Pruning
+		flags.Uint64Var(&builder.registerDBPruneThreshold,
+			"registerdb-pruning-threshold",
+			defaultConfig.registerDBPruneThreshold,
+			fmt.Sprintf("specifies the number of blocks below the latest stored block height to keep in register db. default: %d", defaultConfig.registerDBPruneThreshold))
 	}).ValidateFlags(func() error {
 		if builder.executionDataSyncEnabled {
 			if builder.executionDataConfig.FetchTimeout <= 0 {
@@ -1436,7 +1444,7 @@ func (builder *ObserverServiceBuilder) BuildExecutionSyncComponents() *ObserverS
 				}
 			}
 
-			registers, err := pstorage.NewRegisters(pdb)
+			registers, err := pstorage.NewRegisters(pdb, builder.registerDBPruneThreshold)
 			if err != nil {
 				return nil, fmt.Errorf("could not create registers storage: %w", err)
 			}

engine/access/rpc/backend/script_executor_test.go

@@ -126,7 +126,7 @@ func (s *ScriptExecutorSuite) SetupTest() {
 
 	s.dbDir = unittest.TempDir(s.T())
 	db := pebbleStorage.NewBootstrappedRegistersWithPathForTest(s.T(), s.dbDir, s.height, s.height)
-	pebbleRegisters, err := pebbleStorage.NewRegisters(db)
+	pebbleRegisters, err := pebbleStorage.NewRegisters(db, pebbleStorage.PruningDisabled)
 	s.Require().NoError(err)
 	s.registerIndex = pebbleRegisters
 

engine/common/requester/engine.go

@@ -421,15 +421,6 @@ func (e *Engine) dispatchRequest() (bool, error) {
 	}
 	e.requests[req.Nonce] = req
 
-	if e.log.Debug().Enabled() {
-		e.log.Debug().
-			Hex("provider", logging.ID(providerID)).
-			Uint64("nonce", req.Nonce).
-			Strs("entities", logging.IDs(entityIDs)).
-			TimeDiff("duration", time.Now(), requestStart).
-			Msg("entity request sent")
-	}
-
 	// NOTE: we forget about requests after the expiry of the shortest retry time
 	// from the entities in the list; this means that we purge requests aggressively.
 	// However, most requests should be responded to on the first attempt and clearing
@@ -443,11 +434,15 @@ func (e *Engine) dispatchRequest() (bool, error) {
 		delete(e.requests, req.Nonce)
 	}()
 
+	if e.log.Debug().Enabled() {
+		e.log.Debug().
+			Hex("provider", logging.ID(providerID)).
+			Uint64("nonce", req.Nonce).
+			Strs("entities", logging.IDs(entityIDs)).
+			TimeDiff("duration", time.Now(), requestStart).
+			Msg("entity request sent")
+	}
 	e.metrics.MessageSent(e.channel.String(), metrics.MessageEntityRequest)
-	e.log.Debug().
-		Uint64("nonce", req.Nonce).
-		Strs("entity_ids", flow.IdentifierList(req.EntityIDs).Strings()).
-		Msg("entity request sent")
 
 	return true, nil
 }

engine/testutil/nodes.go

@@ -616,7 +616,7 @@ func ExecutionNode(t *testing.T, hub *stub.Hub, identity bootstrap.NodeInfo, ide
 	checkpointHeight := uint64(0)
 	require.NoError(t, esbootstrap.ImportRegistersFromCheckpoint(node.Log, checkpointFile, checkpointHeight, matchTrie.RootHash(), pebbledb, 2))
 
-	diskStore, err := storagepebble.NewRegisters(pebbledb)
+	diskStore, err := storagepebble.NewRegisters(pebbledb, storagepebble.PruningDisabled)
 	require.NoError(t, err)
 
 	reader := finalizedreader.NewFinalizedReader(headersStorage, checkpointHeight)

fvm/evm/debug/tracer_test.go

@@ -1,4 +1,4 @@
-package debug
+package debug_test
 
 import (
 	"encoding/json"
@@ -11,6 +11,7 @@ import (
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/require"
 
+	"github.com/onflow/flow-go/fvm/evm/debug"
 	"github.com/onflow/flow-go/fvm/evm/testutils"
 	"github.com/onflow/flow-go/model/flow"
 )
@@ -23,13 +24,13 @@ func Test_CallTracer(t *testing.T) {
 
 		mockUpload := &testutils.MockUploader{
 			UploadFunc: func(id string, message json.RawMessage) error {
-				require.Equal(t, TraceID(txID, blockID), id)
+				require.Equal(t, debug.TraceID(txID, blockID), id)
 				require.Equal(t, res, message)
 				return nil
 			},
 		}
 
-		tracer, err := NewEVMCallTracer(mockUpload, zerolog.Nop())
+		tracer, err := debug.NewEVMCallTracer(mockUpload, zerolog.Nop())
 		require.NoError(t, err)
 		tracer.WithBlockID(blockID)
 
@@ -61,13 +62,13 @@ func Test_CallTracer(t *testing.T) {
 
 		mockUpload := &testutils.MockUploader{
 			UploadFunc: func(id string, message json.RawMessage) error {
-				require.Equal(t, TraceID(txID, blockID), id)
+				require.Equal(t, debug.TraceID(txID, blockID), id)
 				require.Equal(t, res, message)
 				return nil
 			},
 		}
 
-		tracer, err := NewEVMCallTracer(mockUpload, zerolog.Nop())
+		tracer, err := debug.NewEVMCallTracer(mockUpload, zerolog.Nop())
 		require.NoError(t, err)
 		tracer.WithBlockID(blockID)
 
@@ -107,7 +108,7 @@ func Test_CallTracer(t *testing.T) {
 			},
 		}
 
-		tracer, err := NewEVMCallTracer(mockUpload, zerolog.Nop())
+		tracer, err := debug.NewEVMCallTracer(mockUpload, zerolog.Nop())
 		require.NoError(t, err)
 		tracer.WithBlockID(blockID)
 
@@ -117,7 +118,7 @@ func Test_CallTracer(t *testing.T) {
 	})
 
 	t.Run("nop tracer", func(t *testing.T) {
-		tracer := nopTracer{}
+		tracer := debug.NopTracer
 		require.Nil(t, tracer.TxTracer())
 	})
 }

fvm/evm/debug/uploader_test.go

@@ -1,4 +1,4 @@
-package debug
+package debug_test
 
 import (
 	"context"
@@ -16,6 +16,7 @@ import (
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/require"
 
+	"github.com/onflow/flow-go/fvm/evm/debug"
 	"github.com/onflow/flow-go/model/flow"
 	testutils "github.com/onflow/flow-go/utils/unittest"
 )
@@ -27,7 +28,7 @@ func Test_Uploader(t *testing.T) {
 	testutils.SkipUnless(t, testutils.TEST_REQUIRES_GCP_ACCESS, "requires GCP Bucket setup")
 
 	t.Run("successfuly upload traces", func(t *testing.T) {
-		uploader, err := NewGCPUploader(bucket)
+		uploader, err := debug.NewGCPUploader(bucket)
 		require.NoError(t, err)
 
 		const testID = "test_p"
@@ -56,10 +57,10 @@ func Test_TracerUploaderIntegration(t *testing.T) {
 	testutils.SkipUnless(t, testutils.TEST_REQUIRES_GCP_ACCESS, "requires GCP Bucket setup")
 
 	t.Run("successfuly uploads traces", func(t *testing.T) {
-		uploader, err := NewGCPUploader(bucket)
+		uploader, err := debug.NewGCPUploader(bucket)
 		require.NoError(t, err)
 
-		tracer, err := NewEVMCallTracer(uploader, zerolog.Nop())
+		tracer, err := debug.NewEVMCallTracer(uploader, zerolog.Nop())
 		require.NoError(t, err)
 
 		tr := tracer.TxTracer()

fvm/evm/emulator/emulator_test.go

@@ -1257,7 +1257,7 @@ func TestTransactionTracing(t *testing.T) {
 					testAccount.Address(),
 					types.Address{0x01, 0x02},
 					testContract.ByteCode,
-					1_000_000,
+					2_000_000,
 					big.NewInt(0),
 					testAccount.Nonce(),
 				),