diff --git a/access/validator.go b/access/validator.go index 6463632fadd..f0dec8071d0 100644 --- a/access/validator.go +++ b/access/validator.go @@ -5,18 +5,21 @@ import ( "errors" "fmt" + "github.com/rs/zerolog/log" + "github.com/onflow/cadence" jsoncdc "github.com/onflow/cadence/encoding/json" "github.com/onflow/cadence/runtime/parser" "github.com/onflow/crypto" "github.com/onflow/flow-core-contracts/lib/go/templates" - "github.com/rs/zerolog/log" cadenceutils "github.com/onflow/flow-go/access/utils" "github.com/onflow/flow-go/fvm" "github.com/onflow/flow-go/fvm/systemcontracts" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/execution" + "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/state" "github.com/onflow/flow-go/state/protocol" ) @@ -87,19 +90,28 @@ type TransactionValidationOptions struct { CheckPayerBalance bool } +type ValidationStep struct { + check func(*flow.TransactionBody) error + failReason string +} + type TransactionValidator struct { - blocks Blocks // for looking up blocks to check transaction expiry - chain flow.Chain // for checking validity of addresses - options TransactionValidationOptions - serviceAccountAddress flow.Address - limiter RateLimiter - scriptExecutor execution.ScriptExecutor - verifyPayerBalanceScript []byte + blocks Blocks // for looking up blocks to check transaction expiry + chain flow.Chain // for checking validity of addresses + options TransactionValidationOptions + serviceAccountAddress flow.Address + limiter RateLimiter + scriptExecutor execution.ScriptExecutor + verifyPayerBalanceScript []byte + transactionValidationMetrics module.TransactionValidationMetrics + + validationSteps []ValidationStep } func NewTransactionValidator( blocks Blocks, chain flow.Chain, + transactionValidationMetrics module.TransactionValidationMetrics, options TransactionValidationOptions, executor execution.ScriptExecutor, ) (*TransactionValidator, error) { @@ -109,80 +121,68 @@ func NewTransactionValidator( env := systemcontracts.SystemContractsForChain(chain.ChainID()).AsTemplateEnv() - return &TransactionValidator{ - blocks: blocks, - chain: chain, - options: options, - serviceAccountAddress: chain.ServiceAddress(), - limiter: NewNoopLimiter(), - scriptExecutor: executor, - verifyPayerBalanceScript: templates.GenerateVerifyPayerBalanceForTxExecution(env), - }, nil + txValidator := &TransactionValidator{ + blocks: blocks, + chain: chain, + options: options, + serviceAccountAddress: chain.ServiceAddress(), + limiter: NewNoopLimiter(), + scriptExecutor: executor, + verifyPayerBalanceScript: templates.GenerateVerifyPayerBalanceForTxExecution(env), + transactionValidationMetrics: transactionValidationMetrics, + } + + txValidator.initValidationSteps() + + return txValidator, nil } func NewTransactionValidatorWithLimiter( blocks Blocks, chain flow.Chain, options TransactionValidationOptions, + transactionValidationMetrics module.TransactionValidationMetrics, rateLimiter RateLimiter, ) *TransactionValidator { - return &TransactionValidator{ - blocks: blocks, - chain: chain, - options: options, - serviceAccountAddress: chain.ServiceAddress(), - limiter: rateLimiter, - } -} - -func (v *TransactionValidator) Validate(ctx context.Context, tx *flow.TransactionBody) (err error) { - // rate limit transactions for specific payers. - // a short term solution to prevent attacks that send too many failed transactions - // if a transaction is from a payer that should be rate limited, all the following - // checks will be skipped - err = v.checkRateLimitPayer(tx) - if err != nil { - return err - } - - err = v.checkTxSizeLimit(tx) - if err != nil { - return err - } - - err = v.checkMissingFields(tx) - if err != nil { - return err + txValidator := &TransactionValidator{ + blocks: blocks, + chain: chain, + options: options, + serviceAccountAddress: chain.ServiceAddress(), + limiter: rateLimiter, + transactionValidationMetrics: transactionValidationMetrics, } - err = v.checkGasLimit(tx) - if err != nil { - return err - } - - err = v.checkExpiry(tx) - if err != nil { - return err - } + txValidator.initValidationSteps() - err = v.checkCanBeParsed(tx) - if err != nil { - return err - } + return txValidator +} - err = v.checkAddresses(tx) - if err != nil { - return err +func (v *TransactionValidator) initValidationSteps() { + v.validationSteps = []ValidationStep{ + // rate limit transactions for specific payers. + // a short term solution to prevent attacks that send too many failed transactions + // if a transaction is from a payer that should be rate limited, all the following + // checks will be skipped + {v.checkRateLimitPayer, metrics.InvalidTransactionRateLimit}, + {v.checkTxSizeLimit, metrics.InvalidTransactionByteSize}, + {v.checkMissingFields, metrics.IncompleteTransaction}, + {v.checkGasLimit, metrics.InvalidGasLimit}, + {v.checkExpiry, metrics.ExpiredTransaction}, + {v.checkCanBeParsed, metrics.InvalidScript}, + {v.checkAddresses, metrics.InvalidAddresses}, + {v.checkSignatureFormat, metrics.InvalidSignature}, + {v.checkSignatureDuplications, metrics.DuplicatedSignature}, } +} - err = v.checkSignatureFormat(tx) - if err != nil { - return err - } +func (v *TransactionValidator) Validate(ctx context.Context, tx *flow.TransactionBody) (err error) { - err = v.checkSignatureDuplications(tx) - if err != nil { - return err + for _, step := range v.validationSteps { + if err = step.check(tx); err != nil { + v.transactionValidationMetrics.TransactionValidationFailed(step.failReason) + return err + } } err = v.checkSufficientBalanceToPayForTransaction(ctx, tx) @@ -192,15 +192,19 @@ func (v *TransactionValidator) Validate(ctx context.Context, tx *flow.Transactio // are 'internal' and related to script execution process. they shouldn't // prevent the transaction from proceeding. if IsInsufficientBalanceError(err) { + v.transactionValidationMetrics.TransactionValidationFailed(metrics.InsufficientBalance) return err } // log and ignore all other errors + v.transactionValidationMetrics.TransactionValidationSkipped() log.Info().Err(err).Msg("check payer validation skipped due to error") } // TODO replace checkSignatureFormat by verifying the account/payer signatures + v.transactionValidationMetrics.TransactionValidated() + return nil } @@ -328,7 +332,6 @@ func (v *TransactionValidator) checkCanBeParsed(tx *flow.TransactionBody) error } func (v *TransactionValidator) checkAddresses(tx *flow.TransactionBody) error { - for _, address := range append(tx.Authorizers, tx.Payer) { // we check whether this is a valid output of the address generator if !v.chain.IsValid(address) { @@ -356,7 +359,6 @@ func (v *TransactionValidator) checkSignatureDuplications(tx *flow.TransactionBo } func (v *TransactionValidator) checkSignatureFormat(tx *flow.TransactionBody) error { - for _, signature := range append(tx.PayloadSignatures, tx.EnvelopeSignatures...) { // check the format of the signature is valid. // a valid signature is an ECDSA signature of either P-256 or secp256k1 curve. diff --git a/access/validator_test.go b/access/validator_test.go index 1554a91cfcb..87b7ade62a9 100644 --- a/access/validator_test.go +++ b/access/validator_test.go @@ -17,7 +17,9 @@ import ( accessmock "github.com/onflow/flow-go/access/mock" "github.com/onflow/flow-go/fvm" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" execmock "github.com/onflow/flow-go/module/execution/mock" + "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/utils/unittest" ) @@ -31,9 +33,11 @@ type TransactionValidatorSuite struct { header *flow.Header chain flow.Chain validatorOptions access.TransactionValidationOptions + metrics module.TransactionValidationMetrics } func (s *TransactionValidatorSuite) SetupTest() { + s.metrics = metrics.NewNoopCollector() s.blocks = accessmock.NewBlocks(s.T()) assert.NotNil(s.T(), s.blocks) @@ -89,7 +93,7 @@ func (s *TransactionValidatorSuite) TestTransactionValidator_ScriptExecutorInter Return(nil, errors.New("script executor internal error")). Once() - validator, err := access.NewTransactionValidator(s.blocks, s.chain, s.validatorOptions, scriptExecutor) + validator, err := access.NewTransactionValidator(s.blocks, s.chain, s.metrics, s.validatorOptions, scriptExecutor) assert.NoError(s.T(), err) assert.NotNil(s.T(), validator) @@ -116,7 +120,7 @@ func (s *TransactionValidatorSuite) TestTransactionValidator_SufficientBalance() Return(actualResponse, nil). Once() - validator, err := access.NewTransactionValidator(s.blocks, s.chain, s.validatorOptions, scriptExecutor) + validator, err := access.NewTransactionValidator(s.blocks, s.chain, s.metrics, s.validatorOptions, scriptExecutor) assert.NoError(s.T(), err) assert.NotNil(s.T(), validator) @@ -147,7 +151,7 @@ func (s *TransactionValidatorSuite) TestTransactionValidator_InsufficientBalance assert.NoError(s.T(), err) assert.NotNil(s.T(), actualAccountResponse) - validator, err := access.NewTransactionValidator(s.blocks, s.chain, s.validatorOptions, scriptExecutor) + validator, err := access.NewTransactionValidator(s.blocks, s.chain, s.metrics, s.validatorOptions, scriptExecutor) assert.NoError(s.T(), err) assert.NotNil(s.T(), validator) diff --git a/cmd/access/node_builder/access_node_builder.go b/cmd/access/node_builder/access_node_builder.go index 6f381a867fb..926b25357ff 100644 --- a/cmd/access/node_builder/access_node_builder.go +++ b/cmd/access/node_builder/access_node_builder.go @@ -287,43 +287,44 @@ type FlowAccessNodeBuilder struct { *AccessNodeConfig // components - FollowerState protocol.FollowerState - SyncCore *chainsync.Core - RpcEng *rpc.Engine - FollowerDistributor *consensuspubsub.FollowerDistributor - CollectionRPC access.AccessAPIClient - TransactionTimings *stdmap.TransactionTimings - CollectionsToMarkFinalized *stdmap.Times - CollectionsToMarkExecuted *stdmap.Times - BlocksToMarkExecuted *stdmap.Times - TransactionMetrics *metrics.TransactionCollector - RestMetrics *metrics.RestCollector - AccessMetrics module.AccessMetrics - PingMetrics module.PingMetrics - Committee hotstuff.DynamicCommittee - Finalized *flow.Header // latest finalized block that the node knows of at startup time - Pending []*flow.Header - FollowerCore module.HotStuffFollower - Validator hotstuff.Validator - ExecutionDataDownloader execution_data.Downloader - PublicBlobService network.BlobService - ExecutionDataRequester state_synchronization.ExecutionDataRequester - ExecutionDataStore execution_data.ExecutionDataStore - ExecutionDataBlobstore blobs.Blobstore - ExecutionDataCache *execdatacache.ExecutionDataCache - ExecutionIndexer *indexer.Indexer - ExecutionIndexerCore *indexer.IndexerCore - ScriptExecutor *backend.ScriptExecutor - RegistersAsyncStore *execution.RegistersAsyncStore - Reporter *index.Reporter - EventsIndex *index.EventsIndex - TxResultsIndex *index.TransactionResultsIndex - IndexerDependencies *cmd.DependencyList - collectionExecutedMetric module.CollectionExecutedMetric - ExecutionDataPruner *pruner.Pruner - ExecutionDatastoreManager edstorage.DatastoreManager - ExecutionDataTracker tracker.Storage - VersionControl *version.VersionControl + FollowerState protocol.FollowerState + SyncCore *chainsync.Core + RpcEng *rpc.Engine + FollowerDistributor *consensuspubsub.FollowerDistributor + CollectionRPC access.AccessAPIClient + TransactionTimings *stdmap.TransactionTimings + CollectionsToMarkFinalized *stdmap.Times + CollectionsToMarkExecuted *stdmap.Times + BlocksToMarkExecuted *stdmap.Times + TransactionMetrics *metrics.TransactionCollector + TransactionValidationMetrics *metrics.TransactionValidationCollector + RestMetrics *metrics.RestCollector + AccessMetrics module.AccessMetrics + PingMetrics module.PingMetrics + Committee hotstuff.DynamicCommittee + Finalized *flow.Header // latest finalized block that the node knows of at startup time + Pending []*flow.Header + FollowerCore module.HotStuffFollower + Validator hotstuff.Validator + ExecutionDataDownloader execution_data.Downloader + PublicBlobService network.BlobService + ExecutionDataRequester state_synchronization.ExecutionDataRequester + ExecutionDataStore execution_data.ExecutionDataStore + ExecutionDataBlobstore blobs.Blobstore + ExecutionDataCache *execdatacache.ExecutionDataCache + ExecutionIndexer *indexer.Indexer + ExecutionIndexerCore *indexer.IndexerCore + ScriptExecutor *backend.ScriptExecutor + RegistersAsyncStore *execution.RegistersAsyncStore + Reporter *index.Reporter + EventsIndex *index.EventsIndex + TxResultsIndex *index.TransactionResultsIndex + IndexerDependencies *cmd.DependencyList + collectionExecutedMetric module.CollectionExecutedMetric + ExecutionDataPruner *pruner.Pruner + ExecutionDatastoreManager edstorage.DatastoreManager + ExecutionDataTracker tracker.Storage + VersionControl *version.VersionControl // The sync engine participants provider is the libp2p peer store for the access node // which is not available until after the network has started. @@ -1665,6 +1666,10 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { ) return nil }). + Module("transaction validation metrics", func(node *cmd.NodeConfig) error { + builder.TransactionValidationMetrics = metrics.NewTransactionValidationCollector() + return nil + }). Module("rest metrics", func(node *cmd.NodeConfig) error { m, err := metrics.NewRestCollector(routes.URLToRoute, node.MetricsRegisterer) if err != nil { @@ -1676,6 +1681,7 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { Module("access metrics", func(node *cmd.NodeConfig) error { builder.AccessMetrics = metrics.NewAccessCollector( metrics.WithTransactionMetrics(builder.TransactionMetrics), + metrics.WithTransactionValidationMetrics(builder.TransactionValidationMetrics), metrics.WithBackendScriptsMetrics(builder.TransactionMetrics), metrics.WithRestMetrics(builder.RestMetrics), ) diff --git a/engine/access/rpc/backend/backend.go b/engine/access/rpc/backend/backend.go index 184c37e0bcd..530c7943a1b 100644 --- a/engine/access/rpc/backend/backend.go +++ b/engine/access/rpc/backend/backend.go @@ -246,7 +246,7 @@ func New(params Params) (*Backend, error) { versionControl: params.VersionControl, } - txValidator, err := configureTransactionValidator(params.State, params.ChainID, params.ScriptExecutor, params.CheckPayerBalance) + txValidator, err := configureTransactionValidator(params.State, params.ChainID, params.AccessMetrics, params.ScriptExecutor, params.CheckPayerBalance) if err != nil { return nil, fmt.Errorf("could not create transaction validator: %w", err) } @@ -310,10 +310,17 @@ func identifierList(ids []string) (flow.IdentifierList, error) { return idList, nil } -func configureTransactionValidator(state protocol.State, chainID flow.ChainID, executor execution.ScriptExecutor, checkPayerBalance bool) (*access.TransactionValidator, error) { +func configureTransactionValidator( + state protocol.State, + chainID flow.ChainID, + transactionMetrics module.TransactionValidationMetrics, + executor execution.ScriptExecutor, + checkPayerBalance bool, +) (*access.TransactionValidator, error) { return access.NewTransactionValidator( access.NewProtocolStateBlocks(state), chainID.Chain(), + transactionMetrics, access.TransactionValidationOptions{ Expiry: flow.DefaultTransactionExpiry, ExpiryBuffer: flow.DefaultTransactionExpiryBuffer, diff --git a/engine/collection/ingest/engine.go b/engine/collection/ingest/engine.go index a8adcedaded..438e6bea88b 100644 --- a/engine/collection/ingest/engine.go +++ b/engine/collection/ingest/engine.go @@ -71,6 +71,7 @@ func New( MaxTransactionByteSize: config.MaxTransactionByteSize, MaxCollectionByteSize: config.MaxCollectionByteSize, }, + colMetrics, limiter, ) diff --git a/go.mod b/go.mod index 6fc6dc77172..35311028dca 100644 --- a/go.mod +++ b/go.mod @@ -255,13 +255,13 @@ require ( github.com/multiformats/go-multicodec v0.9.0 // indirect github.com/multiformats/go-multistream v0.5.0 // indirect github.com/multiformats/go-varint v0.0.7 // indirect - github.com/nxadm/tail v1.4.8 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/onflow/flow-ft/lib/go/contracts v1.0.0 // indirect github.com/onflow/flow-ft/lib/go/templates v1.0.0 // indirect github.com/onflow/flow-nft/lib/go/contracts v1.2.1 // indirect github.com/onflow/flow-nft/lib/go/templates v1.2.0 // indirect github.com/onflow/sdks v0.6.0-preview.1 // indirect + github.com/onsi/ginkgo v1.16.5 // indirect github.com/onsi/ginkgo/v2 v2.13.2 // indirect github.com/opencontainers/runtime-spec v1.1.0 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect diff --git a/go.sum b/go.sum index d750ce109b2..ba0e842329e 100644 --- a/go.sum +++ b/go.sum @@ -1480,6 +1480,7 @@ github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= @@ -2209,8 +2210,9 @@ github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= -github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.13.2 h1:Bi2gGVkfn6gQcjNjZJVO8Gf0FHzMPf2phUei9tejVMs= github.com/onsi/ginkgo/v2 v2.13.2/go.mod h1:XStQ8QcGwLyF4HdfcZB8SFOS/MWCgDuXMSBe6zrvLgM= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= @@ -2991,6 +2993,7 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -3181,6 +3184,7 @@ golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= diff --git a/insecure/go.mod b/insecure/go.mod index 2c31ae9c8cd..4eb97a8aab2 100644 --- a/insecure/go.mod +++ b/insecure/go.mod @@ -200,6 +200,7 @@ require ( github.com/multiformats/go-multihash v0.2.3 // indirect github.com/multiformats/go-multistream v0.5.0 // indirect github.com/multiformats/go-varint v0.0.7 // indirect + github.com/nxadm/tail v1.4.8 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/onflow/atree v0.8.0-rc.6 // indirect github.com/onflow/cadence v1.0.0-preview.50 // indirect @@ -214,7 +215,6 @@ require ( github.com/onflow/go-ethereum v1.14.7 // indirect github.com/onflow/sdks v0.6.0-preview.1 // indirect github.com/onflow/wal v1.0.2 // indirect - github.com/onsi/ginkgo v1.16.5 // indirect github.com/onsi/ginkgo/v2 v2.13.2 // indirect github.com/opencontainers/runtime-spec v1.1.0 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect diff --git a/insecure/go.sum b/insecure/go.sum index 616e2155cd0..e9a54b19be8 100644 --- a/insecure/go.sum +++ b/insecure/go.sum @@ -1475,7 +1475,6 @@ github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= @@ -2977,7 +2976,6 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -3168,7 +3166,6 @@ golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= diff --git a/integration/go.mod b/integration/go.mod index dbc4a985f47..92fb18f5522 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -24,7 +24,7 @@ require ( github.com/onflow/crypto v0.25.2 github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1 github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1 - github.com/onflow/flow-emulator v1.0.0-preview.36.0.20240729195722-d4eb1c30eb9f + github.com/onflow/flow-emulator v1.0.0-preview.41.0.20240829134601-0be55d6970b5 github.com/onflow/flow-go v0.37.7-0.20240826193109-e211841b59f5 github.com/onflow/flow-go-sdk v1.0.0-preview.53 github.com/onflow/flow-go/insecure v0.0.0-00010101000000-000000000000 diff --git a/integration/go.sum b/integration/go.sum index 5200631813b..ddb9b4bb31e 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -2153,8 +2153,8 @@ github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1 h1:q9tXLIALwQ76bO4 github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1/go.mod h1:u/mkP/B+PbV33tEG3qfkhhBlydSvAKxfLZSfB4lsJHg= github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1 h1:FfhMBAb78p6VAWkJ+iqdKLErGQVQgxk5w6DP5ZruWX8= github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1/go.mod h1:NgbMOYnMh0GN48VsNKZuiwK7uyk38Wyo8jN9+C9QE30= -github.com/onflow/flow-emulator v1.0.0-preview.36.0.20240729195722-d4eb1c30eb9f h1:2Ejpmm2Vrl/XLaE6lniE1vNfi6WYhzqHiCk6oomGoFE= -github.com/onflow/flow-emulator v1.0.0-preview.36.0.20240729195722-d4eb1c30eb9f/go.mod h1:0rqp896zEcjNqqDiQNBUlpS/7nzS4+E+yG/4s0P13bQ= +github.com/onflow/flow-emulator v1.0.0-preview.41.0.20240829134601-0be55d6970b5 h1:Z5PC3Sqvl2UemY27uwUwzkLb8EXUf+m/aEimxFzOXuc= +github.com/onflow/flow-emulator v1.0.0-preview.41.0.20240829134601-0be55d6970b5/go.mod h1:gFafyGD4Qxs+XT2BRSIjQJ86ILSmgm1VYUoCr1nVxVI= github.com/onflow/flow-ft/lib/go/contracts v1.0.0 h1:mToacZ5NWqtlWwk/7RgIl/jeKB/Sy/tIXdw90yKHcV0= github.com/onflow/flow-ft/lib/go/contracts v1.0.0/go.mod h1:PwsL8fC81cjnUnTfmyL/HOIyHnyaw/JA474Wfj2tl6A= github.com/onflow/flow-ft/lib/go/templates v1.0.0 h1:6cMS/lUJJ17HjKBfMO/eh0GGvnpElPgBXx7h5aoWJhs= diff --git a/module/metrics.go b/module/metrics.go index cfad9bb11a0..f43c8b9325e 100644 --- a/module/metrics.go +++ b/module/metrics.go @@ -607,6 +607,7 @@ type CruiseCtlMetrics interface { } type CollectionMetrics interface { + TransactionValidationMetrics // TransactionIngested is called when a new transaction is ingested by the // node. It increments the total count of ingested transactions and starts // a tx->col span for the transaction. @@ -907,6 +908,7 @@ type AccessMetrics interface { RestMetrics GRPCConnectionPoolMetrics TransactionMetrics + TransactionValidationMetrics BackendScriptsMetrics // UpdateExecutionReceiptMaxHeight is called whenever we store an execution receipt from a block from a newer height @@ -1085,6 +1087,15 @@ type TransactionMetrics interface { TransactionSubmissionFailed() } +type TransactionValidationMetrics interface { + // TransactionValidated tracks number of successfully validated transactions + TransactionValidated() + // TransactionValidationFailed tracks number of validation failed transactions with reason + TransactionValidationFailed(reason string) + // TransactionValidationSkipped tracks number of skipped transaction validations + TransactionValidationSkipped() +} + type PingMetrics interface { // NodeReachable tracks the round trip time in milliseconds taken to ping a node // The nodeInfo provides additional information about the node such as the name of the node operator diff --git a/module/metrics/access.go b/module/metrics/access.go index 1116f87f433..aacfe316c76 100644 --- a/module/metrics/access.go +++ b/module/metrics/access.go @@ -10,6 +10,12 @@ import ( type AccessCollectorOpts func(*AccessCollector) +func WithTransactionValidationMetrics(m module.TransactionValidationMetrics) AccessCollectorOpts { + return func(ac *AccessCollector) { + ac.TransactionValidationMetrics = m + } +} + func WithTransactionMetrics(m module.TransactionMetrics) AccessCollectorOpts { return func(ac *AccessCollector) { ac.TransactionMetrics = m @@ -31,6 +37,7 @@ func WithRestMetrics(m module.RestMetrics) AccessCollectorOpts { type AccessCollector struct { module.RestMetrics module.TransactionMetrics + module.TransactionValidationMetrics module.BackendScriptsMetrics connectionReused prometheus.Counter diff --git a/module/metrics/collection.go b/module/metrics/collection.go index fa97a28b679..557eab48b77 100644 --- a/module/metrics/collection.go +++ b/module/metrics/collection.go @@ -10,6 +10,7 @@ import ( ) type CollectionCollector struct { + module.TransactionValidationMetrics tracer module.Tracer transactionsIngested prometheus.Counter // tracks the number of ingested transactions finalizedHeight *prometheus.GaugeVec // tracks the finalized height @@ -17,11 +18,13 @@ type CollectionCollector struct { guarantees *prometheus.HistogramVec // counts the number/size of FINALIZED collections } +var _ module.CollectionMetrics = (*CollectionCollector)(nil) + func NewCollectionCollector(tracer module.Tracer) *CollectionCollector { cc := &CollectionCollector{ - tracer: tracer, - + TransactionValidationMetrics: NewTransactionValidationCollector(), + tracer: tracer, transactionsIngested: promauto.NewCounter(prometheus.CounterOpts{ Namespace: namespaceCollection, Name: "ingested_transactions_total", diff --git a/module/metrics/labels.go b/module/metrics/labels.go index 916f3e2bc07..82260ca3c5d 100644 --- a/module/metrics/labels.go +++ b/module/metrics/labels.go @@ -155,6 +155,20 @@ const ( MessageEntityResponse = "entity_response" ) +// transaction validation labels +const ( + InvalidTransactionRateLimit = "payer_exceeded_rate_limit" + InvalidTransactionByteSize = "transaction_exceeded_size_limit" + IncompleteTransaction = "missing_fields" + InvalidGasLimit = "invalid_gas_limit" + ExpiredTransaction = "transaction_expired" + InvalidScript = "invalid_script" + InvalidAddresses = "invalid_address" + InvalidSignature = "invalid_signature" + DuplicatedSignature = "duplicate_signature" + InsufficientBalance = "payer_insufficient_balance" +) + const ExecutionDataRequestRetryable = "retryable" const LabelViolationReason = "reason" diff --git a/module/metrics/namespaces.go b/module/metrics/namespaces.go index a35885db7dc..c0f99af3fcf 100644 --- a/module/metrics/namespaces.go +++ b/module/metrics/namespaces.go @@ -44,6 +44,7 @@ const ( const ( subsystemTransactionTiming = "transaction_timing" subsystemTransactionSubmission = "transaction_submission" + subsystemTransactionValidation = "transaction_validation" subsystemConnectionPool = "connection_pool" subsystemHTTP = "http" ) diff --git a/module/metrics/noop.go b/module/metrics/noop.go index fb5ceeeed81..17460bf460a 100644 --- a/module/metrics/noop.go +++ b/module/metrics/noop.go @@ -37,6 +37,7 @@ func (nc *NoopCollector) BlockProposalDuration(duration time.Duration) // interface check var _ module.BackendScriptsMetrics = (*NoopCollector)(nil) var _ module.TransactionMetrics = (*NoopCollector)(nil) +var _ module.TransactionValidationMetrics = (*NoopCollector)(nil) var _ module.HotstuffMetrics = (*NoopCollector)(nil) var _ module.EngineMetrics = (*NoopCollector)(nil) var _ module.HeroCacheMetrics = (*NoopCollector)(nil) @@ -215,6 +216,9 @@ func (nc *NoopCollector) TransactionReceived(txID flow.Identifier, when time.Tim func (nc *NoopCollector) TransactionFinalized(txID flow.Identifier, when time.Time) {} func (nc *NoopCollector) TransactionExecuted(txID flow.Identifier, when time.Time) {} func (nc *NoopCollector) TransactionExpired(txID flow.Identifier) {} +func (nc *NoopCollector) TransactionValidated() {} +func (nc *NoopCollector) TransactionValidationFailed(reason string) {} +func (nc *NoopCollector) TransactionValidationSkipped() {} func (nc *NoopCollector) TransactionSubmissionFailed() {} func (nc *NoopCollector) UpdateExecutionReceiptMaxHeight(height uint64) {} func (nc *NoopCollector) UpdateLastFullBlockHeight(height uint64) {} diff --git a/module/metrics/transaction_validation.go b/module/metrics/transaction_validation.go new file mode 100644 index 00000000000..70cfe2d7a6a --- /dev/null +++ b/module/metrics/transaction_validation.go @@ -0,0 +1,59 @@ +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + + "github.com/onflow/flow-go/module" +) + +type NamespaceType string + +// TransactionValidationCollector the metrics for transaction validation functionality +type TransactionValidationCollector struct { + transactionValidated prometheus.Counter + transactionValidationSkipped prometheus.Counter + transactionValidationFailed *prometheus.CounterVec +} + +// interface check +var _ module.TransactionValidationMetrics = (*TransactionValidationCollector)(nil) + +// NewTransactionValidationCollector creates new instance of TransactionValidationCollector +func NewTransactionValidationCollector() *TransactionValidationCollector { + return &TransactionValidationCollector{ + transactionValidated: promauto.NewCounter(prometheus.CounterOpts{ + Name: "transaction_validation_successes_total", + Namespace: namespaceAccess, + Subsystem: subsystemTransactionValidation, + Help: "counter for the validated transactions", + }), + transactionValidationSkipped: promauto.NewCounter(prometheus.CounterOpts{ + Name: "transaction_validation_skipped_total", + Namespace: namespaceAccess, + Subsystem: subsystemTransactionValidation, + Help: "counter for the skipped transaction validations", + }), + transactionValidationFailed: promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "transaction_validation_failed_total", + Namespace: namespaceAccess, + Subsystem: subsystemTransactionValidation, + Help: "counter for the failed transactions validation", + }, []string{"reason"}), + } +} + +// TransactionValidated tracks number of successfully validated transactions +func (tc *TransactionValidationCollector) TransactionValidated() { + tc.transactionValidated.Inc() +} + +// TransactionValidationFailed tracks number of validation failed transactions with reason +func (tc *TransactionValidationCollector) TransactionValidationFailed(reason string) { + tc.transactionValidationFailed.WithLabelValues(reason).Inc() +} + +// TransactionValidationSkipped tracks number of skipped transaction validations +func (tc *TransactionValidationCollector) TransactionValidationSkipped() { + tc.transactionValidationSkipped.Inc() +} diff --git a/module/mock/access_metrics.go b/module/mock/access_metrics.go index 9866ad90e02..21ecc03740f 100644 --- a/module/mock/access_metrics.go +++ b/module/mock/access_metrics.go @@ -143,6 +143,21 @@ func (_m *AccessMetrics) TransactionSubmissionFailed() { _m.Called() } +// TransactionValidated provides a mock function with given fields: +func (_m *AccessMetrics) TransactionValidated() { + _m.Called() +} + +// TransactionValidationFailed provides a mock function with given fields: reason +func (_m *AccessMetrics) TransactionValidationFailed(reason string) { + _m.Called(reason) +} + +// TransactionValidationSkipped provides a mock function with given fields: +func (_m *AccessMetrics) TransactionValidationSkipped() { + _m.Called() +} + // UpdateExecutionReceiptMaxHeight provides a mock function with given fields: height func (_m *AccessMetrics) UpdateExecutionReceiptMaxHeight(height uint64) { _m.Called(height) diff --git a/module/mock/collection_metrics.go b/module/mock/collection_metrics.go index bde071e902e..34c5e7efee5 100644 --- a/module/mock/collection_metrics.go +++ b/module/mock/collection_metrics.go @@ -29,6 +29,21 @@ func (_m *CollectionMetrics) TransactionIngested(txID flow.Identifier) { _m.Called(txID) } +// TransactionValidated provides a mock function with given fields: +func (_m *CollectionMetrics) TransactionValidated() { + _m.Called() +} + +// TransactionValidationFailed provides a mock function with given fields: reason +func (_m *CollectionMetrics) TransactionValidationFailed(reason string) { + _m.Called(reason) +} + +// TransactionValidationSkipped provides a mock function with given fields: +func (_m *CollectionMetrics) TransactionValidationSkipped() { + _m.Called() +} + // NewCollectionMetrics creates a new instance of CollectionMetrics. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. func NewCollectionMetrics(t interface { diff --git a/module/mock/transaction_validation_metrics.go b/module/mock/transaction_validation_metrics.go new file mode 100644 index 00000000000..0003f6762f7 --- /dev/null +++ b/module/mock/transaction_validation_metrics.go @@ -0,0 +1,39 @@ +// Code generated by mockery v2.43.2. DO NOT EDIT. + +package mock + +import mock "github.com/stretchr/testify/mock" + +// TransactionValidationMetrics is an autogenerated mock type for the TransactionValidationMetrics type +type TransactionValidationMetrics struct { + mock.Mock +} + +// TransactionValidated provides a mock function with given fields: +func (_m *TransactionValidationMetrics) TransactionValidated() { + _m.Called() +} + +// TransactionValidationFailed provides a mock function with given fields: reason +func (_m *TransactionValidationMetrics) TransactionValidationFailed(reason string) { + _m.Called(reason) +} + +// TransactionValidationSkipped provides a mock function with given fields: +func (_m *TransactionValidationMetrics) TransactionValidationSkipped() { + _m.Called() +} + +// NewTransactionValidationMetrics creates a new instance of TransactionValidationMetrics. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewTransactionValidationMetrics(t interface { + mock.TestingT + Cleanup(func()) +}) *TransactionValidationMetrics { + mock := &TransactionValidationMetrics{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +}