Skip to content

Commit

Permalink
Merge branch 'master' into deriveTransactionStatus-clean-up-blockID-a…
Browse files Browse the repository at this point in the history
…rgument
  • Loading branch information
peterargue authored Apr 23, 2024
2 parents 0dc42cf + eff9395 commit f6814f8
Show file tree
Hide file tree
Showing 119 changed files with 4,318 additions and 2,784 deletions.
1 change: 0 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,6 @@ generate-mocks: install-mock-generators
mockery --name '(Connector|PingInfoProvider)' --dir=network/p2p --case=underscore --output="./network/mocknetwork" --outpkg="mocknetwork"
CGO_CFLAGS=$(CRYPTO_FLAG) mockgen -destination=storage/mocks/storage.go -package=mocks github.com/onflow/flow-go/storage Blocks,Headers,Payloads,Collections,Commits,Events,ServiceEvents,TransactionResults
CGO_CFLAGS=$(CRYPTO_FLAG) mockgen -destination=network/mocknetwork/mock_network.go -package=mocknetwork github.com/onflow/flow-go/network EngineRegistry
mockery --name='.*' --dir=integration/benchmark/mocksiface --case=underscore --output="integration/benchmark/mock" --outpkg="mock"
mockery --name=ExecutionDataStore --dir=module/executiondatasync/execution_data --case=underscore --output="./module/executiondatasync/execution_data/mock" --outpkg="mock"
mockery --name=Downloader --dir=module/executiondatasync/execution_data --case=underscore --output="./module/executiondatasync/execution_data/mock" --outpkg="mock"
mockery --name '(ExecutionDataRequester|IndexReporter)' --dir=module/state_synchronization --case=underscore --output="./module/state_synchronization/mock" --outpkg="state_synchronization"
Expand Down
2 changes: 1 addition & 1 deletion access/api.go
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,7 @@ type API interface {
// SubscribeTransactionStatuses streams transaction statuses starting from the reference block saved in the
// transaction itself until the block containing the transaction becomes sealed or expired. When the transaction
// status becomes TransactionStatusSealed or TransactionStatusExpired, the subscription will automatically shut down.
SubscribeTransactionStatuses(ctx context.Context, tx *flow.TransactionBody) subscription.Subscription
SubscribeTransactionStatuses(ctx context.Context, tx *flow.TransactionBody, requiredEventEncodingVersion entities.EventEncodingVersion) subscription.Subscription
}

// TODO: Combine this with flow.TransactionResult?
Expand Down
27 changes: 20 additions & 7 deletions access/handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
"github.com/onflow/flow-go/engine/common/rpc/convert"
"github.com/onflow/flow-go/model/flow"
"github.com/onflow/flow-go/module"
"github.com/onflow/flow-go/module/counters"

"github.com/onflow/flow/protobuf/go/flow/access"
"github.com/onflow/flow/protobuf/go/flow/entities"
Expand Down Expand Up @@ -708,9 +709,9 @@ func (h *Handler) GetExecutionResultForBlockID(ctx context.Context, req *access.
func (h *Handler) GetExecutionResultByID(ctx context.Context, req *access.GetExecutionResultByIDRequest) (*access.ExecutionResultByIDResponse, error) {
metadata := h.buildMetadataResponse()

blockID := convert.MessageToIdentifier(req.GetId())
resultID := convert.MessageToIdentifier(req.GetId())

result, err := h.api.GetExecutionResultByID(ctx, blockID)
result, err := h.api.GetExecutionResultByID(ctx, resultID)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -1112,11 +1113,23 @@ func (h *Handler) SendAndSubscribeTransactionStatuses(
return err
}

sub := h.api.SubscribeTransactionStatuses(ctx, &tx)
return subscription.HandleSubscription(sub, func(txSubInfo *convert.TransactionSubscribeInfo) error {
err = stream.Send(convert.TransactionSubscribeInfoToMessage(txSubInfo))
if err != nil {
return rpc.ConvertError(err, "could not send response", codes.Internal)
sub := h.api.SubscribeTransactionStatuses(ctx, &tx, request.GetEventEncodingVersion())

messageIndex := counters.NewMonotonousCounter(0)
return subscription.HandleSubscription(sub, func(txResults []*TransactionResult) error {
for i := range txResults {
value := messageIndex.Value()
if ok := messageIndex.Set(value + 1); !ok {
return status.Errorf(codes.Internal, "the message index has already been incremented to %d", messageIndex.Value())
}

err = stream.Send(&access.SendAndSubscribeTransactionStatusesResponse{
TransactionResults: TransactionResultToMessage(txResults[i]),
MessageIndex: value,
})
if err != nil {
return rpc.ConvertError(err, "could not send response", codes.Internal)
}
}

return nil
Expand Down
10 changes: 5 additions & 5 deletions access/mock/api.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

13 changes: 11 additions & 2 deletions admin/commands/storage/read_protocol_snapshot.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ func (s *ProtocolSnapshotCommand) Handler(_ context.Context, req *admin.CommandR

s.logger.Info().Uint("blocksToSkip", blocksToSkip).Msgf("admintool: generating protocol snapshot")

snapshot, sealedHeight, commit, err := common.GenerateProtocolSnapshotForCheckpoint(
snapshot, sealedHeight, commit, checkpointFile, err := common.GenerateProtocolSnapshotForCheckpoint(
s.logger, s.state, s.headers, s.seals, s.checkpointDir, blocksToSkip)
if err != nil {
return nil, fmt.Errorf("could not generate protocol snapshot for checkpoint, checkpointDir %v: %w",
Expand All @@ -79,10 +79,19 @@ func (s *ProtocolSnapshotCommand) Handler(_ context.Context, req *admin.CommandR
Hex("finalized_block_id", logging.Entity(header)).
Uint64("sealed_height", sealedHeight).
Hex("sealed_commit", commit[:]). // not the commit for the finalized height, but for the sealed height
Str("checkpoint_file", checkpointFile).
Uint("blocks_to_skip", blocksToSkip).
Msgf("admintool: protocol snapshot generated successfully")

return commands.ConvertToMap(serializable.Encodable())
return commands.ConvertToMap(protocolSnapshotResponse{
Snapshot: serializable.Encodable(),
Checkpoint: checkpointFile,
})
}

type protocolSnapshotResponse struct {
Snapshot inmem.EncodableSnapshot `json:"snapshot"`
Checkpoint string `json:"checkpoint"`
}

func (s *ProtocolSnapshotCommand) Validator(req *admin.CommandRequest) error {
Expand Down
2 changes: 1 addition & 1 deletion cmd/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ WORKDIR /app

ARG GOARCH=amd64
# TAGS can be overriden to modify the go build tags (e.g. build without netgo)
ARG TAGS="netgo"
ARG TAGS="netgo,osusergo"
# CC flag can be overwritten to specify a C compiler
ARG CC=""
# CGO_FLAG uses ADX instructions by default, flag can be overwritten to build without ADX
Expand Down
2 changes: 1 addition & 1 deletion cmd/access/node_builder/access_node_builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -1315,7 +1315,7 @@ func (builder *FlowAccessNodeBuilder) InitIDProviders() {
filter.And(
filter.HasRole[flow.Identity](flow.RoleConsensus),
filter.Not(filter.HasNodeID[flow.Identity](node.Me.NodeID())),
underlay.NotEjectedFilter,
filter.NotEjectedFilter,
),
builder.IdentityProvider,
)
Expand Down
11 changes: 9 additions & 2 deletions cmd/bootstrap/cmd/check_machine_account.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
sdk "github.com/onflow/flow-go-sdk"
client "github.com/onflow/flow-go-sdk/access/grpc"
"github.com/onflow/flow-go/cmd"
"github.com/onflow/flow-go/cmd/util/cmd/common"
model "github.com/onflow/flow-go/model/bootstrap"
"github.com/onflow/flow-go/module/epochs"
)
Expand Down Expand Up @@ -44,7 +45,10 @@ func checkMachineAccountRun(_ *cobra.Command, _ []string) {

// read the private node information - used to get the role
var nodeInfoPriv model.NodeInfoPriv
readJSON(filepath.Join(flagOutdir, fmt.Sprintf(model.PathNodeInfoPriv, nodeID)), &nodeInfoPriv)
err = common.ReadJSON(filepath.Join(flagOutdir, fmt.Sprintf(model.PathNodeInfoPriv, nodeID)), &nodeInfoPriv)
if err != nil {
log.Fatal().Err(err).Msg("failed to read json")
}

// read the machine account info file
machineAccountInfo := readMachineAccountInfo(nodeID)
Expand Down Expand Up @@ -97,7 +101,10 @@ func readMachineAccountInfo(nodeID string) model.NodeMachineAccountInfo {
var machineAccountInfo model.NodeMachineAccountInfo

path := filepath.Join(flagOutdir, fmt.Sprintf(model.PathNodeMachineAccountInfoPriv, nodeID))
readJSON(path, &machineAccountInfo)
err := common.ReadJSON(path, &machineAccountInfo)
if err != nil {
log.Fatal().Err(err).Msg("failed to read json")
}

return machineAccountInfo
}
125 changes: 0 additions & 125 deletions cmd/bootstrap/cmd/clusters.go

This file was deleted.

10 changes: 8 additions & 2 deletions cmd/bootstrap/cmd/db_encryption_key.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"github.com/spf13/cobra"

"github.com/onflow/flow-go/cmd/bootstrap/utils"
"github.com/onflow/flow-go/cmd/util/cmd/common"
model "github.com/onflow/flow-go/model/bootstrap"
)

Expand Down Expand Up @@ -35,7 +36,7 @@ func dbEncryptionKeyRun(_ *cobra.Command, _ []string) {
log = log.With().Str("path", dbEncryptionKeyPath).Logger()

// check if the key already exists
exists, err := pathExists(path.Join(flagOutdir, dbEncryptionKeyPath))
exists, err := common.PathExists(path.Join(flagOutdir, dbEncryptionKeyPath))
if err != nil {
log.Fatal().Err(err).Msg("could not check if db encryption key already exists")
}
Expand All @@ -50,5 +51,10 @@ func dbEncryptionKeyRun(_ *cobra.Command, _ []string) {
}
log.Info().Msg("generated db encryption key")

writeText(dbEncryptionKeyPath, dbEncryptionKey)
err = common.WriteText(dbEncryptionKeyPath, flagOutdir, dbEncryptionKey)
if err != nil {
log.Fatal().Err(err).Msg("failed to write file")
}

log.Info().Msgf("wrote file %s/%s", flagOutdir, dbEncryptionKeyPath)
}
13 changes: 11 additions & 2 deletions cmd/bootstrap/cmd/dkg.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"github.com/onflow/crypto"

bootstrapDKG "github.com/onflow/flow-go/cmd/bootstrap/dkg"
"github.com/onflow/flow-go/cmd/util/cmd/common"
model "github.com/onflow/flow-go/model/bootstrap"
"github.com/onflow/flow-go/model/dkg"
"github.com/onflow/flow-go/model/encodable"
Expand Down Expand Up @@ -38,17 +39,25 @@ func runBeaconKG(nodes []model.NodeInfo) dkg.DKGData {
encKey := encodable.RandomBeaconPrivKey{PrivateKey: privKey}
privKeyShares = append(privKeyShares, encKey)

writeJSON(fmt.Sprintf(model.PathRandomBeaconPriv, nodeID), encKey)
err = common.WriteJSON(fmt.Sprintf(model.PathRandomBeaconPriv, nodeID), flagOutdir, encKey)
if err != nil {
log.Fatal().Err(err).Msg("failed to write json")
}
log.Info().Msgf("wrote file %s/%s", flagOutdir, fmt.Sprintf(model.PathRandomBeaconPriv, nodeID))
}

// write full DKG info that will be used to construct QC
writeJSON(model.PathRootDKGData, inmem.EncodableFullDKG{
err = common.WriteJSON(model.PathRootDKGData, flagOutdir, inmem.EncodableFullDKG{
GroupKey: encodable.RandomBeaconPubKey{
PublicKey: dkgData.PubGroupKey,
},
PubKeyShares: pubKeyShares,
PrivKeyShares: privKeyShares,
})
if err != nil {
log.Fatal().Err(err).Msg("failed to write json")
}
log.Info().Msgf("wrote file %s/%s", flagOutdir, model.PathRootDKGData)

return dkgData
}
Loading

0 comments on commit f6814f8

Please sign in to comment.