EFM Recovery Integration Tests: Part 1

cmd/bootstrap/cmd/rootblock.go

@@ -187,7 +187,7 @@ func rootBlock(cmd *cobra.Command, args []string) {
 	log.Info().Msg("")
 
 	log.Info().Msg("constructing root QCs for collection node clusters")
-	clusterQCs := common.ConstructRootQCsForClusters(log, clusters, internalNodes, clusterBlocks)
+	clusterQCs := run.ConstructRootQCsForClusters(log, clusters, internalNodes, clusterBlocks)
 	log.Info().Msg("")
 
 	log.Info().Msg("constructing root header")

cmd/bootstrap/run/epochs.go

@@ -0,0 +1,208 @@
+package run
+
+import (
+	"fmt"
+
+	"github.com/rs/zerolog"
+
+	"github.com/onflow/cadence"
+
+	"github.com/onflow/flow-go/cmd/util/cmd/common"
+	"github.com/onflow/flow-go/fvm/systemcontracts"
+	"github.com/onflow/flow-go/model/bootstrap"
+	model "github.com/onflow/flow-go/model/bootstrap"
+	"github.com/onflow/flow-go/model/cluster"
+	"github.com/onflow/flow-go/model/flow"
+	"github.com/onflow/flow-go/model/flow/filter"
+	"github.com/onflow/flow-go/state/protocol/inmem"
+)
+
+// GenerateRecoverEpochTxArgs generates the required transaction arguments for the `recoverEpoch` transaction.
+func GenerateRecoverEpochTxArgs(log zerolog.Logger,
+	internalNodePrivInfoDir string,
+	nodeConfigJson string,
+	collectionClusters int,
+	epochCounter uint64,
+	rootChainID flow.ChainID,
+	numViewsInStakingAuction uint64,
+	numViewsInEpoch uint64,
+	targetDuration uint64,
+	initNewEpoch bool,
+	snapshot *inmem.Snapshot,
+) ([]cadence.Value, error) {
+	epoch := snapshot.Epochs().Current()
+
+	currentEpochIdentities, err := snapshot.Identities(filter.IsValidProtocolParticipant)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get  valid protocol participants from snapshot: %w", err)
+	}
+	// We need canonical ordering here; sanity check to enforce this:
+	if !currentEpochIdentities.Sorted(flow.Canonical[flow.Identity]) {
+		return nil, fmt.Errorf("identies from snapshot not in canonical order")
+	}
+
+	// separate collector nodes by internal and partner nodes
+	collectors := currentEpochIdentities.Filter(filter.HasRole[flow.Identity](flow.RoleCollection))
+	internalCollectors := make(flow.IdentityList, 0)
+	partnerCollectors := make(flow.IdentityList, 0)
+
+	log.Info().Msg("collecting internal node network and staking keys")
+	internalNodes, err := common.ReadFullInternalNodeInfos(log, internalNodePrivInfoDir, nodeConfigJson)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read full internal node infos: %w", err)
+	}
+
+	internalNodesMap := make(map[flow.Identifier]struct{})
+	for _, node := range internalNodes {
+		if !currentEpochIdentities.Exists(node.Identity()) {
+			return nil, fmt.Errorf("node ID found in internal node infos missing from protocol snapshot identities %s: %w", node.NodeID, err)
+		}
+		internalNodesMap[node.NodeID] = struct{}{}
+	}
+	log.Info().Msg("")
+
+	for _, collector := range collectors {
+		if _, ok := internalNodesMap[collector.NodeID]; ok {
+			internalCollectors = append(internalCollectors, collector)
+		} else {
+			partnerCollectors = append(partnerCollectors, collector)
+		}
+	}
+
+	currentEpochDKG, err := epoch.DKG()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get DKG for current epoch: %w", err)
+	}
+
+	log.Info().Msg("computing collection node clusters")
+
+	assignments, clusters, err := common.ConstructClusterAssignment(log, partnerCollectors, internalCollectors, collectionClusters)
+	if err != nil {
+		log.Fatal().Err(err).Msg("unable to generate cluster assignment")
+	}
+	log.Info().Msg("")
+
+	log.Info().Msg("constructing root blocks for collection node clusters")
+	clusterBlocks := GenerateRootClusterBlocks(epochCounter, clusters)
+	log.Info().Msg("")
+
+	log.Info().Msg("constructing root QCs for collection node clusters")
+	clusterQCs := ConstructRootQCsForClusters(log, clusters, internalNodes, clusterBlocks)
+	log.Info().Msg("")
+
+	dkgPubKeys := make([]cadence.Value, 0)
+	nodeIds := make([]cadence.Value, 0)
+
+	// NOTE: The RecoveryEpoch will re-use the last successful DKG output. This means that the consensus
+	// committee in the RecoveryEpoch must be identical to the committee which participated in that DKG.
+	dkgGroupKeyCdc, cdcErr := cadence.NewString(currentEpochDKG.GroupKey().String())
+	if cdcErr != nil {
+		log.Fatal().Err(cdcErr).Msg("failed to get dkg group key cadence string")
+	}
+	dkgPubKeys = append(dkgPubKeys, dkgGroupKeyCdc)
+	for _, id := range currentEpochIdentities {
+		if id.GetRole() == flow.RoleConsensus {
+			dkgPubKey, keyShareErr := currentEpochDKG.KeyShare(id.GetNodeID())
+			if keyShareErr != nil {
+				log.Fatal().Err(keyShareErr).Msg(fmt.Sprintf("failed to get dkg pub key share for node: %s", id.GetNodeID()))
+			}
+			dkgPubKeyCdc, cdcErr := cadence.NewString(dkgPubKey.String())
+			if cdcErr != nil {
+				log.Fatal().Err(cdcErr).Msg(fmt.Sprintf("failed to get dkg pub key cadence string for node: %s", id.GetNodeID()))
+			}
+			dkgPubKeys = append(dkgPubKeys, dkgPubKeyCdc)
+		}
+		nodeIdCdc, err := cadence.NewString(id.GetNodeID().String())
+		if err != nil {
+			log.Fatal().Err(err).Msg(fmt.Sprintf("failed to convert node ID to cadence string: %s", id.GetNodeID()))
+		}
+		nodeIds = append(nodeIds, nodeIdCdc)
+	}
+	clusterQCAddress := systemcontracts.SystemContractsForChain(rootChainID).ClusterQC.Address.String()
+	qcVoteData, err := common.ConvertClusterQcsCdc(clusterQCs, clusters, clusterQCAddress)
+	if err != nil {
+		log.Fatal().Err(err).Msg("failed to convert cluster qcs to cadence type")
+	}
+
+	currEpochFinalView, err := epoch.FinalView()
+	if err != nil {
+		log.Fatal().Err(err).Msg("failed to get final view of current epoch")
+	}
+
+	currEpochTargetEndTime, err := epoch.TargetEndTime()
+	if err != nil {
+		log.Fatal().Err(err).Msg("failed to get target end time of current epoch")
+	}
+
+	args := []cadence.Value{
+		// epoch start view
+		cadence.NewUInt64(currEpochFinalView + 1),
+		// staking phase end view
+		cadence.NewUInt64(currEpochFinalView + numViewsInStakingAuction),
+		// epoch end view
+		cadence.NewUInt64(currEpochFinalView + numViewsInEpoch),
+		// target duration
+		cadence.NewUInt64(targetDuration),
+		// target end time
+		cadence.NewUInt64(currEpochTargetEndTime),
+		// clusters,
+		common.ConvertClusterAssignmentsCdc(assignments),
+		// qcVoteData
+		cadence.NewArray(qcVoteData),
+		// dkg pub keys
+		cadence.NewArray(dkgPubKeys),
+		// node ids
+		cadence.NewArray(nodeIds),
+		// recover the network by initializing a new recover epoch which will increment the smart contract epoch counter
+		// or overwrite the epoch metadata for the current epoch
+		cadence.NewBool(initNewEpoch),
+	}
+
+	return args, nil
+}
+
+// ConstructRootQCsForClusters constructs a root QC for each cluster in the list.
+// Args:
+// - log: the logger instance.
+// - clusterList: list of clusters
+// - nodeInfos: list of NodeInfos (must contain all internal nodes)
+// - clusterBlocks: list of root blocks (one for each cluster)
+// Returns:
+// - flow.AssignmentList: the generated assignment list.
+// - flow.ClusterList: the generate collection cluster list.
+func ConstructRootQCsForClusters(log zerolog.Logger, clusterList flow.ClusterList, nodeInfos []bootstrap.NodeInfo, clusterBlocks []*cluster.Block) []*flow.QuorumCertificate {
+	if len(clusterBlocks) != len(clusterList) {
+		log.Fatal().Int("len(clusterBlocks)", len(clusterBlocks)).Int("len(clusterList)", len(clusterList)).
+			Msg("number of clusters needs to equal number of cluster blocks")
+	}
+
+	qcs := make([]*flow.QuorumCertificate, len(clusterBlocks))
+	for i, cluster := range clusterList {
+		signers := filterClusterSigners(cluster, nodeInfos)
+
+		qc, err := GenerateClusterRootQC(signers, cluster, clusterBlocks[i])
+		if err != nil {
+			log.Fatal().Err(err).Int("cluster index", i).Msg("generating collector cluster root QC failed")
+		}
+		qcs[i] = qc
+	}
+
+	return qcs
+}
+
+// Filters a list of nodes to include only nodes that will sign the QC for the
+// given cluster. The resulting list of nodes is only nodes that are in the
+// given cluster AND are not partner nodes (ie. we have the private keys).
+func filterClusterSigners(cluster flow.IdentitySkeletonList, nodeInfos []model.NodeInfo) []model.NodeInfo {
+	var filtered []model.NodeInfo
+	for _, node := range nodeInfos {
+		_, isInCluster := cluster.ByNodeID(node.NodeID)
+		isPrivateKeyAvailable := node.Type() == model.NodeInfoTypePrivate
+
+		if isInCluster && isPrivateKeyAvailable {
+			filtered = append(filtered, node)
+		}
+	}
+
+	return filtered
+}

cmd/bootstrap/utils/key_generation.go

@@ -7,14 +7,15 @@ import (
 	gohash "hash"
 	"io"
 
-	"github.com/onflow/crypto"
 	"golang.org/x/crypto/hkdf"
 
+	"github.com/onflow/crypto"
+
 	sdk "github.com/onflow/flow-go-sdk"
 	sdkcrypto "github.com/onflow/flow-go-sdk/crypto"
-
 	"github.com/onflow/flow-go/fvm/systemcontracts"
 	"github.com/onflow/flow-go/model/bootstrap"
+	model "github.com/onflow/flow-go/model/bootstrap"
 	"github.com/onflow/flow-go/model/encodable"
 	"github.com/onflow/flow-go/model/flow"
 )
@@ -296,3 +297,21 @@ func WriteStakingNetworkingKeyFiles(nodeInfos []bootstrap.NodeInfo, write WriteJ
 
 	return nil
 }
+
+// WriteNodeInternalPubInfos writes the `node-internal-infos.pub.json` file.
+// In a nutshell, this file contains the Role, address and weight for all authorized nodes.
+func WriteNodeInternalPubInfos(nodeInfos []bootstrap.NodeInfo, write WriteJSONFileFunc) error {
+	configs := make([]model.NodeConfig, len(nodeInfos))
+	for i, nodeInfo := range nodeInfos {
+		configs[i] = model.NodeConfig{
+			Role:    nodeInfo.Role,
+			Address: nodeInfo.Address,
+			Weight:  nodeInfo.Weight,
+		}
+	}
+	err := write(bootstrap.PathNodeInfosPub, configs)
+	if err != nil {
+		return err
+	}
+	return nil
+}

cmd/util/cmd/common/clusters.go

@@ -7,11 +7,8 @@ import (
 	"github.com/rs/zerolog"
 
 	"github.com/onflow/cadence"
+	cdcCommon "github.com/onflow/cadence/runtime/common"
 
-	"github.com/onflow/flow-go/cmd/bootstrap/run"
-	"github.com/onflow/flow-go/model/bootstrap"
-	model "github.com/onflow/flow-go/model/bootstrap"
-	"github.com/onflow/flow-go/model/cluster"
 	"github.com/onflow/flow-go/model/flow"
 	"github.com/onflow/flow-go/model/flow/assignment"
 	"github.com/onflow/flow-go/model/flow/factory"
@@ -114,36 +111,6 @@ func ConstructClusterAssignment(log zerolog.Logger, partnerNodes, internalNodes
 	return assignments, clusters, nil
 }
 
-// ConstructRootQCsForClusters constructs a root QC for each cluster in the list.
-// Args:
-// - log: the logger instance.
-// - clusterList: list of clusters
-// - nodeInfos: list of NodeInfos (must contain all internal nodes)
-// - clusterBlocks: list of root blocks for each cluster
-// Returns:
-// - flow.AssignmentList: the generated assignment list.
-// - flow.ClusterList: the generate collection cluster list.
-func ConstructRootQCsForClusters(log zerolog.Logger, clusterList flow.ClusterList, nodeInfos []bootstrap.NodeInfo, clusterBlocks []*cluster.Block) []*flow.QuorumCertificate {
-
-	if len(clusterBlocks) != len(clusterList) {
-		log.Fatal().Int("len(clusterBlocks)", len(clusterBlocks)).Int("len(clusterList)", len(clusterList)).
-			Msg("number of clusters needs to equal number of cluster blocks")
-	}
-
-	qcs := make([]*flow.QuorumCertificate, len(clusterBlocks))
-	for i, cluster := range clusterList {
-		signers := filterClusterSigners(cluster, nodeInfos)
-
-		qc, err := run.GenerateClusterRootQC(signers, cluster, clusterBlocks[i])
-		if err != nil {
-			log.Fatal().Err(err).Int("cluster index", i).Msg("generating collector cluster root QC failed")
-		}
-		qcs[i] = qc
-	}
-
-	return qcs
-}
-
 // ConvertClusterAssignmentsCdc converts golang cluster assignments type to Cadence type `[[String]]`.
 func ConvertClusterAssignmentsCdc(assignments flow.AssignmentList) cadence.Array {
 	stringArrayType := cadence.NewVariableSizedArrayType(cadence.StringType)
@@ -163,8 +130,18 @@ func ConvertClusterAssignmentsCdc(assignments flow.AssignmentList) cadence.Array
 }
 
 // ConvertClusterQcsCdc converts cluster QCs from `QuorumCertificate` type to `ClusterQCVoteData` type.
-func ConvertClusterQcsCdc(qcs []*flow.QuorumCertificate, clusterList flow.ClusterList) ([]*flow.ClusterQCVoteData, error) {
-	voteData := make([]*flow.ClusterQCVoteData, len(qcs))
+// Args:
+//   - qcs: list of quorum certificates.
+//   - clusterList: the list of cluster lists each used to generate one of the quorum certificates in qcs.
+//   - flowClusterQCAddress: the FlowClusterQC contract address where the ClusterQCVoteData type is defined.
+//
+// Returns:
+//   - []cadence.Value: cadence representation of the list of cluster qcs.
+//   - error: error if the cluster qcs and cluster lists don't match in size or
+//     signature indices decoding fails.
+func ConvertClusterQcsCdc(qcs []*flow.QuorumCertificate, clusterList flow.ClusterList, flowClusterQCAddress string) ([]cadence.Value, error) {
+	voteDataType := newClusterQCVoteDataCdcType(flowClusterQCAddress)
+	qcVoteData := make([]cadence.Value, len(qcs))
 	for i, qc := range qcs {
 		c, ok := clusterList.ByIndex(uint(i))
 		if !ok {
@@ -174,29 +151,42 @@ func ConvertClusterQcsCdc(qcs []*flow.QuorumCertificate, clusterList flow.Cluste
 		if err != nil {
 			return nil, fmt.Errorf("could not decode signer indices: %w", err)
 		}
-		voteData[i] = &flow.ClusterQCVoteData{
-			SigData:  qc.SigData,
-			VoterIDs: voterIds,
+		cdcVoterIds := make([]cadence.Value, len(voterIds))
+		for i, id := range voterIds {
+			cdcVoterIds[i] = cadence.String(id.String())
 		}
-	}
 
-	return voteData, nil
-}
+		qcVoteData[i] = cadence.NewStruct([]cadence.Value{
+			// aggregatedSignature
+			cadence.String(fmt.Sprintf("%#x", qc.SigData)),
+			// Node IDs of signers
+			cadence.NewArray(cdcVoterIds).WithType(cadence.NewVariableSizedArrayType(cadence.StringType)),
+		}).WithType(voteDataType)
 
-// Filters a list of nodes to include only nodes that will sign the QC for the
-// given cluster. The resulting list of nodes is only nodes that are in the
-// given cluster AND are not partner nodes (ie. we have the private keys).
-func filterClusterSigners(cluster flow.IdentitySkeletonList, nodeInfos []model.NodeInfo) []model.NodeInfo {
+	}
 
-	var filtered []model.NodeInfo
-	for _, node := range nodeInfos {
-		_, isInCluster := cluster.ByNodeID(node.NodeID)
-		isNotPartner := node.Type() == model.NodeInfoTypePrivate
+	return qcVoteData, nil
+}
 
-		if isInCluster && isNotPartner {
-			filtered = append(filtered, node)
-		}
+// newClusterQCVoteDataCdcType returns the FlowClusterQC cadence struct type.
+func newClusterQCVoteDataCdcType(clusterQcAddress string) *cadence.StructType {
+
+	// FlowClusterQC.ClusterQCVoteData
+	address, _ := cdcCommon.HexToAddress(clusterQcAddress)
+	location := cdcCommon.NewAddressLocation(nil, address, "FlowClusterQC")
+
+	return &cadence.StructType{
+		Location:            location,
+		QualifiedIdentifier: "FlowClusterQC.ClusterQCVoteData",
+		Fields: []cadence.Field{
+			{
+				Identifier: "aggregatedSignature",
+				Type:       cadence.StringType,
+			},
+			{
+				Identifier: "voterIDs",
+				Type:       cadence.NewVariableSizedArrayType(cadence.StringType),
+			},
+		},
 	}
-
-	return filtered
 }