[Dynamic Protocol State] Changing structure of participants in EpochSetup

Makefile

@@ -151,7 +151,7 @@ generate-fvm-env-wrappers:
 generate-mocks: install-mock-generators
 	mockery --name '(Connector|PingInfoProvider)' --dir=network/p2p --case=underscore --output="./network/mocknetwork" --outpkg="mocknetwork"
 	mockgen -destination=storage/mocks/storage.go -package=mocks github.com/onflow/flow-go/storage Blocks,Headers,Payloads,Collections,Commits,Events,ServiceEvents,TransactionResults
-	mockgen -destination=module/mocks/network.go -package=mocks github.com/onflow/flow-go/module Local,Requester
+	#mockgen -destination=module/mocks/network.go -package=mocks github.com/onflow/flow-go/module Local,Requester
 	mockgen -destination=network/mocknetwork/mock_network.go -package=mocknetwork github.com/onflow/flow-go/network EngineRegistry
 	mockery --name='.*' --dir=integration/benchmark/mocksiface --case=underscore --output="integration/benchmark/mock" --outpkg="mock"
 	mockery --name=ExecutionDataStore --dir=module/executiondatasync/execution_data --case=underscore --output="./module/executiondatasync/execution_data/mock" --outpkg="mock"

cmd/access/node_builder/access_node_builder.go

@@ -838,8 +838,8 @@ func (builder *FlowAccessNodeBuilder) InitIDProviders() {
 		builder.SyncEngineParticipantsProviderFactory = func() module.IdentifierProvider {
 			return id.NewIdentityFilterIdentifierProvider(
 				filter.And(
-					filter.HasRole(flow.RoleConsensus),
-					filter.Not(filter.HasNodeID(node.Me.NodeID())),
+					filter.HasRole[flow.Identity](flow.RoleConsensus),
+					filter.Not(filter.HasNodeID[flow.Identity](node.Me.NodeID())),
 					p2pnet.NotEjectedFilter,
 				),
 				builder.IdentityProvider,
@@ -1129,7 +1129,7 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) {
 				node.Me,
 				node.State,
 				channels.RequestCollections,
-				filter.HasRole(flow.RoleCollection),
+				filter.HasRole[flow.Identity](flow.RoleCollection),
 				func() flow.Entity { return &flow.Collection{} },
 			)
 			if err != nil {

cmd/bootstrap/cmd/clusters.go

@@ -26,8 +26,8 @@ import (
 // of succeeding the assignment by re-running the function without increasing the internal nodes ratio.
 func constructClusterAssignment(partnerNodes, internalNodes []model.NodeInfo) (flow.AssignmentList, flow.ClusterList, error) {
 
-	partners := model.ToIdentityList(partnerNodes).Filter(filter.HasRole(flow.RoleCollection))
-	internals := model.ToIdentityList(internalNodes).Filter(filter.HasRole(flow.RoleCollection))
+	partners := model.ToIdentityList(partnerNodes).Filter(filter.HasRole[flow.Identity](flow.RoleCollection))
+	internals := model.ToIdentityList(internalNodes).Filter(filter.HasRole[flow.Identity](flow.RoleCollection))
 	nClusters := int(flagCollectionClusters)
 	nCollectors := len(partners) + len(internals)
 
@@ -73,7 +73,7 @@ func constructClusterAssignment(partnerNodes, internalNodes []model.NodeInfo) (f
 	assignments := assignment.FromIdentifierLists(identifierLists)
 
 	collectors := append(partners, internals...)
-	clusters, err := factory.NewClusterList(assignments, collectors)
+	clusters, err := factory.NewClusterList(assignments, collectors.ToSkeleton())
 	if err != nil {
 		log.Fatal().Err(err).Msg("could not create cluster list")
 	}
@@ -109,7 +109,7 @@ func constructRootQCsForClusters(
 // Filters a list of nodes to include only nodes that will sign the QC for the
 // given cluster. The resulting list of nodes is only nodes that are in the
 // given cluster AND are not partner nodes (ie. we have the private keys).
-func filterClusterSigners(cluster flow.IdentityList, nodeInfos []model.NodeInfo) []model.NodeInfo {
+func filterClusterSigners(cluster flow.IdentitySkeletonList, nodeInfos []model.NodeInfo) []model.NodeInfo {
 
 	var filtered []model.NodeInfo
 	for _, node := range nodeInfos {

cmd/bootstrap/cmd/constraints.go

@@ -12,7 +12,7 @@ import (
 func ensureUniformNodeWeightsPerRole(allNodes flow.IdentityList) {
 	// ensure all nodes of the same role have equal weight
 	for _, role := range flow.Roles() {
-		withRole := allNodes.Filter(filter.HasRole(role))
+		withRole := allNodes.Filter(filter.HasRole[flow.Identity](role))
 		// each role has at least one node so it's safe to access withRole[0]
 		expectedWeight := withRole[0].Weight
 		for _, node := range withRole {

cmd/bootstrap/cmd/finalize.go

@@ -156,7 +156,7 @@ func finalize(cmd *cobra.Command, args []string) {
 	log.Info().Msg("")
 
 	// create flow.IdentityList representation of participant set
-	participants := model.ToIdentityList(stakingNodes).Sort(order.Canonical)
+	participants := model.ToIdentityList(stakingNodes).Sort(order.Canonical[flow.Identity])
 
 	log.Info().Msg("reading root block data")
 	block := readRootBlock()
@@ -491,7 +491,7 @@ func mergeNodeInfos(internalNodes, partnerNodes []model.NodeInfo) []model.NodeIn
 	}
 
 	// sort nodes using the canonical ordering
-	nodes = model.Sort(nodes, order.Canonical)
+	nodes = model.Sort(nodes, order.Canonical[flow.Identity])
 
 	return nodes
 }

cmd/bootstrap/cmd/keys.go

@@ -49,7 +49,7 @@ func genNetworkAndStakingKeys() []model.NodeInfo {
 		internalNodes = append(internalNodes, nodeInfo)
 	}
 
-	return model.Sort(internalNodes, order.Canonical)
+	return model.Sort(internalNodes, order.Canonical[flow.Identity])
 }
 
 func assembleNodeInfo(nodeConfig model.NodeConfig, networkKey, stakingKey crypto.PrivateKey) model.NodeInfo {

cmd/bootstrap/cmd/seal.go

@@ -39,7 +39,7 @@ func constructRootResultAndSeal(
 		DKGPhase1FinalView: firstView + flagNumViewsInStakingAuction + flagNumViewsInDKGPhase - 1,
 		DKGPhase2FinalView: firstView + flagNumViewsInStakingAuction + flagNumViewsInDKGPhase*2 - 1,
 		DKGPhase3FinalView: firstView + flagNumViewsInStakingAuction + flagNumViewsInDKGPhase*3 - 1,
-		Participants:       participants.Sort(order.Canonical),
+		Participants:       participants.Sort(order.Canonical[flow.Identity]).ToSkeleton(),
 		Assignments:        assignments,
 		RandomSource:       GenerateRandomSeed(flow.EpochSetupRandomSourceLength),
 	}

cmd/bootstrap/run/cluster_qc.go

@@ -19,8 +19,8 @@ import (
 )
 
 // GenerateClusterRootQC creates votes and generates a QC based on participant data
-func GenerateClusterRootQC(signers []bootstrap.NodeInfo, allCommitteeMembers flow.IdentityList, clusterBlock *cluster.Block) (*flow.QuorumCertificate, error) {
-	if !allCommitteeMembers.Sorted(order.Canonical) {
+func GenerateClusterRootQC(signers []bootstrap.NodeInfo, allCommitteeMembers flow.IdentitySkeletonList, clusterBlock *cluster.Block) (*flow.QuorumCertificate, error) {
+	if !allCommitteeMembers.Sorted(order.Canonical[flow.IdentitySkeleton]) {
 		return nil, fmt.Errorf("can't create root cluster QC: committee members are not sorted in canonical order")
 	}
 	clusterRootBlock := model.GenesisBlockFromFlow(clusterBlock.Header)
@@ -31,8 +31,22 @@ func GenerateClusterRootQC(signers []bootstrap.NodeInfo, allCommitteeMembers flo
 		return nil, err
 	}
 
+	// STEP 1.5: patch committee to include dynamic identities. This is a temporary measure until bootstrapping is refactored.
+	// We need to do this since the committee is used to create the QC uses dynamic identities, but clustering for root block contain only
+	// static identities since there no state transitions haven't happened yet.
+	dynamicCommitteeMembers := make(flow.IdentityList, 0, len(allCommitteeMembers))
+	for _, participant := range allCommitteeMembers {
+		dynamicCommitteeMembers = append(dynamicCommitteeMembers, &flow.Identity{
+			IdentitySkeleton: *participant,
+			DynamicIdentity: flow.DynamicIdentity{
+				Weight:  participant.InitialWeight,
+				Ejected: false,
+			},
+		})
+	}
+
 	// STEP 2: create VoteProcessor
-	committee, err := committees.NewStaticCommittee(allCommitteeMembers, flow.Identifier{}, nil, nil)
+	committee, err := committees.NewStaticCommittee(dynamicCommitteeMembers, flow.Identifier{}, nil, nil)
 	if err != nil {
 		return nil, err
 	}

cmd/bootstrap/run/cluster_qc_test.go

@@ -33,7 +33,7 @@ func TestGenerateClusterRootQC(t *testing.T) {
 	payload := cluster.EmptyPayload(flow.ZeroID)
 	clusterBlock.SetPayload(payload)
 
-	orderedParticipants := model.ToIdentityList(participants).Sort(order.Canonical)
+	orderedParticipants := model.ToIdentityList(participants).Sort(order.Canonical[flow.Identity]).ToSkeleton()
 	_, err := GenerateClusterRootQC(participants, orderedParticipants, &clusterBlock)
 	require.NoError(t, err)
 }

cmd/bootstrap/run/qc_test.go

@@ -45,7 +45,7 @@ func TestGenerateRootQCWithSomeInvalidVotes(t *testing.T) {
 }
 
 func createSignerData(t *testing.T, n int) *ParticipantData {
-	identities := unittest.IdentityListFixture(n).Sort(order.Canonical)
+	identities := unittest.IdentityListFixture(n).Sort(order.Canonical[flow.Identity])
 
 	networkingKeys := unittest.NetworkingKeys(n)
 	stakingKeys := unittest.StakingKeys(n)

cmd/collection/main.go

@@ -424,7 +424,7 @@ func main() {
 				channels.ProvideCollections,
 				filter.And(
 					filter.HasWeight(true),
-					filter.HasRole(flow.RoleAccess, flow.RoleExecution),
+					filter.HasRole[flow.Identity](flow.RoleAccess, flow.RoleExecution),
 				),
 				retrieve,
 			)

cmd/consensus/main.go

@@ -484,7 +484,7 @@ func main() {
 				node.Me,
 				node.State,
 				channels.RequestReceiptsByBlockID,
-				filter.HasRole(flow.RoleExecution),
+				filter.HasRole[flow.Identity](flow.RoleExecution),
 				func() flow.Entity { return &flow.ExecutionReceipt{} },
 				requester.WithRetryInitial(2*time.Second),
 				requester.WithRetryMaximum(30*time.Second),

cmd/execution_builder.go

@@ -1056,7 +1056,7 @@ func (exeNode *ExecutionNode) LoadReceiptProviderEngine(
 		channels.ProvideReceiptsByBlockID,
 		filter.And(
 			filter.HasWeight(true),
-			filter.HasRole(flow.RoleConsensus),
+			filter.HasRole[flow.Identity](flow.RoleConsensus),
 		),
 		retrieve,
 	)

cmd/scaffold.go

@@ -1021,8 +1021,8 @@ func (fnb *FlowNodeBuilder) InitIDProviders() {
 
 		node.SyncEngineIdentifierProvider = id.NewIdentityFilterIdentifierProvider(
 			filter.And(
-				filter.HasRole(flow.RoleConsensus),
-				filter.Not(filter.HasNodeID(node.Me.NodeID())),
+				filter.HasRole[flow.Identity](flow.RoleConsensus),
+				filter.Not(filter.HasNodeID[flow.Identity](node.Me.NodeID())),
 				p2pnet.NotEjectedFilter,
 			),
 			node.IdentityProvider,

cmd/util/cmd/common/flow_client.go

@@ -93,7 +93,7 @@ func insecureFlowClient(accessAddress string) (*client.Client, error) {
 func FlowClientConfigs(accessNodeIDS []flow.Identifier, insecureAccessAPI bool, snapshot protocol.Snapshot) ([]*FlowClientConfig, error) {
 	flowClientOpts := make([]*FlowClientConfig, 0)
 
-	identities, err := snapshot.Identities(filter.HasNodeID(accessNodeIDS...))
+	identities, err := snapshot.Identities(filter.HasNodeID[flow.Identity](accessNodeIDS...))
 	if err != nil {
 		return nil, fmt.Errorf("failed get identities access node identities (ids=%v) from snapshot: %w", accessNodeIDS, err)
 	}
@@ -139,7 +139,7 @@ func convertAccessAddrFromState(address string, insecureAccessAPI bool) string {
 
 // DefaultAccessNodeIDS will return all the access node IDS in the protocol state for staked access nodes
 func DefaultAccessNodeIDS(snapshot protocol.Snapshot) ([]flow.Identifier, error) {
-	identities, err := snapshot.Identities(filter.HasRole(flow.RoleAccess))
+	identities, err := snapshot.Identities(filter.HasRole[flow.Identity](flow.RoleAccess))
 	if err != nil {
 		return nil, fmt.Errorf("failed to get staked access node IDs from protocol state %w", err)
 	}

cmd/util/cmd/epochs/cmd/move_machine_acct.go

@@ -72,7 +72,7 @@ func moveMachineAcctRun(cmd *cobra.Command, args []string) {
 	}
 
 	// identities with machine accounts
-	machineAcctIdentities := identities.Filter(filter.HasRole(flow.RoleCollection, flow.RoleConsensus))
+	machineAcctIdentities := identities.Filter(filter.HasRole[flow.Identity](flow.RoleCollection, flow.RoleConsensus))
 
 	machineAcctFiles, err := os.ReadDir(flagMachineAccountsSrcDir)
 	if err != nil {

consensus/follower_test.go

@@ -16,6 +16,7 @@ import (
 	mockhotstuff "github.com/onflow/flow-go/consensus/hotstuff/mocks"
 	"github.com/onflow/flow-go/consensus/hotstuff/model"
 	"github.com/onflow/flow-go/model/flow"
+	"github.com/onflow/flow-go/model/flow/order"
 	"github.com/onflow/flow-go/module/irrecoverable"
 	"github.com/onflow/flow-go/module/metrics"
 	mockmodule "github.com/onflow/flow-go/module/mock"
@@ -75,7 +76,7 @@ type HotStuffFollowerSuite struct {
 // SetupTest initializes all the components needed for the Follower.
 // The follower itself is instantiated in method BeforeTest
 func (s *HotStuffFollowerSuite) SetupTest() {
-	identities := unittest.IdentityListFixture(4, unittest.WithRole(flow.RoleConsensus))
+	identities := unittest.IdentityListFixture(4, unittest.WithRole(flow.RoleConsensus)).Sort(order.Canonical[flow.Identity])
 	s.mockConsensus = &MockConsensus{identities: identities}
 
 	// mock storage headers

consensus/hotstuff/committees/cluster_committee.go

@@ -26,7 +26,7 @@ type Cluster struct {
 	// pre-computed leader selection for the full lifecycle of the cluster
 	selection *leader.LeaderSelection
 	// a filter that returns all members of the cluster committee allowed to vote
-	clusterMemberFilter flow.IdentityFilter
+	clusterMemberFilter flow.IdentityFilter[flow.Identity]
 	// initial set of cluster members, WITHOUT dynamic weight changes
 	initialClusterMembers    flow.IdentitySkeletonList
 	initialClusterIdentities flow.IdentityList
@@ -50,19 +50,34 @@ func NewClusterCommittee(
 		return nil, fmt.Errorf("could not compute leader selection for cluster: %w", err)
 	}
 
-	totalWeight := cluster.Members().ToSkeleton().TotalWeight()
+	initialClusterMembers := cluster.Members()
+	totalWeight := initialClusterMembers.TotalWeight()
+	initialClusterMembersSelector := initialClusterMembers.Selector()
+	// the next section is not very nice, but there are no dynamic identities for root block,
+	// and we need them to specificially handle querying of identities for root block
+	initialClusterIdentities := make(flow.IdentityList, 0, len(cluster.Members()))
+	for _, skeleton := range initialClusterMembers {
+		initialClusterIdentities = append(initialClusterIdentities, &flow.Identity{
+			IdentitySkeleton: *skeleton,
+			DynamicIdentity: flow.DynamicIdentity{
+				Weight:  skeleton.InitialWeight,
+				Ejected: false,
+			},
+		})
+	}
 	com := &Cluster{
 		state:     state,
 		payloads:  payloads,
 		me:        me,
 		selection: selection,
-		clusterMemberFilter: filter.And(
-			cluster.Members().Selector(),
+		clusterMemberFilter: filter.And[flow.Identity](
+			// adapt the identity filter to the identity skeleton filter
+			filter.Adapt(initialClusterMembersSelector),
 			filter.Not(filter.Ejected),
 			filter.HasWeight(true),
 		),
-		initialClusterMembers:    cluster.Members().ToSkeleton(),
-		initialClusterIdentities: cluster.Members(),
+		initialClusterMembers:    initialClusterMembers,
+		initialClusterIdentities: initialClusterIdentities,
 		weightThresholdForQC:     WeightThresholdToBuildQC(totalWeight),
 		weightThresholdForTO:     WeightThresholdToTimeout(totalWeight),
 	}

consensus/hotstuff/committees/cluster_committee_test.go

@@ -49,11 +49,11 @@ func (suite *ClusterSuite) SetupTest() {
 	suite.members = unittest.IdentityListFixture(5, unittest.WithRole(flow.RoleCollection))
 	suite.me = suite.members[0]
 	counter := uint64(1)
-	suite.root = clusterstate.CanonicalRootBlock(counter, suite.members)
+	suite.root = clusterstate.CanonicalRootBlock(counter, suite.members.ToSkeleton())
 
 	suite.cluster.On("EpochCounter").Return(counter)
 	suite.cluster.On("Index").Return(uint(1))
-	suite.cluster.On("Members").Return(suite.members)
+	suite.cluster.On("Members").Return(suite.members.ToSkeleton())
 	suite.cluster.On("RootBlock").Return(suite.root)
 	suite.epoch.On("Counter").Return(counter, nil)
 	suite.epoch.On("RandomSource").Return(unittest.SeedFixture(prg.RandomSourceLength), nil)
@@ -146,12 +146,6 @@ func (suite *ClusterSuite) TestInvalidSigner() {
 	})
 
 	suite.Run("should return ErrInvalidSigner for existent but ejected cluster member", func() {
-		// at the root block, the cluster member is not ejected yet
-		suite.Run("root block", func() {
-			actual, err := suite.com.IdentityByBlock(rootBlockID, realEjectedClusterMember.NodeID)
-			suite.Require().NoError(err)
-			suite.Assert().Equal(realEjectedClusterMember, actual)
-		})
 		suite.Run("non-root block", func() {
 			_, err := suite.com.IdentityByBlock(nonRootBlockID, realEjectedClusterMember.NodeID)
 			suite.Assert().True(model.IsInvalidSignerError(err))
@@ -164,12 +158,6 @@ func (suite *ClusterSuite) TestInvalidSigner() {
 	})
 
 	suite.Run("should return ErrInvalidSigner for existent but zero-weight cluster member", func() {
-		// at the root block, the cluster member has its initial weight
-		suite.Run("root block", func() {
-			actual, err := suite.com.IdentityByBlock(rootBlockID, realNoWeightClusterMember.NodeID)
-			suite.Require().NoError(err)
-			suite.Assert().Equal(realNoWeightClusterMember, actual)
-		})
 		suite.Run("non-root block", func() {
 			_, err := suite.com.IdentityByBlock(nonRootBlockID, realNoWeightClusterMember.NodeID)
 			suite.Assert().True(model.IsInvalidSignerError(err))

consensus/hotstuff/committees/consensus_committee.go

@@ -55,7 +55,7 @@ func newStaticEpochInfo(epoch protocol.Epoch) (*staticEpochInfo, error) {
 	if err != nil {
 		return nil, fmt.Errorf("could not initial identities: %w", err)
 	}
-	initialCommittee := initialIdentities.Filter(filter.IsVotingConsensusCommitteeMember).ToSkeleton()
+	initialCommittee := initialIdentities.Filter(filter.IsAllowedConsensusCommitteeMember).ToSkeleton()
 	dkg, err := epoch.DKG()
 	if err != nil {
 		return nil, fmt.Errorf("could not get dkg: %w", err)

consensus/hotstuff/committees/consensus_committee_test.go

@@ -327,7 +327,8 @@ func (suite *ConsensusSuite) TestIdentitiesByEpoch() {
 	// epoch 1 identities with varying conditions which would disqualify them
 	// from committee participation
 	realIdentity := unittest.IdentityFixture(unittest.WithRole(flow.RoleConsensus))
-	zeroWeightConsensusIdentity := unittest.IdentityFixture(unittest.WithRole(flow.RoleConsensus), unittest.WithWeight(0))
+	zeroWeightConsensusIdentity := unittest.IdentityFixture(unittest.WithRole(flow.RoleConsensus),
+		unittest.WithInitialWeight(0))
 	ejectedConsensusIdentity := unittest.IdentityFixture(unittest.WithRole(flow.RoleConsensus), unittest.WithEjected(true))
 	validNonConsensusIdentity := unittest.IdentityFixture(unittest.WithRole(flow.RoleVerification))
 	epoch1Identities := flow.IdentityList{realIdentity, zeroWeightConsensusIdentity, ejectedConsensusIdentity, validNonConsensusIdentity}
@@ -356,11 +357,6 @@ func (suite *ConsensusSuite) TestIdentitiesByEpoch() {
 				require.True(t, model.IsInvalidSignerError(err))
 			})
 
-			t.Run("ejected consensus node", func(t *testing.T) {
-				_, err := suite.committee.IdentityByEpoch(unittest.Uint64InRange(1, 100), ejectedConsensusIdentity.NodeID)
-				require.True(t, model.IsInvalidSignerError(err))
-			})
-
 			t.Run("otherwise valid non-consensus node", func(t *testing.T) {
 				_, err := suite.committee.IdentityByEpoch(unittest.Uint64InRange(1, 100), validNonConsensusIdentity.NodeID)
 				require.True(t, model.IsInvalidSignerError(err))
@@ -679,7 +675,7 @@ func newMockEpoch(counter uint64, identities flow.IdentityList, firstView uint64
 
 	epoch := new(protocolmock.Epoch)
 	epoch.On("Counter").Return(counter, nil)
-	epoch.On("InitialIdentities").Return(identities, nil)
+	epoch.On("InitialIdentities").Return(identities.ToSkeleton(), nil)
 	epoch.On("FirstView").Return(firstView, nil)
 	epoch.On("FinalView").Return(finalView, nil)
 	if committed {

consensus/hotstuff/committees/leader/consensus.go

@@ -43,7 +43,7 @@ func SelectionForConsensus(epoch protocol.Epoch) (*LeaderSelection, error) {
 		firstView,
 		rng,
 		int(finalView-firstView+1), // add 1 because both first/final view are inclusive
-		identities.Filter(filter.IsVotingConsensusCommitteeMember).ToSkeleton(),
+		identities.Filter(filter.IsAllowedConsensusCommitteeMember),
 	)
 	return leaders, err
 }