From f49070771b5ec74ebd2e6777feb062a5740ca7d9 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Mon, 18 Sep 2023 17:04:00 +0300 Subject: [PATCH 01/39] Updated EpochSetup to contain IdentitySkeletonList instead of IdentitySkeleton --- model/flow/epoch.go | 18 +++++----- model/flow/filter/identity.go | 2 +- model/flow/identity.go | 63 +++++++++++++++++++++++++++++---- model/flow/protocol_state.go | 12 +++---- state/protocol/convert.go | 2 +- state/protocol/epoch.go | 2 +- state/protocol/inmem/epoch.go | 4 +-- state/protocol/invalid/epoch.go | 2 +- state/protocol/mock/epoch.go | 2 +- 9 files changed, 78 insertions(+), 29 deletions(-) diff --git a/model/flow/epoch.go b/model/flow/epoch.go index e2373b1b4af..f1181753e9c 100644 --- a/model/flow/epoch.go +++ b/model/flow/epoch.go @@ -63,15 +63,15 @@ const EpochSetupRandomSourceLength = 16 // for the upcoming epoch. It contains the participants in the epoch, the // length, the cluster assignment, and the seed for leader selection. type EpochSetup struct { - Counter uint64 // the number of the epoch - FirstView uint64 // the first view of the epoch - DKGPhase1FinalView uint64 // the final view of DKG phase 1 - DKGPhase2FinalView uint64 // the final view of DKG phase 2 - DKGPhase3FinalView uint64 // the final view of DKG phase 3 - FinalView uint64 // the final view of the epoch - Participants IdentityList // all participants of the epoch in canonical order - Assignments AssignmentList // cluster assignment for the epoch with node IDs for each cluster in canonical order - RandomSource []byte // source of randomness for epoch-specific setup tasks + Counter uint64 // the number of the epoch + FirstView uint64 // the first view of the epoch + DKGPhase1FinalView uint64 // the final view of DKG phase 1 + DKGPhase2FinalView uint64 // the final view of DKG phase 2 + DKGPhase3FinalView uint64 // the final view of DKG phase 3 + FinalView uint64 // the final view of the epoch + Participants IdentitySkeletonList // all participants of the epoch in canonical order + Assignments AssignmentList // cluster assignment for the epoch with node IDs for each cluster in canonical order + RandomSource []byte // source of randomness for epoch-specific setup tasks } func (setup *EpochSetup) ServiceEvent() ServiceEvent { diff --git a/model/flow/filter/identity.go b/model/flow/filter/identity.go index 2c312c05028..0e8aea06fb9 100644 --- a/model/flow/filter/identity.go +++ b/model/flow/filter/identity.go @@ -116,4 +116,4 @@ var IsVotingConsensusCommitteeMember = And( // IsValidDKGParticipant is an identity filter for all DKG participants. It is // equivalent to the filter for consensus committee members, as these are // the same group for now. -var IsValidDKGParticipant = IsVotingConsensusCommitteeMember +var IsValidDKGParticipant = func(identity *flow.IdentitySkeleton) {} diff --git a/model/flow/identity.go b/model/flow/identity.go index 8a77202f9bf..52b707ac821 100644 --- a/model/flow/identity.go +++ b/model/flow/identity.go @@ -279,7 +279,7 @@ func (iy *Identity) UnmarshalMsgpack(b []byte) error { return nil } -func (iy *Identity) EqualTo(other *Identity) bool { +func (iy *IdentitySkeleton) EqualTo(other *IdentitySkeleton) bool { if iy.NodeID != other.NodeID { return false } @@ -292,12 +292,6 @@ func (iy *Identity) EqualTo(other *Identity) bool { if iy.InitialWeight != other.InitialWeight { return false } - if iy.Weight != other.Weight { - return false - } - if iy.Ejected != other.Ejected { - return false - } if (iy.StakingPubKey != nil && other.StakingPubKey == nil) || (iy.StakingPubKey == nil && other.StakingPubKey != nil) { return false @@ -317,6 +311,26 @@ func (iy *Identity) EqualTo(other *Identity) bool { return true } +func (iy *DynamicIdentity) EqualTo(other *DynamicIdentity) bool { + if iy.Weight != other.Weight { + return false + } + if iy.Ejected != other.Ejected { + return false + } + return true +} + +func (iy *Identity) EqualTo(other *Identity) bool { + if !iy.IdentitySkeleton.EqualTo(&other.IdentitySkeleton) { + return false + } + if !iy.DynamicIdentity.EqualTo(&other.DynamicIdentity) { + return false + } + return true +} + // IdentityFilter is a filter on identities. type IdentityFilter func(*Identity) bool @@ -598,6 +612,33 @@ func (il IdentityList) Union(other IdentityList) IdentityList { return union } +// Union returns a new identity list containing every identity that occurs in +// either `il`, or `other`, or both. There are no duplicates in the output, +// where duplicates are identities with the same node ID. +// Receiver `il` and/or method input `other` can be nil or empty. +// The returned IdentityList is sorted in canonical order. +func (il IdentitySkeletonList) Union(other IdentitySkeletonList) IdentitySkeletonList { + maxLen := len(il) + len(other) + + union := make(IdentitySkeletonList, 0, maxLen) + set := make(map[Identifier]struct{}, maxLen) + + for _, list := range []IdentitySkeletonList{il, other} { + for _, id := range list { + if _, isDuplicate := set[id.NodeID]; !isDuplicate { + set[id.NodeID] = struct{}{} + union = append(union, id) + } + } + } + + slices.SortFunc(union, func(a, b *IdentitySkeleton) bool { + return bytes.Compare(a.NodeID[:], b.NodeID[:]) < 0 + }) + + return union +} + // EqualTo checks if the other list if the same, that it contains the same elements // in the same order func (il IdentityList) EqualTo(other IdentityList) bool { @@ -606,6 +647,14 @@ func (il IdentityList) EqualTo(other IdentityList) bool { }) } +// EqualTo checks if the other list if the same, that it contains the same elements +// in the same order +func (il IdentitySkeletonList) EqualTo(other IdentitySkeletonList) bool { + return slices.EqualFunc(il, other, func(a, b *IdentitySkeleton) bool { + return a.EqualTo(b) + }) +} + // Exists takes a previously sorted Identity list and searches it for the target value // This code is optimized, so the coding style will be different // target: value to search for diff --git a/model/flow/protocol_state.go b/model/flow/protocol_state.go index 717a4932d62..2ae7e30cde1 100644 --- a/model/flow/protocol_state.go +++ b/model/flow/protocol_state.go @@ -42,9 +42,9 @@ type EpochStateContainer struct { SetupID Identifier // ID of commit event for this epoch. Could be ZeroID if epoch was not committed. CommitID Identifier - // Part of identity table that can be changed on a block-by-block basis. + // Part of identity table that can be changed on a block-by-block basis. // Each non-deferred identity-mutating operation is applied independently to each - // relevant epoch's dynamic identity list separately. + // relevant epoch's dynamic identity list separately. // Always sorted in canonical order. Identities DynamicIdentityEntryList } @@ -187,7 +187,7 @@ func NewRichProtocolStateEntry( } else { // if next epoch is not yet created, it means that we are in staking phase, // so we need to build the identity table using previous and current epoch setup events. - var otherIdentities IdentityList + var otherIdentities IdentitySkeletonList if previousEpochSetup != nil { otherIdentities = previousEpochSetup.Participants } @@ -354,8 +354,8 @@ func (ll DynamicIdentityEntryList) Sort(less IdentifierOrder) DynamicIdentityEnt // No errors are expected during normal operation. func buildIdentityTable( targetEpochDynamicIdentities DynamicIdentityEntryList, - targetEpochIdentitySkeletons IdentityList, // TODO: change to `IdentitySkeletonList` - adjacentEpochIdentitySkeletons IdentityList, // TODO: change to `IdentitySkeletonList` + targetEpochIdentitySkeletons IdentitySkeletonList, + adjacentEpochIdentitySkeletons IdentitySkeletonList, ) (IdentityList, error) { // produce a unique set for current and previous epoch participants allEpochParticipants := targetEpochIdentitySkeletons.Union(adjacentEpochIdentitySkeletons) @@ -372,7 +372,7 @@ func buildIdentityTable( return nil, fmt.Errorf("identites in protocol state are not in canonical order: expected %s, got %s", allEpochParticipants[i].NodeID, identity.NodeID) } result = append(result, &Identity{ - IdentitySkeleton: allEpochParticipants[i].IdentitySkeleton, + IdentitySkeleton: *allEpochParticipants[i], DynamicIdentity: identity.Dynamic, }) } diff --git a/state/protocol/convert.go b/state/protocol/convert.go index a28811c15f5..0890475fd18 100644 --- a/state/protocol/convert.go +++ b/state/protocol/convert.go @@ -122,7 +122,7 @@ func ToEpochCommit(epoch Epoch) (*flow.EpochCommit, error) { // participant keys from the DKG. // All errors indicate inconsistent or invalid inputs. // No errors are expected during normal operation. -func GetDKGParticipantKeys(dkg DKG, participants flow.IdentityList) ([]crypto.PublicKey, error) { +func GetDKGParticipantKeys(dkg DKG, participants flow.IdentitySkeletonList) ([]crypto.PublicKey, error) { keys := make([]crypto.PublicKey, 0, len(participants)) for i, identity := range participants { diff --git a/state/protocol/epoch.go b/state/protocol/epoch.go index 17a6f54da66..41abd660fa1 100644 --- a/state/protocol/epoch.go +++ b/state/protocol/epoch.go @@ -112,7 +112,7 @@ type Epoch interface { // * protocol.ErrNoPreviousEpoch - if the epoch represents a previous epoch which does not exist. // * protocol.ErrNextEpochNotSetup - if the epoch represents a next epoch which has not been set up. // * state.ErrUnknownSnapshotReference - if the epoch is queried from an unresolvable snapshot. - InitialIdentities() (flow.IdentityList, error) + InitialIdentities() (flow.IdentitySkeletonList, error) // Clustering returns the cluster assignment for this epoch. // Error returns: diff --git a/state/protocol/inmem/epoch.go b/state/protocol/inmem/epoch.go index 361a5a30553..be6d8e50244 100644 --- a/state/protocol/inmem/epoch.go +++ b/state/protocol/inmem/epoch.go @@ -33,7 +33,7 @@ func (e Epoch) DKGPhase1FinalView() (uint64, error) { return e.enc.DKGPhase1Fina func (e Epoch) DKGPhase2FinalView() (uint64, error) { return e.enc.DKGPhase2FinalView, nil } func (e Epoch) DKGPhase3FinalView() (uint64, error) { return e.enc.DKGPhase3FinalView, nil } func (e Epoch) FinalView() (uint64, error) { return e.enc.FinalView, nil } -func (e Epoch) InitialIdentities() (flow.IdentityList, error) { +func (e Epoch) InitialIdentities() (flow.IdentitySkeletonList, error) { return e.enc.InitialIdentities, nil } func (e Epoch) RandomSource() ([]byte, error) { @@ -150,7 +150,7 @@ func (es *setupEpoch) RandomSource() ([]byte, error) { return es.setupEvent.RandomSource, nil } -func (es *setupEpoch) InitialIdentities() (flow.IdentityList, error) { +func (es *setupEpoch) InitialIdentities() (flow.IdentitySkeletonList, error) { identities := es.setupEvent.Participants.Filter(filter.Any) return identities, nil } diff --git a/state/protocol/invalid/epoch.go b/state/protocol/invalid/epoch.go index cf4777b4f33..cfd4a48cac5 100644 --- a/state/protocol/invalid/epoch.go +++ b/state/protocol/invalid/epoch.go @@ -65,7 +65,7 @@ func (u *Epoch) DKGPhase3FinalView() (uint64, error) { return 0, u.err } -func (u *Epoch) InitialIdentities() (flow.IdentityList, error) { +func (u *Epoch) InitialIdentities() (flow.IdentitySkeletonList, error) { return nil, u.err } diff --git a/state/protocol/mock/epoch.go b/state/protocol/mock/epoch.go index d1bfabce547..76f1cdd75fd 100644 --- a/state/protocol/mock/epoch.go +++ b/state/protocol/mock/epoch.go @@ -311,7 +311,7 @@ func (_m *Epoch) FirstView() (uint64, error) { } // InitialIdentities provides a mock function with given fields: -func (_m *Epoch) InitialIdentities() (flow.IdentityList, error) { +func (_m *Epoch) InitialIdentities() (flow.IdentitySkeletonList, error) { ret := _m.Called() var r0 flow.IdentityList From 9e9887544097623fd354d017e620670ae507228a Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Tue, 19 Sep 2023 15:08:32 +0300 Subject: [PATCH 02/39] Introduced generic types for identity list, identity skeleton, ordering, sorting, filtering and other operations --- .../state/mock/script_execution_state.go | 88 ++++++ model/bootstrap/node_info.go | 2 +- model/flow/cluster.go | 16 +- model/flow/cluster_test.go | 4 +- model/flow/epoch.go | 2 +- model/flow/epoch_test.go | 16 +- model/flow/factory/cluster_list.go | 6 +- model/flow/factory/cluster_list_test.go | 4 +- model/flow/filter/identity.go | 44 +-- model/flow/identity.go | 252 +++++++++--------- model/flow/identity_test.go | 10 +- model/flow/mapfunc/identity.go | 4 +- model/flow/order/identity.go | 4 +- model/flow/order/identity_test.go | 3 +- model/flow/protocol_state.go | 10 +- model/flow/protocol_state_test.go | 57 +++- model/verification/chunkDataPackRequest.go | 4 +- module/id_provider.go | 2 +- module/local.go | 2 +- module/mock/identity_provider.go | 4 +- module/mock/local.go | 8 +- module/mock/requester.go | 4 +- module/mocks/network.go | 168 ------------ module/requester.go | 4 +- state/cluster/root_block.go | 2 +- state/protocol/cluster.go | 2 +- state/protocol/convert.go | 2 +- state/protocol/inmem/cluster.go | 12 +- state/protocol/inmem/convert.go | 16 +- state/protocol/inmem/encodable.go | 4 +- state/protocol/inmem/epoch.go | 5 +- state/protocol/inmem/snapshot.go | 4 +- state/protocol/invalid/snapshot.go | 2 +- state/protocol/mock/cluster.go | 8 +- state/protocol/mock/epoch.go | 8 +- state/protocol/mock/snapshot.go | 8 +- state/protocol/snapshot.go | 2 +- state/protocol/util.go | 4 +- utils/unittest/chain_suite.go | 4 +- utils/unittest/cluster.go | 14 +- utils/unittest/fixtures.go | 92 +++++-- utils/unittest/protocol_state.go | 2 +- utils/unittest/service_events_fixtures.go | 154 ++++------- utils/unittest/updatable_provider.go | 2 +- 44 files changed, 529 insertions(+), 536 deletions(-) create mode 100644 engine/execution/state/mock/script_execution_state.go diff --git a/engine/execution/state/mock/script_execution_state.go b/engine/execution/state/mock/script_execution_state.go new file mode 100644 index 00000000000..904defab7fa --- /dev/null +++ b/engine/execution/state/mock/script_execution_state.go @@ -0,0 +1,88 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mock + +import ( + context "context" + + flow "github.com/onflow/flow-go/model/flow" + mock "github.com/stretchr/testify/mock" + + snapshot "github.com/onflow/flow-go/fvm/storage/snapshot" +) + +// ScriptExecutionState is an autogenerated mock type for the ScriptExecutionState type +type ScriptExecutionState struct { + mock.Mock +} + +// HasState provides a mock function with given fields: _a0 +func (_m *ScriptExecutionState) HasState(_a0 flow.StateCommitment) bool { + ret := _m.Called(_a0) + + var r0 bool + if rf, ok := ret.Get(0).(func(flow.StateCommitment) bool); ok { + r0 = rf(_a0) + } else { + r0 = ret.Get(0).(bool) + } + + return r0 +} + +// NewStorageSnapshot provides a mock function with given fields: _a0 +func (_m *ScriptExecutionState) NewStorageSnapshot(_a0 flow.StateCommitment) snapshot.StorageSnapshot { + ret := _m.Called(_a0) + + var r0 snapshot.StorageSnapshot + if rf, ok := ret.Get(0).(func(flow.StateCommitment) snapshot.StorageSnapshot); ok { + r0 = rf(_a0) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(snapshot.StorageSnapshot) + } + } + + return r0 +} + +// StateCommitmentByBlockID provides a mock function with given fields: _a0, _a1 +func (_m *ScriptExecutionState) StateCommitmentByBlockID(_a0 context.Context, _a1 flow.Identifier) (flow.StateCommitment, error) { + ret := _m.Called(_a0, _a1) + + var r0 flow.StateCommitment + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier) (flow.StateCommitment, error)); ok { + return rf(_a0, _a1) + } + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier) flow.StateCommitment); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(flow.StateCommitment) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, flow.Identifier) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +type mockConstructorTestingTNewScriptExecutionState interface { + mock.TestingT + Cleanup(func()) +} + +// NewScriptExecutionState creates a new instance of ScriptExecutionState. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewScriptExecutionState(t mockConstructorTestingTNewScriptExecutionState) *ScriptExecutionState { + mock := &ScriptExecutionState{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/model/bootstrap/node_info.go b/model/bootstrap/node_info.go index 6dce969478c..46da4612d03 100644 --- a/model/bootstrap/node_info.go +++ b/model/bootstrap/node_info.go @@ -397,7 +397,7 @@ func FilterByRole(nodes []NodeInfo, role flow.Role) []NodeInfo { } // Sort sorts the NodeInfo list using the given ordering. -func Sort(nodes []NodeInfo, order flow.IdentityOrder) []NodeInfo { +func Sort(nodes []NodeInfo, order flow.IdentityOrder[flow.Identity]) []NodeInfo { dup := make([]NodeInfo, len(nodes)) copy(dup, nodes) sort.Slice(dup, func(i, j int) bool { diff --git a/model/flow/cluster.go b/model/flow/cluster.go index 9e4eb289ff6..7696d834184 100644 --- a/model/flow/cluster.go +++ b/model/flow/cluster.go @@ -11,7 +11,7 @@ type AssignmentList []IdentifierList // ClusterList is a list of identity lists. Each `IdentityList` represents the // nodes assigned to a specific cluster. -type ClusterList []IdentityList +type ClusterList []IdentitySkeletonList func (al AssignmentList) EqualTo(other AssignmentList) bool { if len(al) != len(other) { @@ -45,10 +45,10 @@ func (cl ClusterList) Assignments() AssignmentList { // NewClusterList creates a new cluster list based on the given cluster assignment // and the provided list of identities. -func NewClusterList(assignments AssignmentList, collectors IdentityList) (ClusterList, error) { +func NewClusterList(assignments AssignmentList, collectors IdentitySkeletonList) (ClusterList, error) { // build a lookup for all the identities by node identifier - lookup := make(map[Identifier]*Identity) + lookup := make(map[Identifier]*IdentitySkeleton) for _, collector := range collectors { _, ok := lookup[collector.NodeID] if ok { @@ -60,7 +60,7 @@ func NewClusterList(assignments AssignmentList, collectors IdentityList) (Cluste // replicate the identifier list but use identities instead clusters := make(ClusterList, 0, len(assignments)) for _, participants := range assignments { - cluster := make(IdentityList, 0, len(participants)) + cluster := make(IdentitySkeletonList, 0, len(participants)) for _, participantID := range participants { participant, found := lookup[participantID] if !found { @@ -81,7 +81,7 @@ func NewClusterList(assignments AssignmentList, collectors IdentityList) (Cluste } // ByIndex retrieves the list of identities that are part of the given cluster. -func (cl ClusterList) ByIndex(index uint) (IdentityList, bool) { +func (cl ClusterList) ByIndex(index uint) (IdentitySkeletonList, bool) { if index >= uint(len(cl)) { return nil, false } @@ -93,7 +93,7 @@ func (cl ClusterList) ByIndex(index uint) (IdentityList, bool) { // // For evenly distributed transaction IDs, this will evenly distribute // transactions between clusters. -func (cl ClusterList) ByTxID(txID Identifier) (IdentityList, bool) { +func (cl ClusterList) ByTxID(txID Identifier) (IdentitySkeletonList, bool) { bigTxID := new(big.Int).SetBytes(txID[:]) bigIndex := new(big.Int).Mod(bigTxID, big.NewInt(int64(len(cl)))) return cl.ByIndex(uint(bigIndex.Uint64())) @@ -103,7 +103,7 @@ func (cl ClusterList) ByTxID(txID Identifier) (IdentityList, bool) { // // Nodes will be divided into equally sized clusters as far as possible. // The last return value will indicate if the look up was successful -func (cl ClusterList) ByNodeID(nodeID Identifier) (IdentityList, uint, bool) { +func (cl ClusterList) ByNodeID(nodeID Identifier) (IdentitySkeletonList, uint, bool) { for index, cluster := range cl { for _, participant := range cluster { if participant.NodeID == nodeID { @@ -115,7 +115,7 @@ func (cl ClusterList) ByNodeID(nodeID Identifier) (IdentityList, uint, bool) { } // IndexOf returns the index of the given cluster. -func (cl ClusterList) IndexOf(cluster IdentityList) (uint, bool) { +func (cl ClusterList) IndexOf(cluster IdentitySkeletonList) (uint, bool) { clusterFingerprint := cluster.ID() for index, other := range cl { if other.ID() == clusterFingerprint { diff --git a/model/flow/cluster_test.go b/model/flow/cluster_test.go index 52d8f39e72c..9bd245cafb9 100644 --- a/model/flow/cluster_test.go +++ b/model/flow/cluster_test.go @@ -15,10 +15,10 @@ import ( func TestClusterAssignments(t *testing.T) { identities := unittest.IdentityListFixture(100, unittest.WithRole(flow.RoleCollection)) - assignments := unittest.ClusterAssignment(10, identities) + assignments := unittest.ClusterAssignment(10, identities.ToSkeleton()) assert.Len(t, assignments, 10) - clusters, err := factory.NewClusterList(assignments, identities) + clusters, err := factory.NewClusterList(assignments, identities.ToSkeleton()) require.NoError(t, err) assert.Equal(t, assignments, clusters.Assignments()) } diff --git a/model/flow/epoch.go b/model/flow/epoch.go index f1181753e9c..9da90aeba05 100644 --- a/model/flow/epoch.go +++ b/model/flow/epoch.go @@ -312,7 +312,7 @@ func (commit *EpochCommit) EqualTo(other *EpochCommit) bool { // ToDKGParticipantLookup constructs a DKG participant lookup from an identity // list and a key list. The identity list must be EXACTLY the same (order and // contents) as that used when initializing the corresponding DKG instance. -func ToDKGParticipantLookup(participants IdentityList, keys []crypto.PublicKey) (map[Identifier]DKGParticipant, error) { +func ToDKGParticipantLookup(participants IdentitySkeletonList, keys []crypto.PublicKey) (map[Identifier]DKGParticipant, error) { if len(participants) != len(keys) { return nil, fmt.Errorf("participant list (len=%d) does not match key list (len=%d)", len(participants), len(keys)) } diff --git a/model/flow/epoch_test.go b/model/flow/epoch_test.go index 9c9a542540d..30131b54430 100644 --- a/model/flow/epoch_test.go +++ b/model/flow/epoch_test.go @@ -179,8 +179,8 @@ func TestEpochCommit_EqualTo(t *testing.T) { func TestEpochSetup_EqualTo(t *testing.T) { - identityA := unittest.IdentityFixture() - identityB := unittest.IdentityFixture() + identityA := &unittest.IdentityFixture().IdentitySkeleton + identityB := &unittest.IdentityFixture().IdentitySkeleton assignmentA := flow.AssignmentList{[]flow.Identifier{[32]byte{1, 2, 3}, [32]byte{2, 2, 2}}} assignmentB := flow.AssignmentList{[]flow.Identifier{[32]byte{1, 2, 3}, [32]byte{}}} @@ -243,8 +243,8 @@ func TestEpochSetup_EqualTo(t *testing.T) { t.Run("Participants length differ", func(t *testing.T) { - a := &flow.EpochSetup{Participants: flow.IdentityList{identityA}} - b := &flow.EpochSetup{Participants: flow.IdentityList{}} + a := &flow.EpochSetup{Participants: flow.IdentitySkeletonList{identityA}} + b := &flow.EpochSetup{Participants: flow.IdentitySkeletonList{}} require.False(t, a.EqualTo(b)) require.False(t, b.EqualTo(a)) @@ -252,8 +252,8 @@ func TestEpochSetup_EqualTo(t *testing.T) { t.Run("Participants length same but different data", func(t *testing.T) { - a := &flow.EpochSetup{Participants: flow.IdentityList{identityA}} - b := &flow.EpochSetup{Participants: flow.IdentityList{identityB}} + a := &flow.EpochSetup{Participants: flow.IdentitySkeletonList{identityA}} + b := &flow.EpochSetup{Participants: flow.IdentitySkeletonList{identityB}} require.False(t, a.EqualTo(b)) require.False(t, b.EqualTo(a)) @@ -261,8 +261,8 @@ func TestEpochSetup_EqualTo(t *testing.T) { t.Run("Participants length same with same data", func(t *testing.T) { - a := &flow.EpochSetup{Participants: flow.IdentityList{identityA}} - b := &flow.EpochSetup{Participants: flow.IdentityList{identityA}} + a := &flow.EpochSetup{Participants: flow.IdentitySkeletonList{identityA}} + b := &flow.EpochSetup{Participants: flow.IdentitySkeletonList{identityA}} require.True(t, a.EqualTo(b)) require.True(t, b.EqualTo(a)) diff --git a/model/flow/factory/cluster_list.go b/model/flow/factory/cluster_list.go index 29bf374ac23..aa86dcc7146 100644 --- a/model/flow/factory/cluster_list.go +++ b/model/flow/factory/cluster_list.go @@ -12,10 +12,10 @@ import ( // The caller must ensure each assignment contains identities ordered in canonical order, so that // each cluster in the returned cluster list is ordered in canonical order as well. If not, // an error will be returned. -func NewClusterList(assignments flow.AssignmentList, collectors flow.IdentityList) (flow.ClusterList, error) { +func NewClusterList(assignments flow.AssignmentList, collectors flow.IdentitySkeletonList) (flow.ClusterList, error) { // build a lookup for all the identities by node identifier - lookup := make(map[flow.Identifier]*flow.Identity) + lookup := make(map[flow.Identifier]*flow.IdentitySkeleton) for _, collector := range collectors { lookup[collector.NodeID] = collector } @@ -26,7 +26,7 @@ func NewClusterList(assignments flow.AssignmentList, collectors flow.IdentityLis // replicate the identifier list but use identities instead clusters := make(flow.ClusterList, 0, len(assignments)) for i, participants := range assignments { - cluster := make(flow.IdentityList, 0, len(participants)) + cluster := make(flow.IdentitySkeletonList, 0, len(participants)) if len(participants) == 0 { return nil, fmt.Errorf("particpants in assignment list is empty, cluster index %v", i) } diff --git a/model/flow/factory/cluster_list_test.go b/model/flow/factory/cluster_list_test.go index 0c938d5e8da..6a0f11e3f73 100644 --- a/model/flow/factory/cluster_list_test.go +++ b/model/flow/factory/cluster_list_test.go @@ -14,12 +14,12 @@ import ( // This tests verifies that NewClusterList has implemented the check on the assumption. func TestNewClusterListFail(t *testing.T) { identities := unittest.IdentityListFixture(100, unittest.WithRole(flow.RoleCollection)) - assignments := unittest.ClusterAssignment(10, identities) + assignments := unittest.ClusterAssignment(10, identities.ToSkeleton()) tmp := assignments[1][0] assignments[1][0] = assignments[1][1] assignments[1][1] = tmp - _, err := factory.NewClusterList(assignments, identities) + _, err := factory.NewClusterList(assignments, identities.ToSkeleton()) require.Error(t, err) } diff --git a/model/flow/filter/identity.go b/model/flow/filter/identity.go index 0e8aea06fb9..c3137df0ee7 100644 --- a/model/flow/filter/identity.go +++ b/model/flow/filter/identity.go @@ -13,8 +13,8 @@ func Any(*flow.Identity) bool { } // And combines two or more filters that all need to be true. -func And(filters ...flow.IdentityFilter) flow.IdentityFilter { - return func(identity *flow.Identity) bool { +func And[T flow.GenericIdentity](filters ...flow.IdentityFilter[T]) flow.IdentityFilter[T] { + return func(identity *T) bool { for _, filter := range filters { if !filter(identity) { return false @@ -25,8 +25,8 @@ func And(filters ...flow.IdentityFilter) flow.IdentityFilter { } // Or combines two or more filters and only needs one of them to be true. -func Or(filters ...flow.IdentityFilter) flow.IdentityFilter { - return func(identity *flow.Identity) bool { +func Or[T flow.GenericIdentity](filters ...flow.IdentityFilter[T]) flow.IdentityFilter[T] { + return func(identity *T) bool { for _, filter := range filters { if filter(identity) { return true @@ -37,34 +37,34 @@ func Or(filters ...flow.IdentityFilter) flow.IdentityFilter { } // Not returns a filter equivalent to the inverse of the input filter. -func Not(filter flow.IdentityFilter) flow.IdentityFilter { - return func(identity *flow.Identity) bool { +func Not[T flow.GenericIdentity](filter flow.IdentityFilter[T]) flow.IdentityFilter[T] { + return func(identity *T) bool { return !filter(identity) } } // In returns a filter for identities within the input list. This is equivalent // to HasNodeID, but for list-typed inputs. -func In(list flow.IdentityList) flow.IdentityFilter { - return HasNodeID(list.NodeIDs()...) +func In(list flow.IdentityList) flow.IdentityFilter[flow.Identity] { + return HasNodeID[flow.Identity](list.NodeIDs()...) } // HasNodeID returns a filter that returns true for any identity with an ID // matching any of the inputs. -func HasNodeID(nodeIDs ...flow.Identifier) flow.IdentityFilter { +func HasNodeID[T flow.GenericIdentity](nodeIDs ...flow.Identifier) flow.IdentityFilter[T] { lookup := make(map[flow.Identifier]struct{}) for _, nodeID := range nodeIDs { lookup[nodeID] = struct{}{} } - return func(identity *flow.Identity) bool { - _, ok := lookup[identity.NodeID] + return func(identity *T) bool { + _, ok := lookup[(*identity).GetNodeID()] return ok } } // HasNetworkingKey returns a filter that returns true for any identity with a // networking public key matching any of the inputs. -func HasNetworkingKey(keys ...crypto.PublicKey) flow.IdentityFilter { +func HasNetworkingKey(keys ...crypto.PublicKey) flow.IdentityFilter[flow.Identity] { return func(identity *flow.Identity) bool { for _, key := range keys { if key.Equals(identity.NetworkPubKey) { @@ -76,7 +76,7 @@ func HasNetworkingKey(keys ...crypto.PublicKey) flow.IdentityFilter { } // HasWeight returns a filter for nodes with non-zero weight. -func HasWeight(hasWeight bool) flow.IdentityFilter { +func HasWeight(hasWeight bool) flow.IdentityFilter[flow.Identity] { return func(identity *flow.Identity) bool { return (identity.Weight > 0) == hasWeight } @@ -88,13 +88,13 @@ func Ejected(identity *flow.Identity) bool { } // HasRole returns a filter for nodes with one of the input roles. -func HasRole(roles ...flow.Role) flow.IdentityFilter { +func HasRole[T flow.GenericIdentity](roles ...flow.Role) flow.IdentityFilter[T] { lookup := make(map[flow.Role]struct{}) for _, role := range roles { lookup[role] = struct{}{} } - return func(identity *flow.Identity) bool { - _, ok := lookup[identity.Role] + return func(identity *T) bool { + _, ok := lookup[(*identity).GetRole()] return ok } } @@ -109,11 +109,19 @@ var IsValidCurrentEpochParticipant = And( // IsVotingConsensusCommitteeMember is a identity filter for all members of // the consensus committee allowed to vote. var IsVotingConsensusCommitteeMember = And( - HasRole(flow.RoleConsensus), + HasRole[flow.Identity](flow.RoleConsensus), IsValidCurrentEpochParticipant, ) // IsValidDKGParticipant is an identity filter for all DKG participants. It is // equivalent to the filter for consensus committee members, as these are // the same group for now. -var IsValidDKGParticipant = func(identity *flow.IdentitySkeleton) {} +var IsValidDKGParticipant = func(identity *flow.IdentitySkeleton) bool { + if identity.Role != flow.RoleConsensus { + return false + } + if identity.InitialWeight == 0 { + return false + } + return true +} diff --git a/model/flow/identity.go b/model/flow/identity.go index 52b707ac821..e884bba6dc3 100644 --- a/model/flow/identity.go +++ b/model/flow/identity.go @@ -114,6 +114,11 @@ func (iy Identity) String() string { return fmt.Sprintf("%s-%s@%s=%d", iy.Role, iy.NodeID.String(), iy.Address, iy.Weight) } +// String returns a string representation of the identity. +func (iy IdentitySkeleton) String() string { + return fmt.Sprintf("%s-%s@%s", iy.Role, iy.NodeID.String(), iy.Address) +} + // ID returns a unique, persistent identifier for the identity. // CAUTION: the ID may be chosen by a node operator, so long as it is unique. func (iy Identity) ID() Identifier { @@ -331,25 +336,61 @@ func (iy *Identity) EqualTo(other *Identity) bool { return true } +type GenericIdentity interface { + Identity | IdentitySkeleton + GetNodeID() Identifier + GetRole() Role + GetStakingPubKey() crypto.PublicKey + GetNetworkPubKey() crypto.PublicKey + GetInitialWeight() uint64 + GetSkeleton() IdentitySkeleton +} + +func (iy IdentitySkeleton) GetNodeID() Identifier { + return iy.NodeID +} + +func (iy IdentitySkeleton) GetRole() Role { + return iy.Role +} + +func (iy IdentitySkeleton) GetStakingPubKey() crypto.PublicKey { + return iy.StakingPubKey +} + +func (iy IdentitySkeleton) GetNetworkPubKey() crypto.PublicKey { + return iy.NetworkPubKey +} + +func (iy IdentitySkeleton) GetInitialWeight() uint64 { + return iy.InitialWeight +} + +func (iy IdentitySkeleton) GetSkeleton() IdentitySkeleton { + return iy +} + // IdentityFilter is a filter on identities. -type IdentityFilter func(*Identity) bool +type IdentityFilter[T GenericIdentity] func(*T) bool // IdentityOrder is a sort for identities. -type IdentityOrder func(*Identity, *Identity) bool +type IdentityOrder[T GenericIdentity] func(*T, *T) bool // IdentityMapFunc is a modifier function for map operations for identities. // Identities are COPIED from the source slice. -type IdentityMapFunc func(Identity) Identity +type IdentityMapFunc[T GenericIdentity] func(T) T // IdentitySkeletonList is a list of nodes skeletons. -type IdentitySkeletonList []*IdentitySkeleton +type IdentitySkeletonList = GenericIdentityList[IdentitySkeleton] // IdentityList is a list of nodes. -type IdentityList []*Identity +type IdentityList = GenericIdentityList[Identity] + +type GenericIdentityList[T GenericIdentity] []*T // Filter will apply a filter to the identity list. -func (il IdentityList) Filter(filter IdentityFilter) IdentityList { - var dup IdentityList +func (il GenericIdentityList[T]) Filter(filter IdentityFilter[T]) GenericIdentityList[T] { + var dup GenericIdentityList[T] IDLoop: for _, identity := range il { if !filter(identity) { @@ -366,8 +407,8 @@ IDLoop: // CAUTION: this relies on structure copy semantics. Map functions that modify // an object referenced by the input Identity structure will modify identities // in the source slice as well. -func (il IdentityList) Map(f IdentityMapFunc) IdentityList { - dup := make(IdentityList, 0, len(il)) +func (il GenericIdentityList[T]) Map(f IdentityMapFunc[T]) GenericIdentityList[T] { + dup := make(GenericIdentityList[T], 0, len(il)) for _, identity := range il { next := f(*identity) dup = append(dup, &next) @@ -386,8 +427,8 @@ func (il IdentityList) Map(f IdentityMapFunc) IdentityList { // CAUTION: // All Identity fields are deep-copied, _except_ for their keys, which // are copied by reference. -func (il IdentityList) Copy() IdentityList { - dup := make(IdentityList, 0, len(il)) +func (il GenericIdentityList[T]) Copy() GenericIdentityList[T] { + dup := make(GenericIdentityList[T], 0, len(il)) lenList := len(il) @@ -400,29 +441,45 @@ func (il IdentityList) Copy() IdentityList { return dup } +//// Copy returns a copy of IdentitySkeletonList. The resulting slice uses a different +//// backing array, meaning appends and insert operations on either slice are +//// guaranteed to only affect that slice. +//// +//// Copy should be used when modifying an existing identity list by either +//// appending new elements, re-ordering, or inserting new elements in an +//// existing index. +//// +//// CAUTION: +//// All IdentitySkeleton fields are deep-copied, _except_ for their keys, which +//// are copied by reference. +//func (il IdentitySkeletonList) Copy() IdentitySkeletonList { +// dup := make(IdentitySkeletonList, 0, len(il)) +// +// lenList := len(il) +// +// // performance tests show this is faster than 'range' +// for i := 0; i < lenList; i++ { +// // copy the object +// next := *(il[i]) +// dup = append(dup, &next) +// } +// return dup +//} + // Selector returns an identity filter function that selects only identities // within this identity list. -func (il IdentityList) Selector() IdentityFilter { - +func (il GenericIdentityList[T]) Selector() IdentityFilter[T] { lookup := il.Lookup() - return func(identity *Identity) bool { - _, exists := lookup[identity.NodeID] + return func(identity *T) bool { + _, exists := lookup[(*identity).GetNodeID()] return exists } } -func (il IdentitySkeletonList) Lookup() map[Identifier]*IdentitySkeleton { - lookup := make(map[Identifier]*IdentitySkeleton, len(il)) +func (il GenericIdentityList[T]) Lookup() map[Identifier]*T { + lookup := make(map[Identifier]*T, len(il)) for _, identity := range il { - lookup[identity.NodeID] = identity - } - return lookup -} - -func (il IdentityList) Lookup() map[Identifier]*Identity { - lookup := make(map[Identifier]*Identity, len(il)) - for _, identity := range il { - lookup[identity.NodeID] = identity + lookup[(*identity).GetNodeID()] = identity } return lookup } @@ -430,40 +487,31 @@ func (il IdentityList) Lookup() map[Identifier]*Identity { // Sort will sort the list using the given ordering. This is // not recommended for performance. Expand the 'less' function // in place for best performance, and don't use this function. -func (il IdentityList) Sort(less IdentityOrder) IdentityList { +func (il GenericIdentityList[T]) Sort(less IdentityOrder[T]) GenericIdentityList[T] { dup := il.Copy() slices.SortFunc(dup, less) return dup } // Sorted returns whether the list is sorted by the input ordering. -func (il IdentityList) Sorted(less IdentityOrder) bool { +func (il GenericIdentityList[T]) Sorted(less IdentityOrder[T]) bool { return slices.IsSortedFunc(il, less) } // NodeIDs returns the NodeIDs of the nodes in the list. -func (il IdentityList) NodeIDs() IdentifierList { - nodeIDs := make([]Identifier, 0, len(il)) - for _, id := range il { - nodeIDs = append(nodeIDs, id.NodeID) - } - return nodeIDs -} - -// NodeIDs returns the NodeIDs of the nodes in the list. -func (il IdentitySkeletonList) NodeIDs() IdentifierList { +func (il GenericIdentityList[T]) NodeIDs() IdentifierList { nodeIDs := make([]Identifier, 0, len(il)) for _, id := range il { - nodeIDs = append(nodeIDs, id.NodeID) + nodeIDs = append(nodeIDs, (*id).GetNodeID()) } return nodeIDs } // PublicStakingKeys returns a list with the public staking keys (order preserving). -func (il IdentitySkeletonList) PublicStakingKeys() []crypto.PublicKey { +func (il GenericIdentityList[T]) PublicStakingKeys() []crypto.PublicKey { pks := make([]crypto.PublicKey, 0, len(il)) for _, id := range il { - pks = append(pks, id.StakingPubKey) + pks = append(pks, (*id).GetStakingPubKey()) } return pks } @@ -479,32 +527,32 @@ func (il IdentitySkeletonList) PublicStakingKeys() []crypto.PublicKey { // - The outputs of `IdentityList.ID()` and `IdentityList.Checksum()` are both order-sensitive. // Therefore, the `IdentityList` must be in canonical order, unless explicitly specified // otherwise by the protocol. -func (il IdentityList) ID() Identifier { +func (il GenericIdentityList[T]) ID() Identifier { return il.NodeIDs().ID() } // Checksum generates a cryptographic commitment to the full IdentityList, including mutable fields. // The checksum for the same group of identities (by NodeID) may change from block to block. -func (il IdentityList) Checksum() Identifier { +func (il GenericIdentityList[T]) Checksum() Identifier { return MakeID(il) } // TotalWeight returns the total weight of all given identities. -func (il IdentitySkeletonList) TotalWeight() uint64 { +func (il GenericIdentityList[T]) TotalWeight() uint64 { var total uint64 for _, identity := range il { - total += identity.InitialWeight + total += (*identity).GetInitialWeight() } return total } // Count returns the count of identities. -func (il IdentityList) Count() uint { +func (il GenericIdentityList[T]) Count() uint { return uint(len(il)) } // ByIndex returns the node at the given index. -func (il IdentityList) ByIndex(index uint) (*Identity, bool) { +func (il GenericIdentityList[T]) ByIndex(index uint) (*T, bool) { if index >= uint(len(il)) { return nil, false } @@ -512,19 +560,9 @@ func (il IdentityList) ByIndex(index uint) (*Identity, bool) { } // ByNodeID gets a node from the list by node ID. -func (il IdentitySkeletonList) ByNodeID(nodeID Identifier) (*IdentitySkeleton, bool) { +func (il GenericIdentityList[T]) ByNodeID(nodeID Identifier) (*T, bool) { for _, identity := range il { - if identity.NodeID == nodeID { - return identity, true - } - } - return nil, false -} - -// ByNodeID gets a node from the list by node ID. -func (il IdentityList) ByNodeID(nodeID Identifier) (*Identity, bool) { - for _, identity := range il { - if identity.NodeID == nodeID { + if (*identity).GetNodeID() == nodeID { return identity, true } } @@ -532,9 +570,9 @@ func (il IdentityList) ByNodeID(nodeID Identifier) (*Identity, bool) { } // ByNetworkingKey gets a node from the list by network public key. -func (il IdentityList) ByNetworkingKey(key crypto.PublicKey) (*Identity, bool) { +func (il GenericIdentityList[T]) ByNetworkingKey(key crypto.PublicKey) (*T, bool) { for _, identity := range il { - if identity.NetworkPubKey.Equals(key) { + if (*identity).GetNetworkPubKey().Equals(key) { return identity, true } } @@ -542,9 +580,9 @@ func (il IdentityList) ByNetworkingKey(key crypto.PublicKey) (*Identity, bool) { } // Sample returns non-deterministic random sample from the `IdentityList` -func (il IdentityList) Sample(size uint) (IdentityList, error) { +func (il GenericIdentityList[T]) Sample(size uint) (GenericIdentityList[T], error) { n := uint(len(il)) - dup := make([]*Identity, 0, n) + dup := make(GenericIdentityList[T], 0, n) dup = append(dup, il...) if n < size { size = n @@ -561,7 +599,7 @@ func (il IdentityList) Sample(size uint) (IdentityList, error) { // Shuffle randomly shuffles the identity list (non-deterministic), // and returns the shuffled list without modifying the receiver. -func (il IdentityList) Shuffle() (IdentityList, error) { +func (il GenericIdentityList[T]) Shuffle() (GenericIdentityList[T], error) { return il.Sample(uint(len(il))) } @@ -570,9 +608,9 @@ func (il IdentityList) Shuffle() (IdentityList, error) { // if `pct>0`, so this will always select at least one identity. // // NOTE: The input must be between 0-1. -func (il IdentityList) SamplePct(pct float64) (IdentityList, error) { +func (il GenericIdentityList[T]) SamplePct(pct float64) (GenericIdentityList[T], error) { if pct <= 0 { - return IdentityList{}, nil + return GenericIdentityList[T]{}, nil } count := float64(il.Count()) * pct @@ -590,50 +628,24 @@ func (il IdentityList) SamplePct(pct float64) (IdentityList, error) { // where duplicates are identities with the same node ID. // Receiver `il` and/or method input `other` can be nil or empty. // The returned IdentityList is sorted in canonical order. -func (il IdentityList) Union(other IdentityList) IdentityList { - maxLen := len(il) + len(other) - - union := make(IdentityList, 0, maxLen) - set := make(map[Identifier]struct{}, maxLen) - - for _, list := range []IdentityList{il, other} { - for _, id := range list { - if _, isDuplicate := set[id.NodeID]; !isDuplicate { - set[id.NodeID] = struct{}{} - union = append(union, id) - } - } - } - - slices.SortFunc(union, func(a, b *Identity) bool { - return bytes.Compare(a.NodeID[:], b.NodeID[:]) < 0 - }) - - return union -} - -// Union returns a new identity list containing every identity that occurs in -// either `il`, or `other`, or both. There are no duplicates in the output, -// where duplicates are identities with the same node ID. -// Receiver `il` and/or method input `other` can be nil or empty. -// The returned IdentityList is sorted in canonical order. -func (il IdentitySkeletonList) Union(other IdentitySkeletonList) IdentitySkeletonList { +func (il GenericIdentityList[T]) Union(other GenericIdentityList[T]) GenericIdentityList[T] { maxLen := len(il) + len(other) - union := make(IdentitySkeletonList, 0, maxLen) + union := make(GenericIdentityList[T], 0, maxLen) set := make(map[Identifier]struct{}, maxLen) - for _, list := range []IdentitySkeletonList{il, other} { + for _, list := range []GenericIdentityList[T]{il, other} { for _, id := range list { - if _, isDuplicate := set[id.NodeID]; !isDuplicate { - set[id.NodeID] = struct{}{} + if _, isDuplicate := set[(*id).GetNodeID()]; !isDuplicate { + set[(*id).GetNodeID()] = struct{}{} union = append(union, id) } } } - slices.SortFunc(union, func(a, b *IdentitySkeleton) bool { - return bytes.Compare(a.NodeID[:], b.NodeID[:]) < 0 + slices.SortFunc(union, func(a, b *T) bool { + lhs, rhs := (*a).GetNodeID(), (*b).GetNodeID() + return bytes.Compare(lhs[:], rhs[:]) < 0 }) return union @@ -641,43 +653,38 @@ func (il IdentitySkeletonList) Union(other IdentitySkeletonList) IdentitySkeleto // EqualTo checks if the other list if the same, that it contains the same elements // in the same order -func (il IdentityList) EqualTo(other IdentityList) bool { - return slices.EqualFunc(il, other, func(a, b *Identity) bool { - return a.EqualTo(b) - }) -} - -// EqualTo checks if the other list if the same, that it contains the same elements -// in the same order -func (il IdentitySkeletonList) EqualTo(other IdentitySkeletonList) bool { - return slices.EqualFunc(il, other, func(a, b *IdentitySkeleton) bool { - return a.EqualTo(b) - }) +func (il GenericIdentityList[T]) EqualTo(other GenericIdentityList[T]) bool { + // TODO: temporary + //return slices.EqualFunc(il, other, func(a, b *T) bool { + // return (*a)..EqualTo(b) + //}) + return il.ID() == other.ID() } // Exists takes a previously sorted Identity list and searches it for the target value // This code is optimized, so the coding style will be different // target: value to search for // CAUTION: The identity list MUST be sorted prior to calling this method -func (il IdentityList) Exists(target *Identity) bool { - return il.IdentifierExists(target.NodeID) +func (il GenericIdentityList[T]) Exists(target *T) bool { + return il.IdentifierExists((*target).GetNodeID()) } // IdentifierExists takes a previously sorted Identity list and searches it for the target value // target: value to search for // CAUTION: The identity list MUST be sorted prior to calling this method -func (il IdentityList) IdentifierExists(target Identifier) bool { - _, ok := slices.BinarySearchFunc(il, &Identity{IdentitySkeleton: IdentitySkeleton{NodeID: target}}, func(a, b *Identity) int { - return bytes.Compare(a.NodeID[:], b.NodeID[:]) +func (il GenericIdentityList[T]) IdentifierExists(target Identifier) bool { + _, ok := slices.BinarySearchFunc(il, target, func(a *T, b Identifier) int { + lhs := (*a).GetNodeID() + return bytes.Compare(lhs[:], b[:]) }) return ok } // GetIndex returns the index of the identifier in the IdentityList and true // if the identifier is found. -func (il IdentityList) GetIndex(target Identifier) (uint, bool) { - i := slices.IndexFunc(il, func(a *Identity) bool { - return a.NodeID == target +func (il GenericIdentityList[T]) GetIndex(target Identifier) (uint, bool) { + i := slices.IndexFunc(il, func(a *T) bool { + return (*a).GetNodeID() == target }) if i == -1 { return 0, false @@ -686,10 +693,11 @@ func (il IdentityList) GetIndex(target Identifier) (uint, bool) { } // ToSkeleton converts the identity list to a list of identity skeletons. -func (il IdentityList) ToSkeleton() IdentitySkeletonList { +func (il GenericIdentityList[T]) ToSkeleton() IdentitySkeletonList { skeletons := make(IdentitySkeletonList, len(il)) for i, id := range il { - skeletons[i] = &id.IdentitySkeleton + v := (*id).GetSkeleton() + skeletons[i] = &v } return skeletons } diff --git a/model/flow/identity_test.go b/model/flow/identity_test.go index a913ca5477a..e89c949e3d1 100644 --- a/model/flow/identity_test.go +++ b/model/flow/identity_test.go @@ -104,7 +104,7 @@ func TestIdentityList_Exists(t *testing.T) { il2 := unittest.IdentityListFixture(1) // sort the first list - il1 = il1.Sort(order.Canonical) + il1 = il1.Sort(order.Canonical[flow.Identity]) for i := 0; i < 10; i++ { assert.True(t, il1.Exists(il1[i])) @@ -119,7 +119,7 @@ func TestIdentityList_IdentifierExists(t *testing.T) { il2 := unittest.IdentityListFixture(1) // sort the first list - il1 = il1.Sort(order.Canonical) + il1 = il1.Sort(order.Canonical[flow.Identity]) for i := 0; i < 10; i++ { assert.True(t, il1.IdentifierExists(il1[i].NodeID)) @@ -246,10 +246,10 @@ func TestIdentity_Sort(t *testing.T) { il := unittest.IdentityListFixture(20) random, err := il.Shuffle() require.NoError(t, err) - assert.False(t, random.Sorted(order.Canonical)) + assert.False(t, random.Sorted(order.Canonical[flow.Identity])) - canonical := il.Sort(order.Canonical) - assert.True(t, canonical.Sorted(order.Canonical)) + canonical := il.Sort(order.Canonical[flow.Identity]) + assert.True(t, canonical.Sorted(order.Canonical[flow.Identity])) } func TestIdentity_EqualTo(t *testing.T) { diff --git a/model/flow/mapfunc/identity.go b/model/flow/mapfunc/identity.go index aaab9de0ae4..681e21d1088 100644 --- a/model/flow/mapfunc/identity.go +++ b/model/flow/mapfunc/identity.go @@ -4,14 +4,14 @@ import ( "github.com/onflow/flow-go/model/flow" ) -func WithInitialWeight(weight uint64) flow.IdentityMapFunc { +func WithInitialWeight(weight uint64) flow.IdentityMapFunc[flow.Identity] { return func(identity flow.Identity) flow.Identity { identity.InitialWeight = weight return identity } } -func WithWeight(weight uint64) flow.IdentityMapFunc { +func WithWeight(weight uint64) flow.IdentityMapFunc[flow.Identity] { return func(identity flow.Identity) flow.Identity { identity.Weight = weight return identity diff --git a/model/flow/order/identity.go b/model/flow/order/identity.go index 5b78c7a3dd4..c30c9f0cbd3 100644 --- a/model/flow/order/identity.go +++ b/model/flow/order/identity.go @@ -7,8 +7,8 @@ import ( ) // Canonical represents the canonical ordering for identity lists. -func Canonical(identity1 *flow.Identity, identity2 *flow.Identity) bool { - return IdentifierCanonical(identity1.NodeID, identity2.NodeID) +func Canonical[T flow.GenericIdentity](identity1 *T, identity2 *T) bool { + return IdentifierCanonical((*identity1).GetNodeID(), (*identity2).GetNodeID()) } // ByReferenceOrder return a function for sorting identities based on the order diff --git a/model/flow/order/identity_test.go b/model/flow/order/identity_test.go index 2c79b61ab4a..191a6d9dc39 100644 --- a/model/flow/order/identity_test.go +++ b/model/flow/order/identity_test.go @@ -3,6 +3,7 @@ package order_test import ( + "github.com/onflow/flow-go/model/flow" "testing" "github.com/stretchr/testify/require" @@ -14,5 +15,5 @@ import ( // Test the canonical ordering of identity and identifier match func TestCanonicalOrderingMatch(t *testing.T) { identities := unittest.IdentityListFixture(100) - require.Equal(t, identities.Sort(order.Canonical).NodeIDs(), identities.NodeIDs().Sort(order.IdentifierCanonical)) + require.Equal(t, identities.Sort(order.Canonical[flow.Identity]).NodeIDs(), identities.NodeIDs().Sort(order.IdentifierCanonical)) } diff --git a/model/flow/protocol_state.go b/model/flow/protocol_state.go index 2ae7e30cde1..6b3c6072eed 100644 --- a/model/flow/protocol_state.go +++ b/model/flow/protocol_state.go @@ -167,7 +167,7 @@ func NewRichProtocolStateEntry( // if next epoch is available, it means that we have observed epoch setup event and we are not anymore in staking phase, // so we need to build the identity table using current and next epoch setup events. - result.Identities, err = buildIdentityTable( + result.Identities, err = BuildIdentityTable( protocolState.CurrentEpoch.Identities, currentEpochSetup.Participants, nextEpochSetup.Participants, @@ -176,7 +176,7 @@ func NewRichProtocolStateEntry( return nil, fmt.Errorf("could not build identity table for setup/commit phase: %w", err) } - result.NextIdentities, err = buildIdentityTable( + result.NextIdentities, err = BuildIdentityTable( nextEpoch.Identities, nextEpochSetup.Participants, currentEpochSetup.Participants, @@ -191,7 +191,7 @@ func NewRichProtocolStateEntry( if previousEpochSetup != nil { otherIdentities = previousEpochSetup.Participants } - result.Identities, err = buildIdentityTable( + result.Identities, err = BuildIdentityTable( protocolState.CurrentEpoch.Identities, currentEpochSetup.Participants, otherIdentities, @@ -341,7 +341,7 @@ func (ll DynamicIdentityEntryList) Sort(less IdentifierOrder) DynamicIdentityEnt return dup } -// buildIdentityTable constructs the full identity table for the target epoch by combining data from: +// BuildIdentityTable constructs the full identity table for the target epoch by combining data from: // 1. The target epoch's Dynamic Identities. // 2. The target epoch's IdentitySkeletons // (recorded in EpochSetup event and immutable throughout the epoch). @@ -352,7 +352,7 @@ func (ll DynamicIdentityEntryList) Sort(less IdentifierOrder) DynamicIdentityEnt // // It also performs sanity checks to make sure that the data is consistent. // No errors are expected during normal operation. -func buildIdentityTable( +func BuildIdentityTable( targetEpochDynamicIdentities DynamicIdentityEntryList, targetEpochIdentitySkeletons IdentitySkeletonList, adjacentEpochIdentitySkeletons IdentitySkeletonList, diff --git a/model/flow/protocol_state_test.go b/model/flow/protocol_state_test.go index 01383ac128e..f1a26ba71d3 100644 --- a/model/flow/protocol_state_test.go +++ b/model/flow/protocol_state_test.go @@ -16,13 +16,23 @@ func TestNewRichProtocolStateEntry(t *testing.T) { // * no previous epoch exists from the perspective of the freshly-sporked protocol state // * network is currently in the staking phase for the next epoch, hence no service events for the next epoch exist t.Run("staking-root-protocol-state", func(t *testing.T) { - currentEpochSetup := unittest.EpochSetupFixture() + setup := unittest.EpochSetupFixture() currentEpochCommit := unittest.EpochCommitFixture() + identities := make(flow.DynamicIdentityEntryList, 0, len(setup.Participants)) + for _, identity := range setup.Participants { + identities = append(identities, &flow.DynamicIdentityEntry{ + NodeID: identity.NodeID, + Dynamic: flow.DynamicIdentity{ + Weight: identity.InitialWeight, + Ejected: false, + }, + }) + } stateEntry := &flow.ProtocolStateEntry{ CurrentEpoch: flow.EpochStateContainer{ - SetupID: currentEpochSetup.ID(), + SetupID: setup.ID(), CommitID: currentEpochCommit.ID(), - Identities: flow.DynamicIdentityEntryListFromIdentities(currentEpochSetup.Participants), + Identities: identities, }, PreviousEpochEventIDs: flow.EventIDs{}, InvalidStateTransitionAttempted: false, @@ -31,13 +41,15 @@ func TestNewRichProtocolStateEntry(t *testing.T) { stateEntry, nil, nil, - currentEpochSetup, + setup, currentEpochCommit, nil, nil, ) assert.NoError(t, err) - assert.Equal(t, currentEpochSetup.Participants, entry.Identities, "should be equal to current epoch setup participants") + expectedIdentities, err := flow.BuildIdentityTable(identities, setup.Participants, nil) + assert.NoError(t, err) + assert.Equal(t, expectedIdentities, entry.Identities, "should be equal to current epoch setup participants") }) // Common situation during the staking phase for epoch N+1 @@ -56,7 +68,12 @@ func TestNewRichProtocolStateEntry(t *testing.T) { nil, ) assert.NoError(t, err) - expectedIdentities := stateEntry.CurrentEpochSetup.Participants.Union(stateEntry.PreviousEpochSetup.Participants) + expectedIdentities, err := flow.BuildIdentityTable( + stateEntry.CurrentEpoch.Identities, + stateEntry.CurrentEpochSetup.Participants, + stateEntry.PreviousEpochSetup.Participants, + ) + assert.NoError(t, err) assert.Equal(t, expectedIdentities, richEntry.Identities, "should be equal to current epoch setup participants + previous epoch setup participants") assert.Nil(t, richEntry.NextEpoch) }) @@ -81,10 +98,20 @@ func TestNewRichProtocolStateEntry(t *testing.T) { nil, ) assert.NoError(t, err) - expectedIdentities := stateEntry.CurrentEpochSetup.Participants.Union(stateEntry.NextEpochSetup.Participants) + expectedIdentities, err := flow.BuildIdentityTable( + stateEntry.CurrentEpoch.Identities, + stateEntry.CurrentEpochSetup.Participants, + stateEntry.NextEpochSetup.Participants, + ) + assert.NoError(t, err) assert.Equal(t, expectedIdentities, richEntry.Identities, "should be equal to current epoch setup participants + next epoch setup participants") assert.Nil(t, richEntry.NextEpochCommit) - expectedIdentities = stateEntry.NextEpochSetup.Participants.Union(stateEntry.CurrentEpochSetup.Participants) + expectedIdentities, err = flow.BuildIdentityTable( + stateEntry.NextEpoch.Identities, + stateEntry.NextEpochSetup.Participants, + stateEntry.CurrentEpochSetup.Participants, + ) + assert.NoError(t, err) assert.Equal(t, expectedIdentities, richEntry.NextIdentities, "should be equal to next epoch setup participants + current epoch setup participants") }) @@ -107,9 +134,19 @@ func TestNewRichProtocolStateEntry(t *testing.T) { stateEntry.NextEpochCommit, ) assert.NoError(t, err) - expectedIdentities := stateEntry.CurrentEpochSetup.Participants.Union(stateEntry.NextEpochSetup.Participants) + expectedIdentities, err := flow.BuildIdentityTable( + stateEntry.CurrentEpoch.Identities, + stateEntry.CurrentEpochSetup.Participants, + stateEntry.NextEpochSetup.Participants, + ) + assert.NoError(t, err) assert.Equal(t, expectedIdentities, richEntry.Identities, "should be equal to current epoch setup participants + next epoch setup participants") - expectedIdentities = stateEntry.NextEpochSetup.Participants.Union(stateEntry.CurrentEpochSetup.Participants) + expectedIdentities, err = flow.BuildIdentityTable( + stateEntry.NextEpoch.Identities, + stateEntry.NextEpochSetup.Participants, + stateEntry.CurrentEpochSetup.Participants, + ) + assert.NoError(t, err) assert.Equal(t, expectedIdentities, richEntry.NextIdentities, "should be equal to next epoch setup participants + current epoch setup participants") }) diff --git a/model/verification/chunkDataPackRequest.go b/model/verification/chunkDataPackRequest.go index 9f2bf42c52c..f613750cbcb 100644 --- a/model/verification/chunkDataPackRequest.go +++ b/model/verification/chunkDataPackRequest.go @@ -28,7 +28,7 @@ type ChunkDataPackRequestInfo struct { func (c ChunkDataPackRequestInfo) SampleTargets(count int) (flow.IdentifierList, error) { // if there are enough receipts produced the same result (agrees), we sample from them. if len(c.Agrees) >= count { - sample, err := c.Targets.Filter(filter.HasNodeID(c.Agrees...)).Sample(uint(count)) + sample, err := c.Targets.Filter(filter.HasNodeID[flow.Identity](c.Agrees...)).Sample(uint(count)) if err != nil { return nil, fmt.Errorf("sampling target failed: %w", err) } @@ -41,7 +41,7 @@ func (c ChunkDataPackRequestInfo) SampleTargets(count int) (flow.IdentifierList, // fetch from the one produced the same result (the only agree) need := uint(count - len(c.Agrees)) - nonResponders, err := c.Targets.Filter(filter.Not(filter.HasNodeID(c.Disagrees...))).Sample(need) + nonResponders, err := c.Targets.Filter(filter.Not(filter.HasNodeID[flow.Identity](c.Disagrees...))).Sample(need) if err != nil { return nil, fmt.Errorf("sampling target failed: %w", err) } diff --git a/module/id_provider.go b/module/id_provider.go index b5544f09bc9..3b84181fce2 100644 --- a/module/id_provider.go +++ b/module/id_provider.go @@ -20,7 +20,7 @@ type IdentityProvider interface { // protocol that pass the provided filter. Caution, this includes ejected nodes. // Please check the `Ejected` flag in the identities (or provide a filter for // removing ejected nodes). - Identities(flow.IdentityFilter) flow.IdentityList + Identities(flow.IdentityFilter[flow.Identity]) flow.IdentityList // ByNodeID returns the full identity for the node with the given Identifier, // where Identifier is the way the protocol refers to the node. The function diff --git a/module/local.go b/module/local.go index e1fa2f5fa45..a88a97982c2 100644 --- a/module/local.go +++ b/module/local.go @@ -23,7 +23,7 @@ type Local interface { Sign([]byte, hash.Hasher) (crypto.Signature, error) // NotMeFilter returns handy not-me filter for searching identity - NotMeFilter() flow.IdentityFilter + NotMeFilter() flow.IdentityFilter[flow.Identity] // SignFunc provides a signature oracle that given a message, a hasher, and a signing function, it // generates and returns a signature over the message using the node's private key diff --git a/module/mock/identity_provider.go b/module/mock/identity_provider.go index 925583a40d0..9a98a1b6c72 100644 --- a/module/mock/identity_provider.go +++ b/module/mock/identity_provider.go @@ -67,11 +67,11 @@ func (_m *IdentityProvider) ByPeerID(_a0 peer.ID) (*flow.Identity, bool) { } // Identities provides a mock function with given fields: _a0 -func (_m *IdentityProvider) Identities(_a0 flow.IdentityFilter) flow.IdentityList { +func (_m *IdentityProvider) Identities(_a0 flow.IdentityFilter[flow.Identity]) flow.IdentityList { ret := _m.Called(_a0) var r0 flow.IdentityList - if rf, ok := ret.Get(0).(func(flow.IdentityFilter) flow.IdentityList); ok { + if rf, ok := ret.Get(0).(func(flow.IdentityFilter[flow.Identity]) flow.IdentityList); ok { r0 = rf(_a0) } else { if ret.Get(0) != nil { diff --git a/module/mock/local.go b/module/mock/local.go index 37a980da0cd..c9121573b5d 100644 --- a/module/mock/local.go +++ b/module/mock/local.go @@ -47,15 +47,15 @@ func (_m *Local) NodeID() flow.Identifier { } // NotMeFilter provides a mock function with given fields: -func (_m *Local) NotMeFilter() flow.IdentityFilter { +func (_m *Local) NotMeFilter() flow.IdentityFilter[flow.Identity] { ret := _m.Called() - var r0 flow.IdentityFilter - if rf, ok := ret.Get(0).(func() flow.IdentityFilter); ok { + var r0 flow.IdentityFilter[flow.Identity] + if rf, ok := ret.Get(0).(func() flow.IdentityFilter[flow.Identity]); ok { r0 = rf() } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(flow.IdentityFilter) + r0 = ret.Get(0).(flow.IdentityFilter[flow.Identity]) } } diff --git a/module/mock/requester.go b/module/mock/requester.go index d3effd8e215..47256ecf8bf 100644 --- a/module/mock/requester.go +++ b/module/mock/requester.go @@ -13,7 +13,7 @@ type Requester struct { } // EntityByID provides a mock function with given fields: entityID, selector -func (_m *Requester) EntityByID(entityID flow.Identifier, selector flow.IdentityFilter) { +func (_m *Requester) EntityByID(entityID flow.Identifier, selector flow.IdentityFilter[flow.Identity]) { _m.Called(entityID, selector) } @@ -23,7 +23,7 @@ func (_m *Requester) Force() { } // Query provides a mock function with given fields: key, selector -func (_m *Requester) Query(key flow.Identifier, selector flow.IdentityFilter) { +func (_m *Requester) Query(key flow.Identifier, selector flow.IdentityFilter[flow.Identity]) { _m.Called(key, selector) } diff --git a/module/mocks/network.go b/module/mocks/network.go index 3788efaf45a..e69de29bb2d 100644 --- a/module/mocks/network.go +++ b/module/mocks/network.go @@ -1,168 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/onflow/flow-go/module (interfaces: Local,Requester) - -// Package mocks is a generated GoMock package. -package mocks - -import ( - reflect "reflect" - - gomock "github.com/golang/mock/gomock" - crypto "github.com/onflow/flow-go/crypto" - hash "github.com/onflow/flow-go/crypto/hash" - flow "github.com/onflow/flow-go/model/flow" -) - -// MockLocal is a mock of Local interface. -type MockLocal struct { - ctrl *gomock.Controller - recorder *MockLocalMockRecorder -} - -// MockLocalMockRecorder is the mock recorder for MockLocal. -type MockLocalMockRecorder struct { - mock *MockLocal -} - -// NewMockLocal creates a new mock instance. -func NewMockLocal(ctrl *gomock.Controller) *MockLocal { - mock := &MockLocal{ctrl: ctrl} - mock.recorder = &MockLocalMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockLocal) EXPECT() *MockLocalMockRecorder { - return m.recorder -} - -// Address mocks base method. -func (m *MockLocal) Address() string { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Address") - ret0, _ := ret[0].(string) - return ret0 -} - -// Address indicates an expected call of Address. -func (mr *MockLocalMockRecorder) Address() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Address", reflect.TypeOf((*MockLocal)(nil).Address)) -} - -// NodeID mocks base method. -func (m *MockLocal) NodeID() flow.Identifier { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "NodeID") - ret0, _ := ret[0].(flow.Identifier) - return ret0 -} - -// NodeID indicates an expected call of NodeID. -func (mr *MockLocalMockRecorder) NodeID() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NodeID", reflect.TypeOf((*MockLocal)(nil).NodeID)) -} - -// NotMeFilter mocks base method. -func (m *MockLocal) NotMeFilter() flow.IdentityFilter { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "NotMeFilter") - ret0, _ := ret[0].(flow.IdentityFilter) - return ret0 -} - -// NotMeFilter indicates an expected call of NotMeFilter. -func (mr *MockLocalMockRecorder) NotMeFilter() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NotMeFilter", reflect.TypeOf((*MockLocal)(nil).NotMeFilter)) -} - -// Sign mocks base method. -func (m *MockLocal) Sign(arg0 []byte, arg1 hash.Hasher) (crypto.Signature, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Sign", arg0, arg1) - ret0, _ := ret[0].(crypto.Signature) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Sign indicates an expected call of Sign. -func (mr *MockLocalMockRecorder) Sign(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Sign", reflect.TypeOf((*MockLocal)(nil).Sign), arg0, arg1) -} - -// SignFunc mocks base method. -func (m *MockLocal) SignFunc(arg0 []byte, arg1 hash.Hasher, arg2 func(crypto.PrivateKey, []byte, hash.Hasher) (crypto.Signature, error)) (crypto.Signature, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SignFunc", arg0, arg1, arg2) - ret0, _ := ret[0].(crypto.Signature) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// SignFunc indicates an expected call of SignFunc. -func (mr *MockLocalMockRecorder) SignFunc(arg0, arg1, arg2 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SignFunc", reflect.TypeOf((*MockLocal)(nil).SignFunc), arg0, arg1, arg2) -} - -// MockRequester is a mock of Requester interface. -type MockRequester struct { - ctrl *gomock.Controller - recorder *MockRequesterMockRecorder -} - -// MockRequesterMockRecorder is the mock recorder for MockRequester. -type MockRequesterMockRecorder struct { - mock *MockRequester -} - -// NewMockRequester creates a new mock instance. -func NewMockRequester(ctrl *gomock.Controller) *MockRequester { - mock := &MockRequester{ctrl: ctrl} - mock.recorder = &MockRequesterMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockRequester) EXPECT() *MockRequesterMockRecorder { - return m.recorder -} - -// EntityByID mocks base method. -func (m *MockRequester) EntityByID(arg0 flow.Identifier, arg1 flow.IdentityFilter) { - m.ctrl.T.Helper() - m.ctrl.Call(m, "EntityByID", arg0, arg1) -} - -// EntityByID indicates an expected call of EntityByID. -func (mr *MockRequesterMockRecorder) EntityByID(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EntityByID", reflect.TypeOf((*MockRequester)(nil).EntityByID), arg0, arg1) -} - -// Force mocks base method. -func (m *MockRequester) Force() { - m.ctrl.T.Helper() - m.ctrl.Call(m, "Force") -} - -// Force indicates an expected call of Force. -func (mr *MockRequesterMockRecorder) Force() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Force", reflect.TypeOf((*MockRequester)(nil).Force)) -} - -// Query mocks base method. -func (m *MockRequester) Query(arg0 flow.Identifier, arg1 flow.IdentityFilter) { - m.ctrl.T.Helper() - m.ctrl.Call(m, "Query", arg0, arg1) -} - -// Query indicates an expected call of Query. -func (mr *MockRequesterMockRecorder) Query(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Query", reflect.TypeOf((*MockRequester)(nil).Query), arg0, arg1) -} diff --git a/module/requester.go b/module/requester.go index dc5e1baa059..d6746c6199c 100644 --- a/module/requester.go +++ b/module/requester.go @@ -12,14 +12,14 @@ type Requester interface { // if no additional restrictions are required. Data integrity of response // will be checked upon arrival. This function should be used for requesting // entites by their IDs. - EntityByID(entityID flow.Identifier, selector flow.IdentityFilter) + EntityByID(entityID flow.Identifier, selector flow.IdentityFilter[flow.Identity]) // Query will request data through the request engine backing the interface. // The additional selector will be applied to the subset // of valid providers for the data and allows finer-grained control // over which providers to request data from. Doesn't perform integrity check // can be used to get entities without knowing their ID. - Query(key flow.Identifier, selector flow.IdentityFilter) + Query(key flow.Identifier, selector flow.IdentityFilter[flow.Identity]) // Force will force the dispatcher to send all possible batches immediately. // It can be used in cases where responsiveness is of utmost importance, at diff --git a/state/cluster/root_block.go b/state/cluster/root_block.go index 073c8e84322..3391ee31e85 100644 --- a/state/cluster/root_block.go +++ b/state/cluster/root_block.go @@ -19,7 +19,7 @@ var rootBlockPayloadHash = rootBlockPayload.Hash() // CanonicalRootBlock returns the canonical root block for the given // cluster in the given epoch. It contains an empty collection referencing -func CanonicalRootBlock(epoch uint64, participants flow.IdentityList) *cluster.Block { +func CanonicalRootBlock(epoch uint64, participants flow.IdentitySkeletonList) *cluster.Block { chainID := CanonicalClusterID(epoch, participants.NodeIDs()) header := &flow.Header{ diff --git a/state/protocol/cluster.go b/state/protocol/cluster.go index a689adfc033..6a62aa7c050 100644 --- a/state/protocol/cluster.go +++ b/state/protocol/cluster.go @@ -21,7 +21,7 @@ type Cluster interface { EpochCounter() uint64 // Members returns the initial set of collector nodes in this cluster. - Members() flow.IdentityList + Members() flow.IdentitySkeletonList // RootBlock returns the root block for this cluster. RootBlock() *cluster.Block diff --git a/state/protocol/convert.go b/state/protocol/convert.go index 0890475fd18..c9a7a9de373 100644 --- a/state/protocol/convert.go +++ b/state/protocol/convert.go @@ -149,7 +149,7 @@ func GetDKGParticipantKeys(dkg DKG, participants flow.IdentitySkeletonList) ([]c // DKG instance. The participants must exactly match the DKG instance configuration. // All errors indicate inconsistent or invalid inputs. // No errors are expected during normal operation. -func ToDKGParticipantLookup(dkg DKG, participants flow.IdentityList) (map[flow.Identifier]flow.DKGParticipant, error) { +func ToDKGParticipantLookup(dkg DKG, participants flow.IdentitySkeletonList) (map[flow.Identifier]flow.DKGParticipant, error) { lookup := make(map[flow.Identifier]flow.DKGParticipant) for _, identity := range participants { diff --git a/state/protocol/inmem/cluster.go b/state/protocol/inmem/cluster.go index fd2b0b85108..ea3fb4f3e9b 100644 --- a/state/protocol/inmem/cluster.go +++ b/state/protocol/inmem/cluster.go @@ -12,9 +12,9 @@ type Cluster struct { var _ protocol.Cluster = (*Cluster)(nil) -func (c Cluster) Index() uint { return c.enc.Index } -func (c Cluster) ChainID() flow.ChainID { return c.enc.RootBlock.Header.ChainID } -func (c Cluster) EpochCounter() uint64 { return c.enc.Counter } -func (c Cluster) Members() flow.IdentityList { return c.enc.Members } -func (c Cluster) RootBlock() *clustermodel.Block { return c.enc.RootBlock } -func (c Cluster) RootQC() *flow.QuorumCertificate { return c.enc.RootQC } +func (c Cluster) Index() uint { return c.enc.Index } +func (c Cluster) ChainID() flow.ChainID { return c.enc.RootBlock.Header.ChainID } +func (c Cluster) EpochCounter() uint64 { return c.enc.Counter } +func (c Cluster) Members() flow.IdentitySkeletonList { return c.enc.Members } +func (c Cluster) RootBlock() *clustermodel.Block { return c.enc.RootBlock } +func (c Cluster) RootQC() *flow.QuorumCertificate { return c.enc.RootQC } diff --git a/state/protocol/inmem/convert.go b/state/protocol/inmem/convert.go index 5b569f74450..86c62836bf4 100644 --- a/state/protocol/inmem/convert.go +++ b/state/protocol/inmem/convert.go @@ -155,7 +155,7 @@ func FromEpoch(from protocol.Epoch) (*Epoch, error) { if err != nil { return nil, fmt.Errorf("could not get dkg: %w", err) } - convertedDKG, err := FromDKG(dkg, epoch.InitialIdentities.Filter(filter.HasRole(flow.RoleConsensus))) + convertedDKG, err := FromDKG(dkg, epoch.InitialIdentities.Filter(filter.HasRole[flow.IdentitySkeleton](flow.RoleConsensus))) if err != nil { return nil, err } @@ -215,7 +215,7 @@ func FromCluster(from protocol.Cluster) (*Cluster, error) { // The given participant list must exactly match the DKG members. // All errors indicate inconsistent or invalid inputs. // No errors are expected during normal operation. -func FromDKG(from protocol.DKG, participants flow.IdentityList) (*DKG, error) { +func FromDKG(from protocol.DKG, participants flow.IdentitySkeletonList) (*DKG, error) { var dkg EncodableDKG dkg.GroupKey = encodable.RandomBeaconPubKey{PublicKey: from.GroupKey()} @@ -323,12 +323,22 @@ func SnapshotFromBootstrapStateWithParams( EpochCommitSafetyThreshold: epochCommitSafetyThreshold, // see protocol.Params for details } + identities := make(flow.DynamicIdentityEntryList, 0, len(setup.Participants)) + for _, identity := range setup.Participants { + identities = append(identities, &flow.DynamicIdentityEntry{ + NodeID: identity.NodeID, + Dynamic: flow.DynamicIdentity{ + Weight: identity.InitialWeight, + Ejected: false, + }, + }) + } protocolState := &flow.ProtocolStateEntry{ PreviousEpochEventIDs: flow.EventIDs{}, CurrentEpoch: flow.EpochStateContainer{ SetupID: setup.ID(), CommitID: commit.ID(), - Identities: flow.DynamicIdentityEntryListFromIdentities(setup.Participants), + Identities: identities, }, NextEpoch: nil, InvalidStateTransitionAttempted: false, diff --git a/state/protocol/inmem/encodable.go b/state/protocol/inmem/encodable.go index 101a05a49ab..1b85ea661c2 100644 --- a/state/protocol/inmem/encodable.go +++ b/state/protocol/inmem/encodable.go @@ -35,7 +35,7 @@ type EncodableEpoch struct { DKGPhase3FinalView uint64 FinalView uint64 RandomSource []byte - InitialIdentities flow.IdentityList + InitialIdentities flow.IdentitySkeletonList Clustering flow.ClusterList Clusters []EncodableCluster DKG *EncodableDKG @@ -59,7 +59,7 @@ type EncodableFullDKG struct { type EncodableCluster struct { Index uint Counter uint64 - Members flow.IdentityList + Members flow.IdentitySkeletonList RootBlock *cluster.Block RootQC *flow.QuorumCertificate } diff --git a/state/protocol/inmem/epoch.go b/state/protocol/inmem/epoch.go index be6d8e50244..afc608ede20 100644 --- a/state/protocol/inmem/epoch.go +++ b/state/protocol/inmem/epoch.go @@ -151,8 +151,7 @@ func (es *setupEpoch) RandomSource() ([]byte, error) { } func (es *setupEpoch) InitialIdentities() (flow.IdentitySkeletonList, error) { - identities := es.setupEvent.Participants.Filter(filter.Any) - return identities, nil + return es.setupEvent.Participants, nil } func (es *setupEpoch) Clustering() (flow.ClusterList, error) { @@ -160,7 +159,7 @@ func (es *setupEpoch) Clustering() (flow.ClusterList, error) { } func ClusteringFromSetupEvent(setupEvent *flow.EpochSetup) (flow.ClusterList, error) { - collectorFilter := filter.HasRole(flow.RoleCollection) + collectorFilter := filter.HasRole[flow.IdentitySkeleton](flow.RoleCollection) clustering, err := factory.NewClusterList(setupEvent.Assignments, setupEvent.Participants.Filter(collectorFilter)) if err != nil { return nil, fmt.Errorf("failed to generate ClusterList from collector identities: %w", err) diff --git a/state/protocol/inmem/snapshot.go b/state/protocol/inmem/snapshot.go index 2d56b5fa086..74c40643834 100644 --- a/state/protocol/inmem/snapshot.go +++ b/state/protocol/inmem/snapshot.go @@ -27,7 +27,7 @@ func (s Snapshot) QuorumCertificate() (*flow.QuorumCertificate, error) { return s.enc.QuorumCertificate, nil } -func (s Snapshot) Identities(selector flow.IdentityFilter) (flow.IdentityList, error) { +func (s Snapshot) Identities(selector flow.IdentityFilter[flow.Identity]) (flow.IdentityList, error) { protocolState, err := s.ProtocolState() if err != nil { return nil, fmt.Errorf("could not access protocol state: %w", err) @@ -37,7 +37,7 @@ func (s Snapshot) Identities(selector flow.IdentityFilter) (flow.IdentityList, e func (s Snapshot) Identity(nodeID flow.Identifier) (*flow.Identity, error) { // filter identities at snapshot for node ID - identities, err := s.Identities(filter.HasNodeID(nodeID)) + identities, err := s.Identities(filter.HasNodeID[flow.Identity](nodeID)) if err != nil { return nil, fmt.Errorf("could not get identities: %w", err) } diff --git a/state/protocol/invalid/snapshot.go b/state/protocol/invalid/snapshot.go index 248efe6142d..637821307a9 100644 --- a/state/protocol/invalid/snapshot.go +++ b/state/protocol/invalid/snapshot.go @@ -44,7 +44,7 @@ func (u *Snapshot) Phase() (flow.EpochPhase, error) { return 0, u.err } -func (u *Snapshot) Identities(_ flow.IdentityFilter) (flow.IdentityList, error) { +func (u *Snapshot) Identities(_ flow.IdentityFilter[flow.Identity]) (flow.IdentityList, error) { return nil, u.err } diff --git a/state/protocol/mock/cluster.go b/state/protocol/mock/cluster.go index aebb5a2af5b..5835d2d6df9 100644 --- a/state/protocol/mock/cluster.go +++ b/state/protocol/mock/cluster.go @@ -57,15 +57,15 @@ func (_m *Cluster) Index() uint { } // Members provides a mock function with given fields: -func (_m *Cluster) Members() flow.IdentityList { +func (_m *Cluster) Members() flow.IdentitySkeletonList { ret := _m.Called() - var r0 flow.IdentityList - if rf, ok := ret.Get(0).(func() flow.IdentityList); ok { + var r0 flow.IdentitySkeletonList + if rf, ok := ret.Get(0).(func() flow.IdentitySkeletonList); ok { r0 = rf() } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(flow.IdentityList) + r0 = ret.Get(0).(flow.IdentitySkeletonList) } } diff --git a/state/protocol/mock/epoch.go b/state/protocol/mock/epoch.go index 76f1cdd75fd..dc024c697c4 100644 --- a/state/protocol/mock/epoch.go +++ b/state/protocol/mock/epoch.go @@ -314,16 +314,16 @@ func (_m *Epoch) FirstView() (uint64, error) { func (_m *Epoch) InitialIdentities() (flow.IdentitySkeletonList, error) { ret := _m.Called() - var r0 flow.IdentityList + var r0 flow.IdentitySkeletonList var r1 error - if rf, ok := ret.Get(0).(func() (flow.IdentityList, error)); ok { + if rf, ok := ret.Get(0).(func() (flow.IdentitySkeletonList, error)); ok { return rf() } - if rf, ok := ret.Get(0).(func() flow.IdentityList); ok { + if rf, ok := ret.Get(0).(func() flow.IdentitySkeletonList); ok { r0 = rf() } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(flow.IdentityList) + r0 = ret.Get(0).(flow.IdentitySkeletonList) } } diff --git a/state/protocol/mock/snapshot.go b/state/protocol/mock/snapshot.go index 434afaec89f..5e91c4ce370 100644 --- a/state/protocol/mock/snapshot.go +++ b/state/protocol/mock/snapshot.go @@ -109,15 +109,15 @@ func (_m *Snapshot) Head() (*flow.Header, error) { } // Identities provides a mock function with given fields: selector -func (_m *Snapshot) Identities(selector flow.IdentityFilter) (flow.IdentityList, error) { +func (_m *Snapshot) Identities(selector flow.IdentityFilter[flow.Identity]) (flow.IdentityList, error) { ret := _m.Called(selector) var r0 flow.IdentityList var r1 error - if rf, ok := ret.Get(0).(func(flow.IdentityFilter) (flow.IdentityList, error)); ok { + if rf, ok := ret.Get(0).(func(flow.IdentityFilter[flow.Identity]) (flow.IdentityList, error)); ok { return rf(selector) } - if rf, ok := ret.Get(0).(func(flow.IdentityFilter) flow.IdentityList); ok { + if rf, ok := ret.Get(0).(func(flow.IdentityFilter[flow.Identity]) flow.IdentityList); ok { r0 = rf(selector) } else { if ret.Get(0) != nil { @@ -125,7 +125,7 @@ func (_m *Snapshot) Identities(selector flow.IdentityFilter) (flow.IdentityList, } } - if rf, ok := ret.Get(1).(func(flow.IdentityFilter) error); ok { + if rf, ok := ret.Get(1).(func(flow.IdentityFilter[flow.Identity]) error); ok { r1 = rf(selector) } else { r1 = ret.Error(1) diff --git a/state/protocol/snapshot.go b/state/protocol/snapshot.go index 3dfbd96a79f..1863048d83b 100644 --- a/state/protocol/snapshot.go +++ b/state/protocol/snapshot.go @@ -55,7 +55,7 @@ type Snapshot interface { // It allows us to provide optional upfront filters which can be used by the // implementation to speed up database lookups. // TODO document error returns - Identities(selector flow.IdentityFilter) (flow.IdentityList, error) + Identities(selector flow.IdentityFilter[flow.Identity]) (flow.IdentityList, error) // Identity attempts to retrieve the node with the given identifier at the // selected point of the protocol state history. It will error if it doesn't exist. diff --git a/state/protocol/util.go b/state/protocol/util.go index 0e74916a25c..92bcd901f12 100644 --- a/state/protocol/util.go +++ b/state/protocol/util.go @@ -34,7 +34,7 @@ func IsNodeAuthorizedWithRoleAt(snapshot Snapshot, id flow.Identifier, role flow id, filter.HasWeight(true), filter.Not(filter.Ejected), - filter.HasRole(role), + filter.HasRole[flow.Identity](role), ) } @@ -44,7 +44,7 @@ func IsNodeAuthorizedWithRoleAt(snapshot Snapshot, id flow.Identifier, role flow // - state.ErrUnknownSnapshotReference if snapshot references an unknown block // // All other errors are unexpected and potential symptoms of internal state corruption. -func CheckNodeStatusAt(snapshot Snapshot, id flow.Identifier, checks ...flow.IdentityFilter) (bool, error) { +func CheckNodeStatusAt(snapshot Snapshot, id flow.Identifier, checks ...flow.IdentityFilter[flow.Identity]) (bool, error) { identity, err := snapshot.Identity(id) if IsIdentityNotFound(err) { return false, nil diff --git a/utils/unittest/chain_suite.go b/utils/unittest/chain_suite.go index a2ebc59f8d0..ce8c21ee6a1 100644 --- a/utils/unittest/chain_suite.go +++ b/utils/unittest/chain_suite.go @@ -419,7 +419,7 @@ func StateSnapshotForKnownBlock(block *flow.Header, identities map[flow.Identifi }, ) snapshot.On("Identities", mock.Anything).Return( - func(selector flow.IdentityFilter) flow.IdentityList { + func(selector flow.IdentityFilter[flow.Identity]) flow.IdentityList { var idts flow.IdentityList for _, i := range identities { if selector(i) { @@ -428,7 +428,7 @@ func StateSnapshotForKnownBlock(block *flow.Header, identities map[flow.Identifi } return idts }, - func(selector flow.IdentityFilter) error { + func(selector flow.IdentityFilter[flow.Identity]) error { return nil }, ) diff --git a/utils/unittest/cluster.go b/utils/unittest/cluster.go index 80d8627342c..bed70799279 100644 --- a/utils/unittest/cluster.go +++ b/utils/unittest/cluster.go @@ -2,8 +2,6 @@ package unittest import ( "fmt" - "sort" - "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/flow/factory" "github.com/onflow/flow-go/model/flow/filter" @@ -46,14 +44,12 @@ func AlterTransactionForCluster(tx flow.TransactionBody, clusters flow.ClusterLi // ClusterAssignment creates an assignment list with n clusters and with nodes // evenly distributed among clusters. -func ClusterAssignment(n uint, nodes flow.IdentityList) flow.AssignmentList { +func ClusterAssignment(n uint, nodes flow.IdentitySkeletonList) flow.AssignmentList { - collectors := nodes.Filter(filter.HasRole(flow.RoleCollection)) + collectors := nodes.Filter(filter.HasRole[flow.IdentitySkeleton](flow.RoleCollection)) // order, so the same list results in the same - sort.Slice(collectors, func(i, j int) bool { - return order.Canonical(collectors[i], collectors[j]) - }) + collectors = collectors.Sort(order.Canonical[flow.IdentitySkeleton]) assignments := make(flow.AssignmentList, n) for i, collector := range collectors { @@ -64,9 +60,9 @@ func ClusterAssignment(n uint, nodes flow.IdentityList) flow.AssignmentList { return assignments } -func ClusterList(n uint, nodes flow.IdentityList) flow.ClusterList { +func ClusterList(n uint, nodes flow.IdentitySkeletonList) flow.ClusterList { assignments := ClusterAssignment(n, nodes) - clusters, err := factory.NewClusterList(assignments, nodes.Filter(filter.HasRole(flow.RoleCollection))) + clusters, err := factory.NewClusterList(assignments, nodes.Filter(filter.HasRole[flow.IdentitySkeleton](flow.RoleCollection))) if err != nil { panic(err) } diff --git a/utils/unittest/fixtures.go b/utils/unittest/fixtures.go index b31816c8d81..5fc9f6953c8 100644 --- a/utils/unittest/fixtures.go +++ b/utils/unittest/fixtures.go @@ -1955,8 +1955,8 @@ func VoteWithBeaconSig() func(*hotstuff.Vote) { func WithParticipants(participants flow.IdentityList) func(*flow.EpochSetup) { return func(setup *flow.EpochSetup) { - setup.Participants = participants.Sort(order.Canonical) - setup.Assignments = ClusterAssignment(1, participants) + setup.Participants = participants.Sort(order.Canonical[flow.Identity]).ToSkeleton() + setup.Assignments = ClusterAssignment(1, participants.ToSkeleton()) } } @@ -1987,7 +1987,7 @@ func EpochSetupFixture(opts ...func(setup *flow.EpochSetup)) *flow.EpochSetup { Counter: uint64(rand.Uint32()), FirstView: uint64(0), FinalView: uint64(rand.Uint32() + 1000), - Participants: participants.Sort(order.Canonical), + Participants: participants.Sort(order.Canonical[flow.Identity]).ToSkeleton(), RandomSource: SeedFixture(flow.EpochSetupRandomSourceLength), DKGPhase1FinalView: 100, DKGPhase2FinalView: 200, @@ -2027,7 +2027,7 @@ func IndexFixture() *flow.Index { } } -func WithDKGFromParticipants(participants flow.IdentityList) func(*flow.EpochCommit) { +func WithDKGFromParticipants(participants flow.IdentitySkeletonList) func(*flow.EpochCommit) { count := len(participants.Filter(filter.IsValidDKGParticipant)) return func(commit *flow.EpochCommit) { commit.DKGParticipantKeys = PublicKeysFixture(count, crypto.BLSBLS12381) @@ -2046,9 +2046,9 @@ func WithClusterQCsFromAssignments(assignments flow.AssignmentList) func(*flow.E } } -func DKGParticipantLookup(participants flow.IdentityList) map[flow.Identifier]flow.DKGParticipant { +func DKGParticipantLookup(participants flow.IdentitySkeletonList) map[flow.Identifier]flow.DKGParticipant { lookup := make(map[flow.Identifier]flow.DKGParticipant) - for i, node := range participants.Filter(filter.HasRole(flow.RoleConsensus)) { + for i, node := range participants.Filter(filter.HasRole[flow.IdentitySkeleton](flow.RoleConsensus)) { lookup[node.NodeID] = flow.DKGParticipant{ Index: uint(i), KeyShare: KeyFixture(crypto.BLSBLS12381).PublicKey(), @@ -2136,7 +2136,7 @@ func BootstrapFixtureWithChainID( commit := EpochCommitFixture( CommitWithCounter(counter), WithClusterQCsFromAssignments(setup.Assignments), - WithDKGFromParticipants(participants), + WithDKGFromParticipants(participants.ToSkeleton()), ) stateCommit := GenesisStateCommitmentByChainID(chainID) @@ -2165,7 +2165,7 @@ func RootSnapshotFixtureWithChainID( chainID flow.ChainID, opts ...func(*flow.Block), ) *inmem.Snapshot { - block, result, seal := BootstrapFixtureWithChainID(participants.Sort(order.Canonical), chainID, opts...) + block, result, seal := BootstrapFixtureWithChainID(participants.Sort(order.Canonical[flow.Identity]), chainID, opts...) qc := QuorumCertificateFixture(QCWithRootBlockID(block.ID())) root, err := inmem.SnapshotFromBootstrapState(block, result, seal, qc) if err != nil { @@ -2536,8 +2536,16 @@ func RootProtocolStateFixture() *flow.RichProtocolStateEntry { commit.Counter = currentEpochSetup.Counter }) - allIdentities := currentEpochSetup.Participants - + allIdentities := make(flow.IdentityList, 0, len(currentEpochSetup.Participants)) + for _, identity := range currentEpochSetup.Participants { + allIdentities = append(allIdentities, &flow.Identity{ + IdentitySkeleton: *identity, + DynamicIdentity: flow.DynamicIdentity{ + Weight: identity.InitialWeight, + Ejected: false, + }, + }) + } return &flow.RichProtocolStateEntry{ ProtocolStateEntry: &flow.ProtocolStateEntry{ @@ -2590,8 +2598,16 @@ func ProtocolStateFixture(options ...func(*flow.RichProtocolStateEntry)) *flow.R commit.Counter = currentEpochSetup.Counter }) - allIdentities := currentEpochSetup.Participants.Union(prevEpochSetup.Participants) - + allIdentities := make(flow.IdentityList, 0, len(currentEpochSetup.Participants)) + for _, identity := range currentEpochSetup.Participants.Union(prevEpochSetup.Participants) { + allIdentities = append(allIdentities, &flow.Identity{ + IdentitySkeleton: *identity, + DynamicIdentity: flow.DynamicIdentity{ + Weight: identity.InitialWeight, + Ejected: false, + }, + }) + } entry := &flow.RichProtocolStateEntry{ ProtocolStateEntry: &flow.ProtocolStateEntry{ CurrentEpoch: flow.EpochStateContainer{ @@ -2641,19 +2657,59 @@ func WithNextEpochProtocolState() func(entry *flow.RichProtocolStateEntry) { nextEpochCommit := EpochCommitFixture(func(commit *flow.EpochCommit) { commit.Counter = nextEpochSetup.Counter }) - allIdentities := nextEpochSetup.Participants.Union(entry.CurrentEpochSetup.Participants) - entry.Identities = entry.CurrentEpochSetup.Participants.Union(nextEpochSetup.Participants) - entry.ProtocolStateEntry.CurrentEpoch.Identities = flow.DynamicIdentityEntryListFromIdentities(entry.Identities) + entry.Identities = nil + identitiesStateLookup := entry.CurrentEpoch.Identities.Lookup() + for _, identity := range entry.CurrentEpochSetup.Participants { + entry.Identities = append(entry.Identities, &flow.Identity{ + IdentitySkeleton: *identity, + DynamicIdentity: identitiesStateLookup[identity.NodeID].Dynamic, + }) + } + + for _, identity := range nextEpochSetup.Participants { + if _, found := entry.Identities.ByNodeID(identity.NodeID); !found { + entry.Identities = append(entry.Identities, &flow.Identity{ + IdentitySkeleton: *identity, + DynamicIdentity: flow.DynamicIdentity{ + Weight: 0, + Ejected: false, + }, + }) + } + + entry.NextIdentities = append(entry.NextIdentities, &flow.Identity{ + IdentitySkeleton: *identity, + DynamicIdentity: flow.DynamicIdentity{ + Weight: identity.InitialWeight, + Ejected: false, + }, + }) + } + + for _, identity := range entry.CurrentEpochSetup.Participants { + if _, found := entry.NextIdentities.ByNodeID(identity.NodeID); !found { + entry.NextIdentities = append(entry.NextIdentities, &flow.Identity{ + IdentitySkeleton: *identity, + DynamicIdentity: flow.DynamicIdentity{ + Weight: 0, + Ejected: false, + }, + }) + } + } + + entry.Identities = entry.Identities.Sort(order.Canonical[flow.Identity]) + entry.NextIdentities = entry.NextIdentities.Sort(order.Canonical[flow.Identity]) - entry.ProtocolStateEntry.NextEpoch = &flow.EpochStateContainer{ + entry.CurrentEpoch.Identities = flow.DynamicIdentityEntryListFromIdentities(entry.Identities) + entry.NextEpoch = &flow.EpochStateContainer{ SetupID: nextEpochSetup.ID(), CommitID: nextEpochCommit.ID(), - Identities: flow.DynamicIdentityEntryListFromIdentities(allIdentities), + Identities: flow.DynamicIdentityEntryListFromIdentities(entry.NextIdentities), } entry.NextEpochSetup = nextEpochSetup entry.NextEpochCommit = nextEpochCommit - entry.NextIdentities = allIdentities } } diff --git a/utils/unittest/protocol_state.go b/utils/unittest/protocol_state.go index f5dbcb88073..9a9c2938460 100644 --- a/utils/unittest/protocol_state.go +++ b/utils/unittest/protocol_state.go @@ -22,7 +22,7 @@ func FinalizedProtocolStateWithParticipants(participants flow.IdentityList) ( // set up protocol snapshot mock snapshot := &mockprotocol.Snapshot{} snapshot.On("Identities", mock.Anything).Return( - func(filter flow.IdentityFilter) flow.IdentityList { + func(filter flow.IdentityFilter[flow.Identity]) flow.IdentityList { return participants.Filter(filter) }, nil, diff --git a/utils/unittest/service_events_fixtures.go b/utils/unittest/service_events_fixtures.go index c5963a31505..b56bd3cd88c 100644 --- a/utils/unittest/service_events_fixtures.go +++ b/utils/unittest/service_events_fixtures.go @@ -47,104 +47,62 @@ func EpochSetupFixtureByChainID(chain flow.ChainID) (flow.Event, *flow.EpochSetu flow.MustHexStringToIdentifier("0000000000000000000000000000000000000000000000000000000000000004"), }, }, - Participants: flow.IdentityList{ - { - IdentitySkeleton: flow.IdentitySkeleton{ - Role: flow.RoleCollection, - NodeID: flow.MustHexStringToIdentifier("0000000000000000000000000000000000000000000000000000000000000001"), - Address: "1.flow.com", - NetworkPubKey: MustDecodePublicKeyHex(crypto.ECDSAP256, "378dbf45d85c614feb10d8bd4f78f4b6ef8eec7d987b937e123255444657fb3da031f232a507e323df3a6f6b8f50339c51d188e80c0e7a92420945cc6ca893fc"), - StakingPubKey: MustDecodePublicKeyHex(crypto.BLSBLS12381, "af4aade26d76bb2ab15dcc89adcef82a51f6f04b3cb5f4555214b40ec89813c7a5f95776ea4fe449de48166d0bbc59b919b7eabebaac9614cf6f9461fac257765415f4d8ef1376a2365ec9960121888ea5383d88a140c24c29962b0a14e4e4e7"), - InitialWeight: 100, - }, - DynamicIdentity: flow.DynamicIdentity{ - Weight: 100, - Ejected: false, - }, - }, - { - IdentitySkeleton: flow.IdentitySkeleton{ - Role: flow.RoleCollection, - NodeID: flow.MustHexStringToIdentifier("0000000000000000000000000000000000000000000000000000000000000002"), - Address: "2.flow.com", - NetworkPubKey: MustDecodePublicKeyHex(crypto.ECDSAP256, "378dbf45d85c614feb10d8bd4f78f4b6ef8eec7d987b937e123255444657fb3da031f232a507e323df3a6f6b8f50339c51d188e80c0e7a92420945cc6ca893fc"), - StakingPubKey: MustDecodePublicKeyHex(crypto.BLSBLS12381, "af4aade26d76bb2ab15dcc89adcef82a51f6f04b3cb5f4555214b40ec89813c7a5f95776ea4fe449de48166d0bbc59b919b7eabebaac9614cf6f9461fac257765415f4d8ef1376a2365ec9960121888ea5383d88a140c24c29962b0a14e4e4e7"), - InitialWeight: 100, - }, - DynamicIdentity: flow.DynamicIdentity{ - Weight: 100, - Ejected: false, - }, - }, - { - IdentitySkeleton: flow.IdentitySkeleton{ - Role: flow.RoleCollection, - NodeID: flow.MustHexStringToIdentifier("0000000000000000000000000000000000000000000000000000000000000003"), - Address: "3.flow.com", - NetworkPubKey: MustDecodePublicKeyHex(crypto.ECDSAP256, "378dbf45d85c614feb10d8bd4f78f4b6ef8eec7d987b937e123255444657fb3da031f232a507e323df3a6f6b8f50339c51d188e80c0e7a92420945cc6ca893fc"), - StakingPubKey: MustDecodePublicKeyHex(crypto.BLSBLS12381, "af4aade26d76bb2ab15dcc89adcef82a51f6f04b3cb5f4555214b40ec89813c7a5f95776ea4fe449de48166d0bbc59b919b7eabebaac9614cf6f9461fac257765415f4d8ef1376a2365ec9960121888ea5383d88a140c24c29962b0a14e4e4e7"), - InitialWeight: 100, - }, - DynamicIdentity: flow.DynamicIdentity{ - Weight: 100, - Ejected: false, - }, - }, - { - IdentitySkeleton: flow.IdentitySkeleton{ - Role: flow.RoleCollection, - NodeID: flow.MustHexStringToIdentifier("0000000000000000000000000000000000000000000000000000000000000004"), - Address: "4.flow.com", - NetworkPubKey: MustDecodePublicKeyHex(crypto.ECDSAP256, "378dbf45d85c614feb10d8bd4f78f4b6ef8eec7d987b937e123255444657fb3da031f232a507e323df3a6f6b8f50339c51d188e80c0e7a92420945cc6ca893fc"), - StakingPubKey: MustDecodePublicKeyHex(crypto.BLSBLS12381, "af4aade26d76bb2ab15dcc89adcef82a51f6f04b3cb5f4555214b40ec89813c7a5f95776ea4fe449de48166d0bbc59b919b7eabebaac9614cf6f9461fac257765415f4d8ef1376a2365ec9960121888ea5383d88a140c24c29962b0a14e4e4e7"), - InitialWeight: 100, - }, - DynamicIdentity: flow.DynamicIdentity{ - Weight: 100, - Ejected: false, - }, - }, - { - IdentitySkeleton: flow.IdentitySkeleton{ - Role: flow.RoleConsensus, - NodeID: flow.MustHexStringToIdentifier("0000000000000000000000000000000000000000000000000000000000000011"), - Address: "11.flow.com", - NetworkPubKey: MustDecodePublicKeyHex(crypto.ECDSAP256, "cfdfe8e4362c8f79d11772cb7277ab16e5033a63e8dd5d34caf1b041b77e5b2d63c2072260949ccf8907486e4cfc733c8c42ca0e4e208f30470b0d950856cd47"), - StakingPubKey: MustDecodePublicKeyHex(crypto.BLSBLS12381, "8207559cd7136af378bba53a8f0196dee3849a3ab02897c1995c3e3f6ca0c4a776c3ae869d1ddbb473090054be2400ad06d7910aa2c5d1780220fdf3765a3c1764bce10c6fe66a5a2be51a422e878518bd750424bb56b8a0ecf0f8ad2057e83f"), - InitialWeight: 100, - }, - DynamicIdentity: flow.DynamicIdentity{ - Weight: 100, - Ejected: false, - }, - }, - { - IdentitySkeleton: flow.IdentitySkeleton{ - Role: flow.RoleExecution, - NodeID: flow.MustHexStringToIdentifier("0000000000000000000000000000000000000000000000000000000000000021"), - Address: "21.flow.com", - NetworkPubKey: MustDecodePublicKeyHex(crypto.ECDSAP256, "d64318ba0dbf68f3788fc81c41d507c5822bf53154530673127c66f50fe4469ccf1a054a868a9f88506a8999f2386d86fcd2b901779718cba4fb53c2da258f9e"), - StakingPubKey: MustDecodePublicKeyHex(crypto.BLSBLS12381, "880b162b7ec138b36af401d07868cb08d25746d905395edbb4625bdf105d4bb2b2f4b0f4ae273a296a6efefa7ce9ccb914e39947ce0e83745125cab05d62516076ff0173ed472d3791ccef937597c9ea12381d76f547a092a4981d77ff3fba83"), - InitialWeight: 100, - }, - DynamicIdentity: flow.DynamicIdentity{ - Weight: 100, - Ejected: false, - }, - }, - { - IdentitySkeleton: flow.IdentitySkeleton{ - Role: flow.RoleVerification, - NodeID: flow.MustHexStringToIdentifier("0000000000000000000000000000000000000000000000000000000000000031"), - Address: "31.flow.com", - NetworkPubKey: MustDecodePublicKeyHex(crypto.ECDSAP256, "697241208dcc9142b6f53064adc8ff1c95760c68beb2ba083c1d005d40181fd7a1b113274e0163c053a3addd47cd528ec6a1f190cf465aac87c415feaae011ae"), - StakingPubKey: MustDecodePublicKeyHex(crypto.BLSBLS12381, "b1f97d0a06020eca97352e1adde72270ee713c7daf58da7e74bf72235321048b4841bdfc28227964bf18e371e266e32107d238358848bcc5d0977a0db4bda0b4c33d3874ff991e595e0f537c7b87b4ddce92038ebc7b295c9ea20a1492302aa7"), - InitialWeight: 100, - }, - DynamicIdentity: flow.DynamicIdentity{ - Weight: 100, - Ejected: false, - }, + Participants: flow.IdentitySkeletonList{ + { + Role: flow.RoleCollection, + NodeID: flow.MustHexStringToIdentifier("0000000000000000000000000000000000000000000000000000000000000001"), + Address: "1.flow.com", + NetworkPubKey: MustDecodePublicKeyHex(crypto.ECDSAP256, "378dbf45d85c614feb10d8bd4f78f4b6ef8eec7d987b937e123255444657fb3da031f232a507e323df3a6f6b8f50339c51d188e80c0e7a92420945cc6ca893fc"), + StakingPubKey: MustDecodePublicKeyHex(crypto.BLSBLS12381, "af4aade26d76bb2ab15dcc89adcef82a51f6f04b3cb5f4555214b40ec89813c7a5f95776ea4fe449de48166d0bbc59b919b7eabebaac9614cf6f9461fac257765415f4d8ef1376a2365ec9960121888ea5383d88a140c24c29962b0a14e4e4e7"), + InitialWeight: 100, + }, + { + Role: flow.RoleCollection, + NodeID: flow.MustHexStringToIdentifier("0000000000000000000000000000000000000000000000000000000000000002"), + Address: "2.flow.com", + NetworkPubKey: MustDecodePublicKeyHex(crypto.ECDSAP256, "378dbf45d85c614feb10d8bd4f78f4b6ef8eec7d987b937e123255444657fb3da031f232a507e323df3a6f6b8f50339c51d188e80c0e7a92420945cc6ca893fc"), + StakingPubKey: MustDecodePublicKeyHex(crypto.BLSBLS12381, "af4aade26d76bb2ab15dcc89adcef82a51f6f04b3cb5f4555214b40ec89813c7a5f95776ea4fe449de48166d0bbc59b919b7eabebaac9614cf6f9461fac257765415f4d8ef1376a2365ec9960121888ea5383d88a140c24c29962b0a14e4e4e7"), + InitialWeight: 100, + }, + { + Role: flow.RoleCollection, + NodeID: flow.MustHexStringToIdentifier("0000000000000000000000000000000000000000000000000000000000000003"), + Address: "3.flow.com", + NetworkPubKey: MustDecodePublicKeyHex(crypto.ECDSAP256, "378dbf45d85c614feb10d8bd4f78f4b6ef8eec7d987b937e123255444657fb3da031f232a507e323df3a6f6b8f50339c51d188e80c0e7a92420945cc6ca893fc"), + StakingPubKey: MustDecodePublicKeyHex(crypto.BLSBLS12381, "af4aade26d76bb2ab15dcc89adcef82a51f6f04b3cb5f4555214b40ec89813c7a5f95776ea4fe449de48166d0bbc59b919b7eabebaac9614cf6f9461fac257765415f4d8ef1376a2365ec9960121888ea5383d88a140c24c29962b0a14e4e4e7"), + InitialWeight: 100, + }, + { + Role: flow.RoleCollection, + NodeID: flow.MustHexStringToIdentifier("0000000000000000000000000000000000000000000000000000000000000004"), + Address: "4.flow.com", + NetworkPubKey: MustDecodePublicKeyHex(crypto.ECDSAP256, "378dbf45d85c614feb10d8bd4f78f4b6ef8eec7d987b937e123255444657fb3da031f232a507e323df3a6f6b8f50339c51d188e80c0e7a92420945cc6ca893fc"), + StakingPubKey: MustDecodePublicKeyHex(crypto.BLSBLS12381, "af4aade26d76bb2ab15dcc89adcef82a51f6f04b3cb5f4555214b40ec89813c7a5f95776ea4fe449de48166d0bbc59b919b7eabebaac9614cf6f9461fac257765415f4d8ef1376a2365ec9960121888ea5383d88a140c24c29962b0a14e4e4e7"), + InitialWeight: 100, + }, + { + Role: flow.RoleConsensus, + NodeID: flow.MustHexStringToIdentifier("0000000000000000000000000000000000000000000000000000000000000011"), + Address: "11.flow.com", + NetworkPubKey: MustDecodePublicKeyHex(crypto.ECDSAP256, "cfdfe8e4362c8f79d11772cb7277ab16e5033a63e8dd5d34caf1b041b77e5b2d63c2072260949ccf8907486e4cfc733c8c42ca0e4e208f30470b0d950856cd47"), + StakingPubKey: MustDecodePublicKeyHex(crypto.BLSBLS12381, "8207559cd7136af378bba53a8f0196dee3849a3ab02897c1995c3e3f6ca0c4a776c3ae869d1ddbb473090054be2400ad06d7910aa2c5d1780220fdf3765a3c1764bce10c6fe66a5a2be51a422e878518bd750424bb56b8a0ecf0f8ad2057e83f"), + InitialWeight: 100, + }, + { + Role: flow.RoleExecution, + NodeID: flow.MustHexStringToIdentifier("0000000000000000000000000000000000000000000000000000000000000021"), + Address: "21.flow.com", + NetworkPubKey: MustDecodePublicKeyHex(crypto.ECDSAP256, "d64318ba0dbf68f3788fc81c41d507c5822bf53154530673127c66f50fe4469ccf1a054a868a9f88506a8999f2386d86fcd2b901779718cba4fb53c2da258f9e"), + StakingPubKey: MustDecodePublicKeyHex(crypto.BLSBLS12381, "880b162b7ec138b36af401d07868cb08d25746d905395edbb4625bdf105d4bb2b2f4b0f4ae273a296a6efefa7ce9ccb914e39947ce0e83745125cab05d62516076ff0173ed472d3791ccef937597c9ea12381d76f547a092a4981d77ff3fba83"), + InitialWeight: 100, + }, + { + Role: flow.RoleVerification, + NodeID: flow.MustHexStringToIdentifier("0000000000000000000000000000000000000000000000000000000000000031"), + Address: "31.flow.com", + NetworkPubKey: MustDecodePublicKeyHex(crypto.ECDSAP256, "697241208dcc9142b6f53064adc8ff1c95760c68beb2ba083c1d005d40181fd7a1b113274e0163c053a3addd47cd528ec6a1f190cf465aac87c415feaae011ae"), + StakingPubKey: MustDecodePublicKeyHex(crypto.BLSBLS12381, "b1f97d0a06020eca97352e1adde72270ee713c7daf58da7e74bf72235321048b4841bdfc28227964bf18e371e266e32107d238358848bcc5d0977a0db4bda0b4c33d3874ff991e595e0f537c7b87b4ddce92038ebc7b295c9ea20a1492302aa7"), + InitialWeight: 100, }, }, } diff --git a/utils/unittest/updatable_provider.go b/utils/unittest/updatable_provider.go index 9661f7039a6..c2b023f1258 100644 --- a/utils/unittest/updatable_provider.go +++ b/utils/unittest/updatable_provider.go @@ -29,7 +29,7 @@ func (p *UpdatableIDProvider) SetIdentities(identities flow.IdentityList) { p.identities = identities } -func (p *UpdatableIDProvider) Identities(filter flow.IdentityFilter) flow.IdentityList { +func (p *UpdatableIDProvider) Identities(filter flow.IdentityFilter[flow.Identity]) flow.IdentityList { p.mu.RLock() defer p.mu.RUnlock() return p.identities.Filter(filter) From 86418975e07c8ed47ce5dc4b5d8bcebaea66cd7a Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Tue, 19 Sep 2023 16:34:13 +0300 Subject: [PATCH 03/39] Updated godoc, cleanup of commented out code --- model/flow/identity.go | 35 ++++++++--------------------------- 1 file changed, 8 insertions(+), 27 deletions(-) diff --git a/model/flow/identity.go b/model/flow/identity.go index e884bba6dc3..2eda6403429 100644 --- a/model/flow/identity.go +++ b/model/flow/identity.go @@ -336,6 +336,10 @@ func (iy *Identity) EqualTo(other *Identity) bool { return true } +// GenericIdentity defines a constraint for generic identities. +// Golang doesn't support constraint with fields(for time being) so we have to define this interface +// with getter methods. +// Details here: https://github.com/golang/go/issues/51259. type GenericIdentity interface { Identity | IdentitySkeleton GetNodeID() Identifier @@ -380,10 +384,12 @@ type IdentityOrder[T GenericIdentity] func(*T, *T) bool // Identities are COPIED from the source slice. type IdentityMapFunc[T GenericIdentity] func(T) T -// IdentitySkeletonList is a list of nodes skeletons. +// IdentitySkeletonList is a list of nodes skeletons. We use a type alias instead of defining a new type +// since go generics doesn't support implicit conversion between types. type IdentitySkeletonList = GenericIdentityList[IdentitySkeleton] -// IdentityList is a list of nodes. +// IdentityList is a list of nodes. We use a type alias instead of defining a new type +// since go generics doesn't support implicit conversion between types. type IdentityList = GenericIdentityList[Identity] type GenericIdentityList[T GenericIdentity] []*T @@ -441,31 +447,6 @@ func (il GenericIdentityList[T]) Copy() GenericIdentityList[T] { return dup } -//// Copy returns a copy of IdentitySkeletonList. The resulting slice uses a different -//// backing array, meaning appends and insert operations on either slice are -//// guaranteed to only affect that slice. -//// -//// Copy should be used when modifying an existing identity list by either -//// appending new elements, re-ordering, or inserting new elements in an -//// existing index. -//// -//// CAUTION: -//// All IdentitySkeleton fields are deep-copied, _except_ for their keys, which -//// are copied by reference. -//func (il IdentitySkeletonList) Copy() IdentitySkeletonList { -// dup := make(IdentitySkeletonList, 0, len(il)) -// -// lenList := len(il) -// -// // performance tests show this is faster than 'range' -// for i := 0; i < lenList; i++ { -// // copy the object -// next := *(il[i]) -// dup = append(dup, &next) -// } -// return dup -//} - // Selector returns an identity filter function that selects only identities // within this identity list. func (il GenericIdentityList[T]) Selector() IdentityFilter[T] { From 001d6a07a08f047e9339621721a9ba688a2f8f0c Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Thu, 28 Sep 2023 21:37:53 +0300 Subject: [PATCH 04/39] Updated how generic identities are handled in hotstuff committee --- .../hotstuff/committees/cluster_committee.go | 27 ++++++++++++++----- .../committees/consensus_committee.go | 2 +- .../hotstuff/committees/leader/consensus.go | 2 +- engine/consensus/dkg/reactor_engine.go | 2 +- 4 files changed, 24 insertions(+), 9 deletions(-) diff --git a/consensus/hotstuff/committees/cluster_committee.go b/consensus/hotstuff/committees/cluster_committee.go index fd5361c9eec..ea375a68051 100644 --- a/consensus/hotstuff/committees/cluster_committee.go +++ b/consensus/hotstuff/committees/cluster_committee.go @@ -26,7 +26,7 @@ type Cluster struct { // pre-computed leader selection for the full lifecycle of the cluster selection *leader.LeaderSelection // a filter that returns all members of the cluster committee allowed to vote - clusterMemberFilter flow.IdentityFilter + clusterMemberFilter flow.IdentityFilter[flow.Identity] // initial set of cluster members, WITHOUT dynamic weight changes initialClusterMembers flow.IdentitySkeletonList initialClusterIdentities flow.IdentityList @@ -50,19 +50,34 @@ func NewClusterCommittee( return nil, fmt.Errorf("could not compute leader selection for cluster: %w", err) } - totalWeight := cluster.Members().ToSkeleton().TotalWeight() + initialClusterMembers := cluster.Members() + totalWeight := initialClusterMembers.TotalWeight() + initialClusterMembersSelector := initialClusterMembers.Selector() + initialClusterIdentities := make(flow.IdentityList, 0, len(cluster.Members())) + for _, skeleton := range initialClusterMembers { + initialClusterIdentities = append(initialClusterIdentities, &flow.Identity{ + IdentitySkeleton: *skeleton, + DynamicIdentity: flow.DynamicIdentity{ + Weight: skeleton.InitialWeight, + Ejected: false, + }, + }) + } com := &Cluster{ state: state, payloads: payloads, me: me, selection: selection, - clusterMemberFilter: filter.And( - cluster.Members().Selector(), + clusterMemberFilter: filter.And[flow.Identity]( + // adapt the identity filter to the identity skeleton filter + func(f *flow.Identity) bool { + return initialClusterMembersSelector(&f.IdentitySkeleton) + }, filter.Not(filter.Ejected), filter.HasWeight(true), ), - initialClusterMembers: cluster.Members().ToSkeleton(), - initialClusterIdentities: cluster.Members(), + initialClusterMembers: initialClusterMembers, + initialClusterIdentities: initialClusterIdentities, weightThresholdForQC: WeightThresholdToBuildQC(totalWeight), weightThresholdForTO: WeightThresholdToTimeout(totalWeight), } diff --git a/consensus/hotstuff/committees/consensus_committee.go b/consensus/hotstuff/committees/consensus_committee.go index 33f51d61063..172f67cd4e8 100644 --- a/consensus/hotstuff/committees/consensus_committee.go +++ b/consensus/hotstuff/committees/consensus_committee.go @@ -55,7 +55,7 @@ func newStaticEpochInfo(epoch protocol.Epoch) (*staticEpochInfo, error) { if err != nil { return nil, fmt.Errorf("could not initial identities: %w", err) } - initialCommittee := initialIdentities.Filter(filter.IsVotingConsensusCommitteeMember).ToSkeleton() + initialCommittee := initialIdentities.Filter(filter.IsAllowedConsensusCommitteeMember).ToSkeleton() dkg, err := epoch.DKG() if err != nil { return nil, fmt.Errorf("could not get dkg: %w", err) diff --git a/consensus/hotstuff/committees/leader/consensus.go b/consensus/hotstuff/committees/leader/consensus.go index a51ae3b758f..a2c1400b8e0 100644 --- a/consensus/hotstuff/committees/leader/consensus.go +++ b/consensus/hotstuff/committees/leader/consensus.go @@ -43,7 +43,7 @@ func SelectionForConsensus(epoch protocol.Epoch) (*LeaderSelection, error) { firstView, rng, int(finalView-firstView+1), // add 1 because both first/final view are inclusive - identities.Filter(filter.IsVotingConsensusCommitteeMember).ToSkeleton(), + identities.Filter(filter.IsAllowedConsensusCommitteeMember), ) return leaders, err } diff --git a/engine/consensus/dkg/reactor_engine.go b/engine/consensus/dkg/reactor_engine.go index 1704483ef48..fa5208c25f6 100644 --- a/engine/consensus/dkg/reactor_engine.go +++ b/engine/consensus/dkg/reactor_engine.go @@ -181,7 +181,7 @@ func (e *ReactorEngine) startDKGForEpoch(currentEpochCounter uint64, first *flow log.Fatal().Err(err).Msg("could not retrieve epoch info") } - committee := curDKGInfo.identities.Filter(filter.IsVotingConsensusCommitteeMember) + committee := curDKGInfo.identities.Filter(filter.IsAllowedConsensusCommitteeMember) log.Info(). Uint64("phase1", curDKGInfo.phase1FinalView). From c2f112382412e61c9c8cff334cf140512a65a817 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Thu, 28 Sep 2023 21:38:16 +0300 Subject: [PATCH 05/39] Updated state updater and other usages of protocol state --- model/flow/filter/identity.go | 26 +++-- state/protocol/badger/snapshot.go | 4 +- state/protocol/badger/validity.go | 2 +- state/protocol/protocol_state/updater.go | 12 +-- state/protocol/protocol_state/updater_test.go | 99 +++++++------------ 5 files changed, 60 insertions(+), 83 deletions(-) diff --git a/model/flow/filter/identity.go b/model/flow/filter/identity.go index c3137df0ee7..43c303360d6 100644 --- a/model/flow/filter/identity.go +++ b/model/flow/filter/identity.go @@ -75,6 +75,13 @@ func HasNetworkingKey(keys ...crypto.PublicKey) flow.IdentityFilter[flow.Identit } } +// HasInitialWeight returns a filter for nodes with non-zero initial weight. +func HasInitialWeight[T flow.GenericIdentity](hasWeight bool) flow.IdentityFilter[T] { + return func(identity *T) bool { + return ((*identity).GetInitialWeight() > 0) == hasWeight + } +} + // HasWeight returns a filter for nodes with non-zero weight. func HasWeight(hasWeight bool) flow.IdentityFilter[flow.Identity] { return func(identity *flow.Identity) bool { @@ -106,9 +113,16 @@ var IsValidCurrentEpochParticipant = And( Not(Ejected), // ejection will change signer index ) +// IsAllowedConsensusCommitteeMember is a identity filter for all members of +// the consensus committee allowed to participate. +var IsAllowedConsensusCommitteeMember = And( + HasRole[flow.IdentitySkeleton](flow.RoleConsensus), + HasInitialWeight[flow.IdentitySkeleton](true), +) + // IsVotingConsensusCommitteeMember is a identity filter for all members of // the consensus committee allowed to vote. -var IsVotingConsensusCommitteeMember = And( +var IsVotingConsensusCommitteeMember = And[flow.Identity]( HasRole[flow.Identity](flow.RoleConsensus), IsValidCurrentEpochParticipant, ) @@ -116,12 +130,4 @@ var IsVotingConsensusCommitteeMember = And( // IsValidDKGParticipant is an identity filter for all DKG participants. It is // equivalent to the filter for consensus committee members, as these are // the same group for now. -var IsValidDKGParticipant = func(identity *flow.IdentitySkeleton) bool { - if identity.Role != flow.RoleConsensus { - return false - } - if identity.InitialWeight == 0 { - return false - } - return true -} +var IsValidDKGParticipant = IsAllowedConsensusCommitteeMember diff --git a/state/protocol/badger/snapshot.go b/state/protocol/badger/snapshot.go index 8c9f30f56e0..f68ea9cf8d1 100644 --- a/state/protocol/badger/snapshot.go +++ b/state/protocol/badger/snapshot.go @@ -91,7 +91,7 @@ func (s *Snapshot) Phase() (flow.EpochPhase, error) { return phase, err } -func (s *Snapshot) Identities(selector flow.IdentityFilter) (flow.IdentityList, error) { +func (s *Snapshot) Identities(selector flow.IdentityFilter[flow.Identity]) (flow.IdentityList, error) { psSnapshot, err := s.state.protocolState.AtBlockID(s.blockID) if err != nil { return nil, err @@ -104,7 +104,7 @@ func (s *Snapshot) Identities(selector flow.IdentityFilter) (flow.IdentityList, func (s *Snapshot) Identity(nodeID flow.Identifier) (*flow.Identity, error) { // filter identities at snapshot for node ID - identities, err := s.Identities(filter.HasNodeID(nodeID)) + identities, err := s.Identities(filter.HasNodeID[flow.Identity](nodeID)) if err != nil { return nil, fmt.Errorf("could not get identities: %w", err) } diff --git a/state/protocol/badger/validity.go b/state/protocol/badger/validity.go index 06d14483744..c2e4c3c2933 100644 --- a/state/protocol/badger/validity.go +++ b/state/protocol/badger/validity.go @@ -304,7 +304,7 @@ func IsValidRootSnapshotQCs(snap protocol.Snapshot) error { // validateRootQC performs validation of root QC // Returns nil on success func validateRootQC(snap protocol.Snapshot) error { - identities, err := snap.Identities(filter.IsVotingConsensusCommitteeMember) + identities, err := snap.Identities(filter.IsAllowedConsensusCommitteeMember) if err != nil { return fmt.Errorf("could not get root snapshot identities: %w", err) } diff --git a/state/protocol/protocol_state/updater.go b/state/protocol/protocol_state/updater.go index 46b8fbeb415..f846d564063 100644 --- a/state/protocol/protocol_state/updater.go +++ b/state/protocol/protocol_state/updater.go @@ -84,14 +84,14 @@ func (u *Updater) ProcessEpochSetup(epochSetup *flow.EpochSetup) error { // lookup of dynamic data for current protocol state identities // by definition, this will include identities from current epoch + identities from previous epoch with 0 weight. - identitiesStateLookup := u.parentState.CurrentEpoch.ActiveIdentities.Lookup() + activeIdentitiesLookup := u.parentState.CurrentEpoch.ActiveIdentities.Lookup() currentEpochSetupParticipants := u.parentState.CurrentEpochSetup.Participants // construct identities for current epoch: current epoch participants + next epoch participants with 0 weight currentEpochIdentities := make(flow.DynamicIdentityEntryList, 0, len(currentEpochSetupParticipants)) // In this loop, we will perform step 1 from above. for _, identity := range currentEpochSetupParticipants { - identityParentState := identitiesStateLookup[identity.NodeID] + identityParentState := activeIdentitiesLookup[identity.NodeID] currentEpochIdentities = append(currentEpochIdentities, &flow.DynamicIdentityEntry{ NodeID: identity.NodeID, Dynamic: identityParentState.Dynamic, @@ -109,18 +109,18 @@ func (u *Updater) ProcessEpochSetup(epochSetup *flow.EpochSetup) error { NodeID: identity.NodeID, Dynamic: flow.DynamicIdentity{ Weight: 0, - Ejected: identity.Ejected, + Ejected: false, }, }) } // Step 3: for the next epoch we include every identity from its setup event; - // we give authority to epoch smart contract to decide who should be included in the next epoch and with what flags. + identityParentState, found := activeIdentitiesLookup[identity.NodeID] nextEpochIdentities = append(nextEpochIdentities, &flow.DynamicIdentityEntry{ NodeID: identity.NodeID, Dynamic: flow.DynamicIdentity{ Weight: identity.InitialWeight, - Ejected: identity.Ejected, + Ejected: found && identityParentState.Dynamic.Ejected, }, }) } @@ -131,7 +131,7 @@ func (u *Updater) ProcessEpochSetup(epochSetup *flow.EpochSetup) error { // Setup Event for the next epoch is added with 0 weight and the _current_ value of the Ejected flag. for _, identity := range currentEpochSetupParticipants { if _, found := nextEpochIdentitiesLookup[identity.NodeID]; !found { - identityParentState := identitiesStateLookup[identity.NodeID] + identityParentState := activeIdentitiesLookup[identity.NodeID] nextEpochIdentities = append(nextEpochIdentities, &flow.DynamicIdentityEntry{ NodeID: identity.NodeID, Dynamic: flow.DynamicIdentity{ diff --git a/state/protocol/protocol_state/updater_test.go b/state/protocol/protocol_state/updater_test.go index f1f2cefaa8f..0915926b597 100644 --- a/state/protocol/protocol_state/updater_test.go +++ b/state/protocol/protocol_state/updater_test.go @@ -1,6 +1,7 @@ package protocol_state import ( + "github.com/onflow/flow-go/model/flow/order" "testing" "github.com/stretchr/testify/require" @@ -8,7 +9,6 @@ import ( "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/flow/mapfunc" - "github.com/onflow/flow-go/model/flow/order" "github.com/onflow/flow-go/utils/unittest" ) @@ -214,7 +214,7 @@ func (s *UpdaterSuite) TestUpdateIdentityHappyPath() { unittest.WithNextEpochProtocolState()(s.parentProtocolState) s.updater = NewUpdater(s.candidate, s.parentProtocolState) - currentEpochParticipants := s.parentProtocolState.CurrentEpochSetup.Participants.Copy() + currentEpochParticipants := s.parentProtocolState.CurrentEpochIdentityTable.Copy() weightChanges, err := currentEpochParticipants.Sample(2) require.NoError(s.T(), err) ejectedChanges, err := currentEpochParticipants.Sample(2) @@ -289,66 +289,26 @@ func (s *UpdaterSuite) TestProcessEpochSetupInvariants() { // For current epoch we should have identity table with all the nodes from the current epoch + nodes from the next epoch with 0 weight. // For next epoch we should have identity table with all the nodes from the next epoch + nodes from the current epoch with 0 weight. func (s *UpdaterSuite) TestProcessEpochSetupHappyPath() { + setupParticipants := unittest.IdentityListFixture(5).Sort(order.Canonical[flow.Identity]) + setupParticipants[0].InitialWeight = 13 + setupParticipants[0].Weight = setupParticipants[0].InitialWeight setup := unittest.EpochSetupFixture(func(setup *flow.EpochSetup) { setup.Counter = s.parentProtocolState.CurrentEpochSetup.Counter + 1 - setup.Participants[0].InitialWeight = 13 + setup.Participants = setupParticipants.ToSkeleton() }) - // prepare expected identity tables for current and next epochs - expectedCurrentEpochIdentityTable := make(flow.DynamicIdentityEntryList, 0, - len(s.parentProtocolState.CurrentEpochSetup.Participants)+len(setup.Participants)) - expectedNextEpochIdentityTable := make(flow.DynamicIdentityEntryList, 0, - len(expectedCurrentEpochIdentityTable)) - + participantsFromCurrentEpochSetup := s.parentProtocolState.CurrentEpochIdentityTable.Filter(func(i *flow.Identity) bool { + _, exists := s.parentProtocolState.CurrentEpochSetup.Participants.ByNodeID(i.NodeID) + return exists + }) // for current epoch we will have all the nodes from the current epoch + nodes from the next epoch with 0 weight + expectedCurrentEpochIdentityTable := flow.DynamicIdentityEntryListFromIdentities( + participantsFromCurrentEpochSetup.Union(setupParticipants.Map(mapfunc.WithWeight(0))), + ) // for next epoch we will have all the nodes from the next epoch + nodes from the current epoch with 0 weight - for _, participant := range s.parentProtocolState.CurrentEpochSetup.Participants { - expectedCurrentEpochIdentityTable = append(expectedCurrentEpochIdentityTable, &flow.DynamicIdentityEntry{ - NodeID: participant.NodeID, - Dynamic: flow.DynamicIdentity{ - Weight: participant.Weight, - Ejected: participant.Ejected, - }, - }) - - expectedNextEpochIdentityTable = append(expectedNextEpochIdentityTable, &flow.DynamicIdentityEntry{ - NodeID: participant.NodeID, - Dynamic: flow.DynamicIdentity{ - Weight: 0, - Ejected: participant.Ejected, - }, - }) - } - - // in this loop we perform a few extra lookups to avoid duplicates - for _, participant := range setup.Participants { - if _, found := expectedCurrentEpochIdentityTable.ByNodeID(participant.NodeID); !found { - expectedCurrentEpochIdentityTable = append(expectedCurrentEpochIdentityTable, &flow.DynamicIdentityEntry{ - NodeID: participant.NodeID, - Dynamic: flow.DynamicIdentity{ - Weight: 0, - Ejected: participant.Ejected, - }, - }) - } - - entry := &flow.DynamicIdentityEntry{ - NodeID: participant.NodeID, - Dynamic: flow.DynamicIdentity{ - Weight: participant.InitialWeight, - Ejected: participant.Ejected, - }, - } - existing, found := expectedNextEpochIdentityTable.ByNodeID(participant.NodeID) - if found { - existing.Dynamic = entry.Dynamic - } else { - expectedNextEpochIdentityTable = append(expectedNextEpochIdentityTable, entry) - } - } - // finally, sort in canonical order - expectedCurrentEpochIdentityTable = expectedCurrentEpochIdentityTable.Sort(order.IdentifierCanonical) - expectedNextEpochIdentityTable = expectedNextEpochIdentityTable.Sort(order.IdentifierCanonical) + expectedNextEpochIdentityTable := flow.DynamicIdentityEntryListFromIdentities( + setupParticipants.Union(participantsFromCurrentEpochSetup.Map(mapfunc.WithWeight(0))), + ) // process actual event err := s.updater.ProcessEpochSetup(setup) @@ -368,24 +328,32 @@ func (s *UpdaterSuite) TestProcessEpochSetupHappyPath() { // built updated protocol state. It should build a union of participants from current and next epoch for current and // next epoch protocol states respectively. func (s *UpdaterSuite) TestProcessEpochSetupWithSameParticipants() { - overlappingNodes, err := s.parentProtocolState.CurrentEpochSetup.Participants.Sample(2) + participantsFromCurrentEpochSetup := s.parentProtocolState.CurrentEpochIdentityTable.Filter(func(i *flow.Identity) bool { + _, exists := s.parentProtocolState.CurrentEpochSetup.Participants.ByNodeID(i.NodeID) + return exists + }).Sort(order.Canonical[flow.Identity]) + + overlappingNodes, err := participantsFromCurrentEpochSetup.Sample(2) require.NoError(s.T(), err) + setupParticipants := append(unittest.IdentityListFixture(len(s.parentProtocolState.CurrentEpochIdentityTable)), + overlappingNodes...).Sort(order.Canonical[flow.Identity]) setup := unittest.EpochSetupFixture(func(setup *flow.EpochSetup) { setup.Counter = s.parentProtocolState.CurrentEpochSetup.Counter + 1 - setup.Participants = append(setup.Participants, overlappingNodes...) + setup.Participants = setupParticipants.ToSkeleton() }) err = s.updater.ProcessEpochSetup(setup) require.NoError(s.T(), err) updatedState, _, _ := s.updater.Build() + expectedParticipants := flow.DynamicIdentityEntryListFromIdentities( - s.parentProtocolState.CurrentEpochSetup.Participants.Union(setup.Participants.Map(mapfunc.WithWeight(0))), + participantsFromCurrentEpochSetup.Union(setupParticipants.Map(mapfunc.WithWeight(0))), ) require.Equal(s.T(), updatedState.CurrentEpoch.ActiveIdentities, expectedParticipants, "should have all participants from current epoch and next epoch, but without duplicates") nextEpochParticipants := flow.DynamicIdentityEntryListFromIdentities( - setup.Participants.Union(s.parentProtocolState.CurrentEpochSetup.Participants.Map(mapfunc.WithWeight(0))), + setupParticipants.Union(participantsFromCurrentEpochSetup.Map(mapfunc.WithWeight(0))), ) require.Equal(s.T(), updatedState.NextEpoch.ActiveIdentities, nextEpochParticipants, @@ -395,10 +363,13 @@ func (s *UpdaterSuite) TestProcessEpochSetupWithSameParticipants() { // TestEpochSetupAfterIdentityChange tests that after processing epoch an setup event, all previously made changes to the identity table // are preserved and reflected in the resulting protocol state. func (s *UpdaterSuite) TestEpochSetupAfterIdentityChange() { - currentEpochParticipants := s.parentProtocolState.CurrentEpochSetup.Participants.Copy() // DEEP copy of Identity List - weightChanges, err := currentEpochParticipants.Sample(2) + participantsFromCurrentEpochSetup := s.parentProtocolState.CurrentEpochIdentityTable.Filter(func(i *flow.Identity) bool { + _, exists := s.parentProtocolState.CurrentEpochSetup.Participants.ByNodeID(i.NodeID) + return exists + }).Sort(order.Canonical[flow.Identity]) + weightChanges, err := participantsFromCurrentEpochSetup.Sample(2) require.NoError(s.T(), err) - ejectedChanges, err := currentEpochParticipants.Sample(2) + ejectedChanges, err := participantsFromCurrentEpochSetup.Sample(2) require.NoError(s.T(), err) for i, identity := range weightChanges { identity.DynamicIdentity.Weight = uint64(100 * (i + 1)) @@ -441,7 +412,7 @@ func (s *UpdaterSuite) TestEpochSetupAfterIdentityChange() { setup := unittest.EpochSetupFixture(func(setup *flow.EpochSetup) { setup.Counter = s.parentProtocolState.CurrentEpochSetup.Counter + 1 - setup.Participants = append(setup.Participants, allUpdates...) // add those nodes that were changed in previous epoch + setup.Participants = append(setup.Participants, allUpdates.ToSkeleton()...) // add those nodes that were changed in previous epoch }) err = s.updater.ProcessEpochSetup(setup) From d9e78a61ede0fcfcfd352e1a9b932028dc76d44d Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Thu, 28 Sep 2023 21:52:27 +0300 Subject: [PATCH 06/39] Updated how static replicas are implemented --- consensus/hotstuff/committees/static.go | 87 ++++++++++++++++--------- model/flow/order/identity.go | 14 +--- state/protocol/badger/validity.go | 15 +++-- 3 files changed, 67 insertions(+), 49 deletions(-) diff --git a/consensus/hotstuff/committees/static.go b/consensus/hotstuff/committees/static.go index 082335121c0..071a614c432 100644 --- a/consensus/hotstuff/committees/static.go +++ b/consensus/hotstuff/committees/static.go @@ -11,23 +11,20 @@ import ( "github.com/onflow/flow-go/state/protocol" ) -// NewStaticCommittee returns a new committee with a static participant set. -func NewStaticCommittee(participants flow.IdentityList, myID flow.Identifier, dkgParticipants map[flow.Identifier]flow.DKGParticipant, dkgGroupKey crypto.PublicKey) (*Static, error) { - - return NewStaticCommitteeWithDKG(participants, myID, staticDKG{ +func NewStaticReplicas(participants flow.IdentitySkeletonList, myID flow.Identifier, dkgParticipants map[flow.Identifier]flow.DKGParticipant, dkgGroupKey crypto.PublicKey) (*StaticReplicas, error) { + return NewStaticReplicasWithDKG(participants, myID, staticDKG{ dkgParticipants: dkgParticipants, dkgGroupKey: dkgGroupKey, }) } -// NewStaticCommitteeWithDKG returns a new committee with a static participant set. -func NewStaticCommitteeWithDKG(participants flow.IdentityList, myID flow.Identifier, dkg protocol.DKG) (*Static, error) { +func NewStaticReplicasWithDKG(participants flow.IdentitySkeletonList, myID flow.Identifier, dkg protocol.DKG) (*StaticReplicas, error) { valid := order.IdentityListCanonical(participants) if !valid { return nil, fmt.Errorf("participants %v is not in Canonical order", participants) } - static := &Static{ + static := &StaticReplicas{ participants: participants, myID: myID, dkg: dkg, @@ -35,61 +32,89 @@ func NewStaticCommitteeWithDKG(participants flow.IdentityList, myID flow.Identif return static, nil } -// Static represents a committee with a static participant set. It is used for -// bootstrapping purposes. -type Static struct { - participants flow.IdentityList - myID flow.Identifier - dkg protocol.DKG +// NewStaticCommittee returns a new committee with a static participant set. +func NewStaticCommittee(participants flow.IdentityList, myID flow.Identifier, dkgParticipants map[flow.Identifier]flow.DKGParticipant, dkgGroupKey crypto.PublicKey) (*Static, error) { + return NewStaticCommitteeWithDKG(participants, myID, staticDKG{ + dkgParticipants: dkgParticipants, + dkgGroupKey: dkgGroupKey, + }) } -var _ hotstuff.Replicas = (*Static)(nil) -var _ hotstuff.DynamicCommittee = (*Static)(nil) +// NewStaticCommitteeWithDKG returns a new committee with a static participant set. +func NewStaticCommitteeWithDKG(participants flow.IdentityList, myID flow.Identifier, dkg protocol.DKG) (*Static, error) { + replicas, err := NewStaticReplicasWithDKG(participants.ToSkeleton(), myID, dkg) + if err != nil { + return nil, fmt.Errorf("could not create static replicas: %w", err) + } -func (s Static) IdentitiesByBlock(_ flow.Identifier) (flow.IdentityList, error) { - return s.participants, nil + static := &Static{ + StaticReplicas: *replicas, + fullIdentities: participants, + } + return static, nil } -func (s Static) IdentityByBlock(_ flow.Identifier, participantID flow.Identifier) (*flow.Identity, error) { - identity, ok := s.participants.ByNodeID(participantID) - if !ok { - return nil, model.NewInvalidSignerErrorf("unknown participant %x", participantID) - } - return identity, nil +type StaticReplicas struct { + participants flow.IdentitySkeletonList + myID flow.Identifier + dkg protocol.DKG } -func (s Static) IdentitiesByEpoch(view uint64) (flow.IdentitySkeletonList, error) { +var _ hotstuff.Replicas = (*StaticReplicas)(nil) + +func (s StaticReplicas) IdentitiesByEpoch(view uint64) (flow.IdentitySkeletonList, error) { return s.participants.ToSkeleton(), nil } -func (s Static) IdentityByEpoch(view uint64, participantID flow.Identifier) (*flow.IdentitySkeleton, error) { +func (s StaticReplicas) IdentityByEpoch(view uint64, participantID flow.Identifier) (*flow.IdentitySkeleton, error) { identity, ok := s.participants.ByNodeID(participantID) if !ok { return nil, model.NewInvalidSignerErrorf("unknown participant %x", participantID) } - return &identity.IdentitySkeleton, nil + return identity, nil } -func (s Static) LeaderForView(_ uint64) (flow.Identifier, error) { +func (s StaticReplicas) LeaderForView(_ uint64) (flow.Identifier, error) { return flow.ZeroID, fmt.Errorf("invalid for static committee") } -func (s Static) QuorumThresholdForView(_ uint64) (uint64, error) { +func (s StaticReplicas) QuorumThresholdForView(_ uint64) (uint64, error) { return WeightThresholdToBuildQC(s.participants.ToSkeleton().TotalWeight()), nil } -func (s Static) TimeoutThresholdForView(_ uint64) (uint64, error) { +func (s StaticReplicas) TimeoutThresholdForView(_ uint64) (uint64, error) { return WeightThresholdToTimeout(s.participants.ToSkeleton().TotalWeight()), nil } -func (s Static) Self() flow.Identifier { +func (s StaticReplicas) Self() flow.Identifier { return s.myID } -func (s Static) DKG(_ uint64) (hotstuff.DKG, error) { +func (s StaticReplicas) DKG(_ uint64) (hotstuff.DKG, error) { return s.dkg, nil } +// Static represents a committee with a static participant set. It is used for +// bootstrapping purposes. +type Static struct { + StaticReplicas + fullIdentities flow.IdentityList +} + +var _ hotstuff.DynamicCommittee = (*Static)(nil) + +func (s Static) IdentitiesByBlock(_ flow.Identifier) (flow.IdentityList, error) { + return s.fullIdentities, nil +} + +func (s Static) IdentityByBlock(_ flow.Identifier, participantID flow.Identifier) (*flow.Identity, error) { + identity, ok := s.fullIdentities.ByNodeID(participantID) + if !ok { + return nil, model.NewInvalidSignerErrorf("unknown participant %x", participantID) + } + return identity, nil +} + type staticDKG struct { dkgParticipants map[flow.Identifier]flow.DKGParticipant dkgGroupKey crypto.PublicKey diff --git a/model/flow/order/identity.go b/model/flow/order/identity.go index c30c9f0cbd3..9912aedfb23 100644 --- a/model/flow/order/identity.go +++ b/model/flow/order/identity.go @@ -29,19 +29,9 @@ func ByReferenceOrder(nodeIDs []flow.Identifier) func(*flow.Identity, *flow.Iden // IdentityListCanonical takes a list of identities and // check if it's ordered in canonical order. -func IdentityListCanonical(identities flow.IdentityList) bool { +func IdentityListCanonical[T flow.GenericIdentity](identities flow.GenericIdentityList[T]) bool { if len(identities) == 0 { return true } - - prev := identities[0].ID() - for i := 1; i < len(identities); i++ { - id := identities[i].ID() - if !IdentifierCanonical(prev, id) { - return false - } - prev = id - } - - return true + return identities.Sorted(Canonical[T]) } diff --git a/state/protocol/badger/validity.go b/state/protocol/badger/validity.go index c2e4c3c2933..9c1b95aa0a0 100644 --- a/state/protocol/badger/validity.go +++ b/state/protocol/badger/validity.go @@ -87,14 +87,14 @@ func verifyEpochSetup(setup *flow.EpochSetup, verifyNetworkAddress bool) error { } // the participants must be listed in canonical order - if !setup.Participants.Sorted(order.Canonical) { + if !setup.Participants.Sorted(order.Canonical[flow.IdentitySkeleton]) { return fmt.Errorf("participants are not canonically ordered") } // STEP 3: sanity checks for individual roles // IMPORTANT: here we remove all nodes with zero weight, as they are allowed to partake // in communication but not in respective node functions - activeParticipants := setup.Participants.Filter(filter.HasWeight(true)) + activeParticipants := setup.Participants.Filter(filter.HasInitialWeight[flow.IdentitySkeleton](true)) // we need at least one node of each role roles := make(map[flow.Role]uint) @@ -125,7 +125,8 @@ func verifyEpochSetup(setup *flow.EpochSetup, verifyNetworkAddress bool) error { } // the collection cluster assignments need to be valid - _, err := factory.NewClusterList(setup.Assignments, activeParticipants.Filter(filter.HasRole(flow.RoleCollection))) + _, err := factory.NewClusterList(setup.Assignments, + activeParticipants.Filter(filter.HasRole[flow.IdentitySkeleton](flow.RoleCollection))) if err != nil { return fmt.Errorf("invalid cluster assignments: %w", err) } @@ -235,7 +236,7 @@ func IsValidRootSnapshot(snap protocol.Snapshot, verifyResultID bool) error { if err != nil { return fmt.Errorf("could not get identities for root snapshot: %w", err) } - if !identities.Sorted(order.Canonical) { + if !identities.Sorted(order.Canonical[flow.Identity]) { return fmt.Errorf("identities are not canonically ordered") } @@ -304,7 +305,9 @@ func IsValidRootSnapshotQCs(snap protocol.Snapshot) error { // validateRootQC performs validation of root QC // Returns nil on success func validateRootQC(snap protocol.Snapshot) error { - identities, err := snap.Identities(filter.IsAllowedConsensusCommitteeMember) + identities, err := snap.Identities(func(identity *flow.Identity) bool { + return filter.IsAllowedConsensusCommitteeMember(&identity.IdentitySkeleton) + }) if err != nil { return fmt.Errorf("could not get root snapshot identities: %w", err) } @@ -335,7 +338,7 @@ func validateRootQC(snap protocol.Snapshot) error { // validateClusterQC performs QC validation of single collection cluster // Returns nil on success func validateClusterQC(cluster protocol.Cluster) error { - committee, err := committees.NewStaticCommittee(cluster.Members(), flow.Identifier{}, nil, nil) + committee, err := committees.NewStaticReplicas(cluster.Members(), flow.Identifier{}, nil, nil) if err != nil { return fmt.Errorf("could not create static committee: %w", err) } From e408bc19925100c3d97d24dac279e1281add640e Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Thu, 28 Sep 2023 21:56:30 +0300 Subject: [PATCH 07/39] Updated how static consensus committee is implemented. Updated identity filter --- consensus/hotstuff/committees/cluster_committee.go | 4 +--- model/flow/identity.go | 6 ++++++ state/protocol/badger/validity.go | 4 +--- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/consensus/hotstuff/committees/cluster_committee.go b/consensus/hotstuff/committees/cluster_committee.go index ea375a68051..80d21864c34 100644 --- a/consensus/hotstuff/committees/cluster_committee.go +++ b/consensus/hotstuff/committees/cluster_committee.go @@ -70,9 +70,7 @@ func NewClusterCommittee( selection: selection, clusterMemberFilter: filter.And[flow.Identity]( // adapt the identity filter to the identity skeleton filter - func(f *flow.Identity) bool { - return initialClusterMembersSelector(&f.IdentitySkeleton) - }, + flow.AdaptIdentityFilter(initialClusterMembersSelector), filter.Not(filter.Ejected), filter.HasWeight(true), ), diff --git a/model/flow/identity.go b/model/flow/identity.go index 2eda6403429..b1d1c8dadd8 100644 --- a/model/flow/identity.go +++ b/model/flow/identity.go @@ -394,6 +394,12 @@ type IdentityList = GenericIdentityList[Identity] type GenericIdentityList[T GenericIdentity] []*T +func AdaptIdentityFilter(f IdentityFilter[IdentitySkeleton]) IdentityFilter[Identity] { + return func(i *Identity) bool { + return f(&i.IdentitySkeleton) + } +} + // Filter will apply a filter to the identity list. func (il GenericIdentityList[T]) Filter(filter IdentityFilter[T]) GenericIdentityList[T] { var dup GenericIdentityList[T] diff --git a/state/protocol/badger/validity.go b/state/protocol/badger/validity.go index 9c1b95aa0a0..455dd62a340 100644 --- a/state/protocol/badger/validity.go +++ b/state/protocol/badger/validity.go @@ -305,9 +305,7 @@ func IsValidRootSnapshotQCs(snap protocol.Snapshot) error { // validateRootQC performs validation of root QC // Returns nil on success func validateRootQC(snap protocol.Snapshot) error { - identities, err := snap.Identities(func(identity *flow.Identity) bool { - return filter.IsAllowedConsensusCommitteeMember(&identity.IdentitySkeleton) - }) + identities, err := snap.Identities(flow.AdaptIdentityFilter(filter.IsAllowedConsensusCommitteeMember)) if err != nil { return fmt.Errorf("could not get root snapshot identities: %w", err) } From 4f68f383f331d7470f9a35815617e23ffabf1c82 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Thu, 28 Sep 2023 21:58:43 +0300 Subject: [PATCH 08/39] Updated filter adapter --- consensus/hotstuff/committees/cluster_committee.go | 2 +- model/flow/filter/identity.go | 9 +++++++++ model/flow/identity.go | 6 ------ state/protocol/badger/validity.go | 2 +- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/consensus/hotstuff/committees/cluster_committee.go b/consensus/hotstuff/committees/cluster_committee.go index 80d21864c34..1ce44cd730f 100644 --- a/consensus/hotstuff/committees/cluster_committee.go +++ b/consensus/hotstuff/committees/cluster_committee.go @@ -70,7 +70,7 @@ func NewClusterCommittee( selection: selection, clusterMemberFilter: filter.And[flow.Identity]( // adapt the identity filter to the identity skeleton filter - flow.AdaptIdentityFilter(initialClusterMembersSelector), + filter.Adapt(initialClusterMembersSelector), filter.Not(filter.Ejected), filter.HasWeight(true), ), diff --git a/model/flow/filter/identity.go b/model/flow/filter/identity.go index 43c303360d6..dde0bff2fc6 100644 --- a/model/flow/filter/identity.go +++ b/model/flow/filter/identity.go @@ -7,6 +7,15 @@ import ( "github.com/onflow/flow-go/model/flow" ) +// Adapt adapts a filter for a specific identity type. +// +// Converts flow.IdentityFilter[flow.IdentitySkeleton] to flow.IdentityFilter[flow.Identity]. +func Adapt(f flow.IdentityFilter[flow.IdentitySkeleton]) flow.IdentityFilter[flow.Identity] { + return func(i *flow.Identity) bool { + return f(&i.IdentitySkeleton) + } +} + // Any will always be true. func Any(*flow.Identity) bool { return true diff --git a/model/flow/identity.go b/model/flow/identity.go index b1d1c8dadd8..2eda6403429 100644 --- a/model/flow/identity.go +++ b/model/flow/identity.go @@ -394,12 +394,6 @@ type IdentityList = GenericIdentityList[Identity] type GenericIdentityList[T GenericIdentity] []*T -func AdaptIdentityFilter(f IdentityFilter[IdentitySkeleton]) IdentityFilter[Identity] { - return func(i *Identity) bool { - return f(&i.IdentitySkeleton) - } -} - // Filter will apply a filter to the identity list. func (il GenericIdentityList[T]) Filter(filter IdentityFilter[T]) GenericIdentityList[T] { var dup GenericIdentityList[T] diff --git a/state/protocol/badger/validity.go b/state/protocol/badger/validity.go index 455dd62a340..64455331569 100644 --- a/state/protocol/badger/validity.go +++ b/state/protocol/badger/validity.go @@ -305,7 +305,7 @@ func IsValidRootSnapshotQCs(snap protocol.Snapshot) error { // validateRootQC performs validation of root QC // Returns nil on success func validateRootQC(snap protocol.Snapshot) error { - identities, err := snap.Identities(flow.AdaptIdentityFilter(filter.IsAllowedConsensusCommitteeMember)) + identities, err := snap.Identities(filter.Adapt(filter.IsAllowedConsensusCommitteeMember)) if err != nil { return fmt.Errorf("could not get root snapshot identities: %w", err) } From 16c1347175f6c096621cbde60374eece28679c0a Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Thu, 28 Sep 2023 22:11:15 +0300 Subject: [PATCH 09/39] Work in progress on fixing mutator tests --- state/protocol/badger/mutator_test.go | 12 ++++++------ state/protocol/badger/validity_test.go | 2 +- utils/unittest/fixtures.go | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/state/protocol/badger/mutator_test.go b/state/protocol/badger/mutator_test.go index bbc6f43afbe..a0e24041fe9 100644 --- a/state/protocol/badger/mutator_test.go +++ b/state/protocol/badger/mutator_test.go @@ -868,7 +868,7 @@ func TestExtendEpochTransitionValid(t *testing.T) { // add a participant for the next epoch epoch2NewParticipant := unittest.IdentityFixture(unittest.WithRole(flow.RoleVerification)) - epoch2Participants := append(participants, epoch2NewParticipant).Sort(order.Canonical) + epoch2Participants := append(participants, epoch2NewParticipant).Sort(order.Canonical[flow.Identity]).ToSkeleton() // create the epoch setup event for the second epoch epoch2Setup := unittest.EpochSetupFixture( @@ -951,7 +951,7 @@ func TestExtendEpochTransitionValid(t *testing.T) { epoch2Commit := unittest.EpochCommitFixture( unittest.CommitWithCounter(epoch2Setup.Counter), unittest.WithClusterQCsFromAssignments(epoch2Setup.Assignments), - unittest.WithDKGFromParticipants(epoch2Participants), + unittest.WithDKGFromParticipants(epoch2Participants.ToSkeleton()), ) // create receipt and seal for block 2 @@ -1319,7 +1319,7 @@ func TestExtendEpochSetupInvalid(t *testing.T) { // add a participant for the next epoch epoch2NewParticipant := unittest.IdentityFixture(unittest.WithRole(flow.RoleVerification)) - epoch2Participants := append(participants, epoch2NewParticipant).Sort(order.Canonical) + epoch2Participants := append(participants, epoch2NewParticipant).Sort(order.Canonical[flow.Identity]).ToSkeleton() // this function will return a VALID setup event and seal, we will modify // in different ways in each test case @@ -1435,9 +1435,9 @@ func TestExtendEpochCommitInvalid(t *testing.T) { // swap consensus node for a new one for epoch 2 epoch2NewParticipant := unittest.IdentityFixture(unittest.WithRole(flow.RoleConsensus)) epoch2Participants := append( - participants.Filter(filter.Not(filter.HasRole(flow.RoleConsensus))), + participants.Filter(filter.Not[flow.Identity](filter.HasRole[flow.Identity](flow.RoleConsensus))), epoch2NewParticipant, - ).Sort(order.Canonical) + ).Sort(order.Canonical[flow.Identity]).ToSkeleton() // factory method to create a valid EpochSetup method w.r.t. the generated state createSetup := func(block *flow.Block) (*flow.EpochSetup, *flow.ExecutionReceipt, *flow.Seal) { @@ -1620,7 +1620,7 @@ func TestExtendEpochTransitionWithoutCommit(t *testing.T) { // add a participant for the next epoch epoch2NewParticipant := unittest.IdentityFixture(unittest.WithRole(flow.RoleVerification)) - epoch2Participants := append(participants, epoch2NewParticipant).Sort(order.Canonical) + epoch2Participants := append(participants, epoch2NewParticipant).Sort(order.Canonical[flow.Identity]).ToSkeleton() // create the epoch setup event for the second epoch epoch2Setup := unittest.EpochSetupFixture( diff --git a/state/protocol/badger/validity_test.go b/state/protocol/badger/validity_test.go index 30ee94c40d6..c749e62cec6 100644 --- a/state/protocol/badger/validity_test.go +++ b/state/protocol/badger/validity_test.go @@ -41,7 +41,7 @@ func TestEpochSetupValidity(t *testing.T) { _, result, _ := unittest.BootstrapFixture(participants) setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) // create an invalid cluster assignment (node appears in multiple clusters) - collector := participants.Filter(filter.HasRole(flow.RoleCollection))[0] + collector := participants.Filter(filter.HasRole[flow.Identity](flow.RoleCollection))[0] setup.Assignments = append(setup.Assignments, []flow.Identifier{collector.NodeID}) err := verifyEpochSetup(setup, true) diff --git a/utils/unittest/fixtures.go b/utils/unittest/fixtures.go index 36e7d82f038..7f3bcb9c1d5 100644 --- a/utils/unittest/fixtures.go +++ b/utils/unittest/fixtures.go @@ -1953,9 +1953,9 @@ func VoteWithBeaconSig() func(*hotstuff.Vote) { } } -func WithParticipants(participants flow.IdentityList) func(*flow.EpochSetup) { +func WithParticipants(participants flow.IdentitySkeletonList) func(*flow.EpochSetup) { return func(setup *flow.EpochSetup) { - setup.Participants = participants.Sort(order.Canonical[flow.Identity]).ToSkeleton() + setup.Participants = participants.Sort(order.Canonical[flow.IdentitySkeleton]) setup.Assignments = ClusterAssignment(1, participants.ToSkeleton()) } } From 1ea0703f07fc6f7142f0baa8a895ef7248bc2b4c Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Fri, 29 Sep 2023 15:38:14 +0300 Subject: [PATCH 10/39] Fixed more compilation issues in tests --- state/protocol/badger/mutator_test.go | 4 ++-- state/protocol/badger/snapshot_test.go | 20 ++++++++++---------- state/protocol/badger/state_test.go | 2 +- utils/unittest/epoch_builder.go | 2 +- utils/unittest/fixtures.go | 2 +- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/state/protocol/badger/mutator_test.go b/state/protocol/badger/mutator_test.go index a0e24041fe9..71b2fcbdde6 100644 --- a/state/protocol/badger/mutator_test.go +++ b/state/protocol/badger/mutator_test.go @@ -1761,7 +1761,7 @@ func TestEmergencyEpochFallback(t *testing.T) { // add a participant for the next epoch epoch2NewParticipant := unittest.IdentityFixture(unittest.WithRole(flow.RoleVerification)) - epoch2Participants := append(participants, epoch2NewParticipant).Sort(order.Canonical) + epoch2Participants := append(participants, epoch2NewParticipant).Sort(order.Canonical[flow.Identity]).ToSkeleton() // create the epoch setup event for the second epoch epoch2Setup := unittest.EpochSetupFixture( @@ -1851,7 +1851,7 @@ func TestEmergencyEpochFallback(t *testing.T) { // add a participant for the next epoch epoch2NewParticipant := unittest.IdentityFixture(unittest.WithRole(flow.RoleVerification)) - epoch2Participants := append(participants, epoch2NewParticipant).Sort(order.Canonical) + epoch2Participants := append(participants, epoch2NewParticipant).Sort(order.Canonical[flow.Identity]).ToSkeleton() // create the epoch setup event for the second epoch // this event is invalid because it used a non-contiguous first view diff --git a/state/protocol/badger/snapshot_test.go b/state/protocol/badger/snapshot_test.go index 25c9d1cd6f2..d6845ebd2b2 100644 --- a/state/protocol/badger/snapshot_test.go +++ b/state/protocol/badger/snapshot_test.go @@ -193,9 +193,9 @@ func TestIdentities(t *testing.T) { t.Run("filtered", func(t *testing.T) { sample, err := identities.SamplePct(0.1) require.NoError(t, err) - filters := []flow.IdentityFilter{ - filter.HasRole(flow.RoleCollection), - filter.HasNodeID(sample.NodeIDs()...), + filters := []flow.IdentityFilter[flow.Identity]{ + filter.HasRole[flow.Identity](flow.RoleCollection), + filter.HasNodeID[flow.Identity](sample.NodeIDs()...), filter.HasWeight(true), } @@ -220,7 +220,7 @@ func TestClusters(t *testing.T) { qc := unittest.QuorumCertificateFixture(unittest.QCWithRootBlockID(root.ID())) setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) commit := result.ServiceEvents[1].Event.(*flow.EpochCommit) - setup.Assignments = unittest.ClusterAssignment(uint(nClusters), collectors) + setup.Assignments = unittest.ClusterAssignment(uint(nClusters), collectors.ToSkeleton()) clusterQCs := unittest.QuorumCertificatesFromAssignments(setup.Assignments) commit.ClusterQCs = flow.ClusterQCVoteDatasFromQCs(clusterQCs) seal.ResultID = result.ID() @@ -229,7 +229,7 @@ func TestClusters(t *testing.T) { require.NoError(t, err) util.RunWithBootstrapState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.State) { - expectedClusters, err := factory.NewClusterList(setup.Assignments, collectors) + expectedClusters, err := factory.NewClusterList(setup.Assignments, collectors.ToSkeleton()) require.NoError(t, err) actualClusters, err := state.Final().Epochs().Current().Clustering() require.NoError(t, err) @@ -1240,7 +1240,7 @@ func TestSnapshot_CrossEpochIdentities(t *testing.T) { removedAtEpoch2 := epoch1Identities[rand.Intn(len(epoch1Identities))] // epoch 2 has partial overlap with epoch 1 epoch2Identities := append( - epoch1Identities.Filter(filter.Not(filter.HasNodeID(removedAtEpoch2.NodeID))), + epoch1Identities.Filter(filter.Not(filter.HasNodeID[flow.Identity](removedAtEpoch2.NodeID))), addedAtEpoch2) // epoch 3 has no overlap with epoch 2 epoch3Identities := unittest.IdentityListFixture(10, unittest.WithAllRoles()) @@ -1251,12 +1251,12 @@ func TestSnapshot_CrossEpochIdentities(t *testing.T) { epochBuilder := unittest.NewEpochBuilder(t, state) // build epoch 1 (prepare epoch 2) epochBuilder. - UsingSetupOpts(unittest.WithParticipants(epoch2Identities)). + UsingSetupOpts(unittest.WithParticipants(epoch2Identities.ToSkeleton())). BuildEpoch(). CompleteEpoch() // build epoch 2 (prepare epoch 3) epochBuilder. - UsingSetupOpts(unittest.WithParticipants(epoch3Identities)). + UsingSetupOpts(unittest.WithParticipants(epoch3Identities.ToSkeleton())). BuildEpoch(). CompleteEpoch() @@ -1298,7 +1298,7 @@ func TestSnapshot_CrossEpochIdentities(t *testing.T) { assert.ElementsMatch(t, epoch1Identities, identities.Filter(epoch1Identities.Selector())) // should contain single next epoch identity with 0 weight - nextEpochIdentity := identities.Filter(filter.HasNodeID(addedAtEpoch2.NodeID))[0] + nextEpochIdentity := identities.Filter(filter.HasNodeID[flow.Identity](addedAtEpoch2.NodeID))[0] assert.Equal(t, uint64(0), nextEpochIdentity.Weight) // should have 0 weight nextEpochIdentity.Weight = addedAtEpoch2.Weight assert.Equal(t, addedAtEpoch2, nextEpochIdentity) // should be equal besides weight @@ -1319,7 +1319,7 @@ func TestSnapshot_CrossEpochIdentities(t *testing.T) { assert.ElementsMatch(t, epoch2Identities, identities.Filter(epoch2Identities.Selector())) // should contain single previous epoch identity with 0 weight - lastEpochIdentity := identities.Filter(filter.HasNodeID(removedAtEpoch2.NodeID))[0] + lastEpochIdentity := identities.Filter(filter.HasNodeID[flow.Identity](removedAtEpoch2.NodeID))[0] assert.Equal(t, uint64(0), lastEpochIdentity.Weight) // should have 0 weight lastEpochIdentity.Weight = removedAtEpoch2.Weight // overwrite weight assert.Equal(t, removedAtEpoch2, lastEpochIdentity) // should be equal besides weight diff --git a/state/protocol/badger/state_test.go b/state/protocol/badger/state_test.go index 085c8378f49..298cfe4e5f5 100644 --- a/state/protocol/badger/state_test.go +++ b/state/protocol/badger/state_test.go @@ -426,7 +426,7 @@ func TestBootstrap_InvalidIdentities(t *testing.T) { // randomly shuffle the identities so they are not canonically ordered encodable := root.Encodable() var err error - encodable.Epochs.Current.InitialIdentities, err = participants.Shuffle() + encodable.Epochs.Current.InitialIdentities, err = participants.ToSkeleton().Shuffle() require.NoError(t, err) root = inmem.SnapshotFromEncodable(encodable) bootstrap(t, root, func(state *bprotocol.State, err error) { diff --git a/utils/unittest/epoch_builder.go b/utils/unittest/epoch_builder.go index 321522f582a..9cc370a98fe 100644 --- a/utils/unittest/epoch_builder.go +++ b/utils/unittest/epoch_builder.go @@ -201,7 +201,7 @@ func (builder *EpochBuilder) BuildEpoch() *EpochBuilder { // defaults for the EpochSetup event setupDefaults := []func(*flow.EpochSetup){ - WithParticipants(identities), + WithParticipants(identities.ToSkeleton()), SetupWithCounter(counter + 1), WithFirstView(finalView + 1), WithFinalView(finalView + 1_000_000), diff --git a/utils/unittest/fixtures.go b/utils/unittest/fixtures.go index 7f3bcb9c1d5..70b22be6258 100644 --- a/utils/unittest/fixtures.go +++ b/utils/unittest/fixtures.go @@ -2128,7 +2128,7 @@ func BootstrapFixtureWithChainID( counter := uint64(1) setup := EpochSetupFixture( - WithParticipants(participants), + WithParticipants(participants.ToSkeleton()), SetupWithCounter(counter), WithFirstView(root.Header.View), WithFinalView(root.Header.View+1000), From 72349e022e250c873136cff0a67daaf9723f6def Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Fri, 29 Sep 2023 17:10:35 +0300 Subject: [PATCH 11/39] Moved identity list to a separate file --- model/flow/identity.go | 354 +--------------------------------- model/flow/identity_list.go | 370 ++++++++++++++++++++++++++++++++++++ 2 files changed, 371 insertions(+), 353 deletions(-) create mode 100644 model/flow/identity_list.go diff --git a/model/flow/identity.go b/model/flow/identity.go index 2eda6403429..6441583bc85 100644 --- a/model/flow/identity.go +++ b/model/flow/identity.go @@ -1,22 +1,17 @@ package flow import ( - "bytes" "encoding/json" "fmt" "io" - "math" "regexp" "strconv" "github.com/ethereum/go-ethereum/rlp" "github.com/fxamacker/cbor/v2" + "github.com/onflow/flow-go/crypto" "github.com/pkg/errors" "github.com/vmihailenco/msgpack" - "golang.org/x/exp/slices" - - "github.com/onflow/flow-go/crypto" - "github.com/onflow/flow-go/utils/rand" ) // DefaultInitialWeight is the default initial weight for a node identity. @@ -335,350 +330,3 @@ func (iy *Identity) EqualTo(other *Identity) bool { } return true } - -// GenericIdentity defines a constraint for generic identities. -// Golang doesn't support constraint with fields(for time being) so we have to define this interface -// with getter methods. -// Details here: https://github.com/golang/go/issues/51259. -type GenericIdentity interface { - Identity | IdentitySkeleton - GetNodeID() Identifier - GetRole() Role - GetStakingPubKey() crypto.PublicKey - GetNetworkPubKey() crypto.PublicKey - GetInitialWeight() uint64 - GetSkeleton() IdentitySkeleton -} - -func (iy IdentitySkeleton) GetNodeID() Identifier { - return iy.NodeID -} - -func (iy IdentitySkeleton) GetRole() Role { - return iy.Role -} - -func (iy IdentitySkeleton) GetStakingPubKey() crypto.PublicKey { - return iy.StakingPubKey -} - -func (iy IdentitySkeleton) GetNetworkPubKey() crypto.PublicKey { - return iy.NetworkPubKey -} - -func (iy IdentitySkeleton) GetInitialWeight() uint64 { - return iy.InitialWeight -} - -func (iy IdentitySkeleton) GetSkeleton() IdentitySkeleton { - return iy -} - -// IdentityFilter is a filter on identities. -type IdentityFilter[T GenericIdentity] func(*T) bool - -// IdentityOrder is a sort for identities. -type IdentityOrder[T GenericIdentity] func(*T, *T) bool - -// IdentityMapFunc is a modifier function for map operations for identities. -// Identities are COPIED from the source slice. -type IdentityMapFunc[T GenericIdentity] func(T) T - -// IdentitySkeletonList is a list of nodes skeletons. We use a type alias instead of defining a new type -// since go generics doesn't support implicit conversion between types. -type IdentitySkeletonList = GenericIdentityList[IdentitySkeleton] - -// IdentityList is a list of nodes. We use a type alias instead of defining a new type -// since go generics doesn't support implicit conversion between types. -type IdentityList = GenericIdentityList[Identity] - -type GenericIdentityList[T GenericIdentity] []*T - -// Filter will apply a filter to the identity list. -func (il GenericIdentityList[T]) Filter(filter IdentityFilter[T]) GenericIdentityList[T] { - var dup GenericIdentityList[T] -IDLoop: - for _, identity := range il { - if !filter(identity) { - continue IDLoop - } - dup = append(dup, identity) - } - return dup -} - -// Map returns a new identity list with the map function f applied to a copy of -// each identity. -// -// CAUTION: this relies on structure copy semantics. Map functions that modify -// an object referenced by the input Identity structure will modify identities -// in the source slice as well. -func (il GenericIdentityList[T]) Map(f IdentityMapFunc[T]) GenericIdentityList[T] { - dup := make(GenericIdentityList[T], 0, len(il)) - for _, identity := range il { - next := f(*identity) - dup = append(dup, &next) - } - return dup -} - -// Copy returns a copy of IdentityList. The resulting slice uses a different -// backing array, meaning appends and insert operations on either slice are -// guaranteed to only affect that slice. -// -// Copy should be used when modifying an existing identity list by either -// appending new elements, re-ordering, or inserting new elements in an -// existing index. -// -// CAUTION: -// All Identity fields are deep-copied, _except_ for their keys, which -// are copied by reference. -func (il GenericIdentityList[T]) Copy() GenericIdentityList[T] { - dup := make(GenericIdentityList[T], 0, len(il)) - - lenList := len(il) - - // performance tests show this is faster than 'range' - for i := 0; i < lenList; i++ { - // copy the object - next := *(il[i]) - dup = append(dup, &next) - } - return dup -} - -// Selector returns an identity filter function that selects only identities -// within this identity list. -func (il GenericIdentityList[T]) Selector() IdentityFilter[T] { - lookup := il.Lookup() - return func(identity *T) bool { - _, exists := lookup[(*identity).GetNodeID()] - return exists - } -} - -func (il GenericIdentityList[T]) Lookup() map[Identifier]*T { - lookup := make(map[Identifier]*T, len(il)) - for _, identity := range il { - lookup[(*identity).GetNodeID()] = identity - } - return lookup -} - -// Sort will sort the list using the given ordering. This is -// not recommended for performance. Expand the 'less' function -// in place for best performance, and don't use this function. -func (il GenericIdentityList[T]) Sort(less IdentityOrder[T]) GenericIdentityList[T] { - dup := il.Copy() - slices.SortFunc(dup, less) - return dup -} - -// Sorted returns whether the list is sorted by the input ordering. -func (il GenericIdentityList[T]) Sorted(less IdentityOrder[T]) bool { - return slices.IsSortedFunc(il, less) -} - -// NodeIDs returns the NodeIDs of the nodes in the list. -func (il GenericIdentityList[T]) NodeIDs() IdentifierList { - nodeIDs := make([]Identifier, 0, len(il)) - for _, id := range il { - nodeIDs = append(nodeIDs, (*id).GetNodeID()) - } - return nodeIDs -} - -// PublicStakingKeys returns a list with the public staking keys (order preserving). -func (il GenericIdentityList[T]) PublicStakingKeys() []crypto.PublicKey { - pks := make([]crypto.PublicKey, 0, len(il)) - for _, id := range il { - pks = append(pks, (*id).GetStakingPubKey()) - } - return pks -} - -// ID uniquely identifies a list of identities, by node ID. This can be used -// to perpetually identify a group of nodes, even if mutable fields of some nodes -// are changed, as node IDs are immutable. -// CAUTION: -// - An IdentityList's ID is a cryptographic commitment to only node IDs. A node operator -// can freely choose the ID for their node. There is no relationship whatsoever between -// a node's ID and keys. -// - To generate a cryptographic commitment for the full IdentityList, use method `Checksum()`. -// - The outputs of `IdentityList.ID()` and `IdentityList.Checksum()` are both order-sensitive. -// Therefore, the `IdentityList` must be in canonical order, unless explicitly specified -// otherwise by the protocol. -func (il GenericIdentityList[T]) ID() Identifier { - return il.NodeIDs().ID() -} - -// Checksum generates a cryptographic commitment to the full IdentityList, including mutable fields. -// The checksum for the same group of identities (by NodeID) may change from block to block. -func (il GenericIdentityList[T]) Checksum() Identifier { - return MakeID(il) -} - -// TotalWeight returns the total weight of all given identities. -func (il GenericIdentityList[T]) TotalWeight() uint64 { - var total uint64 - for _, identity := range il { - total += (*identity).GetInitialWeight() - } - return total -} - -// Count returns the count of identities. -func (il GenericIdentityList[T]) Count() uint { - return uint(len(il)) -} - -// ByIndex returns the node at the given index. -func (il GenericIdentityList[T]) ByIndex(index uint) (*T, bool) { - if index >= uint(len(il)) { - return nil, false - } - return il[int(index)], true -} - -// ByNodeID gets a node from the list by node ID. -func (il GenericIdentityList[T]) ByNodeID(nodeID Identifier) (*T, bool) { - for _, identity := range il { - if (*identity).GetNodeID() == nodeID { - return identity, true - } - } - return nil, false -} - -// ByNetworkingKey gets a node from the list by network public key. -func (il GenericIdentityList[T]) ByNetworkingKey(key crypto.PublicKey) (*T, bool) { - for _, identity := range il { - if (*identity).GetNetworkPubKey().Equals(key) { - return identity, true - } - } - return nil, false -} - -// Sample returns non-deterministic random sample from the `IdentityList` -func (il GenericIdentityList[T]) Sample(size uint) (GenericIdentityList[T], error) { - n := uint(len(il)) - dup := make(GenericIdentityList[T], 0, n) - dup = append(dup, il...) - if n < size { - size = n - } - swap := func(i, j uint) { - dup[i], dup[j] = dup[j], dup[i] - } - err := rand.Samples(n, size, swap) - if err != nil { - return nil, fmt.Errorf("failed to sample identity list: %w", err) - } - return dup[:size], nil -} - -// Shuffle randomly shuffles the identity list (non-deterministic), -// and returns the shuffled list without modifying the receiver. -func (il GenericIdentityList[T]) Shuffle() (GenericIdentityList[T], error) { - return il.Sample(uint(len(il))) -} - -// SamplePct returns a random sample from the receiver identity list. The -// sample contains `pct` percentage of the list. The sample is rounded up -// if `pct>0`, so this will always select at least one identity. -// -// NOTE: The input must be between 0-1. -func (il GenericIdentityList[T]) SamplePct(pct float64) (GenericIdentityList[T], error) { - if pct <= 0 { - return GenericIdentityList[T]{}, nil - } - - count := float64(il.Count()) * pct - size := uint(math.Round(count)) - // ensure we always select at least 1, for non-zero input - if size == 0 { - size = 1 - } - - return il.Sample(size) -} - -// Union returns a new identity list containing every identity that occurs in -// either `il`, or `other`, or both. There are no duplicates in the output, -// where duplicates are identities with the same node ID. -// Receiver `il` and/or method input `other` can be nil or empty. -// The returned IdentityList is sorted in canonical order. -func (il GenericIdentityList[T]) Union(other GenericIdentityList[T]) GenericIdentityList[T] { - maxLen := len(il) + len(other) - - union := make(GenericIdentityList[T], 0, maxLen) - set := make(map[Identifier]struct{}, maxLen) - - for _, list := range []GenericIdentityList[T]{il, other} { - for _, id := range list { - if _, isDuplicate := set[(*id).GetNodeID()]; !isDuplicate { - set[(*id).GetNodeID()] = struct{}{} - union = append(union, id) - } - } - } - - slices.SortFunc(union, func(a, b *T) bool { - lhs, rhs := (*a).GetNodeID(), (*b).GetNodeID() - return bytes.Compare(lhs[:], rhs[:]) < 0 - }) - - return union -} - -// EqualTo checks if the other list if the same, that it contains the same elements -// in the same order -func (il GenericIdentityList[T]) EqualTo(other GenericIdentityList[T]) bool { - // TODO: temporary - //return slices.EqualFunc(il, other, func(a, b *T) bool { - // return (*a)..EqualTo(b) - //}) - return il.ID() == other.ID() -} - -// Exists takes a previously sorted Identity list and searches it for the target value -// This code is optimized, so the coding style will be different -// target: value to search for -// CAUTION: The identity list MUST be sorted prior to calling this method -func (il GenericIdentityList[T]) Exists(target *T) bool { - return il.IdentifierExists((*target).GetNodeID()) -} - -// IdentifierExists takes a previously sorted Identity list and searches it for the target value -// target: value to search for -// CAUTION: The identity list MUST be sorted prior to calling this method -func (il GenericIdentityList[T]) IdentifierExists(target Identifier) bool { - _, ok := slices.BinarySearchFunc(il, target, func(a *T, b Identifier) int { - lhs := (*a).GetNodeID() - return bytes.Compare(lhs[:], b[:]) - }) - return ok -} - -// GetIndex returns the index of the identifier in the IdentityList and true -// if the identifier is found. -func (il GenericIdentityList[T]) GetIndex(target Identifier) (uint, bool) { - i := slices.IndexFunc(il, func(a *T) bool { - return (*a).GetNodeID() == target - }) - if i == -1 { - return 0, false - } - return uint(i), true -} - -// ToSkeleton converts the identity list to a list of identity skeletons. -func (il GenericIdentityList[T]) ToSkeleton() IdentitySkeletonList { - skeletons := make(IdentitySkeletonList, len(il)) - for i, id := range il { - v := (*id).GetSkeleton() - skeletons[i] = &v - } - return skeletons -} diff --git a/model/flow/identity_list.go b/model/flow/identity_list.go new file mode 100644 index 00000000000..a4ad2efacd4 --- /dev/null +++ b/model/flow/identity_list.go @@ -0,0 +1,370 @@ +package flow + +import ( + "bytes" + "fmt" + "math" + + "golang.org/x/exp/slices" + + "github.com/onflow/flow-go/crypto" + "github.com/onflow/flow-go/utils/rand" +) + +// Notes on using generic types: +// DO NOT pass an interface to a generic function (100x runtime cost as of go 1.20). +// For example, consider the function +// +// func f[T GenericIdentity]() +// +// The call `f(identity)` is completely ok and doesn't introduce overhead when `identity` is a struct type, +// such as `var identity *flow.Identity`. +// In contrast `f(identity)` where identity is declared as an interface `var identity GenericIdentity` is drastically slower, +// since golang involves a global hash table lookup for every method call to dispatch the underlying type behind the interface. + +// GenericIdentity defines a constraint for generic identities. +// Golang doesn't support constraint with fields(for time being) so we have to define this interface +// with getter methods. +// Details here: https://github.com/golang/go/issues/51259. +type GenericIdentity interface { + Identity | IdentitySkeleton + GetNodeID() Identifier + GetRole() Role + GetStakingPubKey() crypto.PublicKey + GetNetworkPubKey() crypto.PublicKey + GetInitialWeight() uint64 + GetSkeleton() IdentitySkeleton +} + +func (iy IdentitySkeleton) GetNodeID() Identifier { + return iy.NodeID +} + +func (iy IdentitySkeleton) GetRole() Role { + return iy.Role +} + +func (iy IdentitySkeleton) GetStakingPubKey() crypto.PublicKey { + return iy.StakingPubKey +} + +func (iy IdentitySkeleton) GetNetworkPubKey() crypto.PublicKey { + return iy.NetworkPubKey +} + +func (iy IdentitySkeleton) GetInitialWeight() uint64 { + return iy.InitialWeight +} + +func (iy IdentitySkeleton) GetSkeleton() IdentitySkeleton { + return iy +} + +// IdentityFilter is a filter on identities. +type IdentityFilter[T GenericIdentity] func(*T) bool + +// IdentityOrder is a sort for identities. +type IdentityOrder[T GenericIdentity] func(*T, *T) bool + +// IdentityMapFunc is a modifier function for map operations for identities. +// Identities are COPIED from the source slice. +type IdentityMapFunc[T GenericIdentity] func(T) T + +// IdentitySkeletonList is a list of nodes skeletons. We use a type alias instead of defining a new type +// since go generics doesn't support implicit conversion between types. +type IdentitySkeletonList = GenericIdentityList[IdentitySkeleton] + +// IdentityList is a list of nodes. We use a type alias instead of defining a new type +// since go generics doesn't support implicit conversion between types. +type IdentityList = GenericIdentityList[Identity] + +type GenericIdentityList[T GenericIdentity] []*T + +// Filter will apply a filter to the identity list. +func (il GenericIdentityList[T]) Filter(filter IdentityFilter[T]) GenericIdentityList[T] { + var dup GenericIdentityList[T] +IDLoop: + for _, identity := range il { + if !filter(identity) { + continue IDLoop + } + dup = append(dup, identity) + } + return dup +} + +// Map returns a new identity list with the map function f applied to a copy of +// each identity. +// +// CAUTION: this relies on structure copy semantics. Map functions that modify +// an object referenced by the input Identity structure will modify identities +// in the source slice as well. +func (il GenericIdentityList[T]) Map(f IdentityMapFunc[T]) GenericIdentityList[T] { + dup := make(GenericIdentityList[T], 0, len(il)) + for _, identity := range il { + next := f(*identity) + dup = append(dup, &next) + } + return dup +} + +// Copy returns a copy of IdentityList. The resulting slice uses a different +// backing array, meaning appends and insert operations on either slice are +// guaranteed to only affect that slice. +// +// Copy should be used when modifying an existing identity list by either +// appending new elements, re-ordering, or inserting new elements in an +// existing index. +// +// CAUTION: +// All Identity fields are deep-copied, _except_ for their keys, which +// are copied by reference. +func (il GenericIdentityList[T]) Copy() GenericIdentityList[T] { + dup := make(GenericIdentityList[T], 0, len(il)) + + lenList := len(il) + + // performance tests show this is faster than 'range' + for i := 0; i < lenList; i++ { + // copy the object + next := *(il[i]) + dup = append(dup, &next) + } + return dup +} + +// Selector returns an identity filter function that selects only identities +// within this identity list. +func (il GenericIdentityList[T]) Selector() IdentityFilter[T] { + lookup := il.Lookup() + return func(identity *T) bool { + _, exists := lookup[(*identity).GetNodeID()] + return exists + } +} + +func (il GenericIdentityList[T]) Lookup() map[Identifier]*T { + lookup := make(map[Identifier]*T, len(il)) + for _, identity := range il { + lookup[(*identity).GetNodeID()] = identity + } + return lookup +} + +// Sort will sort the list using the given ordering. This is +// not recommended for performance. Expand the 'less' function +// in place for best performance, and don't use this function. +func (il GenericIdentityList[T]) Sort(less IdentityOrder[T]) GenericIdentityList[T] { + dup := il.Copy() + slices.SortFunc(dup, less) + return dup +} + +// Sorted returns whether the list is sorted by the input ordering. +func (il GenericIdentityList[T]) Sorted(less IdentityOrder[T]) bool { + return slices.IsSortedFunc(il, less) +} + +// NodeIDs returns the NodeIDs of the nodes in the list. +func (il GenericIdentityList[T]) NodeIDs() IdentifierList { + nodeIDs := make([]Identifier, 0, len(il)) + for _, id := range il { + nodeIDs = append(nodeIDs, (*id).GetNodeID()) + } + return nodeIDs +} + +// PublicStakingKeys returns a list with the public staking keys (order preserving). +func (il GenericIdentityList[T]) PublicStakingKeys() []crypto.PublicKey { + pks := make([]crypto.PublicKey, 0, len(il)) + for _, id := range il { + pks = append(pks, (*id).GetStakingPubKey()) + } + return pks +} + +// ID uniquely identifies a list of identities, by node ID. This can be used +// to perpetually identify a group of nodes, even if mutable fields of some nodes +// are changed, as node IDs are immutable. +// CAUTION: +// - An IdentityList's ID is a cryptographic commitment to only node IDs. A node operator +// can freely choose the ID for their node. There is no relationship whatsoever between +// a node's ID and keys. +// - To generate a cryptographic commitment for the full IdentityList, use method `Checksum()`. +// - The outputs of `IdentityList.ID()` and `IdentityList.Checksum()` are both order-sensitive. +// Therefore, the `IdentityList` must be in canonical order, unless explicitly specified +// otherwise by the protocol. +func (il GenericIdentityList[T]) ID() Identifier { + return il.NodeIDs().ID() +} + +// Checksum generates a cryptographic commitment to the full IdentityList, including mutable fields. +// The checksum for the same group of identities (by NodeID) may change from block to block. +func (il GenericIdentityList[T]) Checksum() Identifier { + return MakeID(il) +} + +// TotalWeight returns the total weight of all given identities. +func (il GenericIdentityList[T]) TotalWeight() uint64 { + var total uint64 + for _, identity := range il { + total += (*identity).GetInitialWeight() + } + return total +} + +// Count returns the count of identities. +func (il GenericIdentityList[T]) Count() uint { + return uint(len(il)) +} + +// ByIndex returns the node at the given index. +func (il GenericIdentityList[T]) ByIndex(index uint) (*T, bool) { + if index >= uint(len(il)) { + return nil, false + } + return il[int(index)], true +} + +// ByNodeID gets a node from the list by node ID. +func (il GenericIdentityList[T]) ByNodeID(nodeID Identifier) (*T, bool) { + for _, identity := range il { + if (*identity).GetNodeID() == nodeID { + return identity, true + } + } + return nil, false +} + +// ByNetworkingKey gets a node from the list by network public key. +func (il GenericIdentityList[T]) ByNetworkingKey(key crypto.PublicKey) (*T, bool) { + for _, identity := range il { + if (*identity).GetNetworkPubKey().Equals(key) { + return identity, true + } + } + return nil, false +} + +// Sample returns non-deterministic random sample from the `IdentityList` +func (il GenericIdentityList[T]) Sample(size uint) (GenericIdentityList[T], error) { + n := uint(len(il)) + dup := make(GenericIdentityList[T], 0, n) + dup = append(dup, il...) + if n < size { + size = n + } + swap := func(i, j uint) { + dup[i], dup[j] = dup[j], dup[i] + } + err := rand.Samples(n, size, swap) + if err != nil { + return nil, fmt.Errorf("failed to sample identity list: %w", err) + } + return dup[:size], nil +} + +// Shuffle randomly shuffles the identity list (non-deterministic), +// and returns the shuffled list without modifying the receiver. +func (il GenericIdentityList[T]) Shuffle() (GenericIdentityList[T], error) { + return il.Sample(uint(len(il))) +} + +// SamplePct returns a random sample from the receiver identity list. The +// sample contains `pct` percentage of the list. The sample is rounded up +// if `pct>0`, so this will always select at least one identity. +// +// NOTE: The input must be between 0-1. +func (il GenericIdentityList[T]) SamplePct(pct float64) (GenericIdentityList[T], error) { + if pct <= 0 { + return GenericIdentityList[T]{}, nil + } + + count := float64(il.Count()) * pct + size := uint(math.Round(count)) + // ensure we always select at least 1, for non-zero input + if size == 0 { + size = 1 + } + + return il.Sample(size) +} + +// Union returns a new identity list containing every identity that occurs in +// either `il`, or `other`, or both. There are no duplicates in the output, +// where duplicates are identities with the same node ID. +// Receiver `il` and/or method input `other` can be nil or empty. +// The returned IdentityList is sorted in canonical order. +func (il GenericIdentityList[T]) Union(other GenericIdentityList[T]) GenericIdentityList[T] { + maxLen := len(il) + len(other) + + union := make(GenericIdentityList[T], 0, maxLen) + set := make(map[Identifier]struct{}, maxLen) + + for _, list := range []GenericIdentityList[T]{il, other} { + for _, id := range list { + if _, isDuplicate := set[(*id).GetNodeID()]; !isDuplicate { + set[(*id).GetNodeID()] = struct{}{} + union = append(union, id) + } + } + } + + slices.SortFunc(union, func(a, b *T) bool { + lhs, rhs := (*a).GetNodeID(), (*b).GetNodeID() + return bytes.Compare(lhs[:], rhs[:]) < 0 + }) + + return union +} + +// EqualTo checks if the other list if the same, that it contains the same elements +// in the same order +func (il GenericIdentityList[T]) EqualTo(other GenericIdentityList[T]) bool { + // TODO: temporary + //return slices.EqualFunc(il, other, func(a, b *T) bool { + // return (*a)..EqualTo(b) + //}) + return il.ID() == other.ID() +} + +// Exists takes a previously sorted Identity list and searches it for the target value +// This code is optimized, so the coding style will be different +// target: value to search for +// CAUTION: The identity list MUST be sorted prior to calling this method +func (il GenericIdentityList[T]) Exists(target *T) bool { + return il.IdentifierExists((*target).GetNodeID()) +} + +// IdentifierExists takes a previously sorted Identity list and searches it for the target value +// target: value to search for +// CAUTION: The identity list MUST be sorted prior to calling this method +func (il GenericIdentityList[T]) IdentifierExists(target Identifier) bool { + _, ok := slices.BinarySearchFunc(il, target, func(a *T, b Identifier) int { + lhs := (*a).GetNodeID() + return bytes.Compare(lhs[:], b[:]) + }) + return ok +} + +// GetIndex returns the index of the identifier in the IdentityList and true +// if the identifier is found. +func (il GenericIdentityList[T]) GetIndex(target Identifier) (uint, bool) { + i := slices.IndexFunc(il, func(a *T) bool { + return (*a).GetNodeID() == target + }) + if i == -1 { + return 0, false + } + return uint(i), true +} + +// ToSkeleton converts the identity list to a list of identity skeletons. +func (il GenericIdentityList[T]) ToSkeleton() IdentitySkeletonList { + skeletons := make(IdentitySkeletonList, len(il)) + for i, id := range il { + v := (*id).GetSkeleton() + skeletons[i] = &v + } + return skeletons +} From 420244fcd5cc8b74d19c56288481f745e5bdf8e1 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Mon, 2 Oct 2023 22:32:12 +0300 Subject: [PATCH 12/39] Added proper encoding for identity and identity skeleton --- model/flow/identity.go | 150 +++++++++++++++++++++++++----------- model/flow/identity_test.go | 13 ---- 2 files changed, 107 insertions(+), 56 deletions(-) diff --git a/model/flow/identity.go b/model/flow/identity.go index 6441583bc85..a368392f031 100644 --- a/model/flow/identity.go +++ b/model/flow/identity.go @@ -125,34 +125,27 @@ func (iy Identity) Checksum() Identifier { return MakeID(iy) } -type encodableIdentity struct { +type encodableIdentitySkeleton struct { NodeID Identifier Address string `json:",omitempty"` Role Role InitialWeight uint64 - Weight uint64 - Ejected bool StakingPubKey []byte NetworkPubKey []byte } -// decodableIdentity provides backward-compatible decoding of old models -// which use the Stake field in place of Weight. -type decodableIdentity struct { - encodableIdentity - // Stake previously was used in place of the Weight field. - // Deprecated: supported in decoding for backward-compatibility - Stake uint64 +type encodableIdentity struct { + encodableIdentitySkeleton + Weight uint64 + Ejected bool } -func encodableFromIdentity(iy Identity) (encodableIdentity, error) { - ie := encodableIdentity{ +func encodableSkeletonFromIdentity(iy IdentitySkeleton) encodableIdentitySkeleton { + ie := encodableIdentitySkeleton{ NodeID: iy.NodeID, Address: iy.Address, Role: iy.Role, InitialWeight: iy.InitialWeight, - Weight: iy.Weight, - Ejected: iy.Ejected, } if iy.StakingPubKey != nil { ie.StakingPubKey = iy.StakingPubKey.Encode() @@ -160,15 +153,19 @@ func encodableFromIdentity(iy Identity) (encodableIdentity, error) { if iy.NetworkPubKey != nil { ie.NetworkPubKey = iy.NetworkPubKey.Encode() } - return ie, nil + return ie } -func (iy Identity) MarshalJSON() ([]byte, error) { - encodable, err := encodableFromIdentity(iy) - if err != nil { - return nil, fmt.Errorf("could not convert identity to encodable: %w", err) +func encodableFromIdentity(iy Identity) encodableIdentity { + return encodableIdentity{ + encodableIdentitySkeleton: encodableSkeletonFromIdentity(iy.IdentitySkeleton), + Weight: iy.Weight, + Ejected: iy.Ejected, } +} +func (iy IdentitySkeleton) MarshalJSON() ([]byte, error) { + encodable := encodableSkeletonFromIdentity(iy) data, err := json.Marshal(encodable) if err != nil { return nil, fmt.Errorf("could not encode json: %w", err) @@ -176,11 +173,44 @@ func (iy Identity) MarshalJSON() ([]byte, error) { return data, nil } -func (iy Identity) MarshalCBOR() ([]byte, error) { - encodable, err := encodableFromIdentity(iy) +func (iy IdentitySkeleton) MarshalCBOR() ([]byte, error) { + encodable := encodableSkeletonFromIdentity(iy) + data, err := cbor.Marshal(encodable) if err != nil { - return nil, fmt.Errorf("could not convert identity to encodable: %w", err) + return nil, fmt.Errorf("could not encode cbor: %w", err) } + return data, nil +} + +func (iy IdentitySkeleton) MarshalMsgpack() ([]byte, error) { + encodable := encodableSkeletonFromIdentity(iy) + data, err := msgpack.Marshal(encodable) + if err != nil { + return nil, fmt.Errorf("could not encode msgpack: %w", err) + } + return data, nil +} + +func (iy IdentitySkeleton) EncodeRLP(w io.Writer) error { + encodable := encodableSkeletonFromIdentity(iy) + err := rlp.Encode(w, encodable) + if err != nil { + return fmt.Errorf("could not encode rlp: %w", err) + } + return nil +} + +func (iy Identity) MarshalJSON() ([]byte, error) { + encodable := encodableFromIdentity(iy) + data, err := json.Marshal(encodable) + if err != nil { + return nil, fmt.Errorf("could not encode json: %w", err) + } + return data, nil +} + +func (iy Identity) MarshalCBOR() ([]byte, error) { + encodable := encodableFromIdentity(iy) data, err := cbor.Marshal(encodable) if err != nil { return nil, fmt.Errorf("could not encode cbor: %w", err) @@ -189,10 +219,7 @@ func (iy Identity) MarshalCBOR() ([]byte, error) { } func (iy Identity) MarshalMsgpack() ([]byte, error) { - encodable, err := encodableFromIdentity(iy) - if err != nil { - return nil, fmt.Errorf("could not convert to encodable: %w", err) - } + encodable := encodableFromIdentity(iy) data, err := msgpack.Marshal(encodable) if err != nil { return nil, fmt.Errorf("could not encode msgpack: %w", err) @@ -201,24 +228,19 @@ func (iy Identity) MarshalMsgpack() ([]byte, error) { } func (iy Identity) EncodeRLP(w io.Writer) error { - encodable, err := encodableFromIdentity(iy) - if err != nil { - return fmt.Errorf("could not convert to encodable: %w", err) - } - err = rlp.Encode(w, encodable) + encodable := encodableFromIdentity(iy) + err := rlp.Encode(w, encodable) if err != nil { return fmt.Errorf("could not encode rlp: %w", err) } return nil } -func identityFromEncodable(ie encodableIdentity, identity *Identity) error { +func identitySkeletonFromEncodable(ie encodableIdentitySkeleton, identity *IdentitySkeleton) error { identity.NodeID = ie.NodeID identity.Address = ie.Address identity.Role = ie.Role identity.InitialWeight = ie.InitialWeight - identity.Weight = ie.Weight - identity.Ejected = ie.Ejected var err error if ie.StakingPubKey != nil { if identity.StakingPubKey, err = crypto.DecodePublicKey(crypto.BLSBLS12381, ie.StakingPubKey); err != nil { @@ -233,20 +255,62 @@ func identityFromEncodable(ie encodableIdentity, identity *Identity) error { return nil } -func (iy *Identity) UnmarshalJSON(b []byte) error { - var decodable decodableIdentity +func identityFromEncodable(ie encodableIdentity, identity *Identity) error { + err := identitySkeletonFromEncodable(ie.encodableIdentitySkeleton, &identity.IdentitySkeleton) + if err != nil { + return fmt.Errorf("could not decode identity skeleton: %w", err) + } + identity.Weight = ie.Weight + identity.Ejected = ie.Ejected + return nil +} + +func (iy *IdentitySkeleton) UnmarshalJSON(b []byte) error { + var decodable encodableIdentitySkeleton err := json.Unmarshal(b, &decodable) if err != nil { return fmt.Errorf("could not decode json: %w", err) } - // compat: translate Stake fields to Weight - if decodable.Stake != 0 { - if decodable.Weight != 0 { - return fmt.Errorf("invalid identity with both Stake and Weight fields") - } - decodable.Weight = decodable.Stake + err = identitySkeletonFromEncodable(decodable, iy) + if err != nil { + return fmt.Errorf("could not convert from encodable json: %w", err) + } + return nil +} + +func (iy *IdentitySkeleton) UnmarshalCBOR(b []byte) error { + var encodable encodableIdentitySkeleton + err := cbor.Unmarshal(b, &encodable) + if err != nil { + return fmt.Errorf("could not decode json: %w", err) + } + err = identitySkeletonFromEncodable(encodable, iy) + if err != nil { + return fmt.Errorf("could not convert from encodable cbor: %w", err) + } + return nil +} + +func (iy *IdentitySkeleton) UnmarshalMsgpack(b []byte) error { + var encodable encodableIdentitySkeleton + err := msgpack.Unmarshal(b, &encodable) + if err != nil { + return fmt.Errorf("could not decode json: %w", err) + } + err = identitySkeletonFromEncodable(encodable, iy) + if err != nil { + return fmt.Errorf("could not convert from encodable msgpack: %w", err) + } + return nil +} + +func (iy *Identity) UnmarshalJSON(b []byte) error { + var decodable encodableIdentity + err := json.Unmarshal(b, &decodable) + if err != nil { + return fmt.Errorf("could not decode json: %w", err) } - err = identityFromEncodable(decodable.encodableIdentity, iy) + err = identityFromEncodable(decodable, iy) if err != nil { return fmt.Errorf("could not convert from encodable json: %w", err) } diff --git a/model/flow/identity_test.go b/model/flow/identity_test.go index e89c949e3d1..fc0b0333fe6 100644 --- a/model/flow/identity_test.go +++ b/model/flow/identity_test.go @@ -73,19 +73,6 @@ func TestIdentityEncodingJSON(t *testing.T) { require.NoError(t, err) require.Equal(t, identity, &dec) }) - - t.Run("compat: should accept old files using Stake field", func(t *testing.T) { - identity := unittest.IdentityFixture(unittest.WithRandomPublicKeys()) - enc, err := json.Marshal(identity) - require.NoError(t, err) - // emulate the old encoding by replacing the new field with old field name - // NOTE: use replace in such way to avoid replacing InitialWeight field. - enc = []byte(strings.Replace(string(enc), "\"Weight", "\"Stake", 1)) - var dec flow.Identity - err = json.Unmarshal(enc, &dec) - require.NoError(t, err) - require.Equal(t, identity, &dec) - }) } func TestIdentityEncodingMsgpack(t *testing.T) { From a9d7587f9442d34e5deebd61d4b11def9e43deb4 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Tue, 3 Oct 2023 13:52:13 +0300 Subject: [PATCH 13/39] Fixed compilation issues regarding generic identity in main codebase --- .../node_builder/access_node_builder.go | 6 ++-- cmd/collection/main.go | 2 +- cmd/consensus/main.go | 2 +- cmd/execution_builder.go | 2 +- cmd/scaffold.go | 4 +-- cmd/util/cmd/common/flow_client.go | 4 +-- .../signature/block_signer_decoder_test.go | 2 +- engine/access/ingestion/engine.go | 2 +- engine/access/ping/engine.go | 2 +- engine/access/rpc/backend/backend.go | 28 ++++++++++--------- engine/access/rpc/backend/backend_accounts.go | 6 ++-- engine/access/rpc/backend/backend_events.go | 12 ++++---- engine/access/rpc/backend/backend_scripts.go | 4 +-- .../rpc/backend/backend_transactions.go | 22 +++++++-------- .../access/rpc/backend/node_communicator.go | 12 ++++---- engine/access/rpc/backend/node_selector.go | 10 +++---- engine/collection/epochmgr/factories/sync.go | 2 +- engine/collection/ingest/engine.go | 4 +-- engine/collection/message_hub/message_hub.go | 6 ++-- engine/collection/pusher/engine.go | 2 +- engine/collection/synchronization/engine.go | 6 ++-- engine/common/provider/engine.go | 8 +++--- engine/common/requester/engine.go | 16 +++++------ engine/common/requester/item.go | 12 ++++---- .../verifying_assignment_collector.go | 2 +- engine/consensus/compliance/core_test.go | 2 +- engine/consensus/dkg/reactor_engine.go | 2 +- engine/consensus/message_hub/message_hub.go | 10 +++---- engine/execution/ingestion/engine.go | 4 +-- engine/execution/provider/engine.go | 2 +- engine/verification/fetcher/engine.go | 2 +- engine/verification/verifier/engine.go | 2 +- model/bootstrap/node_info_test.go | 5 ++-- model/convert/service_event.go | 19 +++++-------- model/flow/filter/identity.go | 4 +-- module/chunks/chunk_assigner.go | 2 +- module/dkg.go | 2 +- module/dkg/broker.go | 4 +-- module/dkg/controller_factory.go | 2 +- module/id/filtered_provider.go | 4 +-- module/id/fixed_provider.go | 2 +- module/local/me.go | 4 +-- module/local/me_nokey.go | 4 +-- module/mocks/network.go | 0 .../execution_data_requester_test.go | 2 +- network/p2p/cache/node_blocklist_wrapper.go | 2 +- network/p2p/cache/protocol_state_provider.go | 2 +- .../identity_provider_translator.go | 2 +- 48 files changed, 130 insertions(+), 132 deletions(-) delete mode 100644 module/mocks/network.go diff --git a/cmd/access/node_builder/access_node_builder.go b/cmd/access/node_builder/access_node_builder.go index d5b0e688cd4..26b75adf61c 100644 --- a/cmd/access/node_builder/access_node_builder.go +++ b/cmd/access/node_builder/access_node_builder.go @@ -838,8 +838,8 @@ func (builder *FlowAccessNodeBuilder) InitIDProviders() { builder.SyncEngineParticipantsProviderFactory = func() module.IdentifierProvider { return id.NewIdentityFilterIdentifierProvider( filter.And( - filter.HasRole(flow.RoleConsensus), - filter.Not(filter.HasNodeID(node.Me.NodeID())), + filter.HasRole[flow.Identity](flow.RoleConsensus), + filter.Not(filter.HasNodeID[flow.Identity](node.Me.NodeID())), p2pnet.NotEjectedFilter, ), builder.IdentityProvider, @@ -1129,7 +1129,7 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { node.Me, node.State, channels.RequestCollections, - filter.HasRole(flow.RoleCollection), + filter.HasRole[flow.Identity](flow.RoleCollection), func() flow.Entity { return &flow.Collection{} }, ) if err != nil { diff --git a/cmd/collection/main.go b/cmd/collection/main.go index f285911bfdd..210045b6d38 100644 --- a/cmd/collection/main.go +++ b/cmd/collection/main.go @@ -424,7 +424,7 @@ func main() { channels.ProvideCollections, filter.And( filter.HasWeight(true), - filter.HasRole(flow.RoleAccess, flow.RoleExecution), + filter.HasRole[flow.Identity](flow.RoleAccess, flow.RoleExecution), ), retrieve, ) diff --git a/cmd/consensus/main.go b/cmd/consensus/main.go index 2090311b1ae..b435c005192 100644 --- a/cmd/consensus/main.go +++ b/cmd/consensus/main.go @@ -484,7 +484,7 @@ func main() { node.Me, node.State, channels.RequestReceiptsByBlockID, - filter.HasRole(flow.RoleExecution), + filter.HasRole[flow.Identity](flow.RoleExecution), func() flow.Entity { return &flow.ExecutionReceipt{} }, requester.WithRetryInitial(2*time.Second), requester.WithRetryMaximum(30*time.Second), diff --git a/cmd/execution_builder.go b/cmd/execution_builder.go index db90b68aa18..a7cccefb5b0 100644 --- a/cmd/execution_builder.go +++ b/cmd/execution_builder.go @@ -1056,7 +1056,7 @@ func (exeNode *ExecutionNode) LoadReceiptProviderEngine( channels.ProvideReceiptsByBlockID, filter.And( filter.HasWeight(true), - filter.HasRole(flow.RoleConsensus), + filter.HasRole[flow.Identity](flow.RoleConsensus), ), retrieve, ) diff --git a/cmd/scaffold.go b/cmd/scaffold.go index 1ff4e38a7cf..23706fe6f57 100644 --- a/cmd/scaffold.go +++ b/cmd/scaffold.go @@ -1021,8 +1021,8 @@ func (fnb *FlowNodeBuilder) InitIDProviders() { node.SyncEngineIdentifierProvider = id.NewIdentityFilterIdentifierProvider( filter.And( - filter.HasRole(flow.RoleConsensus), - filter.Not(filter.HasNodeID(node.Me.NodeID())), + filter.HasRole[flow.Identity](flow.RoleConsensus), + filter.Not(filter.HasNodeID[flow.Identity](node.Me.NodeID())), p2pnet.NotEjectedFilter, ), node.IdentityProvider, diff --git a/cmd/util/cmd/common/flow_client.go b/cmd/util/cmd/common/flow_client.go index e16438da9f6..5056b107a1f 100644 --- a/cmd/util/cmd/common/flow_client.go +++ b/cmd/util/cmd/common/flow_client.go @@ -93,7 +93,7 @@ func insecureFlowClient(accessAddress string) (*client.Client, error) { func FlowClientConfigs(accessNodeIDS []flow.Identifier, insecureAccessAPI bool, snapshot protocol.Snapshot) ([]*FlowClientConfig, error) { flowClientOpts := make([]*FlowClientConfig, 0) - identities, err := snapshot.Identities(filter.HasNodeID(accessNodeIDS...)) + identities, err := snapshot.Identities(filter.HasNodeID[flow.Identity](accessNodeIDS...)) if err != nil { return nil, fmt.Errorf("failed get identities access node identities (ids=%v) from snapshot: %w", accessNodeIDS, err) } @@ -139,7 +139,7 @@ func convertAccessAddrFromState(address string, insecureAccessAPI bool) string { // DefaultAccessNodeIDS will return all the access node IDS in the protocol state for staked access nodes func DefaultAccessNodeIDS(snapshot protocol.Snapshot) ([]flow.Identifier, error) { - identities, err := snapshot.Identities(filter.HasRole(flow.RoleAccess)) + identities, err := snapshot.Identities(filter.HasRole[flow.Identity](flow.RoleAccess)) if err != nil { return nil, fmt.Errorf("failed to get staked access node IDs from protocol state %w", err) } diff --git a/consensus/hotstuff/signature/block_signer_decoder_test.go b/consensus/hotstuff/signature/block_signer_decoder_test.go index f1bf829116b..104197cc0ce 100644 --- a/consensus/hotstuff/signature/block_signer_decoder_test.go +++ b/consensus/hotstuff/signature/block_signer_decoder_test.go @@ -32,7 +32,7 @@ type blockSignerDecoderSuite struct { func (s *blockSignerDecoderSuite) SetupTest() { // the default header fixture creates signerIDs for a committee of 10 nodes, so we prepare a committee same as that - s.allConsensus = unittest.IdentityListFixture(40, unittest.WithRole(flow.RoleConsensus)).Sort(order.Canonical) + s.allConsensus = unittest.IdentityListFixture(40, unittest.WithRole(flow.RoleConsensus)).Sort(order.Canonical[flow.Identity]) // mock consensus committee s.committee = hotstuff.NewDynamicCommittee(s.T()) diff --git a/engine/access/ingestion/engine.go b/engine/access/ingestion/engine.go index 54369ccebdb..6fd8ad724c9 100644 --- a/engine/access/ingestion/engine.go +++ b/engine/access/ingestion/engine.go @@ -814,6 +814,6 @@ func (e *Engine) requestCollectionsInFinalizedBlock(missingColls []*flow.Collect // failed to find guarantors for guarantees contained in a finalized block is fatal error e.log.Fatal().Err(err).Msgf("could not find guarantors for guarantee %v", cg.ID()) } - e.request.EntityByID(cg.ID(), filter.HasNodeID(guarantors...)) + e.request.EntityByID(cg.ID(), filter.HasNodeID[flow.Identity](guarantors...)) } } diff --git a/engine/access/ping/engine.go b/engine/access/ping/engine.go index e85128fccdb..898efad3cc4 100644 --- a/engine/access/ping/engine.go +++ b/engine/access/ping/engine.go @@ -93,7 +93,7 @@ func (e *Engine) Done() <-chan struct{} { func (e *Engine) startPing() { e.unit.LaunchPeriodically(func() { - peers := e.idProvider.Identities(filter.Not(filter.HasNodeID(e.me.NodeID()))) + peers := e.idProvider.Identities(filter.Not(filter.HasNodeID[flow.Identity](e.me.NodeID()))) // for each peer, send a ping every ping interval for _, peer := range peers { diff --git a/engine/access/rpc/backend/backend.go b/engine/access/rpc/backend/backend.go index 1e30b6d3ce2..ea84d5a651a 100644 --- a/engine/access/rpc/backend/backend.go +++ b/engine/access/rpc/backend/backend.go @@ -356,7 +356,7 @@ func executionNodesForBlockID( blockID flow.Identifier, executionReceipts storage.ExecutionReceipts, state protocol.State, - log zerolog.Logger) (flow.IdentityList, error) { + log zerolog.Logger) (flow.IdentitySkeletonList, error) { var executorIDs flow.IdentifierList @@ -368,7 +368,7 @@ func executionNodesForBlockID( } if rootBlock.ID() == blockID { - executorIdentities, err := state.Final().Identities(filter.HasRole(flow.RoleExecution)) + executorIdentities, err := state.Final().Identities(filter.HasRole[flow.Identity](flow.RoleExecution)) if err != nil { return nil, fmt.Errorf("failed to retreive execution IDs for block ID %v: %w", blockID, err) } @@ -405,7 +405,7 @@ func executionNodesForBlockID( receiptCnt := len(executorIDs) // if less than minExecutionNodesCnt execution receipts have been received so far, then return random ENs if receiptCnt < minExecutionNodesCnt { - newExecutorIDs, err := state.AtBlockID(blockID).Identities(filter.HasRole(flow.RoleExecution)) + newExecutorIDs, err := state.AtBlockID(blockID).Identities(filter.HasRole[flow.Identity](flow.RoleExecution)) if err != nil { return nil, fmt.Errorf("failed to retreive execution IDs for block ID %v: %w", blockID, err) } @@ -488,9 +488,9 @@ func findAllExecutionNodes( // If neither preferred nor fixed nodes are defined, then all execution node matching the executor IDs are returned. // e.g. If execution nodes in identity table are {1,2,3,4}, preferred ENs are defined as {2,3,4} // and the executor IDs is {1,2,3}, then {2, 3} is returned as the chosen subset of ENs -func chooseExecutionNodes(state protocol.State, executorIDs flow.IdentifierList) (flow.IdentityList, error) { +func chooseExecutionNodes(state protocol.State, executorIDs flow.IdentifierList) (flow.IdentitySkeletonList, error) { - allENs, err := state.Final().Identities(filter.HasRole(flow.RoleExecution)) + allENs, err := state.Final().Identities(filter.HasRole[flow.Identity](flow.RoleExecution)) if err != nil { return nil, fmt.Errorf("failed to retreive all execution IDs: %w", err) } @@ -499,27 +499,29 @@ func chooseExecutionNodes(state protocol.State, executorIDs flow.IdentifierList) var chosenIDs flow.IdentityList if len(preferredENIdentifiers) > 0 { // find the preferred execution node IDs which have executed the transaction - chosenIDs = allENs.Filter(filter.And(filter.HasNodeID(preferredENIdentifiers...), - filter.HasNodeID(executorIDs...))) + chosenIDs = allENs.Filter(filter.And(filter.HasNodeID[flow.Identity](preferredENIdentifiers...), + filter.HasNodeID[flow.Identity](executorIDs...))) if len(chosenIDs) > 0 { - return chosenIDs, nil + return chosenIDs.ToSkeleton(), nil } } // if no preferred EN ID is found, then choose from the fixed EN IDs if len(fixedENIdentifiers) > 0 { // choose fixed ENs which have executed the transaction - chosenIDs = allENs.Filter(filter.And(filter.HasNodeID(fixedENIdentifiers...), filter.HasNodeID(executorIDs...))) + chosenIDs = allENs.Filter(filter.And( + filter.HasNodeID[flow.Identity](fixedENIdentifiers...), + filter.HasNodeID[flow.Identity](executorIDs...))) if len(chosenIDs) > 0 { - return chosenIDs, nil + return chosenIDs.ToSkeleton(), nil } // if no such ENs are found then just choose all fixed ENs - chosenIDs = allENs.Filter(filter.HasNodeID(fixedENIdentifiers...)) - return chosenIDs, nil + chosenIDs = allENs.Filter(filter.HasNodeID[flow.Identity](fixedENIdentifiers...)) + return chosenIDs.ToSkeleton(), nil } // If no preferred or fixed ENs have been specified, then return all executor IDs i.e. no preference at all - return allENs.Filter(filter.HasNodeID(executorIDs...)), nil + return allENs.Filter(filter.HasNodeID[flow.Identity](executorIDs...)).ToSkeleton(), nil } // Find ports from supplied Address diff --git a/engine/access/rpc/backend/backend_accounts.go b/engine/access/rpc/backend/backend_accounts.go index 289cef941e5..864e8c159b5 100644 --- a/engine/access/rpc/backend/backend_accounts.go +++ b/engine/access/rpc/backend/backend_accounts.go @@ -108,11 +108,11 @@ func (b *backendAccounts) getAccountAtBlockID( // We attempt querying each EN in sequence. If any EN returns a valid response, then errors from // other ENs are logged and swallowed. If all ENs fail to return a valid response, then an // error aggregating all failures is returned. -func (b *backendAccounts) getAccountFromAnyExeNode(ctx context.Context, execNodes flow.IdentityList, req *execproto.GetAccountAtBlockIDRequest) (*execproto.GetAccountAtBlockIDResponse, error) { +func (b *backendAccounts) getAccountFromAnyExeNode(ctx context.Context, execNodes flow.IdentitySkeletonList, req *execproto.GetAccountAtBlockIDRequest) (*execproto.GetAccountAtBlockIDResponse, error) { var resp *execproto.GetAccountAtBlockIDResponse errToReturn := b.nodeCommunicator.CallAvailableNode( execNodes, - func(node *flow.Identity) error { + func(node *flow.IdentitySkeleton) error { var err error start := time.Now() @@ -146,7 +146,7 @@ func (b *backendAccounts) getAccountFromAnyExeNode(ctx context.Context, execNode return resp, errToReturn } -func (b *backendAccounts) tryGetAccount(ctx context.Context, execNode *flow.Identity, req *execproto.GetAccountAtBlockIDRequest) (*execproto.GetAccountAtBlockIDResponse, error) { +func (b *backendAccounts) tryGetAccount(ctx context.Context, execNode *flow.IdentitySkeleton, req *execproto.GetAccountAtBlockIDRequest) (*execproto.GetAccountAtBlockIDResponse, error) { execRPCClient, closer, err := b.connFactory.GetExecutionAPIClient(execNode.Address) if err != nil { return nil, err diff --git a/engine/access/rpc/backend/backend_events.go b/engine/access/rpc/backend/backend_events.go index 40541ceb060..413940e36aa 100644 --- a/engine/access/rpc/backend/backend_events.go +++ b/engine/access/rpc/backend/backend_events.go @@ -137,7 +137,7 @@ func (b *backendEvents) getBlockEventsFromExecutionNode( } var resp *execproto.GetEventsForBlockIDsResponse - var successfulNode *flow.Identity + var successfulNode *flow.IdentitySkeleton resp, successfulNode, err = b.getEventsFromAnyExeNode(ctx, execNodes, req) if err != nil { b.log.Error().Err(err).Msg("failed to retrieve events from execution nodes") @@ -208,13 +208,13 @@ func verifyAndConvertToAccessEvents( // other ENs are logged and swallowed. If all ENs fail to return a valid response, then an // error aggregating all failures is returned. func (b *backendEvents) getEventsFromAnyExeNode(ctx context.Context, - execNodes flow.IdentityList, - req *execproto.GetEventsForBlockIDsRequest) (*execproto.GetEventsForBlockIDsResponse, *flow.Identity, error) { + execNodes flow.IdentitySkeletonList, + req *execproto.GetEventsForBlockIDsRequest) (*execproto.GetEventsForBlockIDsResponse, *flow.IdentitySkeleton, error) { var resp *execproto.GetEventsForBlockIDsResponse - var execNode *flow.Identity + var execNode *flow.IdentitySkeleton errToReturn := b.nodeCommunicator.CallAvailableNode( execNodes, - func(node *flow.Identity) error { + func(node *flow.IdentitySkeleton) error { var err error start := time.Now() resp, err = b.tryGetEvents(ctx, node, req) @@ -244,7 +244,7 @@ func (b *backendEvents) getEventsFromAnyExeNode(ctx context.Context, } func (b *backendEvents) tryGetEvents(ctx context.Context, - execNode *flow.Identity, + execNode *flow.IdentitySkeleton, req *execproto.GetEventsForBlockIDsRequest) (*execproto.GetEventsForBlockIDsResponse, error) { execRPCClient, closer, err := b.connFactory.GetExecutionAPIClient(execNode.Address) if err != nil { diff --git a/engine/access/rpc/backend/backend_scripts.go b/engine/access/rpc/backend/backend_scripts.go index bdff9beb8d7..2267534e8a9 100644 --- a/engine/access/rpc/backend/backend_scripts.go +++ b/engine/access/rpc/backend/backend_scripts.go @@ -253,7 +253,7 @@ func (b *backendScripts) executeScriptOnAvailableExecutionNodes( hasInvalidArgument := false errToReturn := b.nodeCommunicator.CallAvailableNode( executors, - func(node *flow.Identity) error { + func(node *flow.IdentitySkeleton) error { execStartTime := time.Now() result, err = b.tryExecuteScriptOnExecutionNode(ctx, node.Address, blockID, script, arguments) if err == nil { @@ -281,7 +281,7 @@ func (b *backendScripts) executeScriptOnAvailableExecutionNodes( return err }, - func(node *flow.Identity, err error) bool { + func(node *flow.IdentitySkeleton, err error) bool { hasInvalidArgument = status.Code(err) == codes.InvalidArgument if hasInvalidArgument { b.log.Debug().Err(err). diff --git a/engine/access/rpc/backend/backend_transactions.go b/engine/access/rpc/backend/backend_transactions.go index bfa4941ca76..884674beffe 100644 --- a/engine/access/rpc/backend/backend_transactions.go +++ b/engine/access/rpc/backend/backend_transactions.go @@ -103,7 +103,7 @@ func (b *backendTransactions) trySendTransaction(ctx context.Context, tx *flow.T // try sending the transaction to one of the chosen collection nodes sendError = b.nodeCommunicator.CallAvailableNode( collNodes, - func(node *flow.Identity) error { + func(node *flow.IdentitySkeleton) error { err = b.sendTransactionToCollector(ctx, tx, node.Address) if err != nil { return err @@ -118,7 +118,7 @@ func (b *backendTransactions) trySendTransaction(ctx context.Context, tx *flow.T // chooseCollectionNodes finds a random subset of size sampleSize of collection node addresses from the // collection node cluster responsible for the given tx -func (b *backendTransactions) chooseCollectionNodes(tx *flow.TransactionBody) (flow.IdentityList, error) { +func (b *backendTransactions) chooseCollectionNodes(tx *flow.TransactionBody) (flow.IdentitySkeletonList, error) { // retrieve the set of collector clusters clusters, err := b.state.Final().Epochs().Current().Clustering() @@ -781,7 +781,7 @@ func (b *backendTransactions) NotifyFinalizedBlockHeight(height uint64) { func (b *backendTransactions) getTransactionResultFromAnyExeNode( ctx context.Context, - execNodes flow.IdentityList, + execNodes flow.IdentitySkeletonList, req *execproto.GetTransactionResultRequest, ) (*execproto.GetTransactionResultResponse, error) { var errToReturn error @@ -795,7 +795,7 @@ func (b *backendTransactions) getTransactionResultFromAnyExeNode( var resp *execproto.GetTransactionResultResponse errToReturn = b.nodeCommunicator.CallAvailableNode( execNodes, - func(node *flow.Identity) error { + func(node *flow.IdentitySkeleton) error { var err error resp, err = b.tryGetTransactionResult(ctx, node, req) if err == nil { @@ -816,7 +816,7 @@ func (b *backendTransactions) getTransactionResultFromAnyExeNode( func (b *backendTransactions) tryGetTransactionResult( ctx context.Context, - execNode *flow.Identity, + execNode *flow.IdentitySkeleton, req *execproto.GetTransactionResultRequest, ) (*execproto.GetTransactionResultResponse, error) { execRPCClient, closer, err := b.connFactory.GetExecutionAPIClient(execNode.Address) @@ -835,7 +835,7 @@ func (b *backendTransactions) tryGetTransactionResult( func (b *backendTransactions) getTransactionResultsByBlockIDFromAnyExeNode( ctx context.Context, - execNodes flow.IdentityList, + execNodes flow.IdentitySkeletonList, req *execproto.GetTransactionsByBlockIDRequest, ) (*execproto.GetTransactionResultsResponse, error) { var errToReturn error @@ -855,7 +855,7 @@ func (b *backendTransactions) getTransactionResultsByBlockIDFromAnyExeNode( var resp *execproto.GetTransactionResultsResponse errToReturn = b.nodeCommunicator.CallAvailableNode( execNodes, - func(node *flow.Identity) error { + func(node *flow.IdentitySkeleton) error { var err error resp, err = b.tryGetTransactionResultsByBlockID(ctx, node, req) if err == nil { @@ -875,7 +875,7 @@ func (b *backendTransactions) getTransactionResultsByBlockIDFromAnyExeNode( func (b *backendTransactions) tryGetTransactionResultsByBlockID( ctx context.Context, - execNode *flow.Identity, + execNode *flow.IdentitySkeleton, req *execproto.GetTransactionsByBlockIDRequest, ) (*execproto.GetTransactionResultsResponse, error) { execRPCClient, closer, err := b.connFactory.GetExecutionAPIClient(execNode.Address) @@ -894,7 +894,7 @@ func (b *backendTransactions) tryGetTransactionResultsByBlockID( func (b *backendTransactions) getTransactionResultByIndexFromAnyExeNode( ctx context.Context, - execNodes flow.IdentityList, + execNodes flow.IdentitySkeletonList, req *execproto.GetTransactionByIndexRequest, ) (*execproto.GetTransactionResultResponse, error) { var errToReturn error @@ -911,7 +911,7 @@ func (b *backendTransactions) getTransactionResultByIndexFromAnyExeNode( var resp *execproto.GetTransactionResultResponse errToReturn = b.nodeCommunicator.CallAvailableNode( execNodes, - func(node *flow.Identity) error { + func(node *flow.IdentitySkeleton) error { var err error resp, err = b.tryGetTransactionResultByIndex(ctx, node, req) if err == nil { @@ -932,7 +932,7 @@ func (b *backendTransactions) getTransactionResultByIndexFromAnyExeNode( func (b *backendTransactions) tryGetTransactionResultByIndex( ctx context.Context, - execNode *flow.Identity, + execNode *flow.IdentitySkeleton, req *execproto.GetTransactionByIndexRequest, ) (*execproto.GetTransactionResultResponse, error) { execRPCClient, closer, err := b.connFactory.GetExecutionAPIClient(execNode.Address) diff --git a/engine/access/rpc/backend/node_communicator.go b/engine/access/rpc/backend/node_communicator.go index ba4b005a70f..99dcbcdf1ba 100644 --- a/engine/access/rpc/backend/node_communicator.go +++ b/engine/access/rpc/backend/node_communicator.go @@ -15,13 +15,13 @@ const maxFailedRequestCount = 3 type Communicator interface { CallAvailableNode( //List of node identifiers to execute callback on - nodes flow.IdentityList, + nodes flow.IdentitySkeletonList, //Callback function that represents an action to be performed on a node. //It takes a node as input and returns an error indicating the result of the action. - call func(node *flow.Identity) error, + call func(node *flow.IdentitySkeleton) error, // Callback function that determines whether an error should terminate further execution. // It takes an error as input and returns a boolean value indicating whether the error should be considered terminal. - shouldTerminateOnError func(node *flow.Identity, err error) bool, + shouldTerminateOnError func(node *flow.IdentitySkeleton, err error) bool, ) error } @@ -46,13 +46,13 @@ func NewNodeCommunicator(circuitBreakerEnabled bool) *NodeCommunicator { // If the maximum failed request count is reached, it returns the accumulated errors. func (b *NodeCommunicator) CallAvailableNode( //List of node identifiers to execute callback on - nodes flow.IdentityList, + nodes flow.IdentitySkeletonList, //Callback function that determines whether an error should terminate further execution. // It takes an error as input and returns a boolean value indicating whether the error should be considered terminal. - call func(id *flow.Identity) error, + call func(id *flow.IdentitySkeleton) error, // Callback function that determines whether an error should terminate further execution. // It takes an error as input and returns a boolean value indicating whether the error should be considered terminal. - shouldTerminateOnError func(node *flow.Identity, err error) bool, + shouldTerminateOnError func(node *flow.IdentitySkeleton, err error) bool, ) error { var errs *multierror.Error nodeSelector, err := b.nodeSelectorFactory.SelectNodes(nodes) diff --git a/engine/access/rpc/backend/node_selector.go b/engine/access/rpc/backend/node_selector.go index f90f8271b2d..c7d2ada5fb4 100644 --- a/engine/access/rpc/backend/node_selector.go +++ b/engine/access/rpc/backend/node_selector.go @@ -14,7 +14,7 @@ const maxNodesCnt = 3 // of nodes. Implementations of this interface should define the Next method, which returns the next node identity to be // selected. HasNext checks if there is next node available. type NodeSelector interface { - Next() *flow.Identity + Next() *flow.IdentitySkeleton HasNext() bool } @@ -28,7 +28,7 @@ type NodeSelectorFactory struct { // SelectNodes selects the configured number of node identities from the provided list of nodes // and returns the node selector to iterate through them. -func (n *NodeSelectorFactory) SelectNodes(nodes flow.IdentityList) (NodeSelector, error) { +func (n *NodeSelectorFactory) SelectNodes(nodes flow.IdentitySkeletonList) (NodeSelector, error) { var err error // If the circuit breaker is disabled, the legacy logic should be used, which selects only a specified number of nodes. if !n.circuitBreakerEnabled { @@ -44,13 +44,13 @@ func (n *NodeSelectorFactory) SelectNodes(nodes flow.IdentityList) (NodeSelector // MainNodeSelector is a specific implementation of the node selector. // Which performs in-order node selection using fixed list of pre-defined nodes. type MainNodeSelector struct { - nodes flow.IdentityList + nodes flow.IdentitySkeletonList index int } var _ NodeSelector = (*MainNodeSelector)(nil) -func NewMainNodeSelector(nodes flow.IdentityList) *MainNodeSelector { +func NewMainNodeSelector(nodes flow.IdentitySkeletonList) *MainNodeSelector { return &MainNodeSelector{nodes: nodes, index: 0} } @@ -60,7 +60,7 @@ func (e *MainNodeSelector) HasNext() bool { } // Next returns the next node in the selector. -func (e *MainNodeSelector) Next() *flow.Identity { +func (e *MainNodeSelector) Next() *flow.IdentitySkeleton { if e.index < len(e.nodes) { next := e.nodes[e.index] e.index++ diff --git a/engine/collection/epochmgr/factories/sync.go b/engine/collection/epochmgr/factories/sync.go index 020895ee22d..e3dbe15dca6 100644 --- a/engine/collection/epochmgr/factories/sync.go +++ b/engine/collection/epochmgr/factories/sync.go @@ -37,7 +37,7 @@ func NewSyncEngineFactory( } func (f *SyncEngineFactory) Create( - participants flow.IdentityList, + participants flow.IdentitySkeletonList, state cluster.State, blocks storage.ClusterBlocks, core *chainsync.Core, diff --git a/engine/collection/ingest/engine.go b/engine/collection/ingest/engine.go index d635332ba27..dfd1612f5a4 100644 --- a/engine/collection/ingest/engine.go +++ b/engine/collection/ingest/engine.go @@ -295,7 +295,7 @@ func (e *Engine) onTransaction(originID flow.Identifier, tx *flow.TransactionBod // a member of the reference epoch. This is an expected condition and the transaction // should be discarded. // - other error for any other, unexpected error condition. -func (e *Engine) getLocalCluster(refEpoch protocol.Epoch) (flow.IdentityList, error) { +func (e *Engine) getLocalCluster(refEpoch protocol.Epoch) (flow.IdentitySkeletonList, error) { epochCounter, err := refEpoch.Counter() if err != nil { return nil, fmt.Errorf("could not get counter for reference epoch: %w", err) @@ -370,7 +370,7 @@ func (e *Engine) ingestTransaction( // propagateTransaction propagates the transaction to a number of the responsible // cluster's members. Any unexpected networking errors are logged. -func (e *Engine) propagateTransaction(log zerolog.Logger, tx *flow.TransactionBody, txCluster flow.IdentityList) { +func (e *Engine) propagateTransaction(log zerolog.Logger, tx *flow.TransactionBody, txCluster flow.IdentitySkeletonList) { log.Debug().Msg("propagating transaction to cluster") err := e.conduit.Multicast(tx, e.config.PropagationRedundancy+1, txCluster.NodeIDs()...) diff --git a/engine/collection/message_hub/message_hub.go b/engine/collection/message_hub/message_hub.go index d10f4140f2b..f2241dffb73 100644 --- a/engine/collection/message_hub/message_hub.go +++ b/engine/collection/message_hub/message_hub.go @@ -86,7 +86,7 @@ type MessageHub struct { ownOutboundVotes *fifoqueue.FifoQueue // queue for handling outgoing vote transmissions ownOutboundProposals *fifoqueue.FifoQueue // queue for handling outgoing proposal transmissions ownOutboundTimeouts *fifoqueue.FifoQueue // queue for handling outgoing timeout transmissions - clusterIdentityFilter flow.IdentityFilter + clusterIdentityFilter flow.IdentityFilter[flow.Identity] // injected dependencies compliance collection.Compliance // handler of incoming block proposals @@ -150,8 +150,8 @@ func NewMessageHub(log zerolog.Logger, ownOutboundProposals: ownOutboundProposals, ownOutboundTimeouts: ownOutboundTimeouts, clusterIdentityFilter: filter.And( - filter.In(currentCluster), - filter.Not(filter.HasNodeID(me.NodeID())), + filter.Adapt(filter.In(currentCluster)), + filter.Not(filter.HasNodeID[flow.Identity](me.NodeID())), ), } diff --git a/engine/collection/pusher/engine.go b/engine/collection/pusher/engine.go index 5e09ea7418e..317729108dd 100644 --- a/engine/collection/pusher/engine.go +++ b/engine/collection/pusher/engine.go @@ -128,7 +128,7 @@ func (e *Engine) onSubmitCollectionGuarantee(originID flow.Identifier, req *mess // SubmitCollectionGuarantee submits the collection guarantee to all consensus nodes. func (e *Engine) SubmitCollectionGuarantee(guarantee *flow.CollectionGuarantee) error { - consensusNodes, err := e.state.Final().Identities(filter.HasRole(flow.RoleConsensus)) + consensusNodes, err := e.state.Final().Identities(filter.HasRole[flow.Identity](flow.RoleConsensus)) if err != nil { return fmt.Errorf("could not get consensus nodes: %w", err) } diff --git a/engine/collection/synchronization/engine.go b/engine/collection/synchronization/engine.go index 7355ccb75b2..d8e1b41ef61 100644 --- a/engine/collection/synchronization/engine.go +++ b/engine/collection/synchronization/engine.go @@ -42,7 +42,7 @@ type Engine struct { log zerolog.Logger metrics module.EngineMetrics me module.Local - participants flow.IdentityList + participants flow.IdentitySkeletonList con network.Conduit comp collection.Compliance // compliance layer engine @@ -64,7 +64,7 @@ func New( metrics module.EngineMetrics, net network.EngineRegistry, me module.Local, - participants flow.IdentityList, + participants flow.IdentitySkeletonList, state cluster.State, blocks storage.ClusterBlocks, comp collection.Compliance, @@ -88,7 +88,7 @@ func New( log: log.With().Str("engine", "cluster_synchronization").Logger(), metrics: metrics, me: me, - participants: participants.Filter(filter.Not(filter.HasNodeID(me.NodeID()))), + participants: participants.Filter(filter.Not(filter.HasNodeID[flow.IdentitySkeleton](me.NodeID()))), comp: comp, core: core, pollInterval: opt.PollInterval, diff --git a/engine/common/provider/engine.go b/engine/common/provider/engine.go index 3b492bd8788..12593b614ef 100644 --- a/engine/common/provider/engine.go +++ b/engine/common/provider/engine.go @@ -52,7 +52,7 @@ type Engine struct { channel channels.Channel requestHandler *engine.MessageHandler requestQueue engine.MessageStore - selector flow.IdentityFilter + selector flow.IdentityFilter[flow.Identity] retrieve RetrieveFunc // buffered channel for EntityRequest workers to pick and process. requestChannel chan *internal.EntityRequest @@ -72,13 +72,13 @@ func New( requestQueue engine.MessageStore, requestWorkers uint, channel channels.Channel, - selector flow.IdentityFilter, + selector flow.IdentityFilter[flow.Identity], retrieve RetrieveFunc) (*Engine, error) { // make sure we don't respond to request sent by self or unauthorized nodes selector = filter.And( selector, - filter.Not(filter.HasNodeID(me.NodeID())), + filter.Not(filter.HasNodeID[flow.Identity](me.NodeID())), ) handler := engine.NewMessageHandler( @@ -198,7 +198,7 @@ func (e *Engine) onEntityRequest(request *internal.EntityRequest) error { // for the handler to make sure the requester is authorized for this resource requesters, err := e.state.Final().Identities(filter.And( e.selector, - filter.HasNodeID(request.OriginId)), + filter.HasNodeID[flow.Identity](request.OriginId)), ) if err != nil { return fmt.Errorf("could not get requesters: %w", err) diff --git a/engine/common/requester/engine.go b/engine/common/requester/engine.go index 48d8e2bad24..7044d36e12c 100644 --- a/engine/common/requester/engine.go +++ b/engine/common/requester/engine.go @@ -43,7 +43,7 @@ type Engine struct { state protocol.State con network.Conduit channel channels.Channel - selector flow.IdentityFilter + selector flow.IdentityFilter[flow.Identity] create CreateFunc handle HandleFunc @@ -57,7 +57,7 @@ type Engine struct { // within the set obtained by applying the provided selector filter. The options allow customization of the parameters // related to the batch and retry logic. func New(log zerolog.Logger, metrics module.EngineMetrics, net network.EngineRegistry, me module.Local, state protocol.State, - channel channels.Channel, selector flow.IdentityFilter, create CreateFunc, options ...OptionFunc) (*Engine, error) { + channel channels.Channel, selector flow.IdentityFilter[flow.Identity], create CreateFunc, options ...OptionFunc) (*Engine, error) { // initialize the default config cfg := Config{ @@ -89,7 +89,7 @@ func New(log zerolog.Logger, metrics module.EngineMetrics, net network.EngineReg // make sure we don't send requests from self selector = filter.And( selector, - filter.Not(filter.HasNodeID(me.NodeID())), + filter.Not(filter.HasNodeID[flow.Identity](me.NodeID())), filter.Not(filter.Ejected), ) @@ -201,7 +201,7 @@ func (e *Engine) Process(channel channels.Channel, originID flow.Identifier, mes // control over which subset of providers to request a given entity from, such as // selection of a collection cluster. Use `filter.Any` if no additional selection // is required. Checks integrity of response to make sure that we got entity that we were requesting. -func (e *Engine) EntityByID(entityID flow.Identifier, selector flow.IdentityFilter) { +func (e *Engine) EntityByID(entityID flow.Identifier, selector flow.IdentityFilter[flow.Identity]) { e.addEntityRequest(entityID, selector, true) } @@ -210,11 +210,11 @@ func (e *Engine) EntityByID(entityID flow.Identifier, selector flow.IdentityFilt // of valid providers for the data and allows finer-grained control // over which providers to request data from. Doesn't perform integrity check // can be used to get entities without knowing their ID. -func (e *Engine) Query(key flow.Identifier, selector flow.IdentityFilter) { +func (e *Engine) Query(key flow.Identifier, selector flow.IdentityFilter[flow.Identity]) { e.addEntityRequest(key, selector, false) } -func (e *Engine) addEntityRequest(entityID flow.Identifier, selector flow.IdentityFilter, checkIntegrity bool) { +func (e *Engine) addEntityRequest(entityID flow.Identifier, selector flow.IdentityFilter[flow.Identity], checkIntegrity bool) { e.unit.Lock() defer e.unit.Unlock() @@ -349,7 +349,7 @@ func (e *Engine) dispatchRequest() (bool, error) { // for now, so it will be part of the next batch request if providerID != flow.ZeroID { overlap := providers.Filter(filter.And( - filter.HasNodeID(providerID), + filter.HasNodeID[flow.Identity](providerID), item.ExtraSelector, )) if len(overlap) == 0 { @@ -486,7 +486,7 @@ func (e *Engine) onEntityResponse(originID flow.Identifier, res *messages.Entity // check that the response comes from a valid provider providers, err := e.state.Final().Identities(filter.And( e.selector, - filter.HasNodeID(originID), + filter.HasNodeID[flow.Identity](originID), )) if err != nil { return fmt.Errorf("could not get providers: %w", err) diff --git a/engine/common/requester/item.go b/engine/common/requester/item.go index 456a33e881f..06cdf2acb01 100644 --- a/engine/common/requester/item.go +++ b/engine/common/requester/item.go @@ -7,10 +7,10 @@ import ( ) type Item struct { - EntityID flow.Identifier // ID for the entity to be requested - NumAttempts uint // number of times the entity was requested - LastRequested time.Time // approximate timestamp of last request - RetryAfter time.Duration // interval until request should be retried - ExtraSelector flow.IdentityFilter // additional filters for providers of this entity - checkIntegrity bool // check response integrity using `EntityID` + EntityID flow.Identifier // ID for the entity to be requested + NumAttempts uint // number of times the entity was requested + LastRequested time.Time // approximate timestamp of last request + RetryAfter time.Duration // interval until request should be retried + ExtraSelector flow.IdentityFilter[flow.Identity] // additional filters for providers of this entity + checkIntegrity bool // check response integrity using `EntityID` } diff --git a/engine/consensus/approvals/verifying_assignment_collector.go b/engine/consensus/approvals/verifying_assignment_collector.go index a78131783f5..243487d903b 100644 --- a/engine/consensus/approvals/verifying_assignment_collector.go +++ b/engine/consensus/approvals/verifying_assignment_collector.go @@ -396,7 +396,7 @@ func (ac *VerifyingAssignmentCollector) RequestMissingApprovals(observation cons func authorizedVerifiersAtBlock(state protocol.State, blockID flow.Identifier) (map[flow.Identifier]*flow.Identity, error) { authorizedVerifierList, err := state.AtBlockID(blockID).Identities( filter.And( - filter.HasRole(flow.RoleVerification), + filter.HasRole[flow.Identity](flow.RoleVerification), filter.HasWeight(true), filter.Not(filter.Ejected), )) diff --git a/engine/consensus/compliance/core_test.go b/engine/consensus/compliance/core_test.go index 36dddda317d..71e1b68ace8 100644 --- a/engine/consensus/compliance/core_test.go +++ b/engine/consensus/compliance/core_test.go @@ -172,7 +172,7 @@ func (cs *CommonSuite) SetupTest() { // set up protocol snapshot mock cs.snapshot = &protocol.Snapshot{} cs.snapshot.On("Identities", mock.Anything).Return( - func(filter flow.IdentityFilter) flow.IdentityList { + func(filter flow.IdentityFilter[flow.Identity]) flow.IdentityList { return cs.participants.Filter(filter) }, nil, diff --git a/engine/consensus/dkg/reactor_engine.go b/engine/consensus/dkg/reactor_engine.go index fa5208c25f6..e9c0669cf9b 100644 --- a/engine/consensus/dkg/reactor_engine.go +++ b/engine/consensus/dkg/reactor_engine.go @@ -24,7 +24,7 @@ const DefaultPollStep = 10 // dkgInfo consolidates information about the current DKG protocol instance. type dkgInfo struct { - identities flow.IdentityList + identities flow.IdentitySkeletonList phase1FinalView uint64 phase2FinalView uint64 phase3FinalView uint64 diff --git a/engine/consensus/message_hub/message_hub.go b/engine/consensus/message_hub/message_hub.go index 2deb22a7332..99bd7a44d7a 100644 --- a/engine/consensus/message_hub/message_hub.go +++ b/engine/consensus/message_hub/message_hub.go @@ -237,8 +237,8 @@ func (h *MessageHub) sendOwnTimeout(timeout *model.TimeoutObject) error { // TCs might not be constructed at epoch switchover. recipients, err := h.state.Final().Identities(filter.And( filter.Not(filter.Ejected), - filter.HasRole(flow.RoleConsensus), - filter.Not(filter.HasNodeID(h.me.NodeID())), + filter.HasRole[flow.Identity](flow.RoleConsensus), + filter.Not(filter.HasNodeID[flow.Identity](h.me.NodeID())), )) if err != nil { return fmt.Errorf("could not get consensus recipients for broadcasting timeout: %w", err) @@ -329,13 +329,13 @@ func (h *MessageHub) sendOwnProposal(header *flow.Header) error { // `OnOwnTimeout` is directly called by the consensus core logic. allIdentities, err := h.state.AtBlockID(header.ParentID).Identities(filter.And( filter.Not(filter.Ejected), - filter.Not(filter.HasNodeID(h.me.NodeID())), + filter.Not(filter.HasNodeID[flow.Identity](h.me.NodeID())), )) if err != nil { return fmt.Errorf("could not get identities for broadcasting proposal: %w", err) } - consRecipients := allIdentities.Filter(filter.HasRole(flow.RoleConsensus)) + consRecipients := allIdentities.Filter(filter.HasRole[flow.Identity](flow.RoleConsensus)) // NOTE: some fields are not needed for the message // - proposer ID is conveyed over the network message @@ -356,7 +356,7 @@ func (h *MessageHub) sendOwnProposal(header *flow.Header) error { log.Info().Msg("block proposal was broadcast") // submit proposal to non-consensus nodes - h.provideProposal(proposal, allIdentities.Filter(filter.Not(filter.HasRole(flow.RoleConsensus)))) + h.provideProposal(proposal, allIdentities.Filter(filter.Not(filter.HasRole[flow.Identity](flow.RoleConsensus)))) h.engineMetrics.MessageSent(metrics.EngineConsensusMessageHub, metrics.MessageBlockProposal) return nil diff --git a/engine/execution/ingestion/engine.go b/engine/execution/ingestion/engine.go index 4e7706d355a..fb572da1346 100644 --- a/engine/execution/ingestion/engine.go +++ b/engine/execution/ingestion/engine.go @@ -1228,8 +1228,8 @@ func (e *Engine) fetchCollection( return fmt.Errorf("could not find guarantors: %w", err) } - filters := []flow.IdentityFilter{ - filter.HasNodeID(guarantors...), + filters := []flow.IdentityFilter[flow.Identity]{ + filter.HasNodeID[flow.Identity](guarantors...), } // This is included to temporarily work around an issue observed on a small number of ENs. diff --git a/engine/execution/provider/engine.go b/engine/execution/provider/engine.go index a2eb212d56a..0db76a5261f 100644 --- a/engine/execution/provider/engine.go +++ b/engine/execution/provider/engine.go @@ -369,7 +369,7 @@ func (e *Engine) BroadcastExecutionReceipt(ctx context.Context, receipt *flow.Ex Hex("final_state", finalState[:]). Msg("broadcasting execution receipt") - identities, err := e.state.Final().Identities(filter.HasRole(flow.RoleAccess, flow.RoleConsensus, + identities, err := e.state.Final().Identities(filter.HasRole[flow.Identity](flow.RoleAccess, flow.RoleConsensus, flow.RoleVerification)) if err != nil { return fmt.Errorf("could not get consensus and verification identities: %w", err) diff --git a/engine/verification/fetcher/engine.go b/engine/verification/fetcher/engine.go index fd53417b720..20afad04021 100644 --- a/engine/verification/fetcher/engine.go +++ b/engine/verification/fetcher/engine.go @@ -587,7 +587,7 @@ func (e *Engine) requestChunkDataPack(chunkIndex uint64, chunkID flow.Identifier return fmt.Errorf("could not get header for block: %x", blockID) } - allExecutors, err := e.state.AtBlockID(blockID).Identities(filter.HasRole(flow.RoleExecution)) + allExecutors, err := e.state.AtBlockID(blockID).Identities(filter.HasRole[flow.Identity](flow.RoleExecution)) if err != nil { return fmt.Errorf("could not fetch execution node ids at block %x: %w", blockID, err) } diff --git a/engine/verification/verifier/engine.go b/engine/verification/verifier/engine.go index 3c79a8a6057..fdef768625e 100644 --- a/engine/verification/verifier/engine.go +++ b/engine/verification/verifier/engine.go @@ -264,7 +264,7 @@ func (e *Engine) verify(ctx context.Context, originID flow.Identifier, // Extracting consensus node ids // TODO state extraction should be done based on block references consensusNodes, err := e.state.Final(). - Identities(filter.HasRole(flow.RoleConsensus)) + Identities(filter.HasRole[flow.Identity](flow.RoleConsensus)) if err != nil { // TODO this error needs more advance handling after MVP return fmt.Errorf("could not load consensus node IDs: %w", err) diff --git a/model/bootstrap/node_info_test.go b/model/bootstrap/node_info_test.go index 536c0c808f9..506bc85d80f 100644 --- a/model/bootstrap/node_info_test.go +++ b/model/bootstrap/node_info_test.go @@ -2,6 +2,7 @@ package bootstrap_test import ( "encoding/json" + "github.com/onflow/flow-go/model/flow" "strings" "testing" @@ -15,8 +16,8 @@ import ( func TestSort(t *testing.T) { nodes := unittest.NodeInfosFixture(20) - nodes = bootstrap.Sort(nodes, order.Canonical) - require.True(t, bootstrap.ToIdentityList(nodes).Sorted(order.Canonical)) + nodes = bootstrap.Sort(nodes, order.Canonical[flow.Identity]) + require.True(t, bootstrap.ToIdentityList(nodes).Sorted(order.Canonical[flow.Identity])) } func TestNodeConfigEncodingJSON(t *testing.T) { diff --git a/model/convert/service_event.go b/model/convert/service_event.go index cabce39f457..ee46eb28b26 100644 --- a/model/convert/service_event.go +++ b/model/convert/service_event.go @@ -463,8 +463,8 @@ func convertClusterAssignments(cdcClusters []cadence.Value) (flow.AssignmentList // convertParticipants converts the network participants specified in the // EpochSetup event into an IdentityList. // CONVENTION: returned IdentityList is in CANONICAL ORDER -func convertParticipants(cdcParticipants []cadence.Value) (flow.IdentityList, error) { - participants := make(flow.IdentityList, 0, len(cdcParticipants)) +func convertParticipants(cdcParticipants []cadence.Value) (flow.IdentitySkeletonList, error) { + participants := make(flow.IdentitySkeletonList, 0, len(cdcParticipants)) var err error for _, value := range cdcParticipants { @@ -583,15 +583,10 @@ func convertParticipants(cdcParticipants []cadence.Value) (flow.IdentityList, er return nil, fmt.Errorf("invalid role %d", role) } - identity := &flow.Identity{ - IdentitySkeleton: flow.IdentitySkeleton{ - InitialWeight: uint64(initialWeight), - Address: string(address), - Role: flow.Role(role), - }, - DynamicIdentity: flow.DynamicIdentity{ - Weight: uint64(initialWeight), - }, + identity := &flow.IdentitySkeleton{ + InitialWeight: uint64(initialWeight), + Address: string(address), + Role: flow.Role(role), } // convert nodeID string into identifier @@ -636,7 +631,7 @@ func convertParticipants(cdcParticipants []cadence.Value) (flow.IdentityList, er } // IMPORTANT: returned identities must be in *canonical order* - participants = participants.Sort(order.Canonical) + participants = participants.Sort(order.Canonical[flow.IdentitySkeleton]) return participants, nil } diff --git a/model/flow/filter/identity.go b/model/flow/filter/identity.go index dde0bff2fc6..c8b3bac7a96 100644 --- a/model/flow/filter/identity.go +++ b/model/flow/filter/identity.go @@ -54,8 +54,8 @@ func Not[T flow.GenericIdentity](filter flow.IdentityFilter[T]) flow.IdentityFil // In returns a filter for identities within the input list. This is equivalent // to HasNodeID, but for list-typed inputs. -func In(list flow.IdentityList) flow.IdentityFilter[flow.Identity] { - return HasNodeID[flow.Identity](list.NodeIDs()...) +func In[T flow.GenericIdentity](list flow.GenericIdentityList[T]) flow.IdentityFilter[T] { + return HasNodeID[T](list.NodeIDs()...) } // HasNodeID returns a filter that returns true for any identity with an ID diff --git a/module/chunks/chunk_assigner.go b/module/chunks/chunk_assigner.go index dc34816a784..11ce9a0949b 100644 --- a/module/chunks/chunk_assigner.go +++ b/module/chunks/chunk_assigner.go @@ -66,7 +66,7 @@ func (p *ChunkAssigner) Assign(result *flow.ExecutionResult, blockID flow.Identi } // Get a list of verifiers at block that is being sealed - verifiers, err := p.protocolState.AtBlockID(result.BlockID).Identities(filter.And(filter.HasRole(flow.RoleVerification), + verifiers, err := p.protocolState.AtBlockID(result.BlockID).Identities(filter.And(filter.HasRole[flow.Identity](flow.RoleVerification), filter.HasWeight(true), filter.Not(filter.Ejected))) if err != nil { diff --git a/module/dkg.go b/module/dkg.go index 7952fbbdc8d..5519751d020 100644 --- a/module/dkg.go +++ b/module/dkg.go @@ -81,5 +81,5 @@ type DKGController interface { type DKGControllerFactory interface { // Create instantiates a new DKGController. - Create(dkgInstanceID string, participants flow.IdentityList, seed []byte) (DKGController, error) + Create(dkgInstanceID string, participants flow.IdentitySkeletonList, seed []byte) (DKGController, error) } diff --git a/module/dkg/broker.go b/module/dkg/broker.go index f94fbc981fe..f4b319dbdf8 100644 --- a/module/dkg/broker.go +++ b/module/dkg/broker.go @@ -59,7 +59,7 @@ type Broker struct { log zerolog.Logger unit *engine.Unit dkgInstanceID string // unique identifier of the current dkg run (prevent replay attacks) - committee flow.IdentityList // identities of DKG members + committee flow.IdentitySkeletonList // identities of DKG members me module.Local // used for signing broadcast messages myIndex int // index of this instance in the committee dkgContractClients []module.DKGContractClient // array of clients to communicate with the DKG smart contract in priority order for fallbacks during retries @@ -84,7 +84,7 @@ var _ module.DKGBroker = (*Broker)(nil) func NewBroker( log zerolog.Logger, dkgInstanceID string, - committee flow.IdentityList, + committee flow.IdentitySkeletonList, me module.Local, myIndex int, dkgContractClients []module.DKGContractClient, diff --git a/module/dkg/controller_factory.go b/module/dkg/controller_factory.go index ae12219706e..6b19ae20197 100644 --- a/module/dkg/controller_factory.go +++ b/module/dkg/controller_factory.go @@ -46,7 +46,7 @@ func NewControllerFactory( // is capable of communicating with other nodes. func (f *ControllerFactory) Create( dkgInstanceID string, - participants flow.IdentityList, + participants flow.IdentitySkeletonList, seed []byte) (module.DKGController, error) { myIndex, ok := participants.GetIndex(f.me.NodeID()) diff --git a/module/id/filtered_provider.go b/module/id/filtered_provider.go index f3703f0d9ff..7b98c14be06 100644 --- a/module/id/filtered_provider.go +++ b/module/id/filtered_provider.go @@ -8,11 +8,11 @@ import ( // IdentityFilterIdentifierProvider implements an IdentifierProvider which provides the identifiers // resulting from applying a filter to an IdentityProvider. type IdentityFilterIdentifierProvider struct { - filter flow.IdentityFilter + filter flow.IdentityFilter[flow.Identity] identityProvider module.IdentityProvider } -func NewIdentityFilterIdentifierProvider(filter flow.IdentityFilter, identityProvider module.IdentityProvider) *IdentityFilterIdentifierProvider { +func NewIdentityFilterIdentifierProvider(filter flow.IdentityFilter[flow.Identity], identityProvider module.IdentityProvider) *IdentityFilterIdentifierProvider { return &IdentityFilterIdentifierProvider{filter, identityProvider} } diff --git a/module/id/fixed_provider.go b/module/id/fixed_provider.go index d26adc0f375..e6f3713d47c 100644 --- a/module/id/fixed_provider.go +++ b/module/id/fixed_provider.go @@ -34,7 +34,7 @@ func NewFixedIdentityProvider(identities flow.IdentityList) *FixedIdentityProvid return &FixedIdentityProvider{identities} } -func (p *FixedIdentityProvider) Identities(filter flow.IdentityFilter) flow.IdentityList { +func (p *FixedIdentityProvider) Identities(filter flow.IdentityFilter[flow.Identity]) flow.IdentityList { return p.identities.Filter(filter) } diff --git a/module/local/me.go b/module/local/me.go index 6353cdab846..68bba5bcbdf 100644 --- a/module/local/me.go +++ b/module/local/me.go @@ -41,8 +41,8 @@ func (l *Local) Sign(msg []byte, hasher hash.Hasher) (crypto.Signature, error) { return l.sk.Sign(msg, hasher) } -func (l *Local) NotMeFilter() flow.IdentityFilter { - return filter.Not(filter.HasNodeID(l.NodeID())) +func (l *Local) NotMeFilter() flow.IdentityFilter[flow.Identity] { + return filter.Not(filter.HasNodeID[flow.Identity](l.NodeID())) } // SignFunc provides a signature oracle that given a message, a hasher, and a signing function, it diff --git a/module/local/me_nokey.go b/module/local/me_nokey.go index dc9ab729186..7665e3219ba 100644 --- a/module/local/me_nokey.go +++ b/module/local/me_nokey.go @@ -32,8 +32,8 @@ func (l *LocalNoKey) Sign(msg []byte, hasher hash.Hasher) (crypto.Signature, err return nil, fmt.Errorf("no private key") } -func (l *LocalNoKey) NotMeFilter() flow.IdentityFilter { - return filter.Not(filter.HasNodeID(l.NodeID())) +func (l *LocalNoKey) NotMeFilter() flow.IdentityFilter[flow.Identity] { + return filter.Not(filter.HasNodeID[flow.Identity](l.NodeID())) } // SignFunc provides a signature oracle that given a message, a hasher, and a signing function, it diff --git a/module/mocks/network.go b/module/mocks/network.go deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/module/state_synchronization/requester/execution_data_requester_test.go b/module/state_synchronization/requester/execution_data_requester_test.go index 9a12ef9999b..bb7eb903fb3 100644 --- a/module/state_synchronization/requester/execution_data_requester_test.go +++ b/module/state_synchronization/requester/execution_data_requester_test.go @@ -780,7 +780,7 @@ func (m *mockSnapshot) Head() (*flow.Header, error) { // none of these are used in this test func (m *mockSnapshot) QuorumCertificate() (*flow.QuorumCertificate, error) { return nil, nil } -func (m *mockSnapshot) Identities(selector flow.IdentityFilter) (flow.IdentityList, error) { +func (m *mockSnapshot) Identities(selector flow.IdentityFilter[flow.Identity]) (flow.IdentityList, error) { return nil, nil } func (m *mockSnapshot) Identity(nodeID flow.Identifier) (*flow.Identity, error) { return nil, nil } diff --git a/network/p2p/cache/node_blocklist_wrapper.go b/network/p2p/cache/node_blocklist_wrapper.go index fab3d27b56c..c688d35a301 100644 --- a/network/p2p/cache/node_blocklist_wrapper.go +++ b/network/p2p/cache/node_blocklist_wrapper.go @@ -128,7 +128,7 @@ func (w *NodeDisallowListingWrapper) GetDisallowList() flow.IdentifierList { // protocol that pass the provided filter. Caution, this includes ejected nodes. // Please check the `Ejected` flag in the returned identities (or provide a // filter for removing ejected nodes). -func (w *NodeDisallowListingWrapper) Identities(filter flow.IdentityFilter) flow.IdentityList { +func (w *NodeDisallowListingWrapper) Identities(filter flow.IdentityFilter[flow.Identity]) flow.IdentityList { identities := w.identityProvider.Identities(filter) if len(identities) == 0 { return identities diff --git a/network/p2p/cache/protocol_state_provider.go b/network/p2p/cache/protocol_state_provider.go index cf3bba4e49a..6f7a4462b5b 100644 --- a/network/p2p/cache/protocol_state_provider.go +++ b/network/p2p/cache/protocol_state_provider.go @@ -141,7 +141,7 @@ func (p *ProtocolStateIDCache) update(blockID flow.Identifier) { // protocol that pass the provided filter. Caution, this includes ejected nodes. // Please check the `Ejected` flag in the identities (or provide a filter for // removing ejected nodes). -func (p *ProtocolStateIDCache) Identities(filter flow.IdentityFilter) flow.IdentityList { +func (p *ProtocolStateIDCache) Identities(filter flow.IdentityFilter[flow.Identity]) flow.IdentityList { p.mu.RLock() defer p.mu.RUnlock() return p.identities.Filter(filter) diff --git a/network/p2p/translator/identity_provider_translator.go b/network/p2p/translator/identity_provider_translator.go index c2bee0170a3..a052b5f28d5 100644 --- a/network/p2p/translator/identity_provider_translator.go +++ b/network/p2p/translator/identity_provider_translator.go @@ -37,7 +37,7 @@ func (t *IdentityProviderIDTranslator) GetFlowID(p peer.ID) (flow.Identifier, er } func (t *IdentityProviderIDTranslator) GetPeerID(n flow.Identifier) (peer.ID, error) { - ids := t.idProvider.Identities(filter.HasNodeID(n)) + ids := t.idProvider.Identities(filter.HasNodeID[flow.Identity](n)) if len(ids) == 0 { return "", fmt.Errorf("could not find identity with id %v", n.String()) } From 23cec189d9bcce61c1cee3a152924cdedfdb7078 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Tue, 3 Oct 2023 15:26:29 +0300 Subject: [PATCH 14/39] Fixed more compilation issues regarding generic identity --- Makefile | 2 +- cmd/bootstrap/cmd/clusters.go | 4 +- cmd/bootstrap/cmd/constraints.go | 2 +- cmd/bootstrap/cmd/finalize.go | 2 +- cmd/bootstrap/cmd/seal.go | 2 +- cmd/bootstrap/run/cluster_qc.go | 2 +- cmd/bootstrap/run/cluster_qc_test.go | 2 +- cmd/bootstrap/run/qc_test.go | 2 +- cmd/util/cmd/epochs/cmd/move_machine_acct.go | 2 +- .../committees/cluster_committee_test.go | 4 +- .../committees/consensus_committee_test.go | 2 +- consensus/hotstuff/mocks/dynamic_committee.go | 20 +-- consensus/hotstuff/mocks/packer.go | 8 +- consensus/hotstuff/mocks/replicas.go | 10 +- consensus/hotstuff/mocks/verifier.go | 8 +- .../timeout_processor_test.go | 4 +- .../hotstuff/validator/validator_test.go | 2 +- .../staking_vote_processor_test.go | 2 +- consensus/integration/epoch_test.go | 20 +-- consensus/integration/nodes_test.go | 14 +- engine/access/access_test.go | 4 +- engine/access/ingestion/engine_test.go | 6 +- engine/access/rpc/backend/backend_test.go | 12 +- .../access/rpc/backend/mock/communicator.go | 4 +- engine/collection/compliance/engine_test.go | 4 +- engine/collection/ingest/engine_test.go | 12 +- .../message_hub/message_hub_test.go | 4 +- engine/collection/pusher/engine_test.go | 10 +- .../collection/synchronization/engine_test.go | 6 +- .../test/cluster_switchover_test.go | 14 +- engine/common/provider/engine_test.go | 20 +-- engine/common/requester/engine_test.go | 20 +-- engine/common/synchronization/engine_test.go | 8 +- engine/consensus/ingestion/core_test.go | 2 +- .../consensus/message_hub/message_hub_test.go | 2 +- engine/execution/execution_test.go | 2 +- engine/execution/ingestion/engine_test.go | 12 +- engine/testutil/mocklocal/local.go | 4 +- engine/testutil/nodes.go | 10 +- .../assigner/blockconsumer/consumer_test.go | 2 +- engine/verification/utils/unittest/helper.go | 12 +- insecure/wintermute/attackOrchestrator.go | 6 +- .../wintermute/attackOrchestrator_test.go | 26 +-- integration/testnet/network.go | 8 +- integration/tests/collection/suite.go | 2 +- integration/tests/mvp/mvp_test.go | 4 +- model/convert/fixtures_test.go | 128 +++++--------- .../verification/chunkDataPackRequest_test.go | 2 +- module/dkg/broker_test.go | 4 +- module/epochs/qc_voter_test.go | 4 +- .../jobqueue/finalized_block_reader_test.go | 2 +- module/mock/dkg_controller_factory.go | 8 +- module/mock/identity_provider.go | 8 +- module/mocks/network.go | 167 ++++++++++++++++++ module/signature/signer_indices_test.go | 18 +- network/internal/testutils/testUtil.go | 2 +- network/mocknetwork/topology.go | 8 +- .../p2p/cache/node_blocklist_wrapper_test.go | 4 +- .../p2p/cache/protocol_state_provider_test.go | 2 +- network/test/epochtransition_test.go | 6 +- network/test/meshengine_test.go | 6 +- state/protocol/mock/cluster.go | 8 +- state/protocol/mock/dynamic_protocol_state.go | 8 +- state/protocol/mock/epoch.go | 10 +- state/protocol/mock/snapshot.go | 10 +- state/protocol/snapshot.go | 2 +- utils/unittest/cluster.go | 4 +- 67 files changed, 433 insertions(+), 308 deletions(-) create mode 100644 module/mocks/network.go diff --git a/Makefile b/Makefile index 6b54defe9eb..23a4623fbd4 100644 --- a/Makefile +++ b/Makefile @@ -151,7 +151,7 @@ generate-fvm-env-wrappers: generate-mocks: install-mock-generators mockery --name '(Connector|PingInfoProvider)' --dir=network/p2p --case=underscore --output="./network/mocknetwork" --outpkg="mocknetwork" mockgen -destination=storage/mocks/storage.go -package=mocks github.com/onflow/flow-go/storage Blocks,Headers,Payloads,Collections,Commits,Events,ServiceEvents,TransactionResults - mockgen -destination=module/mocks/network.go -package=mocks github.com/onflow/flow-go/module Local,Requester + #mockgen -destination=module/mocks/network.go -package=mocks github.com/onflow/flow-go/module Local,Requester mockgen -destination=network/mocknetwork/mock_network.go -package=mocknetwork github.com/onflow/flow-go/network EngineRegistry mockery --name='.*' --dir=integration/benchmark/mocksiface --case=underscore --output="integration/benchmark/mock" --outpkg="mock" mockery --name=ExecutionDataStore --dir=module/executiondatasync/execution_data --case=underscore --output="./module/executiondatasync/execution_data/mock" --outpkg="mock" diff --git a/cmd/bootstrap/cmd/clusters.go b/cmd/bootstrap/cmd/clusters.go index 441f573f429..30c91db9a9d 100644 --- a/cmd/bootstrap/cmd/clusters.go +++ b/cmd/bootstrap/cmd/clusters.go @@ -26,8 +26,8 @@ import ( // of succeeding the assignment by re-running the function without increasing the internal nodes ratio. func constructClusterAssignment(partnerNodes, internalNodes []model.NodeInfo) (flow.AssignmentList, flow.ClusterList, error) { - partners := model.ToIdentityList(partnerNodes).Filter(filter.HasRole(flow.RoleCollection)) - internals := model.ToIdentityList(internalNodes).Filter(filter.HasRole(flow.RoleCollection)) + partners := model.ToIdentityList(partnerNodes).Filter(filter.HasRole[flow.Identity](flow.RoleCollection)) + internals := model.ToIdentityList(internalNodes).Filter(filter.HasRole[flow.Identity](flow.RoleCollection)) nClusters := int(flagCollectionClusters) nCollectors := len(partners) + len(internals) diff --git a/cmd/bootstrap/cmd/constraints.go b/cmd/bootstrap/cmd/constraints.go index 2b0487b56cc..12973feb47e 100644 --- a/cmd/bootstrap/cmd/constraints.go +++ b/cmd/bootstrap/cmd/constraints.go @@ -12,7 +12,7 @@ import ( func ensureUniformNodeWeightsPerRole(allNodes flow.IdentityList) { // ensure all nodes of the same role have equal weight for _, role := range flow.Roles() { - withRole := allNodes.Filter(filter.HasRole(role)) + withRole := allNodes.Filter(filter.HasRole[flow.Identity](role)) // each role has at least one node so it's safe to access withRole[0] expectedWeight := withRole[0].Weight for _, node := range withRole { diff --git a/cmd/bootstrap/cmd/finalize.go b/cmd/bootstrap/cmd/finalize.go index 6f5507fdcfc..53d398ede7c 100644 --- a/cmd/bootstrap/cmd/finalize.go +++ b/cmd/bootstrap/cmd/finalize.go @@ -156,7 +156,7 @@ func finalize(cmd *cobra.Command, args []string) { log.Info().Msg("") // create flow.IdentityList representation of participant set - participants := model.ToIdentityList(stakingNodes).Sort(order.Canonical) + participants := model.ToIdentityList(stakingNodes).Sort(order.Canonical[flow.Identity]) log.Info().Msg("reading root block data") block := readRootBlock() diff --git a/cmd/bootstrap/cmd/seal.go b/cmd/bootstrap/cmd/seal.go index 1a34c394e13..c04e67fb733 100644 --- a/cmd/bootstrap/cmd/seal.go +++ b/cmd/bootstrap/cmd/seal.go @@ -39,7 +39,7 @@ func constructRootResultAndSeal( DKGPhase1FinalView: firstView + flagNumViewsInStakingAuction + flagNumViewsInDKGPhase - 1, DKGPhase2FinalView: firstView + flagNumViewsInStakingAuction + flagNumViewsInDKGPhase*2 - 1, DKGPhase3FinalView: firstView + flagNumViewsInStakingAuction + flagNumViewsInDKGPhase*3 - 1, - Participants: participants.Sort(order.Canonical), + Participants: participants.Sort(order.Canonical[flow.Identity]), Assignments: assignments, RandomSource: GenerateRandomSeed(flow.EpochSetupRandomSourceLength), } diff --git a/cmd/bootstrap/run/cluster_qc.go b/cmd/bootstrap/run/cluster_qc.go index 0d480cdd0b7..0b1b609e76b 100644 --- a/cmd/bootstrap/run/cluster_qc.go +++ b/cmd/bootstrap/run/cluster_qc.go @@ -20,7 +20,7 @@ import ( // GenerateClusterRootQC creates votes and generates a QC based on participant data func GenerateClusterRootQC(signers []bootstrap.NodeInfo, allCommitteeMembers flow.IdentityList, clusterBlock *cluster.Block) (*flow.QuorumCertificate, error) { - if !allCommitteeMembers.Sorted(order.Canonical) { + if !allCommitteeMembers.Sorted(order.Canonical[flow.Identity]) { return nil, fmt.Errorf("can't create root cluster QC: committee members are not sorted in canonical order") } clusterRootBlock := model.GenesisBlockFromFlow(clusterBlock.Header) diff --git a/cmd/bootstrap/run/cluster_qc_test.go b/cmd/bootstrap/run/cluster_qc_test.go index b422609f266..7533bf54662 100644 --- a/cmd/bootstrap/run/cluster_qc_test.go +++ b/cmd/bootstrap/run/cluster_qc_test.go @@ -33,7 +33,7 @@ func TestGenerateClusterRootQC(t *testing.T) { payload := cluster.EmptyPayload(flow.ZeroID) clusterBlock.SetPayload(payload) - orderedParticipants := model.ToIdentityList(participants).Sort(order.Canonical) + orderedParticipants := model.ToIdentityList(participants).Sort(order.Canonical[flow.Identity]) _, err := GenerateClusterRootQC(participants, orderedParticipants, &clusterBlock) require.NoError(t, err) } diff --git a/cmd/bootstrap/run/qc_test.go b/cmd/bootstrap/run/qc_test.go index 5deed36d1ed..d86222460d2 100644 --- a/cmd/bootstrap/run/qc_test.go +++ b/cmd/bootstrap/run/qc_test.go @@ -45,7 +45,7 @@ func TestGenerateRootQCWithSomeInvalidVotes(t *testing.T) { } func createSignerData(t *testing.T, n int) *ParticipantData { - identities := unittest.IdentityListFixture(n).Sort(order.Canonical) + identities := unittest.IdentityListFixture(n).Sort(order.Canonical[flow.Identity]) networkingKeys := unittest.NetworkingKeys(n) stakingKeys := unittest.StakingKeys(n) diff --git a/cmd/util/cmd/epochs/cmd/move_machine_acct.go b/cmd/util/cmd/epochs/cmd/move_machine_acct.go index 7f51867693b..f3443d63c5c 100644 --- a/cmd/util/cmd/epochs/cmd/move_machine_acct.go +++ b/cmd/util/cmd/epochs/cmd/move_machine_acct.go @@ -72,7 +72,7 @@ func moveMachineAcctRun(cmd *cobra.Command, args []string) { } // identities with machine accounts - machineAcctIdentities := identities.Filter(filter.HasRole(flow.RoleCollection, flow.RoleConsensus)) + machineAcctIdentities := identities.Filter(filter.HasRole[flow.Identity](flow.RoleCollection, flow.RoleConsensus)) machineAcctFiles, err := os.ReadDir(flagMachineAccountsSrcDir) if err != nil { diff --git a/consensus/hotstuff/committees/cluster_committee_test.go b/consensus/hotstuff/committees/cluster_committee_test.go index 51ccb43b010..f2af95558dd 100644 --- a/consensus/hotstuff/committees/cluster_committee_test.go +++ b/consensus/hotstuff/committees/cluster_committee_test.go @@ -49,11 +49,11 @@ func (suite *ClusterSuite) SetupTest() { suite.members = unittest.IdentityListFixture(5, unittest.WithRole(flow.RoleCollection)) suite.me = suite.members[0] counter := uint64(1) - suite.root = clusterstate.CanonicalRootBlock(counter, suite.members) + suite.root = clusterstate.CanonicalRootBlock(counter, suite.members.ToSkeleton()) suite.cluster.On("EpochCounter").Return(counter) suite.cluster.On("Index").Return(uint(1)) - suite.cluster.On("Members").Return(suite.members) + suite.cluster.On("Members").Return(suite.members.ToSkeleton()) suite.cluster.On("RootBlock").Return(suite.root) suite.epoch.On("Counter").Return(counter, nil) suite.epoch.On("RandomSource").Return(unittest.SeedFixture(prg.RandomSourceLength), nil) diff --git a/consensus/hotstuff/committees/consensus_committee_test.go b/consensus/hotstuff/committees/consensus_committee_test.go index afcf738f888..b7e5cc66315 100644 --- a/consensus/hotstuff/committees/consensus_committee_test.go +++ b/consensus/hotstuff/committees/consensus_committee_test.go @@ -679,7 +679,7 @@ func newMockEpoch(counter uint64, identities flow.IdentityList, firstView uint64 epoch := new(protocolmock.Epoch) epoch.On("Counter").Return(counter, nil) - epoch.On("InitialIdentities").Return(identities, nil) + epoch.On("InitialIdentities").Return(identities.ToSkeleton(), nil) epoch.On("FirstView").Return(firstView, nil) epoch.On("FinalView").Return(finalView, nil) if committed { diff --git a/consensus/hotstuff/mocks/dynamic_committee.go b/consensus/hotstuff/mocks/dynamic_committee.go index 580fb350ed2..e2869b3955d 100644 --- a/consensus/hotstuff/mocks/dynamic_committee.go +++ b/consensus/hotstuff/mocks/dynamic_committee.go @@ -41,19 +41,19 @@ func (_m *DynamicCommittee) DKG(view uint64) (hotstuff.DKG, error) { } // IdentitiesByBlock provides a mock function with given fields: blockID -func (_m *DynamicCommittee) IdentitiesByBlock(blockID flow.Identifier) (flow.IdentityList, error) { +func (_m *DynamicCommittee) IdentitiesByBlock(blockID flow.Identifier) (flow.GenericIdentityList[flow.Identity], error) { ret := _m.Called(blockID) - var r0 flow.IdentityList + var r0 flow.GenericIdentityList[flow.Identity] var r1 error - if rf, ok := ret.Get(0).(func(flow.Identifier) (flow.IdentityList, error)); ok { + if rf, ok := ret.Get(0).(func(flow.Identifier) (flow.GenericIdentityList[flow.Identity], error)); ok { return rf(blockID) } - if rf, ok := ret.Get(0).(func(flow.Identifier) flow.IdentityList); ok { + if rf, ok := ret.Get(0).(func(flow.Identifier) flow.GenericIdentityList[flow.Identity]); ok { r0 = rf(blockID) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(flow.IdentityList) + r0 = ret.Get(0).(flow.GenericIdentityList[flow.Identity]) } } @@ -67,19 +67,19 @@ func (_m *DynamicCommittee) IdentitiesByBlock(blockID flow.Identifier) (flow.Ide } // IdentitiesByEpoch provides a mock function with given fields: view -func (_m *DynamicCommittee) IdentitiesByEpoch(view uint64) (flow.IdentitySkeletonList, error) { +func (_m *DynamicCommittee) IdentitiesByEpoch(view uint64) (flow.GenericIdentityList[flow.IdentitySkeleton], error) { ret := _m.Called(view) - var r0 flow.IdentitySkeletonList + var r0 flow.GenericIdentityList[flow.IdentitySkeleton] var r1 error - if rf, ok := ret.Get(0).(func(uint64) (flow.IdentitySkeletonList, error)); ok { + if rf, ok := ret.Get(0).(func(uint64) (flow.GenericIdentityList[flow.IdentitySkeleton], error)); ok { return rf(view) } - if rf, ok := ret.Get(0).(func(uint64) flow.IdentitySkeletonList); ok { + if rf, ok := ret.Get(0).(func(uint64) flow.GenericIdentityList[flow.IdentitySkeleton]); ok { r0 = rf(view) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(flow.IdentitySkeletonList) + r0 = ret.Get(0).(flow.GenericIdentityList[flow.IdentitySkeleton]) } } diff --git a/consensus/hotstuff/mocks/packer.go b/consensus/hotstuff/mocks/packer.go index 8daf1ef76f1..da10b25b3b9 100644 --- a/consensus/hotstuff/mocks/packer.go +++ b/consensus/hotstuff/mocks/packer.go @@ -50,15 +50,15 @@ func (_m *Packer) Pack(view uint64, sig *hotstuff.BlockSignatureData) ([]byte, [ } // Unpack provides a mock function with given fields: signerIdentities, sigData -func (_m *Packer) Unpack(signerIdentities flow.IdentitySkeletonList, sigData []byte) (*hotstuff.BlockSignatureData, error) { +func (_m *Packer) Unpack(signerIdentities flow.GenericIdentityList[flow.IdentitySkeleton], sigData []byte) (*hotstuff.BlockSignatureData, error) { ret := _m.Called(signerIdentities, sigData) var r0 *hotstuff.BlockSignatureData var r1 error - if rf, ok := ret.Get(0).(func(flow.IdentitySkeletonList, []byte) (*hotstuff.BlockSignatureData, error)); ok { + if rf, ok := ret.Get(0).(func(flow.GenericIdentityList[flow.IdentitySkeleton], []byte) (*hotstuff.BlockSignatureData, error)); ok { return rf(signerIdentities, sigData) } - if rf, ok := ret.Get(0).(func(flow.IdentitySkeletonList, []byte) *hotstuff.BlockSignatureData); ok { + if rf, ok := ret.Get(0).(func(flow.GenericIdentityList[flow.IdentitySkeleton], []byte) *hotstuff.BlockSignatureData); ok { r0 = rf(signerIdentities, sigData) } else { if ret.Get(0) != nil { @@ -66,7 +66,7 @@ func (_m *Packer) Unpack(signerIdentities flow.IdentitySkeletonList, sigData []b } } - if rf, ok := ret.Get(1).(func(flow.IdentitySkeletonList, []byte) error); ok { + if rf, ok := ret.Get(1).(func(flow.GenericIdentityList[flow.IdentitySkeleton], []byte) error); ok { r1 = rf(signerIdentities, sigData) } else { r1 = ret.Error(1) diff --git a/consensus/hotstuff/mocks/replicas.go b/consensus/hotstuff/mocks/replicas.go index 702b8b5c90c..8ddba461152 100644 --- a/consensus/hotstuff/mocks/replicas.go +++ b/consensus/hotstuff/mocks/replicas.go @@ -41,19 +41,19 @@ func (_m *Replicas) DKG(view uint64) (hotstuff.DKG, error) { } // IdentitiesByEpoch provides a mock function with given fields: view -func (_m *Replicas) IdentitiesByEpoch(view uint64) (flow.IdentitySkeletonList, error) { +func (_m *Replicas) IdentitiesByEpoch(view uint64) (flow.GenericIdentityList[flow.IdentitySkeleton], error) { ret := _m.Called(view) - var r0 flow.IdentitySkeletonList + var r0 flow.GenericIdentityList[flow.IdentitySkeleton] var r1 error - if rf, ok := ret.Get(0).(func(uint64) (flow.IdentitySkeletonList, error)); ok { + if rf, ok := ret.Get(0).(func(uint64) (flow.GenericIdentityList[flow.IdentitySkeleton], error)); ok { return rf(view) } - if rf, ok := ret.Get(0).(func(uint64) flow.IdentitySkeletonList); ok { + if rf, ok := ret.Get(0).(func(uint64) flow.GenericIdentityList[flow.IdentitySkeleton]); ok { r0 = rf(view) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(flow.IdentitySkeletonList) + r0 = ret.Get(0).(flow.GenericIdentityList[flow.IdentitySkeleton]) } } diff --git a/consensus/hotstuff/mocks/verifier.go b/consensus/hotstuff/mocks/verifier.go index 73f40205ec6..03cd04a934d 100644 --- a/consensus/hotstuff/mocks/verifier.go +++ b/consensus/hotstuff/mocks/verifier.go @@ -14,11 +14,11 @@ type Verifier struct { } // VerifyQC provides a mock function with given fields: signers, sigData, view, blockID -func (_m *Verifier) VerifyQC(signers flow.IdentitySkeletonList, sigData []byte, view uint64, blockID flow.Identifier) error { +func (_m *Verifier) VerifyQC(signers flow.GenericIdentityList[flow.IdentitySkeleton], sigData []byte, view uint64, blockID flow.Identifier) error { ret := _m.Called(signers, sigData, view, blockID) var r0 error - if rf, ok := ret.Get(0).(func(flow.IdentitySkeletonList, []byte, uint64, flow.Identifier) error); ok { + if rf, ok := ret.Get(0).(func(flow.GenericIdentityList[flow.IdentitySkeleton], []byte, uint64, flow.Identifier) error); ok { r0 = rf(signers, sigData, view, blockID) } else { r0 = ret.Error(0) @@ -28,11 +28,11 @@ func (_m *Verifier) VerifyQC(signers flow.IdentitySkeletonList, sigData []byte, } // VerifyTC provides a mock function with given fields: signers, sigData, view, highQCViews -func (_m *Verifier) VerifyTC(signers flow.IdentitySkeletonList, sigData []byte, view uint64, highQCViews []uint64) error { +func (_m *Verifier) VerifyTC(signers flow.GenericIdentityList[flow.IdentitySkeleton], sigData []byte, view uint64, highQCViews []uint64) error { ret := _m.Called(signers, sigData, view, highQCViews) var r0 error - if rf, ok := ret.Get(0).(func(flow.IdentitySkeletonList, []byte, uint64, []uint64) error); ok { + if rf, ok := ret.Get(0).(func(flow.GenericIdentityList[flow.IdentitySkeleton], []byte, uint64, []uint64) error); ok { r0 = rf(signers, sigData, view, highQCViews) } else { r0 = ret.Error(0) diff --git a/consensus/hotstuff/timeoutcollector/timeout_processor_test.go b/consensus/hotstuff/timeoutcollector/timeout_processor_test.go index 7d79a1b868a..5591b8b4adc 100644 --- a/consensus/hotstuff/timeoutcollector/timeout_processor_test.go +++ b/consensus/hotstuff/timeoutcollector/timeout_processor_test.go @@ -55,7 +55,7 @@ func (s *TimeoutProcessorTestSuite) SetupTest() { s.validator = mocks.NewValidator(s.T()) s.sigAggregator = mocks.NewTimeoutSignatureAggregator(s.T()) s.notifier = mocks.NewTimeoutCollectorConsumer(s.T()) - s.participants = unittest.IdentityListFixture(11, unittest.WithWeight(s.sigWeight)).Sort(order.Canonical).ToSkeleton() + s.participants = unittest.IdentityListFixture(11, unittest.WithWeight(s.sigWeight)).Sort(order.Canonical[flow.Identity]).ToSkeleton() s.signer = s.participants[0] s.view = (uint64)(rand.Uint32() + 100) s.totalWeight = *atomic.NewUint64(0) @@ -471,7 +471,7 @@ func TestTimeoutProcessor_BuildVerifyTC(t *testing.T) { require.NoError(t, err) signers[identity.NodeID] = verification.NewStakingSigner(me) - }).Sort(order.Canonical) + }).Sort(order.Canonical[flow.Identity]) // utility function which generates a valid timeout for every signer createTimeouts := func(participants flow.IdentitySkeletonList, view uint64, newestQC *flow.QuorumCertificate, lastViewTC *flow.TimeoutCertificate) []*model.TimeoutObject { diff --git a/consensus/hotstuff/validator/validator_test.go b/consensus/hotstuff/validator/validator_test.go index 43ec36d5f19..f348bb57a9c 100644 --- a/consensus/hotstuff/validator/validator_test.go +++ b/consensus/hotstuff/validator/validator_test.go @@ -69,7 +69,7 @@ func (ps *ProposalSuite) SetupTest() { voterIDs, err := signature.DecodeSignerIndicesToIdentifiers(ps.participants.NodeIDs(), ps.block.QC.SignerIndices) require.NoError(ps.T(), err) - ps.voters = ps.participants.Filter(filter.HasNodeID(voterIDs...)).ToSkeleton() + ps.voters = ps.participants.Filter(filter.HasNodeID[flow.Identity](voterIDs...)).ToSkeleton() ps.proposal = &model.Proposal{Block: ps.block} ps.vote = ps.proposal.ProposerVote() ps.voter = ps.leader diff --git a/consensus/hotstuff/votecollector/staking_vote_processor_test.go b/consensus/hotstuff/votecollector/staking_vote_processor_test.go index 082aee074e0..26a2e42764a 100644 --- a/consensus/hotstuff/votecollector/staking_vote_processor_test.go +++ b/consensus/hotstuff/votecollector/staking_vote_processor_test.go @@ -265,7 +265,7 @@ func TestStakingVoteProcessorV2_BuildVerifyQC(t *testing.T) { require.NoError(t, err) signers[identity.NodeID] = verification.NewStakingSigner(me) - }).Sort(order.Canonical) + }).Sort(order.Canonical[flow.Identity]) leader := stakingSigners[0] block := helper.MakeBlock(helper.WithBlockView(view), helper.WithBlockProposer(leader.NodeID)) diff --git a/consensus/integration/epoch_test.go b/consensus/integration/epoch_test.go index 65bea994ca8..e874cf7589e 100644 --- a/consensus/integration/epoch_test.go +++ b/consensus/integration/epoch_test.go @@ -33,7 +33,7 @@ func TestUnweightedNode(t *testing.T) { // * same collection node from epoch 1, so cluster QCs are consistent // * 1 new consensus node, joining at epoch 2 // * random nodes with other roles - currentEpochCollectionNodes, err := rootSnapshot.Identities(filter.HasRole(flow.RoleCollection)) + currentEpochCollectionNodes, err := rootSnapshot.Identities(filter.HasRole[flow.Identity](flow.RoleCollection)) require.NoError(t, err) nextEpochIdentities := unittest.CompleteIdentitySet( append( @@ -121,7 +121,7 @@ func TestEpochTransition_IdentitiesOverlap(t *testing.T) { removedIdentity := privateNodeInfos[0].Identity() newIdentity := privateNodeInfos[3].Identity() nextEpochIdentities := append( - firstEpochIdentities.Filter(filter.Not(filter.HasNodeID(removedIdentity.NodeID))), + firstEpochIdentities.Filter(filter.Not(filter.HasNodeID[flow.Identity](removedIdentity.NodeID))), newIdentity, ) @@ -172,8 +172,8 @@ func TestEpochTransition_IdentitiesDisjoint(t *testing.T) { nextEpochParticipantData := createConsensusIdentities(t, 3) nextEpochIdentities := append( - firstEpochIdentities.Filter(filter.Not(filter.HasRole(flow.RoleConsensus))), // remove all consensus nodes - nextEpochParticipantData.Identities()..., // add new consensus nodes + firstEpochIdentities.Filter(filter.Not(filter.HasRole[flow.Identity](flow.RoleConsensus))), // remove all consensus nodes + nextEpochParticipantData.Identities()..., // add new consensus nodes ) rootSnapshot = withNextEpoch( @@ -221,7 +221,7 @@ func withNextEpoch( currEpoch := &encodableSnapshot.Epochs.Current // take pointer so assignments apply currentEpochIdentities := currEpoch.InitialIdentities - nextEpochIdentities = nextEpochIdentities.Sort(order.Canonical) + nextEpochIdentities = nextEpochIdentities.Sort(order.Canonical[flow.Identity]) currEpoch.FinalView = currEpoch.FirstView + curEpochViews - 1 // first epoch lasts curEpochViews encodableSnapshot.Epochs.Next = &inmem.EncodableEpoch{ @@ -229,10 +229,10 @@ func withNextEpoch( FirstView: currEpoch.FinalView + 1, FinalView: currEpoch.FinalView + 1 + 10000, RandomSource: unittest.SeedFixture(flow.EpochSetupRandomSourceLength), - InitialIdentities: nextEpochIdentities, + InitialIdentities: nextEpochIdentities.ToSkeleton(), // must include info corresponding to EpochCommit event, since we are // starting in committed phase - Clustering: unittest.ClusterList(1, nextEpochIdentities), + Clustering: unittest.ClusterList(1, nextEpochIdentities.ToSkeleton()), Clusters: currEpoch.Clusters, DKG: &inmem.EncodableDKG{ GroupKey: encodable.RandomBeaconPubKey{ @@ -257,9 +257,9 @@ func withNextEpoch( currentEpochIdentities, // and all the NEW identities in next epoch, with 0 weight nextEpochIdentities. - Filter(filter.Not(filter.In(currentEpochIdentities))). + Filter(filter.Not(filter.In[flow.Identity](currentEpochIdentities))). Map(mapfunc.WithWeight(0))..., - ).Sort(order.Canonical)) + ).Sort(order.Canonical[flow.Identity])) nextEpochIdentities = append( // all the next epoch identities @@ -268,7 +268,7 @@ func withNextEpoch( currentEpochIdentities. Filter(filter.Not(filter.In(nextEpochIdentities))). Map(mapfunc.WithWeight(0))..., - ).Sort(order.Canonical) + ).Sort(order.Canonical[flow.Identity]) // setup ID has changed, need to update it convertedEpochSetup, _ := protocol.ToEpochSetup(inmem.NewEpoch(*currEpoch)) diff --git a/consensus/integration/nodes_test.go b/consensus/integration/nodes_test.go index b069f812ca8..4124d3d6618 100644 --- a/consensus/integration/nodes_test.go +++ b/consensus/integration/nodes_test.go @@ -188,7 +188,7 @@ func buildEpochLookupList(epochs ...protocol.Epoch) []epochInfo { // The list of created nodes, the common network hub, and a function which starts // all the nodes together, is returned. func createNodes(t *testing.T, participants *ConsensusParticipants, rootSnapshot protocol.Snapshot, stopper *Stopper) (nodes []*Node, hub *Hub, runFor func(time.Duration)) { - consensus, err := rootSnapshot.Identities(filter.HasRole(flow.RoleConsensus)) + consensus, err := rootSnapshot.Identities(filter.HasRole[flow.Identity](flow.RoleConsensus)) require.NoError(t, err) epochViewLookup := buildEpochLookupList(rootSnapshot.Epochs().Current(), @@ -257,10 +257,10 @@ func createRootBlockData(participantData *run.ParticipantData) (*flow.Block, *fl // add other roles to create a complete identity list participants := unittest.CompleteIdentitySet(consensusParticipants...) - participants.Sort(order.Canonical) + participants.Sort(order.Canonical[flow.Identity]) dkgParticipantsKeys := make([]crypto.PublicKey, 0, len(consensusParticipants)) - for _, participant := range participants.Filter(filter.HasRole(flow.RoleConsensus)) { + for _, participant := range participants.Filter(filter.HasRole[flow.Identity](flow.RoleConsensus)) { dkgParticipantsKeys = append(dkgParticipantsKeys, participantData.Lookup[participant.NodeID].KeyShare) } @@ -289,7 +289,7 @@ func createRootBlockData(participantData *run.ParticipantData) (*flow.Block, *fl } func createPrivateNodeIdentities(n int) []bootstrap.NodeInfo { - consensus := unittest.IdentityListFixture(n, unittest.WithRole(flow.RoleConsensus)).Sort(order.Canonical) + consensus := unittest.IdentityListFixture(n, unittest.WithRole(flow.RoleConsensus)).Sort(order.Canonical[flow.Identity]) infos := make([]bootstrap.NodeInfo, 0, n) for _, node := range consensus { networkPrivKey := unittest.NetworkingPrivKeyFixture() @@ -472,7 +472,7 @@ func createNode( rootQC, err := rootSnapshot.QuorumCertificate() require.NoError(t, err) - // selector := filter.HasRole(flow.RoleConsensus) + // selector := filter.HasRole[flow.Identity](flow.RoleConsensus) committee, err := committees.NewConsensusCommittee(state, localID) require.NoError(t, err) protocolStateEvents.AddConsumer(committee) @@ -621,8 +621,8 @@ func createNode( require.NoError(t, err) identities, err := state.Final().Identities(filter.And( - filter.HasRole(flow.RoleConsensus), - filter.Not(filter.HasNodeID(me.NodeID())), + filter.HasRole[flow.Identity](flow.RoleConsensus), + filter.Not(filter.HasNodeID[flow.Identity](me.NodeID())), )) require.NoError(t, err) idProvider := id.NewFixedIdentifierProvider(identities.NodeIDs()) diff --git a/engine/access/access_test.go b/engine/access/access_test.go index d09835a4395..3fd32324ebc 100644 --- a/engine/access/access_test.go +++ b/engine/access/access_test.go @@ -271,7 +271,7 @@ func (suite *Suite) TestSendTransactionToRandomCollectionNode() { // create collection node cluster count := 2 - collNodes := unittest.IdentityListFixture(count, unittest.WithRole(flow.RoleCollection)) + collNodes := unittest.IdentityListFixture(count, unittest.WithRole(flow.RoleCollection)).ToSkeleton() assignments := unittest.ClusterAssignment(uint(count), collNodes) clusters, err := factory.NewClusterList(assignments, collNodes) suite.Require().Nil(err) @@ -1168,7 +1168,7 @@ func (suite *Suite) createChain() (*flow.Block, *flow.Collection) { collection := unittest.CollectionFixture(10) refBlockID := unittest.IdentifierFixture() // prepare cluster committee members - clusterCommittee := unittest.IdentityListFixture(32 * 4).Filter(filter.HasRole(flow.RoleCollection)) + clusterCommittee := unittest.IdentityListFixture(32 * 4).Filter(filter.HasRole[flow.Identity](flow.RoleCollection)) // guarantee signers must be cluster committee members, so that access will fetch collection from // the signers that are specified by guarantee.SignerIndices indices, err := signature.EncodeSignersToIndices(clusterCommittee.NodeIDs(), clusterCommittee.NodeIDs()) diff --git a/engine/access/ingestion/engine_test.go b/engine/access/ingestion/engine_test.go index c4d0fb72141..996e4d94343 100644 --- a/engine/access/ingestion/engine_test.go +++ b/engine/access/ingestion/engine_test.go @@ -139,7 +139,7 @@ func (suite *Suite) TestOnFinalizedBlock() { )) // prepare cluster committee members - clusterCommittee := unittest.IdentityListFixture(32 * 4).Filter(filter.HasRole(flow.RoleCollection)) + clusterCommittee := unittest.IdentityListFixture(32 * 4).Filter(filter.HasRole[flow.Identity](flow.RoleCollection)) refBlockID := unittest.IdentifierFixture() for _, guarantee := range block.Payload.Guarantees { guarantee.ReferenceBlockID = refBlockID @@ -326,7 +326,7 @@ func (suite *Suite) TestRequestMissingCollections() { heightMap := make(map[uint64]*flow.Block, blkCnt) // prepare cluster committee members - clusterCommittee := unittest.IdentityListFixture(32 * 4).Filter(filter.HasRole(flow.RoleCollection)) + clusterCommittee := unittest.IdentityListFixture(32 * 4).Filter(filter.HasRole[flow.Identity](flow.RoleCollection)) // generate the test blocks and collections var collIDs []flow.Identifier @@ -468,7 +468,7 @@ func (suite *Suite) TestUpdateLastFullBlockReceivedIndex() { collMap := make(map[flow.Identifier]*flow.LightCollection, blkCnt*collPerBlk) // prepare cluster committee members - clusterCommittee := unittest.IdentityListFixture(32 * 4).Filter(filter.HasRole(flow.RoleCollection)) + clusterCommittee := unittest.IdentityListFixture(32 * 4).Filter(filter.HasRole[flow.Identity](flow.RoleCollection)) refBlockID := unittest.IdentifierFixture() // generate the test blocks, cgs and collections diff --git a/engine/access/rpc/backend/backend_test.go b/engine/access/rpc/backend/backend_test.go index 6bedde98329..cee5194cb0c 100644 --- a/engine/access/rpc/backend/backend_test.go +++ b/engine/access/rpc/backend/backend_test.go @@ -1413,10 +1413,10 @@ func (suite *Suite) TestGetEventsForHeightRange() { func() error { return nil }, ) snapshot.On("Identities", mock.Anything).Return( - func(_ flow.IdentityFilter) flow.IdentityList { + func(_ flow.IdentityFilter[flow.Identity]) flow.IdentityList { return nodeIdentities }, - func(flow.IdentityFilter) error { return nil }, + func(flow.IdentityFilter[flow.Identity]) error { return nil }, ) // mock Headers to pull from Headers backend @@ -1859,11 +1859,11 @@ func (suite *Suite) TestExecutionNodesForBlockID() { func(id flow.Identifier) error { return nil }) suite.snapshot.On("Identities", mock.Anything).Return( - func(filter flow.IdentityFilter) flow.IdentityList { + func(filter flow.IdentityFilter[flow.Identity]) flow.IdentityList { // apply the filter passed in to the list of all the execution nodes return allExecutionNodes.Filter(filter) }, - func(flow.IdentityFilter) error { return nil }) + func(flow.IdentityFilter[flow.Identity]) error { return nil }) suite.state.On("Final").Return(suite.snapshot, nil).Maybe() testExecutionNodesForBlockID := func(preferredENs, fixedENs, expectedENs flow.IdentityList) { @@ -1886,7 +1886,7 @@ func (suite *Suite) TestExecutionNodesForBlockID() { execSelector, err := execNodeSelectorFactory.SelectNodes(allExecNodes) require.NoError(suite.T(), err) - actualList := flow.IdentityList{} + actualList := flow.IdentitySkeletonList{} for actual := execSelector.Next(); actual != nil; actual = execSelector.Next() { actualList = append(actualList, actual) } @@ -1914,7 +1914,7 @@ func (suite *Suite) TestExecutionNodesForBlockID() { execSelector, err := execNodeSelectorFactory.SelectNodes(allExecNodes) require.NoError(suite.T(), err) - actualList := flow.IdentityList{} + actualList := flow.IdentitySkeletonList{} for actual := execSelector.Next(); actual != nil; actual = execSelector.Next() { actualList = append(actualList, actual) } diff --git a/engine/access/rpc/backend/mock/communicator.go b/engine/access/rpc/backend/mock/communicator.go index ab7498ac8f8..5544bd426ef 100644 --- a/engine/access/rpc/backend/mock/communicator.go +++ b/engine/access/rpc/backend/mock/communicator.go @@ -13,11 +13,11 @@ type Communicator struct { } // CallAvailableNode provides a mock function with given fields: nodes, call, shouldTerminateOnError -func (_m *Communicator) CallAvailableNode(nodes flow.IdentityList, call func(*flow.Identity) error, shouldTerminateOnError func(*flow.Identity, error) bool) error { +func (_m *Communicator) CallAvailableNode(nodes flow.GenericIdentityList[flow.IdentitySkeleton], call func(*flow.IdentitySkeleton) error, shouldTerminateOnError func(*flow.IdentitySkeleton, error) bool) error { ret := _m.Called(nodes, call, shouldTerminateOnError) var r0 error - if rf, ok := ret.Get(0).(func(flow.IdentityList, func(*flow.Identity) error, func(*flow.Identity, error) bool) error); ok { + if rf, ok := ret.Get(0).(func(flow.GenericIdentityList[flow.IdentitySkeleton], func(*flow.IdentitySkeleton) error, func(*flow.IdentitySkeleton, error) bool) error); ok { r0 = rf(nodes, call, shouldTerminateOnError) } else { r0 = ret.Error(0) diff --git a/engine/collection/compliance/engine_test.go b/engine/collection/compliance/engine_test.go index 3c760ed05c3..495bc26f764 100644 --- a/engine/collection/compliance/engine_test.go +++ b/engine/collection/compliance/engine_test.go @@ -62,7 +62,7 @@ func (cs *EngineSuite) SetupTest() { cs.myID = cs.cluster[0].NodeID protoEpoch := &protocol.Epoch{} - clusters := flow.ClusterList{cs.cluster} + clusters := flow.ClusterList{cs.cluster.ToSkeleton()} protoEpoch.On("Clustering").Return(clusters, nil) protoQuery := &protocol.EpochQuery{} @@ -71,7 +71,7 @@ func (cs *EngineSuite) SetupTest() { protoSnapshot := &protocol.Snapshot{} protoSnapshot.On("Epochs").Return(protoQuery) protoSnapshot.On("Identities", mock.Anything).Return( - func(selector flow.IdentityFilter) flow.IdentityList { + func(selector flow.IdentityFilter[flow.Identity]) flow.IdentityList { return cs.cluster.Filter(selector) }, nil, diff --git a/engine/collection/ingest/engine_test.go b/engine/collection/ingest/engine_test.go index cdaa33eb7db..a1d5c60e7a2 100644 --- a/engine/collection/ingest/engine_test.go +++ b/engine/collection/ingest/engine_test.go @@ -384,7 +384,7 @@ func (suite *Suite) TestRoutingLocalClusterFromOtherNode() { suite.Require().True(ok) // another node will send us the transaction - sender := local.Filter(filter.Not(filter.HasNodeID(suite.me.NodeID())))[0] + sender := local.Filter(filter.Not(filter.HasNodeID[flow.Identity](suite.me.NodeID())))[0] // get a transaction that will be routed to local cluster tx := unittest.TransactionBodyFixture() @@ -475,8 +475,8 @@ func (suite *Suite) TestRouting_ClusterAssignmentRemoved() { // remove ourselves from the cluster assignment for epoch 2 withoutMe := suite.identities. - Filter(filter.Not(filter.HasNodeID(suite.me.NodeID()))). - Filter(filter.HasRole(flow.RoleCollection)) + Filter(filter.Not(filter.HasNodeID[flow.Identity](suite.me.NodeID()))). + Filter(filter.HasRole[flow.Identity](flow.RoleCollection)) epoch2Assignment := unittest.ClusterAssignment(suite.N_CLUSTERS, withoutMe) epoch2Clusters, err := factory.NewClusterList(epoch2Assignment, withoutMe) suite.Require().NoError(err) @@ -514,8 +514,8 @@ func (suite *Suite) TestRouting_ClusterAssignmentAdded() { // remove ourselves from the cluster assignment for epoch 2 withoutMe := suite.identities. - Filter(filter.Not(filter.HasNodeID(suite.me.NodeID()))). - Filter(filter.HasRole(flow.RoleCollection)) + Filter(filter.Not(filter.HasNodeID[flow.Identity](suite.me.NodeID()))). + Filter(filter.HasRole[flow.Identity](flow.RoleCollection)) epoch2Assignment := unittest.ClusterAssignment(suite.N_CLUSTERS, withoutMe) epoch2Clusters, err := factory.NewClusterList(epoch2Assignment, withoutMe) suite.Require().NoError(err) @@ -544,7 +544,7 @@ func (suite *Suite) TestRouting_ClusterAssignmentAdded() { // EPOCH 3: // include ourselves in cluster assignment - withMe := suite.identities.Filter(filter.HasRole(flow.RoleCollection)) + withMe := suite.identities.Filter(filter.HasRole[flow.Identity](flow.RoleCollection)) epoch3Assignment := unittest.ClusterAssignment(suite.N_CLUSTERS, withMe) epoch3Clusters, err := factory.NewClusterList(epoch3Assignment, withMe) suite.Require().NoError(err) diff --git a/engine/collection/message_hub/message_hub_test.go b/engine/collection/message_hub/message_hub_test.go index 7e60e4d7877..5195c1c8361 100644 --- a/engine/collection/message_hub/message_hub_test.go +++ b/engine/collection/message_hub/message_hub_test.go @@ -89,7 +89,7 @@ func (s *MessageHubSuite) SetupTest() { // set up proto state mock protoEpoch := &protocol.Epoch{} - clusters := flow.ClusterList{s.cluster} + clusters := flow.ClusterList{s.cluster.ToSkeleton()} protoEpoch.On("Clustering").Return(clusters, nil) protoQuery := &protocol.EpochQuery{} @@ -98,7 +98,7 @@ func (s *MessageHubSuite) SetupTest() { protoSnapshot := &protocol.Snapshot{} protoSnapshot.On("Epochs").Return(protoQuery) protoSnapshot.On("Identities", mock.Anything).Return( - func(selector flow.IdentityFilter) flow.IdentityList { + func(selector flow.IdentityFilter[flow.Identity]) flow.IdentityList { return s.cluster.Filter(selector) }, nil, diff --git a/engine/collection/pusher/engine_test.go b/engine/collection/pusher/engine_test.go index fec34346ad9..fde6d9696dc 100644 --- a/engine/collection/pusher/engine_test.go +++ b/engine/collection/pusher/engine_test.go @@ -40,13 +40,13 @@ func (suite *Suite) SetupTest() { // add some dummy identities so we have one of each role suite.identities = unittest.IdentityListFixture(5, unittest.WithAllRoles()) - me := suite.identities.Filter(filter.HasRole(flow.RoleCollection))[0] + me := suite.identities.Filter(filter.HasRole[flow.Identity](flow.RoleCollection))[0] suite.state = new(protocol.State) suite.snapshot = new(protocol.Snapshot) - suite.snapshot.On("Identities", mock.Anything).Return(func(filter flow.IdentityFilter) flow.IdentityList { + suite.snapshot.On("Identities", mock.Anything).Return(func(filter flow.IdentityFilter[flow.Identity]) flow.IdentityList { return suite.identities.Filter(filter) - }, func(filter flow.IdentityFilter) error { + }, func(filter flow.IdentityFilter[flow.Identity]) error { return nil }) suite.state.On("Final").Return(suite.snapshot) @@ -86,7 +86,7 @@ func (suite *Suite) TestSubmitCollectionGuarantee() { guarantee := unittest.CollectionGuaranteeFixture() // should submit the collection to consensus nodes - consensus := suite.identities.Filter(filter.HasRole(flow.RoleConsensus)) + consensus := suite.identities.Filter(filter.HasRole[flow.Identity](flow.RoleConsensus)) suite.conduit.On("Publish", guarantee, consensus[0].NodeID).Return(nil) msg := &messages.SubmitCollectionGuarantee{ @@ -104,7 +104,7 @@ func (suite *Suite) TestSubmitCollectionGuaranteeNonLocal() { guarantee := unittest.CollectionGuaranteeFixture() // send from a non-allowed role - sender := suite.identities.Filter(filter.HasRole(flow.RoleVerification))[0] + sender := suite.identities.Filter(filter.HasRole[flow.Identity](flow.RoleVerification))[0] msg := &messages.SubmitCollectionGuarantee{ Guarantee: *guarantee, diff --git a/engine/collection/synchronization/engine_test.go b/engine/collection/synchronization/engine_test.go index a637a9eedec..067119a196e 100644 --- a/engine/collection/synchronization/engine_test.go +++ b/engine/collection/synchronization/engine_test.go @@ -116,7 +116,7 @@ func (ss *SyncSuite) SetupTest() { nil, ) ss.snapshot.On("Identities", mock.Anything).Return( - func(selector flow.IdentityFilter) flow.IdentityList { + func(selector flow.IdentityFilter[flow.Identity]) flow.IdentityList { return ss.participants.Filter(selector) }, nil, @@ -159,7 +159,7 @@ func (ss *SyncSuite) SetupTest() { log := zerolog.New(io.Discard) metrics := metrics.NewNoopCollector() - e, err := New(log, metrics, ss.net, ss.me, ss.participants, ss.state, ss.blocks, ss.comp, ss.core) + e, err := New(log, metrics, ss.net, ss.me, ss.participants.ToSkeleton(), ss.state, ss.blocks, ss.comp, ss.core) require.NoError(ss.T(), err, "should pass engine initialization") ss.e = e @@ -456,7 +456,7 @@ func (ss *SyncSuite) TestOnBlockResponse() { func (ss *SyncSuite) TestPollHeight() { // check that we send to three nodes from our total list - others := ss.participants.Filter(filter.HasNodeID(ss.participants[1:].NodeIDs()...)) + others := ss.participants.Filter(filter.HasNodeID[flow.Identity](ss.participants[1:].NodeIDs()...)) ss.con.On("Multicast", mock.Anything, synccore.DefaultPollNodes, others[0].NodeID, others[1].NodeID).Return(nil).Run( func(args mock.Arguments) { req := args.Get(0).(*messages.SyncRequest) diff --git a/engine/collection/test/cluster_switchover_test.go b/engine/collection/test/cluster_switchover_test.go index ec582fadfc4..13cd6fe063d 100644 --- a/engine/collection/test/cluster_switchover_test.go +++ b/engine/collection/test/cluster_switchover_test.go @@ -60,8 +60,8 @@ func NewClusterSwitchoverTestCase(t *testing.T, conf ClusterSwitchoverTestConf) nodeInfos := unittest.PrivateNodeInfosFixture(int(conf.collectors), unittest.WithRole(flow.RoleCollection)) collectors := model.ToIdentityList(nodeInfos) tc.identities = unittest.CompleteIdentitySet(collectors...) - assignment := unittest.ClusterAssignment(tc.conf.clusters, collectors) - clusters, err := factory.NewClusterList(assignment, collectors) + assignment := unittest.ClusterAssignment(tc.conf.clusters, collectors.ToSkeleton()) + clusters, err := factory.NewClusterList(assignment, collectors.ToSkeleton()) require.NoError(t, err) rootClusterBlocks := run.GenerateRootClusterBlocks(1, clusters) rootClusterQCs := make([]flow.ClusterQCVoteData, len(rootClusterBlocks)) @@ -72,7 +72,7 @@ func NewClusterSwitchoverTestCase(t *testing.T, conf ClusterSwitchoverTestConf) signers = append(signers, identity) } } - signerIdentities := model.ToIdentityList(signers).Sort(order.Canonical) + signerIdentities := model.ToIdentityList(signers).Sort(order.Canonical[flow.Identity]) qc, err := run.GenerateClusterRootQC(signers, signerIdentities, rootClusterBlocks[i]) require.NoError(t, err) rootClusterQCs[i] = flow.ClusterQCVoteDataFromQC(&flow.QuorumCertificateWithSignerIDs{ @@ -92,7 +92,7 @@ func NewClusterSwitchoverTestCase(t *testing.T, conf ClusterSwitchoverTestConf) setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) commit := result.ServiceEvents[1].Event.(*flow.EpochCommit) - setup.Assignments = unittest.ClusterAssignment(tc.conf.clusters, tc.identities) + setup.Assignments = unittest.ClusterAssignment(tc.conf.clusters, tc.identities.ToSkeleton()) commit.ClusterQCs = rootClusterQCs seal.ResultID = result.ID() @@ -109,7 +109,7 @@ func NewClusterSwitchoverTestCase(t *testing.T, conf ClusterSwitchoverTestConf) consensus := testutil.GenericNode( tc.T(), tc.hub, - tc.identities.Filter(filter.HasRole(flow.RoleConsensus))[0], + tc.identities.Filter(filter.HasRole[flow.Identity](flow.RoleConsensus))[0], tc.root, ) tc.sn = new(mocknetwork.Engine) @@ -140,7 +140,7 @@ func NewClusterSwitchoverTestCase(t *testing.T, conf ClusterSwitchoverTestConf) } // generate root cluster block - rootClusterBlock := cluster.CanonicalRootBlock(commit.Counter, model.ToIdentityList(signers)) + rootClusterBlock := cluster.CanonicalRootBlock(commit.Counter, model.ToIdentityList(signers).ToSkeleton()) // generate cluster root qc qc, err := run.GenerateClusterRootQC(signers, model.ToIdentityList(signers), rootClusterBlock) require.NoError(t, err) @@ -360,7 +360,7 @@ func (tc *ClusterSwitchoverTestCase) SubmitTransactionToCluster( // cluster) and asserts that only transaction specified by ExpectTransaction are // included. func (tc *ClusterSwitchoverTestCase) CheckClusterState( - identity *flow.Identity, + identity *flow.IdentitySkeleton, clusterInfo protocol.Cluster, ) { node := tc.Collector(identity.NodeID) diff --git a/engine/common/provider/engine_test.go b/engine/common/provider/engine_test.go index 8af1c41a18f..58f57c82ef8 100644 --- a/engine/common/provider/engine_test.go +++ b/engine/common/provider/engine_test.go @@ -34,7 +34,7 @@ func TestOnEntityRequestFull(t *testing.T) { entities := make(map[flow.Identifier]flow.Entity) identities := unittest.IdentityListFixture(8) - selector := filter.HasNodeID(identities.NodeIDs()...) + selector := filter.HasNodeID[flow.Identity](identities.NodeIDs()...) originID := identities[0].NodeID coll1 := unittest.CollectionFixture(1) @@ -59,7 +59,7 @@ func TestOnEntityRequestFull(t *testing.T) { final := protocol.NewSnapshot(t) final.On("Identities", mock.Anything).Return( - func(selector flow.IdentityFilter) flow.IdentityList { + func(selector flow.IdentityFilter[flow.Identity]) flow.IdentityList { return identities.Filter(selector) }, nil, @@ -128,7 +128,7 @@ func TestOnEntityRequestPartial(t *testing.T) { entities := make(map[flow.Identifier]flow.Entity) identities := unittest.IdentityListFixture(8) - selector := filter.HasNodeID(identities.NodeIDs()...) + selector := filter.HasNodeID[flow.Identity](identities.NodeIDs()...) originID := identities[0].NodeID coll1 := unittest.CollectionFixture(1) @@ -153,7 +153,7 @@ func TestOnEntityRequestPartial(t *testing.T) { final := protocol.NewSnapshot(t) final.On("Identities", mock.Anything).Return( - func(selector flow.IdentityFilter) flow.IdentityList { + func(selector flow.IdentityFilter[flow.Identity]) flow.IdentityList { return identities.Filter(selector) }, nil, @@ -220,7 +220,7 @@ func TestOnEntityRequestDuplicates(t *testing.T) { entities := make(map[flow.Identifier]flow.Entity) identities := unittest.IdentityListFixture(8) - selector := filter.HasNodeID(identities.NodeIDs()...) + selector := filter.HasNodeID[flow.Identity](identities.NodeIDs()...) originID := identities[0].NodeID coll1 := unittest.CollectionFixture(1) @@ -241,7 +241,7 @@ func TestOnEntityRequestDuplicates(t *testing.T) { final := protocol.NewSnapshot(t) final.On("Identities", mock.Anything).Return( - func(selector flow.IdentityFilter) flow.IdentityList { + func(selector flow.IdentityFilter[flow.Identity]) flow.IdentityList { return identities.Filter(selector) }, nil, @@ -307,7 +307,7 @@ func TestOnEntityRequestEmpty(t *testing.T) { entities := make(map[flow.Identifier]flow.Entity) identities := unittest.IdentityListFixture(8) - selector := filter.HasNodeID(identities.NodeIDs()...) + selector := filter.HasNodeID[flow.Identity](identities.NodeIDs()...) originID := identities[0].NodeID coll1 := unittest.CollectionFixture(1) @@ -326,7 +326,7 @@ func TestOnEntityRequestEmpty(t *testing.T) { final := protocol.NewSnapshot(t) final.On("Identities", mock.Anything).Return( - func(selector flow.IdentityFilter) flow.IdentityList { + func(selector flow.IdentityFilter[flow.Identity]) flow.IdentityList { return identities.Filter(selector) }, nil, @@ -385,7 +385,7 @@ func TestOnEntityRequestInvalidOrigin(t *testing.T) { entities := make(map[flow.Identifier]flow.Entity) identities := unittest.IdentityListFixture(8) - selector := filter.HasNodeID(identities.NodeIDs()...) + selector := filter.HasNodeID[flow.Identity](identities.NodeIDs()...) originID := unittest.IdentifierFixture() coll1 := unittest.CollectionFixture(1) @@ -410,7 +410,7 @@ func TestOnEntityRequestInvalidOrigin(t *testing.T) { final := protocol.NewSnapshot(t) final.On("Identities", mock.Anything).Return( - func(selector flow.IdentityFilter) flow.IdentityList { + func(selector flow.IdentityFilter[flow.Identity]) flow.IdentityList { defer cancel() return identities.Filter(selector) }, diff --git a/engine/common/requester/engine_test.go b/engine/common/requester/engine_test.go index 553386c85d6..31e3accf448 100644 --- a/engine/common/requester/engine_test.go +++ b/engine/common/requester/engine_test.go @@ -54,7 +54,7 @@ func TestDispatchRequestVarious(t *testing.T) { final := &protocol.Snapshot{} final.On("Identities", mock.Anything).Return( - func(selector flow.IdentityFilter) flow.IdentityList { + func(selector flow.IdentityFilter[flow.Identity]) flow.IdentityList { return identities.Filter(selector) }, nil, @@ -134,7 +134,7 @@ func TestDispatchRequestVarious(t *testing.T) { con: con, items: items, requests: make(map[uint64]*messages.EntityRequest), - selector: filter.HasNodeID(targetID), + selector: filter.HasNodeID[flow.Identity](targetID), } dispatched, err := request.dispatchRequest() require.NoError(t, err) @@ -163,7 +163,7 @@ func TestDispatchRequestBatchSize(t *testing.T) { final := &protocol.Snapshot{} final.On("Identities", mock.Anything).Return( - func(selector flow.IdentityFilter) flow.IdentityList { + func(selector flow.IdentityFilter[flow.Identity]) flow.IdentityList { return identities.Filter(selector) }, nil, @@ -226,7 +226,7 @@ func TestOnEntityResponseValid(t *testing.T) { final := &protocol.Snapshot{} final.On("Identities", mock.Anything).Return( - func(selector flow.IdentityFilter) flow.IdentityList { + func(selector flow.IdentityFilter[flow.Identity]) flow.IdentityList { return identities.Filter(selector) }, nil, @@ -283,7 +283,7 @@ func TestOnEntityResponseValid(t *testing.T) { state: state, items: make(map[flow.Identifier]*Item), requests: make(map[uint64]*messages.EntityRequest), - selector: filter.HasNodeID(targetID), + selector: filter.HasNodeID[flow.Identity](targetID), create: func() flow.Entity { return &flow.Collection{} }, handle: func(flow.Identifier, flow.Entity) { if called.Inc() >= 2 { @@ -324,7 +324,7 @@ func TestOnEntityIntegrityCheck(t *testing.T) { final := &protocol.Snapshot{} final.On("Identities", mock.Anything).Return( - func(selector flow.IdentityFilter) flow.IdentityList { + func(selector flow.IdentityFilter[flow.Identity]) flow.IdentityList { return identities.Filter(selector) }, nil, @@ -370,7 +370,7 @@ func TestOnEntityIntegrityCheck(t *testing.T) { state: state, items: make(map[flow.Identifier]*Item), requests: make(map[uint64]*messages.EntityRequest), - selector: filter.HasNodeID(targetID), + selector: filter.HasNodeID[flow.Identity](targetID), create: func() flow.Entity { return &flow.Collection{} }, handle: func(flow.Identifier, flow.Entity) { close(called) }, } @@ -408,7 +408,7 @@ func TestOriginValidation(t *testing.T) { final := &protocol.Snapshot{} final.On("Identities", mock.Anything).Return( - func(selector flow.IdentityFilter) flow.IdentityList { + func(selector flow.IdentityFilter[flow.Identity]) flow.IdentityList { return identities.Filter(selector) }, nil, @@ -430,7 +430,7 @@ func TestOriginValidation(t *testing.T) { iwanted := &Item{ EntityID: wanted.ID(), LastRequested: now, - ExtraSelector: filter.HasNodeID(targetID), + ExtraSelector: filter.HasNodeID[flow.Identity](targetID), checkIntegrity: true, } @@ -458,7 +458,7 @@ func TestOriginValidation(t *testing.T) { me, state, "", - filter.HasNodeID(targetID), + filter.HasNodeID[flow.Identity](targetID), func() flow.Entity { return &flow.Collection{} }, ) assert.NoError(t, err) diff --git a/engine/common/synchronization/engine_test.go b/engine/common/synchronization/engine_test.go index 0c81d3fda5e..31da48149c3 100644 --- a/engine/common/synchronization/engine_test.go +++ b/engine/common/synchronization/engine_test.go @@ -123,7 +123,7 @@ func (ss *SyncSuite) SetupTest() { nil, ) ss.snapshot.On("Identities", mock.Anything).Return( - func(selector flow.IdentityFilter) flow.IdentityList { + func(selector flow.IdentityFilter[flow.Identity]) flow.IdentityList { return ss.participants.Filter(selector) }, nil, @@ -172,8 +172,8 @@ func (ss *SyncSuite) SetupTest() { e, err := New(log, metrics, ss.net, ss.me, ss.state, ss.blocks, ss.comp, ss.core, id.NewIdentityFilterIdentifierProvider( filter.And( - filter.HasRole(flow.RoleConsensus), - filter.Not(filter.HasNodeID(ss.me.NodeID())), + filter.HasRole[flow.Identity](flow.RoleConsensus), + filter.Not(filter.HasNodeID[flow.Identity](ss.me.NodeID())), ), idCache, ), @@ -698,7 +698,7 @@ func (ss *SyncSuite) TestOnBlockResponse() { func (ss *SyncSuite) TestPollHeight() { // check that we send to three nodes from our total list - others := ss.participants.Filter(filter.HasNodeID(ss.participants[1:].NodeIDs()...)) + others := ss.participants.Filter(filter.HasNodeID[flow.Identity](ss.participants[1:].NodeIDs()...)) ss.con.On("Multicast", mock.Anything, synccore.DefaultPollNodes, others[0].NodeID, others[1].NodeID).Return(nil).Run( func(args mock.Arguments) { req := args.Get(0).(*messages.SyncRequest) diff --git a/engine/consensus/ingestion/core_test.go b/engine/consensus/ingestion/core_test.go index 6167f6d55ee..336703676a7 100644 --- a/engine/consensus/ingestion/core_test.go +++ b/engine/consensus/ingestion/core_test.go @@ -108,7 +108,7 @@ func (suite *IngestionCoreSuite) SetupTest() { }, ) final.On("Identities", mock.Anything).Return( - func(selector flow.IdentityFilter) flow.IdentityList { + func(selector flow.IdentityFilter[flow.Identity]) flow.IdentityList { return suite.finalIdentities.Filter(selector) }, nil, diff --git a/engine/consensus/message_hub/message_hub_test.go b/engine/consensus/message_hub/message_hub_test.go index a68ce9eeb7a..c254087a77f 100644 --- a/engine/consensus/message_hub/message_hub_test.go +++ b/engine/consensus/message_hub/message_hub_test.go @@ -121,7 +121,7 @@ func (s *MessageHubSuite) SetupTest() { // set up protocol snapshot mock s.snapshot = &protocol.Snapshot{} s.snapshot.On("Identities", mock.Anything).Return( - func(filter flow.IdentityFilter) flow.IdentityList { + func(filter flow.IdentityFilter[flow.Identity]) flow.IdentityList { return s.participants.Filter(filter) }, nil, diff --git a/engine/execution/execution_test.go b/engine/execution/execution_test.go index f52796f85e5..47972940429 100644 --- a/engine/execution/execution_test.go +++ b/engine/execution/execution_test.go @@ -60,7 +60,7 @@ func TestExecutionFlow(t *testing.T) { unittest.WithKeys, ) - identities := unittest.CompleteIdentitySet(colID, conID, exeID, verID).Sort(order.Canonical) + identities := unittest.CompleteIdentitySet(colID, conID, exeID, verID).Sort(order.Canonical[flow.Identity]) // create execution node exeNode := testutil.ExecutionNode(t, hub, exeID, identities, 21, chainID) diff --git a/engine/execution/ingestion/engine_test.go b/engine/execution/ingestion/engine_test.go index 16f2f2713f6..0dbd0799abd 100644 --- a/engine/execution/ingestion/engine_test.go +++ b/engine/execution/ingestion/engine_test.go @@ -176,9 +176,9 @@ func runWithEngine(t *testing.T, f func(testingContext)) { }() identityListUnsorted := flow.IdentityList{myIdentity, collection1Identity, collection2Identity, collection3Identity} - identityList := identityListUnsorted.Sort(order.Canonical) + identityList := identityListUnsorted.Sort(order.Canonical[flow.Identity]) - snapshot.On("Identities", mock.Anything).Return(func(selector flow.IdentityFilter) flow.IdentityList { + snapshot.On("Identities", mock.Anything).Return(func(selector flow.IdentityFilter[flow.Identity]) flow.IdentityList { return identityList.Filter(selector) }, nil) @@ -373,7 +373,7 @@ func (ctx testingContext) mockSnapshot(header *flow.Header, identities flow.Iden func (ctx testingContext) mockSnapshotWithBlockID(blockID flow.Identifier, identities flow.IdentityList) { cluster := new(protocol.Cluster) // filter only collections as cluster members - cluster.On("Members").Return(identities.Filter(filter.HasRole(flow.RoleCollection))) + cluster.On("Members").Return(identities.Filter(filter.HasRole[flow.Identity](flow.RoleCollection))) epoch := new(protocol.Epoch) epoch.On("ClusterByChainID", mock.Anything).Return(cluster, nil) @@ -750,7 +750,7 @@ func TestBlocksArentExecutedMultipleTimes_multipleBlockEnqueue(t *testing.T) { }), ) - ctx.collectionRequester.EXPECT().EntityByID(gomock.Any(), gomock.Any()).DoAndReturn(func(_ flow.Identifier, _ flow.IdentityFilter) { + ctx.collectionRequester.EXPECT().EntityByID(gomock.Any(), gomock.Any()).DoAndReturn(func(_ flow.Identifier, _ flow.IdentityFilter[flow.Identity]) { // parallel run to avoid deadlock, ingestion engine is thread-safe go func() { err := ctx.engine.handleCollection(unittest.IdentifierFixture(), &collection) @@ -800,7 +800,7 @@ func TestBlocksArentExecutedMultipleTimes_collectionArrival(t *testing.T) { blockA := unittest.BlockHeaderFixture() blockB := unittest.ExecutableBlockFixtureWithParent(nil, blockA, unittest.StateCommitmentPointerFixture()) - collectionIdentities := ctx.identities.Filter(filter.HasRole(flow.RoleCollection)) + collectionIdentities := ctx.identities.Filter(filter.HasRole[flow.Identity](flow.RoleCollection)) colSigner := collectionIdentities[0].ID() // blocks are empty, so no state change is expected blockC := unittest.ExecutableBlockFixtureWithParent([][]flow.Identifier{{colSigner}}, blockB.Block.Header, blockB.StartState) @@ -897,7 +897,7 @@ func TestBlocksArentExecutedMultipleTimes_collectionArrival(t *testing.T) { }), ) - ctx.collectionRequester.EXPECT().EntityByID(gomock.Any(), gomock.Any()).DoAndReturn(func(_ flow.Identifier, _ flow.IdentityFilter) { + ctx.collectionRequester.EXPECT().EntityByID(gomock.Any(), gomock.Any()).DoAndReturn(func(_ flow.Identifier, _ flow.IdentityFilter[flow.Identity]) { // parallel run to avoid deadlock, ingestion engine is thread-safe go func() { // OnCollection is official callback for collection requester engine diff --git a/engine/testutil/mocklocal/local.go b/engine/testutil/mocklocal/local.go index a4a819797b5..213cb31368f 100644 --- a/engine/testutil/mocklocal/local.go +++ b/engine/testutil/mocklocal/local.go @@ -44,8 +44,8 @@ func (m *MockLocal) MockNodeID(id flow.Identifier) { m.id = id } -func (m *MockLocal) NotMeFilter() flow.IdentityFilter { - return filter.Not(filter.HasNodeID(m.id)) +func (m *MockLocal) NotMeFilter() flow.IdentityFilter[flow.Identity] { + return filter.Not(filter.HasNodeID[flow.Identity](m.id)) } func (m *MockLocal) SignFunc(data []byte, hasher hash.Hasher, f func(crypto.PrivateKey, []byte, hash.Hasher) (crypto.Signature, diff --git a/engine/testutil/nodes.go b/engine/testutil/nodes.go index 420135ec8a0..3af10b7c5b3 100644 --- a/engine/testutil/nodes.go +++ b/engine/testutil/nodes.go @@ -299,7 +299,7 @@ func CollectionNode(t *testing.T, hub *stub.Hub, identity bootstrap.NodeInfo, ro ingestionEngine, err := collectioningest.New(node.Log, node.Net, node.State, node.Metrics, node.Metrics, node.Metrics, node.Me, node.ChainID.Chain(), pools, collectioningest.DefaultConfig()) require.NoError(t, err) - selector := filter.HasRole(flow.RoleAccess, flow.RoleVerification) + selector := filter.HasRole[flow.Identity](flow.RoleAccess, flow.RoleVerification) retrieve := func(collID flow.Identifier) (flow.Entity, error) { coll, err := collections.ByID(collID) return coll, err @@ -622,7 +622,7 @@ func ExecutionNode(t *testing.T, hub *stub.Hub, identity *flow.Identity, identit requestEngine, err := requester.New( node.Log, node.Metrics, node.Net, node.Me, node.State, channels.RequestCollections, - filter.HasRole(flow.RoleCollection), + filter.HasRole[flow.Identity](flow.RoleCollection), func() flow.Entity { return &flow.Collection{} }, ) require.NoError(t, err) @@ -785,8 +785,8 @@ func ExecutionNode(t *testing.T, hub *stub.Hub, identity *flow.Identity, identit syncCore, id.NewIdentityFilterIdentifierProvider( filter.And( - filter.HasRole(flow.RoleConsensus), - filter.Not(filter.HasNodeID(node.Me.NodeID())), + filter.HasRole[flow.Identity](flow.RoleConsensus), + filter.Not(filter.HasNodeID[flow.Identity](node.Me.NodeID())), ), idCache, ), @@ -822,7 +822,7 @@ func getRoot(t *testing.T, node *testmock.GenericNode) (*flow.Header, *flow.Quor rootHead, err := node.State.Params().FinalizedRoot() require.NoError(t, err) - signers, err := node.State.AtHeight(0).Identities(filter.HasRole(flow.RoleConsensus)) + signers, err := node.State.AtHeight(0).Identities(filter.HasRole[flow.Identity](flow.RoleConsensus)) require.NoError(t, err) signerIDs := signers.NodeIDs() diff --git a/engine/verification/assigner/blockconsumer/consumer_test.go b/engine/verification/assigner/blockconsumer/consumer_test.go index 67ea6773194..a905d927253 100644 --- a/engine/verification/assigner/blockconsumer/consumer_test.go +++ b/engine/verification/assigner/blockconsumer/consumer_test.go @@ -148,7 +148,7 @@ func withConsumer( // hold any guarantees. root, err := s.State.Params().FinalizedRoot() require.NoError(t, err) - clusterCommittee := participants.Filter(filter.HasRole(flow.RoleCollection)) + clusterCommittee := participants.Filter(filter.HasRole[flow.Identity](flow.RoleCollection)) sources := unittest.RandomSourcesFixture(110) results := vertestutils.CompleteExecutionReceiptChainFixture(t, root, blockCount/2, sources, vertestutils.WithClusterCommittee(clusterCommittee)) blocks := vertestutils.ExtendStateWithFinalizedBlocks(t, results, s.State) diff --git a/engine/verification/utils/unittest/helper.go b/engine/verification/utils/unittest/helper.go index 62f26cd7f70..1f0d67ea361 100644 --- a/engine/verification/utils/unittest/helper.go +++ b/engine/verification/utils/unittest/helper.go @@ -75,7 +75,7 @@ func SetupChunkDataPackProvider(t *testing.T, originID, ok := args[1].(flow.Identifier) require.True(t, ok) // request should be dispatched by a verification node. - require.Contains(t, participants.Filter(filter.HasRole(flow.RoleVerification)).NodeIDs(), originID) + require.Contains(t, participants.Filter(filter.HasRole[flow.Identity](flow.RoleVerification)).NodeIDs(), originID) req, ok := args[2].(*messages.ChunkDataRequest) require.True(t, ok) @@ -479,8 +479,8 @@ func withConsumers(t *testing.T, // bootstraps system with one node of each role. s, verID, participants := bootstrapSystem(t, log, tracer, authorized) - exeID := participants.Filter(filter.HasRole(flow.RoleExecution))[0] - conID := participants.Filter(filter.HasRole(flow.RoleConsensus))[0] + exeID := participants.Filter(filter.HasRole[flow.Identity](flow.RoleExecution))[0] + conID := participants.Filter(filter.HasRole[flow.Identity](flow.RoleConsensus))[0] // generates a chain of blocks in the form of root <- R1 <- C1 <- R2 <- C2 <- ... where Rs are distinct reference // blocks (i.e., containing guarantees), and Cs are container blocks for their preceding reference block, // Container blocks only contain receipts of their preceding reference blocks. But they do not @@ -489,9 +489,9 @@ func withConsumers(t *testing.T, require.NoError(t, err) chainID := root.ChainID ops = append(ops, WithExecutorIDs( - participants.Filter(filter.HasRole(flow.RoleExecution)).NodeIDs()), func(builder *CompleteExecutionReceiptBuilder) { + participants.Filter(filter.HasRole[flow.Identity](flow.RoleExecution)).NodeIDs()), func(builder *CompleteExecutionReceiptBuilder) { // needed for the guarantees to have the correct chainID and signer indices - builder.clusterCommittee = participants.Filter(filter.HasRole(flow.RoleCollection)) + builder.clusterCommittee = participants.Filter(filter.HasRole[flow.Identity](flow.RoleCollection)) }) // random sources for all blocks: @@ -633,7 +633,7 @@ func bootstrapSystem( epochBuilder := unittest.NewEpochBuilder(t, stateFixture.State) epochBuilder. - UsingSetupOpts(unittest.WithParticipants(identities)). + UsingSetupOpts(unittest.WithParticipants(identities.ToSkeleton())). BuildEpoch() } diff --git a/insecure/wintermute/attackOrchestrator.go b/insecure/wintermute/attackOrchestrator.go index 40b7b60616e..9ff3bd3c1f5 100644 --- a/insecure/wintermute/attackOrchestrator.go +++ b/insecure/wintermute/attackOrchestrator.go @@ -217,8 +217,8 @@ func (o *Orchestrator) handleExecutionReceiptEvent(receiptEvent *insecure.Egress corruptedResult := o.corruptExecutionResult(receipt) corruptedExecutionIds := o.allNodeIds.Filter( - filter.And(filter.HasRole(flow.RoleExecution), - filter.HasNodeID(o.corruptedNodeIds...))).NodeIDs() + filter.And(filter.HasRole[flow.Identity](flow.RoleExecution), + filter.HasNodeID[flow.Identity](o.corruptedNodeIds...))).NodeIDs() // sends corrupted execution result to all corrupted execution nodes. for _, corruptedExecutionId := range corruptedExecutionIds { @@ -394,7 +394,7 @@ func (o *Orchestrator) replyWithAttestation(chunkDataPackRequestEvent *insecure. } // sends an attestation on behalf of verification node to all consensus nodes - consensusIds := o.allNodeIds.Filter(filter.HasRole(flow.RoleConsensus)).NodeIDs() + consensusIds := o.allNodeIds.Filter(filter.HasRole[flow.Identity](flow.RoleConsensus)).NodeIDs() err = o.network.SendEgress(&insecure.EgressEvent{ CorruptOriginId: chunkDataPackRequestEvent.CorruptOriginId, Channel: channels.PushApprovals, diff --git a/insecure/wintermute/attackOrchestrator_test.go b/insecure/wintermute/attackOrchestrator_test.go index 1c5d46f6899..bb1c70e5f73 100644 --- a/insecure/wintermute/attackOrchestrator_test.go +++ b/insecure/wintermute/attackOrchestrator_test.go @@ -27,13 +27,13 @@ func TestSingleExecutionReceipt(t *testing.T) { rootStateFixture, allIds, corruptedIds := bootstrapWintermuteFlowSystem(t) // identities of nodes who are expected targets of an execution receipt. - receiptTargetIds, err := rootStateFixture.State.Final().Identities(filter.HasRole(flow.RoleAccess, flow.RoleConsensus, flow.RoleVerification)) + receiptTargetIds, err := rootStateFixture.State.Final().Identities(filter.HasRole[flow.Identity](flow.RoleAccess, flow.RoleConsensus, flow.RoleVerification)) require.NoError(t, err) corruptedExecutionIds := flow.IdentifierList( allIds.Filter( - filter.And(filter.HasRole(flow.RoleExecution), - filter.HasNodeID(corruptedIds...)), + filter.And(filter.HasRole[flow.Identity](flow.RoleExecution), + filter.HasNodeID[flow.Identity](corruptedIds...)), ).NodeIDs()) eventMap, receipts := receiptsWithSameResultFixture(t, 1, corruptedExecutionIds[0:1], receiptTargetIds.NodeIDs()) @@ -140,11 +140,11 @@ func testConcurrentExecutionReceipts(t *testing.T, rootStateFixture, allIds, corruptedIds := bootstrapWintermuteFlowSystem(t) corruptedExecutionIds := flow.IdentifierList( allIds.Filter( - filter.And(filter.HasRole(flow.RoleExecution), - filter.HasNodeID(corruptedIds...)), + filter.And(filter.HasRole[flow.Identity](flow.RoleExecution), + filter.HasNodeID[flow.Identity](corruptedIds...)), ).NodeIDs()) // identities of nodes who are expected targets of an execution receipt. - receiptTargetIds, err := rootStateFixture.State.Final().Identities(filter.HasRole(flow.RoleAccess, flow.RoleConsensus, flow.RoleVerification)) + receiptTargetIds, err := rootStateFixture.State.Final().Identities(filter.HasRole[flow.Identity](flow.RoleAccess, flow.RoleConsensus, flow.RoleVerification)) require.NoError(t, err) var eventMap map[flow.Identifier]*insecure.EgressEvent @@ -270,8 +270,8 @@ func TestRespondingWithCorruptedAttestation(t *testing.T) { _, allIds, corruptedIds := bootstrapWintermuteFlowSystem(t) corruptedVerIds := flow.IdentifierList( allIds.Filter( - filter.And(filter.HasRole(flow.RoleVerification), - filter.HasNodeID(corruptedIds...)), + filter.And(filter.HasRole[flow.Identity](flow.RoleVerification), + filter.HasNodeID[flow.Identity](corruptedIds...)), ).NodeIDs()) wintermuteOrchestrator := NewOrchestrator(unittest.Logger(), corruptedIds, allIds) @@ -351,8 +351,8 @@ func TestPassingThroughChunkDataRequests(t *testing.T) { _, allIds, corruptedIds := bootstrapWintermuteFlowSystem(t) corruptedVerIds := flow.IdentifierList( allIds.Filter( - filter.And(filter.HasRole(flow.RoleVerification), - filter.HasNodeID(corruptedIds...)), + filter.And(filter.HasRole[flow.Identity](flow.RoleVerification), + filter.HasNodeID[flow.Identity](corruptedIds...)), ).NodeIDs()) wintermuteOrchestrator := NewOrchestrator(unittest.Logger(), corruptedIds, allIds) @@ -440,7 +440,7 @@ func TestPassingThroughChunkDataResponse_WithAttack(t *testing.T) { func testPassingThroughChunkDataResponse(t *testing.T, state *attackState) { totalChunks := 10 _, allIds, corruptedIds := bootstrapWintermuteFlowSystem(t) - verIds := flow.IdentifierList(allIds.Filter(filter.HasRole(flow.RoleVerification)).NodeIDs()) + verIds := flow.IdentifierList(allIds.Filter(filter.HasRole[flow.Identity](flow.RoleVerification)).NodeIDs()) wintermuteOrchestrator := NewOrchestrator(unittest.Logger(), corruptedIds, allIds) wintermuteOrchestrator.state = state @@ -510,8 +510,8 @@ func TestWintermuteChunkResponseForCorruptedChunks(t *testing.T) { _, allIds, corruptedIds := bootstrapWintermuteFlowSystem(t) honestVnIds := flow.IdentifierList( allIds.Filter(filter.And( - filter.HasRole(flow.RoleVerification), - filter.Not(filter.HasNodeID(corruptedIds...)))).NodeIDs()) + filter.HasRole[flow.Identity](flow.RoleVerification), + filter.Not(filter.HasNodeID[flow.Identity](corruptedIds...)))).NodeIDs()) wintermuteOrchestrator := NewOrchestrator(unittest.Logger(), corruptedIds, allIds) originalResult := unittest.ExecutionResultFixture() diff --git a/integration/testnet/network.go b/integration/testnet/network.go index 993ee49b1dc..ece67666cee 100644 --- a/integration/testnet/network.go +++ b/integration/testnet/network.go @@ -266,7 +266,7 @@ func (net *FlowNetwork) RemoveContainers() { // DropDBs resets the protocol state database for all containers in the network // matching the given filter. -func (net *FlowNetwork) DropDBs(filter flow.IdentityFilter) { +func (net *FlowNetwork) DropDBs(filter flow.IdentityFilter[flow.Identity]) { if net == nil || net.suite == nil { return } @@ -1305,8 +1305,8 @@ func runBeaconKG(confs []ContainerConfig) (dkgmod.DKGData, error) { func setupClusterGenesisBlockQCs(nClusters uint, epochCounter uint64, confs []ContainerConfig) ([]*cluster.Block, flow.AssignmentList, []*flow.QuorumCertificate, error) { participantsUnsorted := toParticipants(confs) - participants := participantsUnsorted.Sort(order.Canonical) - collectors := participants.Filter(filter.HasRole(flow.RoleCollection)) + participants := participantsUnsorted.Sort(order.Canonical[flow.Identity]) + collectors := participants.Filter(filter.HasRole[flow.Identity](flow.RoleCollection)) assignments := unittest.ClusterAssignment(nClusters, collectors) clusters, err := factory.NewClusterList(assignments, collectors) if err != nil { @@ -1338,7 +1338,7 @@ func setupClusterGenesisBlockQCs(nClusters uint, epochCounter uint64, confs []Co } // must order in canonical ordering otherwise decoding signer indices from cluster QC would fail - clusterCommittee := bootstrap.ToIdentityList(clusterNodeInfos).Sort(order.Canonical) + clusterCommittee := bootstrap.ToIdentityList(clusterNodeInfos).Sort(order.Canonical[flow.Identity]) qc, err := run.GenerateClusterRootQC(clusterNodeInfos, clusterCommittee, block) if err != nil { return nil, nil, nil, fmt.Errorf("fail to generate cluster root QC with clusterNodeInfos %v, %w", diff --git a/integration/tests/collection/suite.go b/integration/tests/collection/suite.go index 608f8cdf4fb..59dd62d3980 100644 --- a/integration/tests/collection/suite.go +++ b/integration/tests/collection/suite.go @@ -142,7 +142,7 @@ func (suite *CollectorSuite) Clusters() flow.ClusterList { setup, ok := result.ServiceEvents[0].Event.(*flow.EpochSetup) suite.Require().True(ok) - collectors := suite.net.Identities().Filter(filter.HasRole(flow.RoleCollection)) + collectors := suite.net.Identities().Filter(filter.HasRole[flow.Identity](flow.RoleCollection)) clusters, err := factory.NewClusterList(setup.Assignments, collectors) suite.Require().Nil(err) return clusters diff --git a/integration/tests/mvp/mvp_test.go b/integration/tests/mvp/mvp_test.go index cb6d6fb6c4f..09ec186efa0 100644 --- a/integration/tests/mvp/mvp_test.go +++ b/integration/tests/mvp/mvp_test.go @@ -90,14 +90,14 @@ func TestMVP_Bootstrap(t *testing.T) { flowNetwork.RemoveContainers() // pick 1 consensus node to restart with empty database and downloaded snapshot - cons := flowNetwork.Identities().Filter(filter.HasRole(flow.RoleConsensus)) + cons := flowNetwork.Identities().Filter(filter.HasRole[flow.Identity](flow.RoleConsensus)) random, err := rand.Uintn(uint(len(cons))) require.NoError(t, err) con1 := cons[random] t.Log("@@ booting from non-root state on consensus node ", con1.NodeID) - flowNetwork.DropDBs(filter.HasNodeID(con1.NodeID)) + flowNetwork.DropDBs(filter.HasNodeID[flow.Identity](con1.NodeID)) con1Container := flowNetwork.ContainerByID(con1.NodeID) con1Container.DropDB() con1Container.WriteRootSnapshot(snapshot) diff --git a/model/convert/fixtures_test.go b/model/convert/fixtures_test.go index 56c32ec93e3..74b2890202b 100644 --- a/model/convert/fixtures_test.go +++ b/model/convert/fixtures_test.go @@ -45,104 +45,62 @@ func EpochSetupFixture(chain flow.ChainID) (flow.Event, *flow.EpochSetup) { flow.MustHexStringToIdentifier("0000000000000000000000000000000000000000000000000000000000000004"), }, }, - Participants: flow.IdentityList{ + Participants: flow.IdentitySkeletonList{ { - IdentitySkeleton: flow.IdentitySkeleton{ - Role: flow.RoleCollection, - NodeID: flow.MustHexStringToIdentifier("0000000000000000000000000000000000000000000000000000000000000001"), - Address: "1.flow.com", - NetworkPubKey: unittest.MustDecodePublicKeyHex(crypto.ECDSAP256, "378dbf45d85c614feb10d8bd4f78f4b6ef8eec7d987b937e123255444657fb3da031f232a507e323df3a6f6b8f50339c51d188e80c0e7a92420945cc6ca893fc"), - StakingPubKey: unittest.MustDecodePublicKeyHex(crypto.BLSBLS12381, "af4aade26d76bb2ab15dcc89adcef82a51f6f04b3cb5f4555214b40ec89813c7a5f95776ea4fe449de48166d0bbc59b919b7eabebaac9614cf6f9461fac257765415f4d8ef1376a2365ec9960121888ea5383d88a140c24c29962b0a14e4e4e7"), - InitialWeight: 100, - }, - DynamicIdentity: flow.DynamicIdentity{ - Weight: 100, - Ejected: false, - }, + Role: flow.RoleCollection, + NodeID: flow.MustHexStringToIdentifier("0000000000000000000000000000000000000000000000000000000000000001"), + Address: "1.flow.com", + NetworkPubKey: unittest.MustDecodePublicKeyHex(crypto.ECDSAP256, "378dbf45d85c614feb10d8bd4f78f4b6ef8eec7d987b937e123255444657fb3da031f232a507e323df3a6f6b8f50339c51d188e80c0e7a92420945cc6ca893fc"), + StakingPubKey: unittest.MustDecodePublicKeyHex(crypto.BLSBLS12381, "af4aade26d76bb2ab15dcc89adcef82a51f6f04b3cb5f4555214b40ec89813c7a5f95776ea4fe449de48166d0bbc59b919b7eabebaac9614cf6f9461fac257765415f4d8ef1376a2365ec9960121888ea5383d88a140c24c29962b0a14e4e4e7"), + InitialWeight: 100, }, { - IdentitySkeleton: flow.IdentitySkeleton{ - Role: flow.RoleCollection, - NodeID: flow.MustHexStringToIdentifier("0000000000000000000000000000000000000000000000000000000000000002"), - Address: "2.flow.com", - NetworkPubKey: unittest.MustDecodePublicKeyHex(crypto.ECDSAP256, "378dbf45d85c614feb10d8bd4f78f4b6ef8eec7d987b937e123255444657fb3da031f232a507e323df3a6f6b8f50339c51d188e80c0e7a92420945cc6ca893fc"), - StakingPubKey: unittest.MustDecodePublicKeyHex(crypto.BLSBLS12381, "af4aade26d76bb2ab15dcc89adcef82a51f6f04b3cb5f4555214b40ec89813c7a5f95776ea4fe449de48166d0bbc59b919b7eabebaac9614cf6f9461fac257765415f4d8ef1376a2365ec9960121888ea5383d88a140c24c29962b0a14e4e4e7"), - InitialWeight: 100, - }, - DynamicIdentity: flow.DynamicIdentity{ - Weight: 100, - Ejected: false, - }, + Role: flow.RoleCollection, + NodeID: flow.MustHexStringToIdentifier("0000000000000000000000000000000000000000000000000000000000000002"), + Address: "2.flow.com", + NetworkPubKey: unittest.MustDecodePublicKeyHex(crypto.ECDSAP256, "378dbf45d85c614feb10d8bd4f78f4b6ef8eec7d987b937e123255444657fb3da031f232a507e323df3a6f6b8f50339c51d188e80c0e7a92420945cc6ca893fc"), + StakingPubKey: unittest.MustDecodePublicKeyHex(crypto.BLSBLS12381, "af4aade26d76bb2ab15dcc89adcef82a51f6f04b3cb5f4555214b40ec89813c7a5f95776ea4fe449de48166d0bbc59b919b7eabebaac9614cf6f9461fac257765415f4d8ef1376a2365ec9960121888ea5383d88a140c24c29962b0a14e4e4e7"), + InitialWeight: 100, }, { - IdentitySkeleton: flow.IdentitySkeleton{ - Role: flow.RoleCollection, - NodeID: flow.MustHexStringToIdentifier("0000000000000000000000000000000000000000000000000000000000000003"), - Address: "3.flow.com", - NetworkPubKey: unittest.MustDecodePublicKeyHex(crypto.ECDSAP256, "378dbf45d85c614feb10d8bd4f78f4b6ef8eec7d987b937e123255444657fb3da031f232a507e323df3a6f6b8f50339c51d188e80c0e7a92420945cc6ca893fc"), - StakingPubKey: unittest.MustDecodePublicKeyHex(crypto.BLSBLS12381, "af4aade26d76bb2ab15dcc89adcef82a51f6f04b3cb5f4555214b40ec89813c7a5f95776ea4fe449de48166d0bbc59b919b7eabebaac9614cf6f9461fac257765415f4d8ef1376a2365ec9960121888ea5383d88a140c24c29962b0a14e4e4e7"), - InitialWeight: 100, - }, - DynamicIdentity: flow.DynamicIdentity{ - Weight: 100, - Ejected: false, - }, + Role: flow.RoleCollection, + NodeID: flow.MustHexStringToIdentifier("0000000000000000000000000000000000000000000000000000000000000003"), + Address: "3.flow.com", + NetworkPubKey: unittest.MustDecodePublicKeyHex(crypto.ECDSAP256, "378dbf45d85c614feb10d8bd4f78f4b6ef8eec7d987b937e123255444657fb3da031f232a507e323df3a6f6b8f50339c51d188e80c0e7a92420945cc6ca893fc"), + StakingPubKey: unittest.MustDecodePublicKeyHex(crypto.BLSBLS12381, "af4aade26d76bb2ab15dcc89adcef82a51f6f04b3cb5f4555214b40ec89813c7a5f95776ea4fe449de48166d0bbc59b919b7eabebaac9614cf6f9461fac257765415f4d8ef1376a2365ec9960121888ea5383d88a140c24c29962b0a14e4e4e7"), + InitialWeight: 100, }, { - IdentitySkeleton: flow.IdentitySkeleton{ - Role: flow.RoleCollection, - NodeID: flow.MustHexStringToIdentifier("0000000000000000000000000000000000000000000000000000000000000004"), - Address: "4.flow.com", - NetworkPubKey: unittest.MustDecodePublicKeyHex(crypto.ECDSAP256, "378dbf45d85c614feb10d8bd4f78f4b6ef8eec7d987b937e123255444657fb3da031f232a507e323df3a6f6b8f50339c51d188e80c0e7a92420945cc6ca893fc"), - StakingPubKey: unittest.MustDecodePublicKeyHex(crypto.BLSBLS12381, "af4aade26d76bb2ab15dcc89adcef82a51f6f04b3cb5f4555214b40ec89813c7a5f95776ea4fe449de48166d0bbc59b919b7eabebaac9614cf6f9461fac257765415f4d8ef1376a2365ec9960121888ea5383d88a140c24c29962b0a14e4e4e7"), - InitialWeight: 100, - }, - DynamicIdentity: flow.DynamicIdentity{ - Weight: 100, - Ejected: false, - }, + Role: flow.RoleCollection, + NodeID: flow.MustHexStringToIdentifier("0000000000000000000000000000000000000000000000000000000000000004"), + Address: "4.flow.com", + NetworkPubKey: unittest.MustDecodePublicKeyHex(crypto.ECDSAP256, "378dbf45d85c614feb10d8bd4f78f4b6ef8eec7d987b937e123255444657fb3da031f232a507e323df3a6f6b8f50339c51d188e80c0e7a92420945cc6ca893fc"), + StakingPubKey: unittest.MustDecodePublicKeyHex(crypto.BLSBLS12381, "af4aade26d76bb2ab15dcc89adcef82a51f6f04b3cb5f4555214b40ec89813c7a5f95776ea4fe449de48166d0bbc59b919b7eabebaac9614cf6f9461fac257765415f4d8ef1376a2365ec9960121888ea5383d88a140c24c29962b0a14e4e4e7"), + InitialWeight: 100, }, { - IdentitySkeleton: flow.IdentitySkeleton{ - Role: flow.RoleConsensus, - NodeID: flow.MustHexStringToIdentifier("0000000000000000000000000000000000000000000000000000000000000011"), - Address: "11.flow.com", - NetworkPubKey: unittest.MustDecodePublicKeyHex(crypto.ECDSAP256, "cfdfe8e4362c8f79d11772cb7277ab16e5033a63e8dd5d34caf1b041b77e5b2d63c2072260949ccf8907486e4cfc733c8c42ca0e4e208f30470b0d950856cd47"), - StakingPubKey: unittest.MustDecodePublicKeyHex(crypto.BLSBLS12381, "8207559cd7136af378bba53a8f0196dee3849a3ab02897c1995c3e3f6ca0c4a776c3ae869d1ddbb473090054be2400ad06d7910aa2c5d1780220fdf3765a3c1764bce10c6fe66a5a2be51a422e878518bd750424bb56b8a0ecf0f8ad2057e83f"), - InitialWeight: 100, - }, - DynamicIdentity: flow.DynamicIdentity{ - Weight: 100, - Ejected: false, - }, + Role: flow.RoleConsensus, + NodeID: flow.MustHexStringToIdentifier("0000000000000000000000000000000000000000000000000000000000000011"), + Address: "11.flow.com", + NetworkPubKey: unittest.MustDecodePublicKeyHex(crypto.ECDSAP256, "cfdfe8e4362c8f79d11772cb7277ab16e5033a63e8dd5d34caf1b041b77e5b2d63c2072260949ccf8907486e4cfc733c8c42ca0e4e208f30470b0d950856cd47"), + StakingPubKey: unittest.MustDecodePublicKeyHex(crypto.BLSBLS12381, "8207559cd7136af378bba53a8f0196dee3849a3ab02897c1995c3e3f6ca0c4a776c3ae869d1ddbb473090054be2400ad06d7910aa2c5d1780220fdf3765a3c1764bce10c6fe66a5a2be51a422e878518bd750424bb56b8a0ecf0f8ad2057e83f"), + InitialWeight: 100, }, { - IdentitySkeleton: flow.IdentitySkeleton{ - Role: flow.RoleExecution, - NodeID: flow.MustHexStringToIdentifier("0000000000000000000000000000000000000000000000000000000000000021"), - Address: "21.flow.com", - NetworkPubKey: unittest.MustDecodePublicKeyHex(crypto.ECDSAP256, "d64318ba0dbf68f3788fc81c41d507c5822bf53154530673127c66f50fe4469ccf1a054a868a9f88506a8999f2386d86fcd2b901779718cba4fb53c2da258f9e"), - StakingPubKey: unittest.MustDecodePublicKeyHex(crypto.BLSBLS12381, "880b162b7ec138b36af401d07868cb08d25746d905395edbb4625bdf105d4bb2b2f4b0f4ae273a296a6efefa7ce9ccb914e39947ce0e83745125cab05d62516076ff0173ed472d3791ccef937597c9ea12381d76f547a092a4981d77ff3fba83"), - InitialWeight: 100, - }, - DynamicIdentity: flow.DynamicIdentity{ - Weight: 100, - Ejected: false, - }, + Role: flow.RoleExecution, + NodeID: flow.MustHexStringToIdentifier("0000000000000000000000000000000000000000000000000000000000000021"), + Address: "21.flow.com", + NetworkPubKey: unittest.MustDecodePublicKeyHex(crypto.ECDSAP256, "d64318ba0dbf68f3788fc81c41d507c5822bf53154530673127c66f50fe4469ccf1a054a868a9f88506a8999f2386d86fcd2b901779718cba4fb53c2da258f9e"), + StakingPubKey: unittest.MustDecodePublicKeyHex(crypto.BLSBLS12381, "880b162b7ec138b36af401d07868cb08d25746d905395edbb4625bdf105d4bb2b2f4b0f4ae273a296a6efefa7ce9ccb914e39947ce0e83745125cab05d62516076ff0173ed472d3791ccef937597c9ea12381d76f547a092a4981d77ff3fba83"), + InitialWeight: 100, }, { - IdentitySkeleton: flow.IdentitySkeleton{ - Role: flow.RoleVerification, - NodeID: flow.MustHexStringToIdentifier("0000000000000000000000000000000000000000000000000000000000000031"), - Address: "31.flow.com", - NetworkPubKey: unittest.MustDecodePublicKeyHex(crypto.ECDSAP256, "697241208dcc9142b6f53064adc8ff1c95760c68beb2ba083c1d005d40181fd7a1b113274e0163c053a3addd47cd528ec6a1f190cf465aac87c415feaae011ae"), - StakingPubKey: unittest.MustDecodePublicKeyHex(crypto.BLSBLS12381, "b1f97d0a06020eca97352e1adde72270ee713c7daf58da7e74bf72235321048b4841bdfc28227964bf18e371e266e32107d238358848bcc5d0977a0db4bda0b4c33d3874ff991e595e0f537c7b87b4ddce92038ebc7b295c9ea20a1492302aa7"), - InitialWeight: 100, - }, - DynamicIdentity: flow.DynamicIdentity{ - Weight: 100, - Ejected: false, - }, + Role: flow.RoleVerification, + NodeID: flow.MustHexStringToIdentifier("0000000000000000000000000000000000000000000000000000000000000031"), + Address: "31.flow.com", + NetworkPubKey: unittest.MustDecodePublicKeyHex(crypto.ECDSAP256, "697241208dcc9142b6f53064adc8ff1c95760c68beb2ba083c1d005d40181fd7a1b113274e0163c053a3addd47cd528ec6a1f190cf465aac87c415feaae011ae"), + StakingPubKey: unittest.MustDecodePublicKeyHex(crypto.BLSBLS12381, "b1f97d0a06020eca97352e1adde72270ee713c7daf58da7e74bf72235321048b4841bdfc28227964bf18e371e266e32107d238358848bcc5d0977a0db4bda0b4c33d3874ff991e595e0f537c7b87b4ddce92038ebc7b295c9ea20a1492302aa7"), + InitialWeight: 100, }, }, } diff --git a/model/verification/chunkDataPackRequest_test.go b/model/verification/chunkDataPackRequest_test.go index cb64b7ec502..9da53a2258c 100644 --- a/model/verification/chunkDataPackRequest_test.go +++ b/model/verification/chunkDataPackRequest_test.go @@ -56,7 +56,7 @@ func TestChunkDataPackRequestList_UniqueRequestInfo(t *testing.T) { return bytes.Compare(thisChunkIDReqInfo.Disagrees[p][:], thisChunkIDReqInfo.Disagrees[q][:]) < 0 }) - thisChunkIDReqInfo.Targets = thisChunkIDReqInfo.Targets.Sort(order.Canonical) + thisChunkIDReqInfo.Targets = thisChunkIDReqInfo.Targets.Sort(order.Canonical[flow.Identity]) require.Equal(t, thisChunkIDReqInfo.Agrees, thisReq1.Agrees.Union(thisReq2.Agrees)) require.Equal(t, thisChunkIDReqInfo.Disagrees, thisReq1.Disagrees.Union(thisReq2.Disagrees)) diff --git a/module/dkg/broker_test.go b/module/dkg/broker_test.go index 38c2e048a93..95d517e5e01 100644 --- a/module/dkg/broker_test.go +++ b/module/dkg/broker_test.go @@ -27,11 +27,11 @@ var ( dkgInstanceID = "flow-testnet-42" // dkg instance identifier ) -func initCommittee(n int) (identities flow.IdentityList, locals []module.Local) { +func initCommittee(n int) (identities flow.IdentitySkeletonList, locals []module.Local) { privateStakingKeys := unittest.StakingKeys(n) for i, key := range privateStakingKeys { id := unittest.IdentityFixture(unittest.WithStakingPubKey(key.PublicKey())) - identities = append(identities, id) + identities = append(identities, &id.IdentitySkeleton) local, _ := local.New(id.IdentitySkeleton, privateStakingKeys[i]) locals = append(locals, local) } diff --git a/module/epochs/qc_voter_test.go b/module/epochs/qc_voter_test.go index 47a54483200..b8a16641207 100644 --- a/module/epochs/qc_voter_test.go +++ b/module/epochs/qc_voter_test.go @@ -75,8 +75,8 @@ func (suite *Suite) SetupTest() { }) var err error - assignments := unittest.ClusterAssignment(2, suite.nodes) - suite.clustering, err = factory.NewClusterList(assignments, suite.nodes) + assignments := unittest.ClusterAssignment(2, suite.nodes.ToSkeleton()) + suite.clustering, err = factory.NewClusterList(assignments, suite.nodes.ToSkeleton()) suite.Require().NoError(err) suite.epoch.On("Counter").Return(suite.counter, nil) diff --git a/module/jobqueue/finalized_block_reader_test.go b/module/jobqueue/finalized_block_reader_test.go index 8349828d272..d08d80c1d94 100644 --- a/module/jobqueue/finalized_block_reader_test.go +++ b/module/jobqueue/finalized_block_reader_test.go @@ -64,7 +64,7 @@ func withReader( // hold any guarantees. root, err := s.State.Params().FinalizedRoot() require.NoError(t, err) - clusterCommittee := participants.Filter(filter.HasRole(flow.RoleCollection)) + clusterCommittee := participants.Filter(filter.HasRole[flow.Identity](flow.RoleCollection)) sources := unittest.RandomSourcesFixture(10) results := vertestutils.CompleteExecutionReceiptChainFixture(t, root, blockCount/2, sources, vertestutils.WithClusterCommittee(clusterCommittee)) blocks := vertestutils.ExtendStateWithFinalizedBlocks(t, results, s.State) diff --git a/module/mock/dkg_controller_factory.go b/module/mock/dkg_controller_factory.go index df4c29971de..b2253370f52 100644 --- a/module/mock/dkg_controller_factory.go +++ b/module/mock/dkg_controller_factory.go @@ -15,15 +15,15 @@ type DKGControllerFactory struct { } // Create provides a mock function with given fields: dkgInstanceID, participants, seed -func (_m *DKGControllerFactory) Create(dkgInstanceID string, participants flow.IdentityList, seed []byte) (module.DKGController, error) { +func (_m *DKGControllerFactory) Create(dkgInstanceID string, participants flow.GenericIdentityList[flow.IdentitySkeleton], seed []byte) (module.DKGController, error) { ret := _m.Called(dkgInstanceID, participants, seed) var r0 module.DKGController var r1 error - if rf, ok := ret.Get(0).(func(string, flow.IdentityList, []byte) (module.DKGController, error)); ok { + if rf, ok := ret.Get(0).(func(string, flow.GenericIdentityList[flow.IdentitySkeleton], []byte) (module.DKGController, error)); ok { return rf(dkgInstanceID, participants, seed) } - if rf, ok := ret.Get(0).(func(string, flow.IdentityList, []byte) module.DKGController); ok { + if rf, ok := ret.Get(0).(func(string, flow.GenericIdentityList[flow.IdentitySkeleton], []byte) module.DKGController); ok { r0 = rf(dkgInstanceID, participants, seed) } else { if ret.Get(0) != nil { @@ -31,7 +31,7 @@ func (_m *DKGControllerFactory) Create(dkgInstanceID string, participants flow.I } } - if rf, ok := ret.Get(1).(func(string, flow.IdentityList, []byte) error); ok { + if rf, ok := ret.Get(1).(func(string, flow.GenericIdentityList[flow.IdentitySkeleton], []byte) error); ok { r1 = rf(dkgInstanceID, participants, seed) } else { r1 = ret.Error(1) diff --git a/module/mock/identity_provider.go b/module/mock/identity_provider.go index 9a98a1b6c72..8711a8e8efb 100644 --- a/module/mock/identity_provider.go +++ b/module/mock/identity_provider.go @@ -67,15 +67,15 @@ func (_m *IdentityProvider) ByPeerID(_a0 peer.ID) (*flow.Identity, bool) { } // Identities provides a mock function with given fields: _a0 -func (_m *IdentityProvider) Identities(_a0 flow.IdentityFilter[flow.Identity]) flow.IdentityList { +func (_m *IdentityProvider) Identities(_a0 flow.IdentityFilter[flow.Identity]) flow.GenericIdentityList[flow.Identity] { ret := _m.Called(_a0) - var r0 flow.IdentityList - if rf, ok := ret.Get(0).(func(flow.IdentityFilter[flow.Identity]) flow.IdentityList); ok { + var r0 flow.GenericIdentityList[flow.Identity] + if rf, ok := ret.Get(0).(func(flow.IdentityFilter[flow.Identity]) flow.GenericIdentityList[flow.Identity]); ok { r0 = rf(_a0) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(flow.IdentityList) + r0 = ret.Get(0).(flow.GenericIdentityList[flow.Identity]) } } diff --git a/module/mocks/network.go b/module/mocks/network.go new file mode 100644 index 00000000000..799954db349 --- /dev/null +++ b/module/mocks/network.go @@ -0,0 +1,167 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/onflow/flow-go/module (interfaces: Local,Requester) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + gomock "github.com/golang/mock/gomock" + crypto "github.com/onflow/flow-go/crypto" + hash "github.com/onflow/flow-go/crypto/hash" + flow "github.com/onflow/flow-go/model/flow" + reflect "reflect" +) + +// MockLocal is a mock of Local interface. +type MockLocal struct { + ctrl *gomock.Controller + recorder *MockLocalMockRecorder +} + +// MockLocalMockRecorder is the mock recorder for MockLocal. +type MockLocalMockRecorder struct { + mock *MockLocal +} + +// NewMockLocal creates a new mock instance. +func NewMockLocal(ctrl *gomock.Controller) *MockLocal { + mock := &MockLocal{ctrl: ctrl} + mock.recorder = &MockLocalMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockLocal) EXPECT() *MockLocalMockRecorder { + return m.recorder +} + +// Address mocks base method. +func (m *MockLocal) Address() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Address") + ret0, _ := ret[0].(string) + return ret0 +} + +// Address indicates an expected call of Address. +func (mr *MockLocalMockRecorder) Address() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Address", reflect.TypeOf((*MockLocal)(nil).Address)) +} + +// NodeID mocks base method. +func (m *MockLocal) NodeID() flow.Identifier { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "NodeID") + ret0, _ := ret[0].(flow.Identifier) + return ret0 +} + +// NodeID indicates an expected call of NodeID. +func (mr *MockLocalMockRecorder) NodeID() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NodeID", reflect.TypeOf((*MockLocal)(nil).NodeID)) +} + +// NotMeFilter mocks base method. +func (m *MockLocal) NotMeFilter() flow.IdentityFilter[flow.Identity] { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "NotMeFilter") + ret0, _ := ret[0].(flow.IdentityFilter[flow.Identity]) + return ret0 +} + +// NotMeFilter indicates an expected call of NotMeFilter. +func (mr *MockLocalMockRecorder) NotMeFilter() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NotMeFilter", reflect.TypeOf((*MockLocal)(nil).NotMeFilter)) +} + +// Sign mocks base method. +func (m *MockLocal) Sign(arg0 []byte, arg1 hash.Hasher) (crypto.Signature, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Sign", arg0, arg1) + ret0, _ := ret[0].(crypto.Signature) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Sign indicates an expected call of Sign. +func (mr *MockLocalMockRecorder) Sign(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Sign", reflect.TypeOf((*MockLocal)(nil).Sign), arg0, arg1) +} + +// SignFunc mocks base method. +func (m *MockLocal) SignFunc(arg0 []byte, arg1 hash.Hasher, arg2 func(crypto.PrivateKey, []byte, hash.Hasher) (crypto.Signature, error)) (crypto.Signature, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SignFunc", arg0, arg1, arg2) + ret0, _ := ret[0].(crypto.Signature) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// SignFunc indicates an expected call of SignFunc. +func (mr *MockLocalMockRecorder) SignFunc(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SignFunc", reflect.TypeOf((*MockLocal)(nil).SignFunc), arg0, arg1, arg2) +} + +// MockRequester is a mock of Requester interface. +type MockRequester struct { + ctrl *gomock.Controller + recorder *MockRequesterMockRecorder +} + +// MockRequesterMockRecorder is the mock recorder for MockRequester. +type MockRequesterMockRecorder struct { + mock *MockRequester +} + +// NewMockRequester creates a new mock instance. +func NewMockRequester(ctrl *gomock.Controller) *MockRequester { + mock := &MockRequester{ctrl: ctrl} + mock.recorder = &MockRequesterMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockRequester) EXPECT() *MockRequesterMockRecorder { + return m.recorder +} + +// EntityByID mocks base method. +func (m *MockRequester) EntityByID(arg0 flow.Identifier, arg1 flow.IdentityFilter[flow.Identity]) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "EntityByID", arg0, arg1) +} + +// EntityByID indicates an expected call of EntityByID. +func (mr *MockRequesterMockRecorder) EntityByID(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EntityByID", reflect.TypeOf((*MockRequester)(nil).EntityByID), arg0, arg1) +} + +// Force mocks base method. +func (m *MockRequester) Force() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Force") +} + +// Force indicates an expected call of Force. +func (mr *MockRequesterMockRecorder) Force() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Force", reflect.TypeOf((*MockRequester)(nil).Force)) +} + +// Query mocks base method. +func (m *MockRequester) Query(arg0 flow.Identifier, arg1 flow.IdentityFilter[flow.Identity]) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Query", arg0, arg1) +} + +// Query indicates an expected call of Query. +func (mr *MockRequesterMockRecorder) Query(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Query", reflect.TypeOf((*MockRequester)(nil).Query), arg0, arg1) +} diff --git a/module/signature/signer_indices_test.go b/module/signature/signer_indices_test.go index 3357d092cca..5dae391e5c9 100644 --- a/module/signature/signer_indices_test.go +++ b/module/signature/signer_indices_test.go @@ -111,7 +111,7 @@ func Test_EncodeSignerToIndicesAndSigType(t *testing.T) { numRandomBeaconSigners := rapid.IntRange(0, committeeSize-numStakingSigners).Draw(t, "numRandomBeaconSigners").(int) // create committee - committeeIdentities := unittest.IdentityListFixture(committeeSize, unittest.WithRole(flow.RoleConsensus)).Sort(order.Canonical) + committeeIdentities := unittest.IdentityListFixture(committeeSize, unittest.WithRole(flow.RoleConsensus)).Sort(order.Canonical[flow.Identity]) committee := committeeIdentities.NodeIDs() stakingSigners, beaconSigners := sampleSigners(t, committee, numStakingSigners, numRandomBeaconSigners) @@ -127,7 +127,7 @@ func Test_EncodeSignerToIndicesAndSigType(t *testing.T) { correctEncoding(t, signerIndices, committee, unorderedSigners) // check sigTypes - canSigners := committeeIdentities.Filter(filter.HasNodeID(unorderedSigners...)).NodeIDs() // generates list of signer IDs in canonical order + canSigners := committeeIdentities.Filter(filter.HasNodeID[flow.Identity](unorderedSigners...)).NodeIDs() // generates list of signer IDs in canonical order correctEncoding(t, sigTypes, canSigners, beaconSigners) }) } @@ -150,7 +150,7 @@ func Test_DecodeSigTypeToStakingAndBeaconSigners(t *testing.T) { // create committee committeeIdentities := unittest.IdentityListFixture(committeeSize, unittest.WithRole(flow.RoleConsensus)). - Sort(order.Canonical) + Sort(order.Canonical[flow.Identity]) committee := committeeIdentities.NodeIDs() stakingSigners, beaconSigners := sampleSigners(t, committee, numStakingSigners, numRandomBeaconSigners) @@ -167,19 +167,19 @@ func Test_DecodeSigTypeToStakingAndBeaconSigners(t *testing.T) { // verify; note that there is a slightly different convention between Filter and the decoding logic: // Filter returns nil for an empty list, while the decoding logic returns an instance of an empty slice sigIdentities := committeeIdentities.Filter( - filter.Or(filter.HasNodeID(stakingSigners...), filter.HasNodeID(beaconSigners...))).ToSkeleton() // signer identities in canonical order + filter.Or(filter.HasNodeID[flow.Identity](stakingSigners...), filter.HasNodeID[flow.Identity](beaconSigners...))).ToSkeleton() // signer identities in canonical order if len(stakingSigners)+len(decBeaconSigners) > 0 { require.Equal(t, sigIdentities, decSignerIdentites) } if len(stakingSigners) == 0 { require.Empty(t, decStakingSigners) } else { - require.Equal(t, committeeIdentities.Filter(filter.HasNodeID(stakingSigners...)).ToSkeleton(), decStakingSigners) + require.Equal(t, committeeIdentities.Filter(filter.HasNodeID[flow.Identity](stakingSigners...)).ToSkeleton(), decStakingSigners) } if len(decBeaconSigners) == 0 { require.Empty(t, decBeaconSigners) } else { - require.Equal(t, committeeIdentities.Filter(filter.HasNodeID(beaconSigners...)).ToSkeleton(), decBeaconSigners) + require.Equal(t, committeeIdentities.Filter(filter.HasNodeID[flow.Identity](beaconSigners...)).ToSkeleton(), decBeaconSigners) } }) } @@ -276,7 +276,7 @@ func Test_EncodeSignersToIndices(t *testing.T) { numSigners := rapid.IntRange(0, committeeSize).Draw(t, "numSigners").(int) // create committee - identities := unittest.IdentityListFixture(committeeSize, unittest.WithRole(flow.RoleConsensus)).Sort(order.Canonical) + identities := unittest.IdentityListFixture(committeeSize, unittest.WithRole(flow.RoleConsensus)).Sort(order.Canonical[flow.Identity]) committee := identities.NodeIDs() signers, err := committee.Sample(uint(numSigners)) require.NoError(t, err) @@ -306,7 +306,7 @@ func Test_DecodeSignerIndicesToIdentifiers(t *testing.T) { numSigners := rapid.IntRange(0, committeeSize).Draw(t, "numSigners").(int) // create committee - identities := unittest.IdentityListFixture(committeeSize, unittest.WithRole(flow.RoleConsensus)).Sort(order.Canonical) + identities := unittest.IdentityListFixture(committeeSize, unittest.WithRole(flow.RoleConsensus)).Sort(order.Canonical[flow.Identity]) committee := identities.NodeIDs() signers, err := committee.Sample(uint(numSigners)) require.NoError(t, err) @@ -342,7 +342,7 @@ func Test_DecodeSignerIndicesToIdentities(t *testing.T) { numSigners := rapid.IntRange(0, committeeSize).Draw(t, "numSigners").(int) // create committee - identities := unittest.IdentityListFixture(committeeSize, unittest.WithRole(flow.RoleConsensus)).Sort(order.Canonical) + identities := unittest.IdentityListFixture(committeeSize, unittest.WithRole(flow.RoleConsensus)).Sort(order.Canonical[flow.Identity]) fullSigners, err := identities.Sample(uint(numSigners)) require.NoError(t, err) signers := fullSigners.ToSkeleton() diff --git a/network/internal/testutils/testUtil.go b/network/internal/testutils/testUtil.go index c817fba98f8..cd45d1af38a 100644 --- a/network/internal/testutils/testUtil.go +++ b/network/internal/testutils/testUtil.go @@ -183,7 +183,7 @@ func NetworkConfigFixture( me := mock.NewLocal(t) me.On("NodeID").Return(myId.NodeID).Maybe() - me.On("NotMeFilter").Return(filter.Not(filter.HasNodeID(me.NodeID()))).Maybe() + me.On("NotMeFilter").Return(filter.Not(filter.HasNodeID[flow.Identity](me.NodeID()))).Maybe() me.On("Address").Return(myId.Address).Maybe() defaultFlowConfig, err := config.DefaultConfig() diff --git a/network/mocknetwork/topology.go b/network/mocknetwork/topology.go index 04a0dec6f17..d5513bde846 100644 --- a/network/mocknetwork/topology.go +++ b/network/mocknetwork/topology.go @@ -13,15 +13,15 @@ type Topology struct { } // Fanout provides a mock function with given fields: ids -func (_m *Topology) Fanout(ids flow.IdentityList) flow.IdentityList { +func (_m *Topology) Fanout(ids flow.GenericIdentityList[flow.Identity]) flow.GenericIdentityList[flow.Identity] { ret := _m.Called(ids) - var r0 flow.IdentityList - if rf, ok := ret.Get(0).(func(flow.IdentityList) flow.IdentityList); ok { + var r0 flow.GenericIdentityList[flow.Identity] + if rf, ok := ret.Get(0).(func(flow.GenericIdentityList[flow.Identity]) flow.GenericIdentityList[flow.Identity]); ok { r0 = rf(ids) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(flow.IdentityList) + r0 = ret.Get(0).(flow.GenericIdentityList[flow.Identity]) } } diff --git a/network/p2p/cache/node_blocklist_wrapper_test.go b/network/p2p/cache/node_blocklist_wrapper_test.go index bdeb50ffd27..055892fe1fd 100644 --- a/network/p2p/cache/node_blocklist_wrapper_test.go +++ b/network/p2p/cache/node_blocklist_wrapper_test.go @@ -72,7 +72,7 @@ func (s *NodeDisallowListWrapperTestSuite) TestHonestNode() { f := filter.In(identities[3:4]) expectedFilteredIdentities := identities.Filter(f) s.provider.On("Identities", mock.Anything).Return( - func(filter flow.IdentityFilter) flow.IdentityList { + func(filter flow.IdentityFilter[flow.Identity]) flow.IdentityList { return identities.Filter(filter) }, nil, @@ -149,7 +149,7 @@ func (s *NodeDisallowListWrapperTestSuite) TestDisallowListNode() { s.provider.On("Identities", mock.Anything).Return(combinedIdentities) - noFilter := filter.Not(filter.In(nil)) + noFilter := filter.Not(filter.In[flow.Identity](nil)) identities := s.wrapper.Identities(noFilter) require.Equal(s.T(), numIdentities, len(identities)) // expected number resulting identities have the diff --git a/network/p2p/cache/protocol_state_provider_test.go b/network/p2p/cache/protocol_state_provider_test.go index 4aa593ef2a3..eca979de299 100644 --- a/network/p2p/cache/protocol_state_provider_test.go +++ b/network/p2p/cache/protocol_state_provider_test.go @@ -73,7 +73,7 @@ func (suite *ProtocolStateProviderTestSuite) triggerUpdate() { // set up protocol snapshot mock snapshot := &mockprotocol.Snapshot{} snapshot.On("Identities", mock.Anything).Return( - func(filter flow.IdentityFilter) flow.IdentityList { + func(filter flow.IdentityFilter[flow.Identity]) flow.IdentityList { return suite.participants.Filter(filter) }, nil, diff --git a/network/test/epochtransition_test.go b/network/test/epochtransition_test.go index b3132d5b398..3af4f9c6670 100644 --- a/network/test/epochtransition_test.go +++ b/network/test/epochtransition_test.go @@ -180,10 +180,10 @@ func (suite *MutableIdentityTableSuite) setupStateMock() { suite.snapshot.On("Phase").Return(flow.EpochPhaseCommitted, nil) // return all the current list of ids for the state.Final.Identities call made by the network suite.snapshot.On("Identities", mock.Anything).Return( - func(flow.IdentityFilter) flow.IdentityList { + func(flow.IdentityFilter[flow.Identity]) flow.IdentityList { return suite.testNodes.ids() }, - func(flow.IdentityFilter) error { return nil }) + func(flow.IdentityFilter[flow.Identity]) error { return nil }) suite.state.On("Final").Return(suite.snapshot, nil) } @@ -397,7 +397,7 @@ func (suite *MutableIdentityTableSuite) exchangeMessages( for i, allowedEng := range allowedEngs { fromID := allowedIDs[i].NodeID - targetIDs := allowedIDs.Filter(filter.Not(filter.HasNodeID(allowedIDs[i].NodeID))) + targetIDs := allowedIDs.Filter(filter.Not(filter.HasNodeID[flow.Identity](allowedIDs[i].NodeID))) err := suite.sendMessage(fromID, allowedEng, targetIDs, send) require.NoError(suite.T(), err) diff --git a/network/test/meshengine_test.go b/network/test/meshengine_test.go index 48800ab609e..b60b6b15e1e 100644 --- a/network/test/meshengine_test.go +++ b/network/test/meshengine_test.go @@ -248,7 +248,7 @@ func (suite *MeshEngineTestSuite) allToAllScenario(send testutils.ConduitSendWra } // others keeps the identifier of all nodes except ith node - others := suite.ids.Filter(filter.Not(filter.HasNodeID(suite.ids[i].NodeID))).NodeIDs() + others := suite.ids.Filter(filter.Not(filter.HasNodeID[flow.Identity](suite.ids[i].NodeID))).NodeIDs() require.NoError(suite.Suite.T(), send(event, engs[i].Con, others...)) wg.Add(count - 1) } @@ -380,7 +380,7 @@ func (suite *MeshEngineTestSuite) messageSizeScenario(send testutils.ConduitSend } } // others keeps the identifier of all nodes except node that is sender. - others := suite.ids.Filter(filter.Not(filter.HasNodeID(suite.ids[0].NodeID))).NodeIDs() + others := suite.ids.Filter(filter.Not(filter.HasNodeID[flow.Identity](suite.ids[0].NodeID))).NodeIDs() // generates and sends an event of custom size to the network payload := testutils.NetworkPayloadFixture(suite.T(), size) @@ -451,7 +451,7 @@ func (suite *MeshEngineTestSuite) conduitCloseScenario(send testutils.ConduitSen // others keeps the identifier of all nodes except ith node and the node that unregistered from the topic. // nodes without valid topic registration for a channel will reject messages on that channel via unicast. - others := suite.ids.Filter(filter.Not(filter.HasNodeID(suite.ids[i].NodeID, suite.ids[unregisterIndex].NodeID))).NodeIDs() + others := suite.ids.Filter(filter.Not(filter.HasNodeID[flow.Identity](suite.ids[i].NodeID, suite.ids[unregisterIndex].NodeID))).NodeIDs() if i == unregisterIndex { // assert that unsubscribed engine cannot publish on that topic diff --git a/state/protocol/mock/cluster.go b/state/protocol/mock/cluster.go index 5835d2d6df9..00c6080c76e 100644 --- a/state/protocol/mock/cluster.go +++ b/state/protocol/mock/cluster.go @@ -57,15 +57,15 @@ func (_m *Cluster) Index() uint { } // Members provides a mock function with given fields: -func (_m *Cluster) Members() flow.IdentitySkeletonList { +func (_m *Cluster) Members() flow.GenericIdentityList[flow.IdentitySkeleton] { ret := _m.Called() - var r0 flow.IdentitySkeletonList - if rf, ok := ret.Get(0).(func() flow.IdentitySkeletonList); ok { + var r0 flow.GenericIdentityList[flow.IdentitySkeleton] + if rf, ok := ret.Get(0).(func() flow.GenericIdentityList[flow.IdentitySkeleton]); ok { r0 = rf() } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(flow.IdentitySkeletonList) + r0 = ret.Get(0).(flow.GenericIdentityList[flow.IdentitySkeleton]) } } diff --git a/state/protocol/mock/dynamic_protocol_state.go b/state/protocol/mock/dynamic_protocol_state.go index 14840b58630..64a1d55646b 100644 --- a/state/protocol/mock/dynamic_protocol_state.go +++ b/state/protocol/mock/dynamic_protocol_state.go @@ -161,15 +161,15 @@ func (_m *DynamicProtocolState) GlobalParams() protocol.GlobalParams { } // Identities provides a mock function with given fields: -func (_m *DynamicProtocolState) Identities() flow.IdentityList { +func (_m *DynamicProtocolState) Identities() flow.GenericIdentityList[flow.Identity] { ret := _m.Called() - var r0 flow.IdentityList - if rf, ok := ret.Get(0).(func() flow.IdentityList); ok { + var r0 flow.GenericIdentityList[flow.Identity] + if rf, ok := ret.Get(0).(func() flow.GenericIdentityList[flow.Identity]); ok { r0 = rf() } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(flow.IdentityList) + r0 = ret.Get(0).(flow.GenericIdentityList[flow.Identity]) } } diff --git a/state/protocol/mock/epoch.go b/state/protocol/mock/epoch.go index dc024c697c4..191e8e0cd28 100644 --- a/state/protocol/mock/epoch.go +++ b/state/protocol/mock/epoch.go @@ -311,19 +311,19 @@ func (_m *Epoch) FirstView() (uint64, error) { } // InitialIdentities provides a mock function with given fields: -func (_m *Epoch) InitialIdentities() (flow.IdentitySkeletonList, error) { +func (_m *Epoch) InitialIdentities() (flow.GenericIdentityList[flow.IdentitySkeleton], error) { ret := _m.Called() - var r0 flow.IdentitySkeletonList + var r0 flow.GenericIdentityList[flow.IdentitySkeleton] var r1 error - if rf, ok := ret.Get(0).(func() (flow.IdentitySkeletonList, error)); ok { + if rf, ok := ret.Get(0).(func() (flow.GenericIdentityList[flow.IdentitySkeleton], error)); ok { return rf() } - if rf, ok := ret.Get(0).(func() flow.IdentitySkeletonList); ok { + if rf, ok := ret.Get(0).(func() flow.GenericIdentityList[flow.IdentitySkeleton]); ok { r0 = rf() } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(flow.IdentitySkeletonList) + r0 = ret.Get(0).(flow.GenericIdentityList[flow.IdentitySkeleton]) } } diff --git a/state/protocol/mock/snapshot.go b/state/protocol/mock/snapshot.go index 5e91c4ce370..15aa41b035c 100644 --- a/state/protocol/mock/snapshot.go +++ b/state/protocol/mock/snapshot.go @@ -109,19 +109,19 @@ func (_m *Snapshot) Head() (*flow.Header, error) { } // Identities provides a mock function with given fields: selector -func (_m *Snapshot) Identities(selector flow.IdentityFilter[flow.Identity]) (flow.IdentityList, error) { +func (_m *Snapshot) Identities(selector flow.IdentityFilter[flow.Identity]) (flow.GenericIdentityList[flow.Identity], error) { ret := _m.Called(selector) - var r0 flow.IdentityList + var r0 flow.GenericIdentityList[flow.Identity] var r1 error - if rf, ok := ret.Get(0).(func(flow.IdentityFilter[flow.Identity]) (flow.IdentityList, error)); ok { + if rf, ok := ret.Get(0).(func(flow.IdentityFilter[flow.Identity]) (flow.GenericIdentityList[flow.Identity], error)); ok { return rf(selector) } - if rf, ok := ret.Get(0).(func(flow.IdentityFilter[flow.Identity]) flow.IdentityList); ok { + if rf, ok := ret.Get(0).(func(flow.IdentityFilter[flow.Identity]) flow.GenericIdentityList[flow.Identity]); ok { r0 = rf(selector) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(flow.IdentityList) + r0 = ret.Get(0).(flow.GenericIdentityList[flow.Identity]) } } diff --git a/state/protocol/snapshot.go b/state/protocol/snapshot.go index 1863048d83b..899d6bed3af 100644 --- a/state/protocol/snapshot.go +++ b/state/protocol/snapshot.go @@ -50,7 +50,7 @@ type Snapshot interface { // epoch. At the end of an epoch, this includes identities scheduled to join // in the next epoch but are not active yet. // - // Identities are guaranteed to be returned in canonical order (order.Canonical). + // Identities are guaranteed to be returned in canonical order (order.Canonical[flow.Identity]). // // It allows us to provide optional upfront filters which can be used by the // implementation to speed up database lookups. diff --git a/utils/unittest/cluster.go b/utils/unittest/cluster.go index bed70799279..70ca17d5910 100644 --- a/utils/unittest/cluster.go +++ b/utils/unittest/cluster.go @@ -10,7 +10,7 @@ import ( // TransactionForCluster generates a transaction that will be assigned to the // target cluster ID. -func TransactionForCluster(clusters flow.ClusterList, target flow.IdentityList) flow.TransactionBody { +func TransactionForCluster(clusters flow.ClusterList, target flow.IdentitySkeletonList) flow.TransactionBody { tx := TransactionBodyFixture() return AlterTransactionForCluster(tx, clusters, target, func(*flow.TransactionBody) {}) } @@ -20,7 +20,7 @@ func TransactionForCluster(clusters flow.ClusterList, target flow.IdentityList) // // The `after` function is run after each modification to allow for any content // dependent changes to the transaction (eg. signing it). -func AlterTransactionForCluster(tx flow.TransactionBody, clusters flow.ClusterList, target flow.IdentityList, after func(tx *flow.TransactionBody)) flow.TransactionBody { +func AlterTransactionForCluster(tx flow.TransactionBody, clusters flow.ClusterList, target flow.IdentitySkeletonList, after func(tx *flow.TransactionBody)) flow.TransactionBody { // Bound to avoid infinite loop in case the routing algorithm is broken for i := 0; i < 10000; i++ { From 38ffeeef10b6b960d922f57a2345b9788855ac03 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Wed, 4 Oct 2023 12:30:03 +0300 Subject: [PATCH 15/39] Fixed compilation and implementation issues with tests --- engine/collection/ingest/engine_test.go | 14 ++++++------- engine/consensus/ingestion/core_test.go | 2 +- storage/badger/protocol_state_test.go | 26 ++++++++++++++++++++----- utils/unittest/fixtures.go | 6 ++++++ 4 files changed, 35 insertions(+), 13 deletions(-) diff --git a/engine/collection/ingest/engine_test.go b/engine/collection/ingest/engine_test.go index a1d5c60e7a2..efc52461fcc 100644 --- a/engine/collection/ingest/engine_test.go +++ b/engine/collection/ingest/engine_test.go @@ -88,8 +88,8 @@ func (suite *Suite) SetupTest() { return herocache.NewTransactions(1000, log, metrics) }) - assignments := unittest.ClusterAssignment(suite.N_CLUSTERS, collectors) - suite.clusters, err = factory.NewClusterList(assignments, collectors) + assignments := unittest.ClusterAssignment(suite.N_CLUSTERS, collectors.ToSkeleton()) + suite.clusters, err = factory.NewClusterList(assignments, collectors.ToSkeleton()) suite.Require().NoError(err) suite.root = unittest.GenesisFixture() @@ -352,7 +352,7 @@ func (suite *Suite) TestRoutingToRemoteClusterWithNoNodes() { suite.Require().True(ok) // set the next cluster to be empty - emptyIdentityList := flow.IdentityList{} + emptyIdentityList := flow.IdentitySkeletonList{} nextClusterIndex := (index + 1) % suite.N_CLUSTERS suite.clusters[nextClusterIndex] = emptyIdentityList @@ -384,7 +384,7 @@ func (suite *Suite) TestRoutingLocalClusterFromOtherNode() { suite.Require().True(ok) // another node will send us the transaction - sender := local.Filter(filter.Not(filter.HasNodeID[flow.Identity](suite.me.NodeID())))[0] + sender := local.Filter(filter.Not(filter.HasNodeID[flow.IdentitySkeleton](suite.me.NodeID())))[0] // get a transaction that will be routed to local cluster tx := unittest.TransactionBodyFixture() @@ -476,7 +476,7 @@ func (suite *Suite) TestRouting_ClusterAssignmentRemoved() { // remove ourselves from the cluster assignment for epoch 2 withoutMe := suite.identities. Filter(filter.Not(filter.HasNodeID[flow.Identity](suite.me.NodeID()))). - Filter(filter.HasRole[flow.Identity](flow.RoleCollection)) + Filter(filter.HasRole[flow.Identity](flow.RoleCollection)).ToSkeleton() epoch2Assignment := unittest.ClusterAssignment(suite.N_CLUSTERS, withoutMe) epoch2Clusters, err := factory.NewClusterList(epoch2Assignment, withoutMe) suite.Require().NoError(err) @@ -515,7 +515,7 @@ func (suite *Suite) TestRouting_ClusterAssignmentAdded() { // remove ourselves from the cluster assignment for epoch 2 withoutMe := suite.identities. Filter(filter.Not(filter.HasNodeID[flow.Identity](suite.me.NodeID()))). - Filter(filter.HasRole[flow.Identity](flow.RoleCollection)) + Filter(filter.HasRole[flow.Identity](flow.RoleCollection)).ToSkeleton() epoch2Assignment := unittest.ClusterAssignment(suite.N_CLUSTERS, withoutMe) epoch2Clusters, err := factory.NewClusterList(epoch2Assignment, withoutMe) suite.Require().NoError(err) @@ -544,7 +544,7 @@ func (suite *Suite) TestRouting_ClusterAssignmentAdded() { // EPOCH 3: // include ourselves in cluster assignment - withMe := suite.identities.Filter(filter.HasRole[flow.Identity](flow.RoleCollection)) + withMe := suite.identities.Filter(filter.HasRole[flow.Identity](flow.RoleCollection)).ToSkeleton() epoch3Assignment := unittest.ClusterAssignment(suite.N_CLUSTERS, withMe) epoch3Clusters, err := factory.NewClusterList(epoch3Assignment, withMe) suite.Require().NoError(err) diff --git a/engine/consensus/ingestion/core_test.go b/engine/consensus/ingestion/core_test.go index 336703676a7..04737be00fb 100644 --- a/engine/consensus/ingestion/core_test.go +++ b/engine/consensus/ingestion/core_test.go @@ -115,7 +115,7 @@ func (suite *IngestionCoreSuite) SetupTest() { ) ref.On("Epochs").Return(suite.query) suite.query.On("Current").Return(suite.epoch) - cluster.On("Members").Return(suite.clusterMembers) + cluster.On("Members").Return(suite.clusterMembers.ToSkeleton()) suite.epoch.On("ClusterByChainID", mock.Anything).Return( func(chainID flow.ChainID) protocol.Cluster { if chainID == suite.clusterID { diff --git a/storage/badger/protocol_state_test.go b/storage/badger/protocol_state_test.go index 7a1d1060e92..e6e54cf3ea0 100644 --- a/storage/badger/protocol_state_test.go +++ b/storage/badger/protocol_state_test.go @@ -1,6 +1,7 @@ package badger import ( + "github.com/onflow/flow-go/model/flow/mapfunc" "testing" "github.com/dgraph-io/badger/v2" @@ -107,6 +108,8 @@ func TestProtocolStateMergeParticipants(t *testing.T) { nodeID := stateEntry.CurrentEpochSetup.Participants[1].NodeID stateEntry.CurrentEpochSetup.Participants[1].Address = newAddress stateEntry.CurrentEpoch.SetupID = stateEntry.CurrentEpochSetup.ID() + identity, _ := stateEntry.CurrentEpochIdentityTable.ByNodeID(nodeID) + identity.Address = newAddress protocolStateID := stateEntry.ID() // store protocol state and auxiliary info @@ -201,15 +204,28 @@ func assertRichProtocolStateValidity(t *testing.T, state *flow.RichProtocolState assert.Equal(t, state.PreviousEpochSetup.ID(), state.ProtocolStateEntry.PreviousEpochEventIDs.SetupID, "epoch setup should be for correct event ID") assert.Equal(t, state.PreviousEpochCommit.ID(), state.ProtocolStateEntry.PreviousEpochEventIDs.CommitID, "epoch commit should be for correct event ID") - previousEpochParticipants = state.PreviousEpochSetup.Participants + for _, participant := range state.PreviousEpochSetup.Participants { + if identity, found := state.CurrentEpochIdentityTable.ByNodeID(participant.NodeID); found { + previousEpochParticipants = append(previousEpochParticipants, identity) + } + } } + participantsFromCurrentEpochSetup := state.CurrentEpochIdentityTable.Filter(func(i *flow.Identity) bool { + _, exists := state.CurrentEpochSetup.Participants.ByNodeID(i.NodeID) + return exists + }) + // invariant: Identities is a full identity table for the current epoch. Identities are sorted in canonical order. Without duplicates. Never nil. - var allIdentities flow.IdentityList + var allIdentities, participantsFromNextEpochSetup flow.IdentityList if state.NextEpoch != nil { - allIdentities = state.CurrentEpochSetup.Participants.Union(state.NextEpochSetup.Participants) + participantsFromNextEpochSetup = state.NextEpochIdentityTable.Filter(func(i *flow.Identity) bool { + _, exists := state.NextEpochSetup.Participants.ByNodeID(i.NodeID) + return exists + }) + allIdentities = participantsFromCurrentEpochSetup.Union(participantsFromNextEpochSetup.Map(mapfunc.WithWeight(0))) } else { - allIdentities = state.CurrentEpochSetup.Participants.Union(previousEpochParticipants) + allIdentities = participantsFromCurrentEpochSetup.Union(previousEpochParticipants.Map(mapfunc.WithWeight(0))) } assert.Equal(t, allIdentities, state.CurrentEpochIdentityTable, "identities should be a full identity table for the current epoch, without duplicates") @@ -230,7 +246,7 @@ func assertRichProtocolStateValidity(t *testing.T, state *flow.RichProtocolState assert.Equal(t, state.NextEpochCommit.ID(), nextEpoch.CommitID, "epoch commit should be for correct event ID") // invariant: Identities is a full identity table for the current epoch. Identities are sorted in canonical order. Without duplicates. Never nil. - allIdentities = state.NextEpochSetup.Participants.Union(state.CurrentEpochSetup.Participants) + allIdentities = participantsFromNextEpochSetup.Union(participantsFromCurrentEpochSetup.Map(mapfunc.WithWeight(0))) assert.Equal(t, allIdentities, state.NextEpochIdentityTable, "identities should be a full identity table for the next epoch, without duplicates") } diff --git a/utils/unittest/fixtures.go b/utils/unittest/fixtures.go index 70b22be6258..db22618e478 100644 --- a/utils/unittest/fixtures.go +++ b/utils/unittest/fixtures.go @@ -2608,6 +2608,12 @@ func ProtocolStateFixture(options ...func(*flow.RichProtocolStateEntry)) *flow.R }, }) } + allIdentities = allIdentities.Map(func(identity flow.Identity) flow.Identity { + if _, found := prevEpochSetup.Participants.ByNodeID(identity.NodeID); found { + identity.Weight = 0 + } + return identity + }) entry := &flow.RichProtocolStateEntry{ ProtocolStateEntry: &flow.ProtocolStateEntry{ CurrentEpoch: flow.EpochStateContainer{ From 6e650185cb55bd8f78064b6e854e0c92184a4399 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Wed, 4 Oct 2023 13:33:37 +0300 Subject: [PATCH 16/39] Removed outdated tests. Fixed compilation for other tests --- .../hotstuff/committees/cluster_committee_test.go | 12 ------------ .../hotstuff/committees/consensus_committee_test.go | 8 ++------ consensus/integration/epoch_test.go | 2 +- consensus/integration/nodes_test.go | 2 +- engine/access/rpc/backend/backend_test.go | 13 ++++++++----- engine/consensus/dkg/reactor_engine_test.go | 4 ++-- 6 files changed, 14 insertions(+), 27 deletions(-) diff --git a/consensus/hotstuff/committees/cluster_committee_test.go b/consensus/hotstuff/committees/cluster_committee_test.go index f2af95558dd..ffee2c794ed 100644 --- a/consensus/hotstuff/committees/cluster_committee_test.go +++ b/consensus/hotstuff/committees/cluster_committee_test.go @@ -146,12 +146,6 @@ func (suite *ClusterSuite) TestInvalidSigner() { }) suite.Run("should return ErrInvalidSigner for existent but ejected cluster member", func() { - // at the root block, the cluster member is not ejected yet - suite.Run("root block", func() { - actual, err := suite.com.IdentityByBlock(rootBlockID, realEjectedClusterMember.NodeID) - suite.Require().NoError(err) - suite.Assert().Equal(realEjectedClusterMember, actual) - }) suite.Run("non-root block", func() { _, err := suite.com.IdentityByBlock(nonRootBlockID, realEjectedClusterMember.NodeID) suite.Assert().True(model.IsInvalidSignerError(err)) @@ -164,12 +158,6 @@ func (suite *ClusterSuite) TestInvalidSigner() { }) suite.Run("should return ErrInvalidSigner for existent but zero-weight cluster member", func() { - // at the root block, the cluster member has its initial weight - suite.Run("root block", func() { - actual, err := suite.com.IdentityByBlock(rootBlockID, realNoWeightClusterMember.NodeID) - suite.Require().NoError(err) - suite.Assert().Equal(realNoWeightClusterMember, actual) - }) suite.Run("non-root block", func() { _, err := suite.com.IdentityByBlock(nonRootBlockID, realNoWeightClusterMember.NodeID) suite.Assert().True(model.IsInvalidSignerError(err)) diff --git a/consensus/hotstuff/committees/consensus_committee_test.go b/consensus/hotstuff/committees/consensus_committee_test.go index b7e5cc66315..a35854a75de 100644 --- a/consensus/hotstuff/committees/consensus_committee_test.go +++ b/consensus/hotstuff/committees/consensus_committee_test.go @@ -327,7 +327,8 @@ func (suite *ConsensusSuite) TestIdentitiesByEpoch() { // epoch 1 identities with varying conditions which would disqualify them // from committee participation realIdentity := unittest.IdentityFixture(unittest.WithRole(flow.RoleConsensus)) - zeroWeightConsensusIdentity := unittest.IdentityFixture(unittest.WithRole(flow.RoleConsensus), unittest.WithWeight(0)) + zeroWeightConsensusIdentity := unittest.IdentityFixture(unittest.WithRole(flow.RoleConsensus), + unittest.WithInitialWeight(0)) ejectedConsensusIdentity := unittest.IdentityFixture(unittest.WithRole(flow.RoleConsensus), unittest.WithEjected(true)) validNonConsensusIdentity := unittest.IdentityFixture(unittest.WithRole(flow.RoleVerification)) epoch1Identities := flow.IdentityList{realIdentity, zeroWeightConsensusIdentity, ejectedConsensusIdentity, validNonConsensusIdentity} @@ -356,11 +357,6 @@ func (suite *ConsensusSuite) TestIdentitiesByEpoch() { require.True(t, model.IsInvalidSignerError(err)) }) - t.Run("ejected consensus node", func(t *testing.T) { - _, err := suite.committee.IdentityByEpoch(unittest.Uint64InRange(1, 100), ejectedConsensusIdentity.NodeID) - require.True(t, model.IsInvalidSignerError(err)) - }) - t.Run("otherwise valid non-consensus node", func(t *testing.T) { _, err := suite.committee.IdentityByEpoch(unittest.Uint64InRange(1, 100), validNonConsensusIdentity.NodeID) require.True(t, model.IsInvalidSignerError(err)) diff --git a/consensus/integration/epoch_test.go b/consensus/integration/epoch_test.go index e874cf7589e..4381e7fbe48 100644 --- a/consensus/integration/epoch_test.go +++ b/consensus/integration/epoch_test.go @@ -220,7 +220,7 @@ func withNextEpoch( encodableSnapshot := snapshot.Encodable() currEpoch := &encodableSnapshot.Epochs.Current // take pointer so assignments apply - currentEpochIdentities := currEpoch.InitialIdentities + currentEpochIdentities, _ := snapshot.Identities(filter.Any) nextEpochIdentities = nextEpochIdentities.Sort(order.Canonical[flow.Identity]) currEpoch.FinalView = currEpoch.FirstView + curEpochViews - 1 // first epoch lasts curEpochViews diff --git a/consensus/integration/nodes_test.go b/consensus/integration/nodes_test.go index 4124d3d6618..940556bea75 100644 --- a/consensus/integration/nodes_test.go +++ b/consensus/integration/nodes_test.go @@ -266,7 +266,7 @@ func createRootBlockData(participantData *run.ParticipantData) (*flow.Block, *fl counter := uint64(1) setup := unittest.EpochSetupFixture( - unittest.WithParticipants(participants), + unittest.WithParticipants(participants.ToSkeleton()), unittest.SetupWithCounter(counter), unittest.WithFirstView(root.Header.View), unittest.WithFinalView(root.Header.View+1000), diff --git a/engine/access/rpc/backend/backend_test.go b/engine/access/rpc/backend/backend_test.go index cee5194cb0c..cada2acad26 100644 --- a/engine/access/rpc/backend/backend_test.go +++ b/engine/access/rpc/backend/backend_test.go @@ -1891,12 +1891,15 @@ func (suite *Suite) TestExecutionNodesForBlockID() { actualList = append(actualList, actual) } - if len(expectedENs) > maxNodesCnt { - for _, actual := range actualList { - require.Contains(suite.T(), expectedENs, actual) + { + expectedENs := expectedENs.ToSkeleton() + if len(expectedENs) > maxNodesCnt { + for _, actual := range actualList { + require.Contains(suite.T(), expectedENs, actual) + } + } else { + require.ElementsMatch(suite.T(), actualList, expectedENs) } - } else { - require.ElementsMatch(suite.T(), actualList, expectedENs) } } // if we don't find sufficient receipts, executionNodesForBlockID should return a list of random ENs diff --git a/engine/consensus/dkg/reactor_engine_test.go b/engine/consensus/dkg/reactor_engine_test.go index 48e2707188d..5580bf2c7a8 100644 --- a/engine/consensus/dkg/reactor_engine_test.go +++ b/engine/consensus/dkg/reactor_engine_test.go @@ -123,7 +123,7 @@ func (suite *ReactorEngineSuite_SetupPhase) SetupTest() { suite.currentEpoch.On("DKGPhase3FinalView").Return(suite.dkgPhase3FinalView, nil) suite.nextEpoch = new(protocol.Epoch) suite.nextEpoch.On("Counter").Return(suite.NextEpochCounter(), nil) - suite.nextEpoch.On("InitialIdentities").Return(suite.committee, nil) + suite.nextEpoch.On("InitialIdentities").Return(suite.committee.ToSkeleton(), nil) suite.epochQuery = mocks.NewEpochQuery(suite.T(), suite.epochCounter) suite.epochQuery.Add(suite.currentEpoch) @@ -162,7 +162,7 @@ func (suite *ReactorEngineSuite_SetupPhase) SetupTest() { suite.factory = new(module.DKGControllerFactory) suite.factory.On("Create", dkgmodule.CanonicalInstanceID(suite.firstBlock.ChainID, suite.NextEpochCounter()), - suite.committee, + suite.committee.ToSkeleton(), mock.Anything, ).Return(suite.controller, nil) From ea3ef4403c954b12dd59c95d788b3c09752c50cb Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Wed, 4 Oct 2023 13:44:13 +0300 Subject: [PATCH 17/39] Fixed execution test --- engine/execution/ingestion/engine_test.go | 4 ++-- state/protocol/badger/state_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/engine/execution/ingestion/engine_test.go b/engine/execution/ingestion/engine_test.go index 0dbd0799abd..1fa4b5f113a 100644 --- a/engine/execution/ingestion/engine_test.go +++ b/engine/execution/ingestion/engine_test.go @@ -167,7 +167,7 @@ func runWithEngine(t *testing.T, f func(testingContext)) { var engine *Engine defer func() { - <-engine.Done() + unittest.AssertClosesBefore(t, engine.Done(), 5*time.Second, "expect to stop before timeout") ctrl.Finish() computationManager.AssertExpectations(t) protocolState.AssertExpectations(t) @@ -373,7 +373,7 @@ func (ctx testingContext) mockSnapshot(header *flow.Header, identities flow.Iden func (ctx testingContext) mockSnapshotWithBlockID(blockID flow.Identifier, identities flow.IdentityList) { cluster := new(protocol.Cluster) // filter only collections as cluster members - cluster.On("Members").Return(identities.Filter(filter.HasRole[flow.Identity](flow.RoleCollection))) + cluster.On("Members").Return(identities.Filter(filter.HasRole[flow.Identity](flow.RoleCollection)).ToSkeleton()) epoch := new(protocol.Epoch) epoch.On("ClusterByChainID", mock.Anything).Return(cluster, nil) diff --git a/state/protocol/badger/state_test.go b/state/protocol/badger/state_test.go index 298cfe4e5f5..68fc8560fdf 100644 --- a/state/protocol/badger/state_test.go +++ b/state/protocol/badger/state_test.go @@ -382,7 +382,7 @@ func TestBootstrap_InvalidIdentities(t *testing.T) { }) t.Run("zero weight", func(t *testing.T) { - zeroWeightIdentity := unittest.IdentityFixture(unittest.WithRole(flow.RoleVerification), unittest.WithWeight(0)) + zeroWeightIdentity := unittest.IdentityFixture(unittest.WithRole(flow.RoleVerification), unittest.WithInitialWeight(0)) participants := unittest.CompleteIdentitySet(zeroWeightIdentity) root := unittest.RootSnapshotFixture(participants) bootstrap(t, root, func(state *bprotocol.State, err error) { From 8f9156246d2c2a90c5c91e9628bc87e7e87ede04 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Wed, 4 Oct 2023 14:10:51 +0300 Subject: [PATCH 18/39] Fixed last batch of broken tests --- cmd/bootstrap/cmd/clusters.go | 4 ++-- cmd/bootstrap/cmd/finalize.go | 2 +- cmd/bootstrap/cmd/keys.go | 2 +- cmd/bootstrap/cmd/seal.go | 2 +- cmd/bootstrap/run/cluster_qc.go | 20 ++++++++++++++++--- cmd/bootstrap/run/cluster_qc_test.go | 2 +- engine/access/access_test.go | 2 +- engine/access/ingestion/engine_test.go | 6 +++--- .../test/cluster_switchover_test.go | 4 ++-- 9 files changed, 29 insertions(+), 15 deletions(-) diff --git a/cmd/bootstrap/cmd/clusters.go b/cmd/bootstrap/cmd/clusters.go index 30c91db9a9d..27ab1c52605 100644 --- a/cmd/bootstrap/cmd/clusters.go +++ b/cmd/bootstrap/cmd/clusters.go @@ -73,7 +73,7 @@ func constructClusterAssignment(partnerNodes, internalNodes []model.NodeInfo) (f assignments := assignment.FromIdentifierLists(identifierLists) collectors := append(partners, internals...) - clusters, err := factory.NewClusterList(assignments, collectors) + clusters, err := factory.NewClusterList(assignments, collectors.ToSkeleton()) if err != nil { log.Fatal().Err(err).Msg("could not create cluster list") } @@ -109,7 +109,7 @@ func constructRootQCsForClusters( // Filters a list of nodes to include only nodes that will sign the QC for the // given cluster. The resulting list of nodes is only nodes that are in the // given cluster AND are not partner nodes (ie. we have the private keys). -func filterClusterSigners(cluster flow.IdentityList, nodeInfos []model.NodeInfo) []model.NodeInfo { +func filterClusterSigners(cluster flow.IdentitySkeletonList, nodeInfos []model.NodeInfo) []model.NodeInfo { var filtered []model.NodeInfo for _, node := range nodeInfos { diff --git a/cmd/bootstrap/cmd/finalize.go b/cmd/bootstrap/cmd/finalize.go index 53d398ede7c..a4680508569 100644 --- a/cmd/bootstrap/cmd/finalize.go +++ b/cmd/bootstrap/cmd/finalize.go @@ -491,7 +491,7 @@ func mergeNodeInfos(internalNodes, partnerNodes []model.NodeInfo) []model.NodeIn } // sort nodes using the canonical ordering - nodes = model.Sort(nodes, order.Canonical) + nodes = model.Sort(nodes, order.Canonical[flow.Identity]) return nodes } diff --git a/cmd/bootstrap/cmd/keys.go b/cmd/bootstrap/cmd/keys.go index 9624ade3a1a..d12a950ecfb 100644 --- a/cmd/bootstrap/cmd/keys.go +++ b/cmd/bootstrap/cmd/keys.go @@ -49,7 +49,7 @@ func genNetworkAndStakingKeys() []model.NodeInfo { internalNodes = append(internalNodes, nodeInfo) } - return model.Sort(internalNodes, order.Canonical) + return model.Sort(internalNodes, order.Canonical[flow.Identity]) } func assembleNodeInfo(nodeConfig model.NodeConfig, networkKey, stakingKey crypto.PrivateKey) model.NodeInfo { diff --git a/cmd/bootstrap/cmd/seal.go b/cmd/bootstrap/cmd/seal.go index c04e67fb733..f5a7a5c72d6 100644 --- a/cmd/bootstrap/cmd/seal.go +++ b/cmd/bootstrap/cmd/seal.go @@ -39,7 +39,7 @@ func constructRootResultAndSeal( DKGPhase1FinalView: firstView + flagNumViewsInStakingAuction + flagNumViewsInDKGPhase - 1, DKGPhase2FinalView: firstView + flagNumViewsInStakingAuction + flagNumViewsInDKGPhase*2 - 1, DKGPhase3FinalView: firstView + flagNumViewsInStakingAuction + flagNumViewsInDKGPhase*3 - 1, - Participants: participants.Sort(order.Canonical[flow.Identity]), + Participants: participants.Sort(order.Canonical[flow.Identity]).ToSkeleton(), Assignments: assignments, RandomSource: GenerateRandomSeed(flow.EpochSetupRandomSourceLength), } diff --git a/cmd/bootstrap/run/cluster_qc.go b/cmd/bootstrap/run/cluster_qc.go index 0b1b609e76b..689a65a339c 100644 --- a/cmd/bootstrap/run/cluster_qc.go +++ b/cmd/bootstrap/run/cluster_qc.go @@ -19,8 +19,8 @@ import ( ) // GenerateClusterRootQC creates votes and generates a QC based on participant data -func GenerateClusterRootQC(signers []bootstrap.NodeInfo, allCommitteeMembers flow.IdentityList, clusterBlock *cluster.Block) (*flow.QuorumCertificate, error) { - if !allCommitteeMembers.Sorted(order.Canonical[flow.Identity]) { +func GenerateClusterRootQC(signers []bootstrap.NodeInfo, allCommitteeMembers flow.IdentitySkeletonList, clusterBlock *cluster.Block) (*flow.QuorumCertificate, error) { + if !allCommitteeMembers.Sorted(order.Canonical[flow.IdentitySkeleton]) { return nil, fmt.Errorf("can't create root cluster QC: committee members are not sorted in canonical order") } clusterRootBlock := model.GenesisBlockFromFlow(clusterBlock.Header) @@ -31,8 +31,22 @@ func GenerateClusterRootQC(signers []bootstrap.NodeInfo, allCommitteeMembers flo return nil, err } + // STEP 1.5: patch committee to include dynamic identities. This is a temporary measure until bootstrapping is refactored. + // We need to do this since the committee is used to create the QC uses dynamic identities, but clustering for root block contain only + // static identities since there no state transitions haven't happened yet. + dynamicCommitteeMembers := make(flow.IdentityList, 0, len(allCommitteeMembers)) + for _, participant := range allCommitteeMembers { + dynamicCommitteeMembers = append(dynamicCommitteeMembers, &flow.Identity{ + IdentitySkeleton: *participant, + DynamicIdentity: flow.DynamicIdentity{ + Weight: participant.InitialWeight, + Ejected: false, + }, + }) + } + // STEP 2: create VoteProcessor - committee, err := committees.NewStaticCommittee(allCommitteeMembers, flow.Identifier{}, nil, nil) + committee, err := committees.NewStaticCommittee(dynamicCommitteeMembers, flow.Identifier{}, nil, nil) if err != nil { return nil, err } diff --git a/cmd/bootstrap/run/cluster_qc_test.go b/cmd/bootstrap/run/cluster_qc_test.go index 7533bf54662..d7a2ef9eaed 100644 --- a/cmd/bootstrap/run/cluster_qc_test.go +++ b/cmd/bootstrap/run/cluster_qc_test.go @@ -33,7 +33,7 @@ func TestGenerateClusterRootQC(t *testing.T) { payload := cluster.EmptyPayload(flow.ZeroID) clusterBlock.SetPayload(payload) - orderedParticipants := model.ToIdentityList(participants).Sort(order.Canonical[flow.Identity]) + orderedParticipants := model.ToIdentityList(participants).Sort(order.Canonical[flow.Identity]).ToSkeleton() _, err := GenerateClusterRootQC(participants, orderedParticipants, &clusterBlock) require.NoError(t, err) } diff --git a/engine/access/access_test.go b/engine/access/access_test.go index 3fd32324ebc..792e18c0d07 100644 --- a/engine/access/access_test.go +++ b/engine/access/access_test.go @@ -1183,7 +1183,7 @@ func (suite *Suite) createChain() (*flow.Block, *flow.Collection) { block.SetPayload(unittest.PayloadFixture(unittest.WithGuarantees(guarantee))) cluster := new(protocol.Cluster) - cluster.On("Members").Return(clusterCommittee, nil) + cluster.On("Members").Return(clusterCommittee.ToSkeleton(), nil) epoch := new(protocol.Epoch) epoch.On("ClusterByChainID", mock.Anything).Return(cluster, nil) epochs := new(protocol.EpochQuery) diff --git a/engine/access/ingestion/engine_test.go b/engine/access/ingestion/engine_test.go index 996e4d94343..895afa8db10 100644 --- a/engine/access/ingestion/engine_test.go +++ b/engine/access/ingestion/engine_test.go @@ -139,7 +139,7 @@ func (suite *Suite) TestOnFinalizedBlock() { )) // prepare cluster committee members - clusterCommittee := unittest.IdentityListFixture(32 * 4).Filter(filter.HasRole[flow.Identity](flow.RoleCollection)) + clusterCommittee := unittest.IdentityListFixture(32 * 4).Filter(filter.HasRole[flow.Identity](flow.RoleCollection)).ToSkeleton() refBlockID := unittest.IdentifierFixture() for _, guarantee := range block.Payload.Guarantees { guarantee.ReferenceBlockID = refBlockID @@ -326,7 +326,7 @@ func (suite *Suite) TestRequestMissingCollections() { heightMap := make(map[uint64]*flow.Block, blkCnt) // prepare cluster committee members - clusterCommittee := unittest.IdentityListFixture(32 * 4).Filter(filter.HasRole[flow.Identity](flow.RoleCollection)) + clusterCommittee := unittest.IdentityListFixture(32 * 4).Filter(filter.HasRole[flow.Identity](flow.RoleCollection)).ToSkeleton() // generate the test blocks and collections var collIDs []flow.Identifier @@ -468,7 +468,7 @@ func (suite *Suite) TestUpdateLastFullBlockReceivedIndex() { collMap := make(map[flow.Identifier]*flow.LightCollection, blkCnt*collPerBlk) // prepare cluster committee members - clusterCommittee := unittest.IdentityListFixture(32 * 4).Filter(filter.HasRole[flow.Identity](flow.RoleCollection)) + clusterCommittee := unittest.IdentityListFixture(32 * 4).Filter(filter.HasRole[flow.Identity](flow.RoleCollection)).ToSkeleton() refBlockID := unittest.IdentifierFixture() // generate the test blocks, cgs and collections diff --git a/engine/collection/test/cluster_switchover_test.go b/engine/collection/test/cluster_switchover_test.go index 13cd6fe063d..f87da6e8798 100644 --- a/engine/collection/test/cluster_switchover_test.go +++ b/engine/collection/test/cluster_switchover_test.go @@ -72,7 +72,7 @@ func NewClusterSwitchoverTestCase(t *testing.T, conf ClusterSwitchoverTestConf) signers = append(signers, identity) } } - signerIdentities := model.ToIdentityList(signers).Sort(order.Canonical[flow.Identity]) + signerIdentities := model.ToIdentityList(signers).Sort(order.Canonical[flow.Identity]).ToSkeleton() qc, err := run.GenerateClusterRootQC(signers, signerIdentities, rootClusterBlocks[i]) require.NoError(t, err) rootClusterQCs[i] = flow.ClusterQCVoteDataFromQC(&flow.QuorumCertificateWithSignerIDs{ @@ -142,7 +142,7 @@ func NewClusterSwitchoverTestCase(t *testing.T, conf ClusterSwitchoverTestConf) // generate root cluster block rootClusterBlock := cluster.CanonicalRootBlock(commit.Counter, model.ToIdentityList(signers).ToSkeleton()) // generate cluster root qc - qc, err := run.GenerateClusterRootQC(signers, model.ToIdentityList(signers), rootClusterBlock) + qc, err := run.GenerateClusterRootQC(signers, model.ToIdentityList(signers).ToSkeleton(), rootClusterBlock) require.NoError(t, err) signerIDs := toSignerIDs(signers) qcWithSignerIDs := &flow.QuorumCertificateWithSignerIDs{ From 092927a02acea57b0c4a731cdf66271d76eef844 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Wed, 4 Oct 2023 14:13:57 +0300 Subject: [PATCH 19/39] Linted --- model/bootstrap/node_info_test.go | 2 +- model/flow/identity.go | 3 ++- model/flow/order/identity_test.go | 2 +- state/protocol/protocol_state/updater_test.go | 2 +- storage/badger/protocol_state_test.go | 2 +- utils/unittest/cluster.go | 1 + 6 files changed, 7 insertions(+), 5 deletions(-) diff --git a/model/bootstrap/node_info_test.go b/model/bootstrap/node_info_test.go index 506bc85d80f..fbe02d86569 100644 --- a/model/bootstrap/node_info_test.go +++ b/model/bootstrap/node_info_test.go @@ -2,7 +2,6 @@ package bootstrap_test import ( "encoding/json" - "github.com/onflow/flow-go/model/flow" "strings" "testing" @@ -10,6 +9,7 @@ import ( "github.com/stretchr/testify/require" "github.com/onflow/flow-go/model/bootstrap" + "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/flow/order" "github.com/onflow/flow-go/utils/unittest" ) diff --git a/model/flow/identity.go b/model/flow/identity.go index a368392f031..c2940f5bec2 100644 --- a/model/flow/identity.go +++ b/model/flow/identity.go @@ -9,9 +9,10 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/fxamacker/cbor/v2" - "github.com/onflow/flow-go/crypto" "github.com/pkg/errors" "github.com/vmihailenco/msgpack" + + "github.com/onflow/flow-go/crypto" ) // DefaultInitialWeight is the default initial weight for a node identity. diff --git a/model/flow/order/identity_test.go b/model/flow/order/identity_test.go index 191a6d9dc39..05c5704f515 100644 --- a/model/flow/order/identity_test.go +++ b/model/flow/order/identity_test.go @@ -3,11 +3,11 @@ package order_test import ( - "github.com/onflow/flow-go/model/flow" "testing" "github.com/stretchr/testify/require" + "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/flow/order" "github.com/onflow/flow-go/utils/unittest" ) diff --git a/state/protocol/protocol_state/updater_test.go b/state/protocol/protocol_state/updater_test.go index 0915926b597..3c3feab7a1b 100644 --- a/state/protocol/protocol_state/updater_test.go +++ b/state/protocol/protocol_state/updater_test.go @@ -1,7 +1,6 @@ package protocol_state import ( - "github.com/onflow/flow-go/model/flow/order" "testing" "github.com/stretchr/testify/require" @@ -9,6 +8,7 @@ import ( "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/flow/mapfunc" + "github.com/onflow/flow-go/model/flow/order" "github.com/onflow/flow-go/utils/unittest" ) diff --git a/storage/badger/protocol_state_test.go b/storage/badger/protocol_state_test.go index e6e54cf3ea0..62388c5b9a8 100644 --- a/storage/badger/protocol_state_test.go +++ b/storage/badger/protocol_state_test.go @@ -1,7 +1,6 @@ package badger import ( - "github.com/onflow/flow-go/model/flow/mapfunc" "testing" "github.com/dgraph-io/badger/v2" @@ -9,6 +8,7 @@ import ( "github.com/stretchr/testify/require" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/model/flow/mapfunc" "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/storage/badger/transaction" "github.com/onflow/flow-go/utils/unittest" diff --git a/utils/unittest/cluster.go b/utils/unittest/cluster.go index 70ca17d5910..71eeb751655 100644 --- a/utils/unittest/cluster.go +++ b/utils/unittest/cluster.go @@ -2,6 +2,7 @@ package unittest import ( "fmt" + "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/flow/factory" "github.com/onflow/flow-go/model/flow/filter" From a5ca17d15706d64e50394c3c3122e57e8764c61c Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Wed, 4 Oct 2023 14:32:58 +0300 Subject: [PATCH 20/39] Updated godoc --- consensus/hotstuff/committees/cluster_committee.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/consensus/hotstuff/committees/cluster_committee.go b/consensus/hotstuff/committees/cluster_committee.go index 1ce44cd730f..f1d388af2df 100644 --- a/consensus/hotstuff/committees/cluster_committee.go +++ b/consensus/hotstuff/committees/cluster_committee.go @@ -53,6 +53,8 @@ func NewClusterCommittee( initialClusterMembers := cluster.Members() totalWeight := initialClusterMembers.TotalWeight() initialClusterMembersSelector := initialClusterMembers.Selector() + // the next section is not very nice, but there are no dynamic identities for root block, + // and we need them to specificially handle querying of identities for root block initialClusterIdentities := make(flow.IdentityList, 0, len(cluster.Members())) for _, skeleton := range initialClusterMembers { initialClusterIdentities = append(initialClusterIdentities, &flow.Identity{ From 5acf22706bb785402b9d7b74fe139bf9a7a3f500 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Wed, 4 Oct 2023 18:44:08 +0300 Subject: [PATCH 21/39] Fixed compilation in DKG tests --- integration/dkg/dkg_emulator_test.go | 4 ++-- integration/dkg/dkg_whiteboard_test.go | 4 ++-- integration/epochs/cluster_epoch_test.go | 6 +++--- integration/testnet/network.go | 8 ++++---- integration/tests/collection/suite.go | 6 +++--- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/integration/dkg/dkg_emulator_test.go b/integration/dkg/dkg_emulator_test.go index 8d349bd8899..bd4a7a4f84f 100644 --- a/integration/dkg/dkg_emulator_test.go +++ b/integration/dkg/dkg_emulator_test.go @@ -50,7 +50,7 @@ func (s *EmulatorSuite) runTest(goodNodes int, emulatorProblems bool) { DKGPhase2FinalView: 200, DKGPhase3FinalView: 250, FinalView: 300, - Participants: s.netIDs, + Participants: s.netIDs.ToSkeleton(), RandomSource: []byte("random bytes for seed"), } @@ -58,7 +58,7 @@ func (s *EmulatorSuite) runTest(goodNodes int, emulatorProblems bool) { // desired parameters nextEpochSetup := flow.EpochSetup{ Counter: currentCounter + 1, - Participants: s.netIDs, + Participants: s.netIDs.ToSkeleton(), RandomSource: []byte("random bytes for seed"), FirstView: 301, FinalView: 600, diff --git a/integration/dkg/dkg_whiteboard_test.go b/integration/dkg/dkg_whiteboard_test.go index a7b00fa1172..6de192160a9 100644 --- a/integration/dkg/dkg_whiteboard_test.go +++ b/integration/dkg/dkg_whiteboard_test.go @@ -227,7 +227,7 @@ func TestWithWhiteboard(t *testing.T) { DKGPhase2FinalView: 200, DKGPhase3FinalView: 250, FinalView: 300, - Participants: conIdentities, + Participants: conIdentities.ToSkeleton(), RandomSource: []byte("random bytes for seed"), } @@ -235,7 +235,7 @@ func TestWithWhiteboard(t *testing.T) { // desired parameters nextEpochSetup := flow.EpochSetup{ Counter: currentCounter + 1, - Participants: conIdentities, + Participants: conIdentities.ToSkeleton(), RandomSource: []byte("random bytes for seed"), } diff --git a/integration/epochs/cluster_epoch_test.go b/integration/epochs/cluster_epoch_test.go index 5ddb2cf1073..bb2ebdf332b 100644 --- a/integration/epochs/cluster_epoch_test.go +++ b/integration/epochs/cluster_epoch_test.go @@ -78,10 +78,10 @@ func (s *Suite) deployEpochQCContract() { } // CreateClusterList creates a clustering with the nodes split evenly and returns the resulting `ClusterList` -func (s *Suite) CreateClusterList(clusterCount, nodesPerCluster int) (flow.ClusterList, flow.IdentityList) { +func (s *Suite) CreateClusterList(clusterCount, nodesPerCluster int) (flow.ClusterList, flow.IdentitySkeletonList) { // create list of nodes to be used for the clustering - nodes := unittest.IdentityListFixture(clusterCount*nodesPerCluster, unittest.WithRole(flow.RoleCollection)) + nodes := unittest.IdentityListFixture(clusterCount*nodesPerCluster, unittest.WithRole(flow.RoleCollection)).ToSkeleton() // create cluster assignment clusterAssignment := unittest.ClusterAssignment(uint(clusterCount), nodes) @@ -142,7 +142,7 @@ func (s *Suite) StartVoting(clustering flow.ClusterList, clusterCount, nodesPerC cdcNodeID, err := cadence.NewString(node.NodeID.String()) require.NoError(s.T(), err) nodeIDs = append(nodeIDs, cdcNodeID) - nodeWeights = append(nodeWeights, cadence.NewUInt64(node.Weight)) + nodeWeights = append(nodeWeights, cadence.NewUInt64(node.InitialWeight)) } clusterNodeIDs[index] = cadence.NewArray(nodeIDs) diff --git a/integration/testnet/network.go b/integration/testnet/network.go index ece67666cee..94f910776d7 100644 --- a/integration/testnet/network.go +++ b/integration/testnet/network.go @@ -1038,7 +1038,7 @@ func BootstrapNetwork(networkConf NetworkConfig, bootstrapDir string, chainID fl // IMPORTANT: we must use this ordering when writing the DKG keys as // this ordering defines the DKG participant's indices - stakedNodeInfos := bootstrap.Sort(toNodeInfos(stakedConfs), order.Canonical) + stakedNodeInfos := bootstrap.Sort(toNodeInfos(stakedConfs), order.Canonical[flow.Identity]) dkg, err := runBeaconKG(stakedConfs) if err != nil { @@ -1145,7 +1145,7 @@ func BootstrapNetwork(networkConf NetworkConfig, bootstrapDir string, chainID fl DKGPhase2FinalView: dkgOffsetView + networkConf.ViewsInDKGPhase*2, DKGPhase3FinalView: dkgOffsetView + networkConf.ViewsInDKGPhase*3, FinalView: root.Header.View + networkConf.ViewsInEpoch - 1, - Participants: participants, + Participants: participants.ToSkeleton(), Assignments: clusterAssignments, RandomSource: randomSource, } @@ -1306,7 +1306,7 @@ func setupClusterGenesisBlockQCs(nClusters uint, epochCounter uint64, confs []Co participantsUnsorted := toParticipants(confs) participants := participantsUnsorted.Sort(order.Canonical[flow.Identity]) - collectors := participants.Filter(filter.HasRole[flow.Identity](flow.RoleCollection)) + collectors := participants.Filter(filter.HasRole[flow.Identity](flow.RoleCollection)).ToSkeleton() assignments := unittest.ClusterAssignment(nClusters, collectors) clusters, err := factory.NewClusterList(assignments, collectors) if err != nil { @@ -1338,7 +1338,7 @@ func setupClusterGenesisBlockQCs(nClusters uint, epochCounter uint64, confs []Co } // must order in canonical ordering otherwise decoding signer indices from cluster QC would fail - clusterCommittee := bootstrap.ToIdentityList(clusterNodeInfos).Sort(order.Canonical[flow.Identity]) + clusterCommittee := bootstrap.ToIdentityList(clusterNodeInfos).Sort(order.Canonical[flow.Identity]).ToSkeleton() qc, err := run.GenerateClusterRootQC(clusterNodeInfos, clusterCommittee, block) if err != nil { return nil, nil, nil, fmt.Errorf("fail to generate cluster root QC with clusterNodeInfos %v, %w", diff --git a/integration/tests/collection/suite.go b/integration/tests/collection/suite.go index 59dd62d3980..75ecda1854f 100644 --- a/integration/tests/collection/suite.go +++ b/integration/tests/collection/suite.go @@ -142,7 +142,7 @@ func (suite *CollectorSuite) Clusters() flow.ClusterList { setup, ok := result.ServiceEvents[0].Event.(*flow.EpochSetup) suite.Require().True(ok) - collectors := suite.net.Identities().Filter(filter.HasRole[flow.Identity](flow.RoleCollection)) + collectors := suite.net.Identities().Filter(filter.HasRole[flow.Identity](flow.RoleCollection)).ToSkeleton() clusters, err := factory.NewClusterList(setup.Assignments, collectors) suite.Require().Nil(err) return clusters @@ -170,7 +170,7 @@ func (suite *CollectorSuite) NextTransaction(opts ...func(*sdk.Transaction)) *sd return tx } -func (suite *CollectorSuite) TxForCluster(target flow.IdentityList) *sdk.Transaction { +func (suite *CollectorSuite) TxForCluster(target flow.IdentitySkeletonList) *sdk.Transaction { acct := suite.acct tx := suite.NextTransaction() @@ -331,7 +331,7 @@ func (suite *CollectorSuite) Collector(clusterIdx, nodeIdx uint) *testnet.Contai node, ok := cluster.ByIndex(nodeIdx) require.True(suite.T(), ok, "invalid node index") - return suite.net.ContainerByID(node.ID()) + return suite.net.ContainerByID(node.NodeID) } // ClusterStateFor returns a cluster state instance for the collector node with the given ID. From d4cba515e2adf4a7dbf13690695754a0b8bb7555 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Wed, 4 Oct 2023 18:44:08 +0300 Subject: [PATCH 22/39] Fixed compilation in integration tests --- integration/dkg/dkg_emulator_test.go | 4 ++-- integration/dkg/dkg_whiteboard_test.go | 4 ++-- integration/epochs/cluster_epoch_test.go | 6 +++--- integration/testnet/network.go | 8 ++++---- integration/tests/collection/suite.go | 6 +++--- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/integration/dkg/dkg_emulator_test.go b/integration/dkg/dkg_emulator_test.go index 8d349bd8899..bd4a7a4f84f 100644 --- a/integration/dkg/dkg_emulator_test.go +++ b/integration/dkg/dkg_emulator_test.go @@ -50,7 +50,7 @@ func (s *EmulatorSuite) runTest(goodNodes int, emulatorProblems bool) { DKGPhase2FinalView: 200, DKGPhase3FinalView: 250, FinalView: 300, - Participants: s.netIDs, + Participants: s.netIDs.ToSkeleton(), RandomSource: []byte("random bytes for seed"), } @@ -58,7 +58,7 @@ func (s *EmulatorSuite) runTest(goodNodes int, emulatorProblems bool) { // desired parameters nextEpochSetup := flow.EpochSetup{ Counter: currentCounter + 1, - Participants: s.netIDs, + Participants: s.netIDs.ToSkeleton(), RandomSource: []byte("random bytes for seed"), FirstView: 301, FinalView: 600, diff --git a/integration/dkg/dkg_whiteboard_test.go b/integration/dkg/dkg_whiteboard_test.go index a7b00fa1172..6de192160a9 100644 --- a/integration/dkg/dkg_whiteboard_test.go +++ b/integration/dkg/dkg_whiteboard_test.go @@ -227,7 +227,7 @@ func TestWithWhiteboard(t *testing.T) { DKGPhase2FinalView: 200, DKGPhase3FinalView: 250, FinalView: 300, - Participants: conIdentities, + Participants: conIdentities.ToSkeleton(), RandomSource: []byte("random bytes for seed"), } @@ -235,7 +235,7 @@ func TestWithWhiteboard(t *testing.T) { // desired parameters nextEpochSetup := flow.EpochSetup{ Counter: currentCounter + 1, - Participants: conIdentities, + Participants: conIdentities.ToSkeleton(), RandomSource: []byte("random bytes for seed"), } diff --git a/integration/epochs/cluster_epoch_test.go b/integration/epochs/cluster_epoch_test.go index 5ddb2cf1073..bb2ebdf332b 100644 --- a/integration/epochs/cluster_epoch_test.go +++ b/integration/epochs/cluster_epoch_test.go @@ -78,10 +78,10 @@ func (s *Suite) deployEpochQCContract() { } // CreateClusterList creates a clustering with the nodes split evenly and returns the resulting `ClusterList` -func (s *Suite) CreateClusterList(clusterCount, nodesPerCluster int) (flow.ClusterList, flow.IdentityList) { +func (s *Suite) CreateClusterList(clusterCount, nodesPerCluster int) (flow.ClusterList, flow.IdentitySkeletonList) { // create list of nodes to be used for the clustering - nodes := unittest.IdentityListFixture(clusterCount*nodesPerCluster, unittest.WithRole(flow.RoleCollection)) + nodes := unittest.IdentityListFixture(clusterCount*nodesPerCluster, unittest.WithRole(flow.RoleCollection)).ToSkeleton() // create cluster assignment clusterAssignment := unittest.ClusterAssignment(uint(clusterCount), nodes) @@ -142,7 +142,7 @@ func (s *Suite) StartVoting(clustering flow.ClusterList, clusterCount, nodesPerC cdcNodeID, err := cadence.NewString(node.NodeID.String()) require.NoError(s.T(), err) nodeIDs = append(nodeIDs, cdcNodeID) - nodeWeights = append(nodeWeights, cadence.NewUInt64(node.Weight)) + nodeWeights = append(nodeWeights, cadence.NewUInt64(node.InitialWeight)) } clusterNodeIDs[index] = cadence.NewArray(nodeIDs) diff --git a/integration/testnet/network.go b/integration/testnet/network.go index ece67666cee..94f910776d7 100644 --- a/integration/testnet/network.go +++ b/integration/testnet/network.go @@ -1038,7 +1038,7 @@ func BootstrapNetwork(networkConf NetworkConfig, bootstrapDir string, chainID fl // IMPORTANT: we must use this ordering when writing the DKG keys as // this ordering defines the DKG participant's indices - stakedNodeInfos := bootstrap.Sort(toNodeInfos(stakedConfs), order.Canonical) + stakedNodeInfos := bootstrap.Sort(toNodeInfos(stakedConfs), order.Canonical[flow.Identity]) dkg, err := runBeaconKG(stakedConfs) if err != nil { @@ -1145,7 +1145,7 @@ func BootstrapNetwork(networkConf NetworkConfig, bootstrapDir string, chainID fl DKGPhase2FinalView: dkgOffsetView + networkConf.ViewsInDKGPhase*2, DKGPhase3FinalView: dkgOffsetView + networkConf.ViewsInDKGPhase*3, FinalView: root.Header.View + networkConf.ViewsInEpoch - 1, - Participants: participants, + Participants: participants.ToSkeleton(), Assignments: clusterAssignments, RandomSource: randomSource, } @@ -1306,7 +1306,7 @@ func setupClusterGenesisBlockQCs(nClusters uint, epochCounter uint64, confs []Co participantsUnsorted := toParticipants(confs) participants := participantsUnsorted.Sort(order.Canonical[flow.Identity]) - collectors := participants.Filter(filter.HasRole[flow.Identity](flow.RoleCollection)) + collectors := participants.Filter(filter.HasRole[flow.Identity](flow.RoleCollection)).ToSkeleton() assignments := unittest.ClusterAssignment(nClusters, collectors) clusters, err := factory.NewClusterList(assignments, collectors) if err != nil { @@ -1338,7 +1338,7 @@ func setupClusterGenesisBlockQCs(nClusters uint, epochCounter uint64, confs []Co } // must order in canonical ordering otherwise decoding signer indices from cluster QC would fail - clusterCommittee := bootstrap.ToIdentityList(clusterNodeInfos).Sort(order.Canonical[flow.Identity]) + clusterCommittee := bootstrap.ToIdentityList(clusterNodeInfos).Sort(order.Canonical[flow.Identity]).ToSkeleton() qc, err := run.GenerateClusterRootQC(clusterNodeInfos, clusterCommittee, block) if err != nil { return nil, nil, nil, fmt.Errorf("fail to generate cluster root QC with clusterNodeInfos %v, %w", diff --git a/integration/tests/collection/suite.go b/integration/tests/collection/suite.go index 59dd62d3980..75ecda1854f 100644 --- a/integration/tests/collection/suite.go +++ b/integration/tests/collection/suite.go @@ -142,7 +142,7 @@ func (suite *CollectorSuite) Clusters() flow.ClusterList { setup, ok := result.ServiceEvents[0].Event.(*flow.EpochSetup) suite.Require().True(ok) - collectors := suite.net.Identities().Filter(filter.HasRole[flow.Identity](flow.RoleCollection)) + collectors := suite.net.Identities().Filter(filter.HasRole[flow.Identity](flow.RoleCollection)).ToSkeleton() clusters, err := factory.NewClusterList(setup.Assignments, collectors) suite.Require().Nil(err) return clusters @@ -170,7 +170,7 @@ func (suite *CollectorSuite) NextTransaction(opts ...func(*sdk.Transaction)) *sd return tx } -func (suite *CollectorSuite) TxForCluster(target flow.IdentityList) *sdk.Transaction { +func (suite *CollectorSuite) TxForCluster(target flow.IdentitySkeletonList) *sdk.Transaction { acct := suite.acct tx := suite.NextTransaction() @@ -331,7 +331,7 @@ func (suite *CollectorSuite) Collector(clusterIdx, nodeIdx uint) *testnet.Contai node, ok := cluster.ByIndex(nodeIdx) require.True(suite.T(), ok, "invalid node index") - return suite.net.ContainerByID(node.ID()) + return suite.net.ContainerByID(node.NodeID) } // ClusterStateFor returns a cluster state instance for the collector node with the given ID. From fe04abc4ad50d7dff79a422723d702e6f46701a8 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Wed, 4 Oct 2023 20:35:55 +0300 Subject: [PATCH 23/39] Fixed test fixtures. Updated implementation of EqualTo --- model/flow/epoch.go | 2 +- model/flow/identity_list.go | 24 ++++++++++++++++-------- model/flow/identity_test.go | 16 ++++++++-------- utils/unittest/fixtures.go | 20 ++++++++++++++------ 4 files changed, 39 insertions(+), 23 deletions(-) diff --git a/model/flow/epoch.go b/model/flow/epoch.go index 9da90aeba05..4733a59156e 100644 --- a/model/flow/epoch.go +++ b/model/flow/epoch.go @@ -105,7 +105,7 @@ func (setup *EpochSetup) EqualTo(other *EpochSetup) bool { if setup.FinalView != other.FinalView { return false } - if !setup.Participants.EqualTo(other.Participants) { + if !IdentitySkeletonListEqualTo(setup.Participants, other.Participants) { return false } if !setup.Assignments.EqualTo(other.Assignments) { diff --git a/model/flow/identity_list.go b/model/flow/identity_list.go index a4ad2efacd4..fc76a2cbcc8 100644 --- a/model/flow/identity_list.go +++ b/model/flow/identity_list.go @@ -318,14 +318,22 @@ func (il GenericIdentityList[T]) Union(other GenericIdentityList[T]) GenericIden return union } -// EqualTo checks if the other list if the same, that it contains the same elements -// in the same order -func (il GenericIdentityList[T]) EqualTo(other GenericIdentityList[T]) bool { - // TODO: temporary - //return slices.EqualFunc(il, other, func(a, b *T) bool { - // return (*a)..EqualTo(b) - //}) - return il.ID() == other.ID() +// IdentityListEqualTo checks if the other list if the same, that it contains the same elements +// in the same order. +// NOTE: currently a generic comparison is not possible, so we have to use a specific function. +func IdentityListEqualTo(lhs, rhs IdentityList) bool { + return slices.EqualFunc(lhs, rhs, func(a, b *Identity) bool { + return a.EqualTo(b) + }) +} + +// IdentitySkeletonListEqualTo checks if the other list if the same, that it contains the same elements +// in the same order. +// NOTE: currently a generic comparison is not possible, so we have to use a specific function. +func IdentitySkeletonListEqualTo(lhs, rhs IdentitySkeletonList) bool { + return slices.EqualFunc(lhs, rhs, func(a, b *IdentitySkeleton) bool { + return a.EqualTo(b) + }) } // Exists takes a previously sorted Identity list and searches it for the target value diff --git a/model/flow/identity_test.go b/model/flow/identity_test.go index fc0b0333fe6..84b7b0ed4b7 100644 --- a/model/flow/identity_test.go +++ b/model/flow/identity_test.go @@ -356,8 +356,8 @@ func TestIdentityList_EqualTo(t *testing.T) { a := flow.IdentityList{} b := flow.IdentityList{} - require.True(t, a.EqualTo(b)) - require.True(t, b.EqualTo(a)) + require.True(t, flow.IdentityListEqualTo(a, b)) + require.True(t, flow.IdentityListEqualTo(b, a)) }) t.Run("different len arent equal", func(t *testing.T) { @@ -366,8 +366,8 @@ func TestIdentityList_EqualTo(t *testing.T) { a := flow.IdentityList{identityA} b := flow.IdentityList{} - require.False(t, a.EqualTo(b)) - require.False(t, b.EqualTo(a)) + require.False(t, flow.IdentityListEqualTo(a, b)) + require.False(t, flow.IdentityListEqualTo(b, a)) }) t.Run("different data means not equal", func(t *testing.T) { @@ -377,8 +377,8 @@ func TestIdentityList_EqualTo(t *testing.T) { a := flow.IdentityList{identityA} b := flow.IdentityList{identityB} - require.False(t, a.EqualTo(b)) - require.False(t, b.EqualTo(a)) + require.False(t, flow.IdentityListEqualTo(a, b)) + require.False(t, flow.IdentityListEqualTo(b, a)) }) t.Run("same data means equal", func(t *testing.T) { @@ -387,8 +387,8 @@ func TestIdentityList_EqualTo(t *testing.T) { a := flow.IdentityList{identityA, identityA} b := flow.IdentityList{identityA, identityA} - require.True(t, a.EqualTo(b)) - require.True(t, b.EqualTo(a)) + require.True(t, flow.IdentityListEqualTo(a, b)) + require.True(t, flow.IdentityListEqualTo(b, a)) }) } diff --git a/utils/unittest/fixtures.go b/utils/unittest/fixtures.go index db22618e478..f9db1f3bd67 100644 --- a/utils/unittest/fixtures.go +++ b/utils/unittest/fixtures.go @@ -2599,7 +2599,7 @@ func ProtocolStateFixture(options ...func(*flow.RichProtocolStateEntry)) *flow.R }) allIdentities := make(flow.IdentityList, 0, len(currentEpochSetup.Participants)) - for _, identity := range currentEpochSetup.Participants.Union(prevEpochSetup.Participants) { + for _, identity := range currentEpochSetup.Participants { allIdentities = append(allIdentities, &flow.Identity{ IdentitySkeleton: *identity, DynamicIdentity: flow.DynamicIdentity{ @@ -2608,12 +2608,20 @@ func ProtocolStateFixture(options ...func(*flow.RichProtocolStateEntry)) *flow.R }, }) } - allIdentities = allIdentities.Map(func(identity flow.Identity) flow.Identity { - if _, found := prevEpochSetup.Participants.ByNodeID(identity.NodeID); found { - identity.Weight = 0 + for _, identity := range prevEpochSetup.Participants { + if _, found := allIdentities.ByNodeID(identity.NodeID); !found { + allIdentities = append(allIdentities, &flow.Identity{ + IdentitySkeleton: *identity, + DynamicIdentity: flow.DynamicIdentity{ + Weight: 0, + Ejected: false, + }, + }) } - return identity - }) + } + + allIdentities = allIdentities.Sort(order.Canonical[flow.Identity]) + entry := &flow.RichProtocolStateEntry{ ProtocolStateEntry: &flow.ProtocolStateEntry{ CurrentEpoch: flow.EpochStateContainer{ From 3dd3c4d38b9b4315e3179b1852c89b3867cf8f18 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Wed, 4 Oct 2023 22:16:08 +0300 Subject: [PATCH 24/39] Updated identities ordering for some tests. Changed how root QC are validated --- consensus/follower_test.go | 3 ++- consensus/hotstuff/signature/packer_test.go | 3 ++- consensus/hotstuff/validator/validator_test.go | 7 ++++--- module/signature/signer_indices_test.go | 2 +- state/protocol/badger/validity.go | 2 +- 5 files changed, 10 insertions(+), 7 deletions(-) diff --git a/consensus/follower_test.go b/consensus/follower_test.go index 06e81b70f25..d478993a6c2 100644 --- a/consensus/follower_test.go +++ b/consensus/follower_test.go @@ -16,6 +16,7 @@ import ( mockhotstuff "github.com/onflow/flow-go/consensus/hotstuff/mocks" "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/model/flow/order" "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/metrics" mockmodule "github.com/onflow/flow-go/module/mock" @@ -75,7 +76,7 @@ type HotStuffFollowerSuite struct { // SetupTest initializes all the components needed for the Follower. // The follower itself is instantiated in method BeforeTest func (s *HotStuffFollowerSuite) SetupTest() { - identities := unittest.IdentityListFixture(4, unittest.WithRole(flow.RoleConsensus)) + identities := unittest.IdentityListFixture(4, unittest.WithRole(flow.RoleConsensus)).Sort(order.Canonical[flow.Identity]) s.mockConsensus = &MockConsensus{identities: identities} // mock storage headers diff --git a/consensus/hotstuff/signature/packer_test.go b/consensus/hotstuff/signature/packer_test.go index 042495eb6d0..0910d4420ee 100644 --- a/consensus/hotstuff/signature/packer_test.go +++ b/consensus/hotstuff/signature/packer_test.go @@ -12,6 +12,7 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff/mocks" "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/model/flow/order" "github.com/onflow/flow-go/module/signature" "github.com/onflow/flow-go/utils/unittest" ) @@ -54,7 +55,7 @@ func makeBlockSigData(committee flow.IdentitySkeletonList) *hotstuff.BlockSignat // aggregated random beacon sigs are from [D,F] func TestPackUnpack(t *testing.T) { // prepare data for testing - committee := unittest.IdentityListFixture(6, unittest.WithRole(flow.RoleConsensus)).ToSkeleton() + committee := unittest.IdentityListFixture(6, unittest.WithRole(flow.RoleConsensus)).Sort(order.Canonical[flow.Identity]).ToSkeleton() view := rand.Uint64() blockSigData := makeBlockSigData(committee) diff --git a/consensus/hotstuff/validator/validator_test.go b/consensus/hotstuff/validator/validator_test.go index f348bb57a9c..c82c7a71738 100644 --- a/consensus/hotstuff/validator/validator_test.go +++ b/consensus/hotstuff/validator/validator_test.go @@ -19,6 +19,7 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/flow/filter" + "github.com/onflow/flow-go/model/flow/order" "github.com/onflow/flow-go/utils/unittest" ) @@ -46,7 +47,7 @@ type ProposalSuite struct { func (ps *ProposalSuite) SetupTest() { // the leader is a random node for now ps.finalized = uint64(rand.Uint32() + 1) - ps.participants = unittest.IdentityListFixture(8, unittest.WithRole(flow.RoleConsensus)) + ps.participants = unittest.IdentityListFixture(8, unittest.WithRole(flow.RoleConsensus)).Sort(order.Canonical[flow.Identity]) ps.leader = &ps.participants[0].IdentitySkeleton // the parent is the last finalized block, followed directly by a block from the leader @@ -584,7 +585,7 @@ func (qs *QCSuite) SetupTest() { qs.participants = unittest.IdentityListFixture(10, unittest.WithRole(flow.RoleConsensus), unittest.WithWeight(1), - ).ToSkeleton() + ).Sort(order.Canonical[flow.Identity]).ToSkeleton() // signers are a qualified majority at 7 qs.signers = qs.participants[:7] @@ -742,7 +743,7 @@ func (s *TCSuite) SetupTest() { s.participants = unittest.IdentityListFixture(10, unittest.WithRole(flow.RoleConsensus), unittest.WithWeight(1), - ).ToSkeleton() + ).Sort(order.Canonical[flow.Identity]).ToSkeleton() // signers are a qualified majority at 7 s.signers = s.participants[:7] diff --git a/module/signature/signer_indices_test.go b/module/signature/signer_indices_test.go index 5dae391e5c9..d33cd14a708 100644 --- a/module/signature/signer_indices_test.go +++ b/module/signature/signer_indices_test.go @@ -23,7 +23,7 @@ import ( // 2. for the decoding step, we offer an optimized convenience function to directly // decode to full identities: Indices --decode--> Identities func TestEncodeDecodeIdentities(t *testing.T) { - canonicalIdentities := unittest.IdentityListFixture(20).ToSkeleton() + canonicalIdentities := unittest.IdentityListFixture(20).Sort(order.Canonical[flow.Identity]).ToSkeleton() canonicalIdentifiers := canonicalIdentities.NodeIDs() for s := 0; s < 20; s++ { for e := s; e < 20; e++ { diff --git a/state/protocol/badger/validity.go b/state/protocol/badger/validity.go index 64455331569..7c0583f67df 100644 --- a/state/protocol/badger/validity.go +++ b/state/protocol/badger/validity.go @@ -305,7 +305,7 @@ func IsValidRootSnapshotQCs(snap protocol.Snapshot) error { // validateRootQC performs validation of root QC // Returns nil on success func validateRootQC(snap protocol.Snapshot) error { - identities, err := snap.Identities(filter.Adapt(filter.IsAllowedConsensusCommitteeMember)) + identities, err := snap.Identities(filter.IsVotingConsensusCommitteeMember) if err != nil { return fmt.Errorf("could not get root snapshot identities: %w", err) } From d3ee95bf10a4a34abfc5726167f0ae79f97862ee Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Thu, 5 Oct 2023 14:24:48 +0300 Subject: [PATCH 25/39] Updated integration test fixtures to work with BootstrapInfo instead of Identity --- .../test/cluster_switchover_test.go | 53 ++++++------ engine/execution/execution_test.go | 82 ++++++++++++------- engine/testutil/nodes.go | 63 +++----------- engine/verification/utils/unittest/helper.go | 54 ++++++++---- insecure/corruptnet/network_test_helper.go | 12 ++- integration/dkg/dkg_emulator_suite.go | 8 +- integration/dkg/dkg_whiteboard_test.go | 26 ++++-- integration/dkg/node.go | 2 +- module/dkg/broker.go | 4 +- 9 files changed, 164 insertions(+), 140 deletions(-) diff --git a/engine/collection/test/cluster_switchover_test.go b/engine/collection/test/cluster_switchover_test.go index f87da6e8798..1e72d754f79 100644 --- a/engine/collection/test/cluster_switchover_test.go +++ b/engine/collection/test/cluster_switchover_test.go @@ -37,12 +37,12 @@ type ClusterSwitchoverTestCase struct { t *testing.T conf ClusterSwitchoverTestConf - identities flow.IdentityList // identity table - hub *stub.Hub // mock network hub - root protocol.Snapshot // shared root snapshot - nodes []testmock.CollectionNode // collection nodes - sn *mocknetwork.Engine // fake consensus node engine for receiving guarantees - builder *unittest.EpochBuilder // utility for building epochs + nodeInfos []model.NodeInfo // identity table + hub *stub.Hub // mock network hub + root protocol.Snapshot // shared root snapshot + nodes []testmock.CollectionNode // collection nodes + sn *mocknetwork.Engine // fake consensus node engine for receiving guarantees + builder *unittest.EpochBuilder // utility for building epochs // epoch counter -> cluster index -> transaction IDs sentTransactions map[uint64]map[uint]flow.IdentifierList // track submitted transactions @@ -57,17 +57,22 @@ func NewClusterSwitchoverTestCase(t *testing.T, conf ClusterSwitchoverTestConf) conf: conf, } - nodeInfos := unittest.PrivateNodeInfosFixture(int(conf.collectors), unittest.WithRole(flow.RoleCollection)) - collectors := model.ToIdentityList(nodeInfos) - tc.identities = unittest.CompleteIdentitySet(collectors...) - assignment := unittest.ClusterAssignment(tc.conf.clusters, collectors.ToSkeleton()) - clusters, err := factory.NewClusterList(assignment, collectors.ToSkeleton()) + identityRoles := unittest.CompleteIdentitySet(unittest.IdentityListFixture(int(conf.collectors), unittest.WithRole(flow.RoleCollection))...) + identities := flow.IdentityList{} + for _, missingRole := range identityRoles { + nodeInfo := unittest.PrivateNodeInfosFixture(1, unittest.WithRole(missingRole.Role))[0] + tc.nodeInfos = append(tc.nodeInfos, nodeInfo) + identities = append(identities, nodeInfo.Identity()) + } + collectors := identities.Filter(filter.HasRole[flow.Identity](flow.RoleCollection)).ToSkeleton() + assignment := unittest.ClusterAssignment(tc.conf.clusters, collectors) + clusters, err := factory.NewClusterList(assignment, collectors) require.NoError(t, err) rootClusterBlocks := run.GenerateRootClusterBlocks(1, clusters) rootClusterQCs := make([]flow.ClusterQCVoteData, len(rootClusterBlocks)) for i, cluster := range clusters { signers := make([]model.NodeInfo, 0) - for _, identity := range nodeInfos { + for _, identity := range tc.nodeInfos { if _, inCluster := cluster.ByNodeID(identity.NodeID); inCluster { signers = append(signers, identity) } @@ -87,21 +92,28 @@ func NewClusterSwitchoverTestCase(t *testing.T, conf ClusterSwitchoverTestConf) tc.hub = stub.NewNetworkHub() // create a root snapshot with the given number of initial clusters - root, result, seal := unittest.BootstrapFixture(tc.identities) + root, result, seal := unittest.BootstrapFixture(identities) qc := unittest.QuorumCertificateFixture(unittest.QCWithRootBlockID(root.ID())) setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) commit := result.ServiceEvents[1].Event.(*flow.EpochCommit) - setup.Assignments = unittest.ClusterAssignment(tc.conf.clusters, tc.identities.ToSkeleton()) + setup.Assignments = unittest.ClusterAssignment(tc.conf.clusters, identities.ToSkeleton()) commit.ClusterQCs = rootClusterQCs seal.ResultID = result.ID() tc.root, err = inmem.SnapshotFromBootstrapState(root, result, seal, qc) require.NoError(t, err) + // build a lookup table for node infos + nodeInfoLookup := make(map[flow.Identifier]model.NodeInfo) + for _, nodeInfo := range tc.nodeInfos { + nodeInfoLookup[nodeInfo.NodeID] = nodeInfo + } + // create a mock node for each collector identity - for _, collector := range nodeInfos { - node := testutil.CollectionNode(tc.T(), tc.hub, collector, tc.root) + for _, collector := range collectors { + nodeInfo := nodeInfoLookup[collector.NodeID] + node := testutil.CollectionNode(tc.T(), tc.hub, nodeInfo, tc.root) tc.nodes = append(tc.nodes, node) } @@ -109,7 +121,7 @@ func NewClusterSwitchoverTestCase(t *testing.T, conf ClusterSwitchoverTestConf) consensus := testutil.GenericNode( tc.T(), tc.hub, - tc.identities.Filter(filter.HasRole[flow.Identity](flow.RoleConsensus))[0], + nodeInfoLookup[identities.Filter(filter.HasRole[flow.Identity](flow.RoleConsensus))[0].NodeID], tc.root, ) tc.sn = new(mocknetwork.Engine) @@ -117,18 +129,13 @@ func NewClusterSwitchoverTestCase(t *testing.T, conf ClusterSwitchoverTestConf) require.NoError(tc.T(), err) // create an epoch builder hooked to each collector's protocol state - states := make([]protocol.FollowerState, 0, len(collectors)) + states := make([]protocol.FollowerState, 0) for _, node := range tc.nodes { states = append(states, node.State) } // when building new epoch we would like to replace fixture cluster QCs with real ones, for that we need // to generate them using node infos tc.builder = unittest.NewEpochBuilder(tc.T(), states...).UsingCommitOpts(func(commit *flow.EpochCommit) { - // build a lookup table for node infos - nodeInfoLookup := make(map[flow.Identifier]model.NodeInfo) - for _, nodeInfo := range nodeInfos { - nodeInfoLookup[nodeInfo.NodeID] = nodeInfo - } // replace cluster QCs, with real data for i, clusterQC := range commit.ClusterQCs { diff --git a/engine/execution/execution_test.go b/engine/execution/execution_test.go index 47972940429..cb4228842a7 100644 --- a/engine/execution/execution_test.go +++ b/engine/execution/execution_test.go @@ -43,24 +43,29 @@ func TestExecutionFlow(t *testing.T) { chainID := flow.Testnet - colID := unittest.IdentityFixture( + colID := unittest.PrivateNodeInfosFixture( + 1, unittest.WithRole(flow.RoleCollection), unittest.WithKeys, - ) - conID := unittest.IdentityFixture( + )[0] + conID := unittest.PrivateNodeInfosFixture( + 1, unittest.WithRole(flow.RoleConsensus), unittest.WithKeys, - ) - exeID := unittest.IdentityFixture( + )[0] + exeID := unittest.PrivateNodeInfosFixture( + 1, unittest.WithRole(flow.RoleExecution), unittest.WithKeys, - ) - verID := unittest.IdentityFixture( + )[0] + verID := unittest.PrivateNodeInfosFixture( + 1, unittest.WithRole(flow.RoleVerification), unittest.WithKeys, - ) + )[0] - identities := unittest.CompleteIdentitySet(colID, conID, exeID, verID).Sort(order.Canonical[flow.Identity]) + identities := unittest.CompleteIdentitySet(colID.Identity(), conID.Identity(), exeID.Identity(), verID.Identity()). + Sort(order.Canonical[flow.Identity]) // create execution node exeNode := testutil.ExecutionNode(t, hub, exeID, identities, 21, chainID) @@ -98,7 +103,7 @@ func TestExecutionFlow(t *testing.T) { col2.ID(): &col2, } - clusterChainID := cluster.CanonicalClusterID(1, flow.IdentityList{colID}.NodeIDs()) + clusterChainID := cluster.CanonicalClusterID(1, flow.IdentityList{colID.Identity()}.NodeIDs()) // signed by the only collector block := unittest.BlockWithParentAndProposerFixture(t, genesis, conID.NodeID) @@ -352,28 +357,35 @@ func TestFailedTxWillNotChangeStateCommitment(t *testing.T) { chainID := flow.Emulator - colID := unittest.IdentityFixture( + colNodeInfo := unittest.PrivateNodeInfosFixture( + 1, unittest.WithRole(flow.RoleCollection), unittest.WithKeys, - ) - conID := unittest.IdentityFixture( + )[0] + conNodeInfo := unittest.PrivateNodeInfosFixture( + 1, unittest.WithRole(flow.RoleConsensus), unittest.WithKeys, - ) - exe1ID := unittest.IdentityFixture( + )[0] + exe1NodeInfo := unittest.PrivateNodeInfosFixture( + 1, unittest.WithRole(flow.RoleExecution), unittest.WithKeys, - ) + )[0] + + colID := colNodeInfo.Identity() + conID := conNodeInfo.Identity() + exe1ID := exe1NodeInfo.Identity() identities := unittest.CompleteIdentitySet(colID, conID, exe1ID) key := unittest.NetworkingPrivKeyFixture() identities[3].NetworkPubKey = key.PublicKey() - collectionNode := testutil.GenericNodeFromParticipants(t, hub, colID, identities, chainID) + collectionNode := testutil.GenericNodeFromParticipants(t, hub, colNodeInfo, identities, chainID) defer collectionNode.Done() - consensusNode := testutil.GenericNodeFromParticipants(t, hub, conID, identities, chainID) + consensusNode := testutil.GenericNodeFromParticipants(t, hub, conNodeInfo, identities, chainID) defer consensusNode.Done() - exe1Node := testutil.ExecutionNode(t, hub, exe1ID, identities, 27, chainID) + exe1Node := testutil.ExecutionNode(t, hub, exe1NodeInfo, identities, 27, chainID) ctx, cancel := context.WithCancel(context.Background()) unittest.RequireReturnsBefore(t, func() { @@ -504,28 +516,38 @@ func TestBroadcastToMultipleVerificationNodes(t *testing.T) { chainID := flow.Emulator - colID := unittest.IdentityFixture( + colID := unittest.PrivateNodeInfosFixture( + 1, unittest.WithRole(flow.RoleCollection), unittest.WithKeys, - ) - conID := unittest.IdentityFixture( + )[0] + conID := unittest.PrivateNodeInfosFixture( + 1, unittest.WithRole(flow.RoleConsensus), unittest.WithKeys, - ) - exeID := unittest.IdentityFixture( + )[0] + exeID := unittest.PrivateNodeInfosFixture( + 1, unittest.WithRole(flow.RoleExecution), unittest.WithKeys, - ) - ver1ID := unittest.IdentityFixture( + )[0] + ver1ID := unittest.PrivateNodeInfosFixture( + 1, unittest.WithRole(flow.RoleVerification), unittest.WithKeys, - ) - ver2ID := unittest.IdentityFixture( + )[0] + ver2ID := unittest.PrivateNodeInfosFixture( + 1, unittest.WithRole(flow.RoleVerification), unittest.WithKeys, - ) + )[0] - identities := unittest.CompleteIdentitySet(colID, conID, exeID, ver1ID, ver2ID) + identities := unittest.CompleteIdentitySet(colID.Identity(), + conID.Identity(), + exeID.Identity(), + ver1ID.Identity(), + ver2ID.Identity(), + ) exeNode := testutil.ExecutionNode(t, hub, exeID, identities, 21, chainID) ctx, cancel := context.WithCancel(context.Background()) diff --git a/engine/testutil/nodes.go b/engine/testutil/nodes.go index 3af10b7c5b3..0fb81de05df 100644 --- a/engine/testutil/nodes.go +++ b/engine/testutil/nodes.go @@ -2,7 +2,6 @@ package testutil import ( "context" - "encoding/json" "fmt" "math" "path/filepath" @@ -26,7 +25,6 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/consensus/hotstuff/notifications" "github.com/onflow/flow-go/consensus/hotstuff/notifications/pubsub" - "github.com/onflow/flow-go/crypto" "github.com/onflow/flow-go/engine" "github.com/onflow/flow-go/engine/collection/epochmgr" "github.com/onflow/flow-go/engine/collection/epochmgr/factories" @@ -108,7 +106,7 @@ import ( // // CAUTION: Please use GenericNode instead for most use-cases so that multiple nodes // may share the same root state snapshot. -func GenericNodeFromParticipants(t testing.TB, hub *stub.Hub, identity *flow.Identity, participants []*flow.Identity, chainID flow.ChainID, +func GenericNodeFromParticipants(t testing.TB, hub *stub.Hub, identity bootstrap.NodeInfo, participants []*flow.Identity, chainID flow.ChainID, options ...func(protocol.State)) testmock.GenericNode { var i int var participant *flow.Identity @@ -142,7 +140,7 @@ func GenericNodeFromParticipants(t testing.TB, hub *stub.Hub, identity *flow.Ide func GenericNode( t testing.TB, hub *stub.Hub, - identity *flow.Identity, + identity bootstrap.NodeInfo, root protocol.Snapshot, ) testmock.GenericNode { @@ -165,13 +163,17 @@ func GenericNode( func GenericNodeWithStateFixture(t testing.TB, stateFixture *testmock.StateFixture, hub *stub.Hub, - identity *flow.Identity, + bootstrapInfo bootstrap.NodeInfo, log zerolog.Logger, metrics *metrics.NoopCollector, tracer module.Tracer, chainID flow.ChainID) testmock.GenericNode { - me := LocalFixture(t, identity) + identity := bootstrapInfo.Identity() + privateKeys, err := bootstrapInfo.PrivateKeys() + require.NoError(t, err) + me, err := local.New(identity.IdentitySkeleton, privateKeys.StakingKey) + require.NoError(t, err) net := stub.NewNetwork(t, identity.NodeID, hub) parentCtx, cancel := context.WithCancel(context.Background()) @@ -201,28 +203,6 @@ func GenericNodeWithStateFixture(t testing.TB, } } -// LocalFixture creates and returns a Local module for given identity. -func LocalFixture(t testing.TB, identity *flow.Identity) module.Local { - - // Generates test signing oracle for the nodes - // Disclaimer: it should not be used for practical applications - // - // uses identity of node as its seed - seed, err := json.Marshal(identity) - require.NoError(t, err) - // creates signing key of the node - sk, err := crypto.GeneratePrivateKey(crypto.BLSBLS12381, seed[:64]) - require.NoError(t, err) - - // sets staking public key of the node - identity.StakingPubKey = sk.PublicKey() - - me, err := local.New(identity.IdentitySkeleton, sk) - require.NoError(t, err) - - return me -} - // CompleteStateFixture is a test helper that creates, bootstraps, and returns a StateFixture for sake of unit testing. func CompleteStateFixture( t testing.TB, @@ -282,7 +262,7 @@ func CompleteStateFixture( // CollectionNode returns a mock collection node. func CollectionNode(t *testing.T, hub *stub.Hub, identity bootstrap.NodeInfo, rootSnapshot protocol.Snapshot) testmock.CollectionNode { - node := GenericNode(t, hub, identity.Identity(), rootSnapshot) + node := GenericNode(t, hub, identity, rootSnapshot) privKeys, err := identity.PrivateKeys() require.NoError(t, err) node.Me, err = local.New(identity.Identity().IdentitySkeleton, privKeys.StakingKey) @@ -427,7 +407,7 @@ func CollectionNode(t *testing.T, hub *stub.Hub, identity bootstrap.NodeInfo, ro } } -func ConsensusNode(t *testing.T, hub *stub.Hub, identity *flow.Identity, identities []*flow.Identity, chainID flow.ChainID) testmock.ConsensusNode { +func ConsensusNode(t *testing.T, hub *stub.Hub, identity bootstrap.NodeInfo, identities []*flow.Identity, chainID flow.ChainID) testmock.ConsensusNode { node := GenericNodeFromParticipants(t, hub, identity, identities, chainID) @@ -519,30 +499,11 @@ func ConsensusNode(t *testing.T, hub *stub.Hub, identity *flow.Identity, identit } } -func ConsensusNodes(t *testing.T, hub *stub.Hub, nNodes int, chainID flow.ChainID) []testmock.ConsensusNode { - conIdentities := unittest.IdentityListFixture(nNodes, unittest.WithRole(flow.RoleConsensus)) - for _, id := range conIdentities { - t.Log(id.String()) - } - - // add some extra dummy identities so we have one of each role - others := unittest.IdentityListFixture(5, unittest.WithAllRolesExcept(flow.RoleConsensus)) - - identities := append(conIdentities, others...) - - nodes := make([]testmock.ConsensusNode, 0, len(conIdentities)) - for _, identity := range conIdentities { - nodes = append(nodes, ConsensusNode(t, hub, identity, identities, chainID)) - } - - return nodes -} - type CheckerMock struct { notifications.NoopConsumer // satisfy the FinalizationConsumer interface } -func ExecutionNode(t *testing.T, hub *stub.Hub, identity *flow.Identity, identities []*flow.Identity, syncThreshold int, chainID flow.ChainID) testmock.ExecutionNode { +func ExecutionNode(t *testing.T, hub *stub.Hub, identity bootstrap.NodeInfo, identities []*flow.Identity, syncThreshold int, chainID flow.ChainID) testmock.ExecutionNode { node := GenericNodeFromParticipants(t, hub, identity, identities, chainID) transactionsStorage := storage.NewTransactions(node.Metrics, node.PublicDB) @@ -938,7 +899,7 @@ func WithGenericNode(genericNode *testmock.GenericNode) VerificationOpt { // (integration) testing. func VerificationNode(t testing.TB, hub *stub.Hub, - verIdentity *flow.Identity, // identity of this verification node. + verIdentity bootstrap.NodeInfo, // identity of this verification node. participants flow.IdentityList, // identity of all nodes in system including this verification node. assigner module.ChunkAssigner, chunksLimit uint, diff --git a/engine/verification/utils/unittest/helper.go b/engine/verification/utils/unittest/helper.go index 1f0d67ea361..92be40d3774 100644 --- a/engine/verification/utils/unittest/helper.go +++ b/engine/verification/utils/unittest/helper.go @@ -3,6 +3,8 @@ package verificationtest import ( "context" "fmt" + "github.com/onflow/flow-go/model/bootstrap" + "golang.org/x/exp/slices" "sync" "testing" "time" @@ -46,7 +48,7 @@ type MockChunkDataProviderFunc func(*testing.T, CompleteExecutionReceiptList, fl // requests should come from a verification node, and should has one of the assigned chunk IDs. Otherwise, it fails the test. func SetupChunkDataPackProvider(t *testing.T, hub *stub.Hub, - exeIdentity *flow.Identity, + exeIdentity bootstrap.NodeInfo, participants flow.IdentityList, chainID flow.ChainID, completeERs CompleteExecutionReceiptList, @@ -150,7 +152,7 @@ func RespondChunkDataPackRequestAfterNTrials(n int) MockChunkDataProviderFunc { func SetupMockConsensusNode(t *testing.T, log zerolog.Logger, hub *stub.Hub, - conIdentity *flow.Identity, + conIdentity bootstrap.NodeInfo, verIdentities flow.IdentityList, othersIdentity flow.IdentityList, completeERs CompleteExecutionReceiptList, @@ -478,9 +480,15 @@ func withConsumers(t *testing.T, log := zerolog.Nop() // bootstraps system with one node of each role. - s, verID, participants := bootstrapSystem(t, log, tracer, authorized) - exeID := participants.Filter(filter.HasRole[flow.Identity](flow.RoleExecution))[0] - conID := participants.Filter(filter.HasRole[flow.Identity](flow.RoleConsensus))[0] + s, verID, bootstrapNodesInfo := bootstrapSystem(t, log, tracer, authorized) + + participants := bootstrap.ToIdentityList(bootstrapNodesInfo) + exeIndex := slices.IndexFunc(bootstrapNodesInfo, func(info bootstrap.NodeInfo) bool { + return info.Role == flow.RoleExecution + }) + conIndex := slices.IndexFunc(bootstrapNodesInfo, func(info bootstrap.NodeInfo) bool { + return info.Role == flow.RoleConsensus + }) // generates a chain of blocks in the form of root <- R1 <- C1 <- R2 <- C2 <- ... where Rs are distinct reference // blocks (i.e., containing guarantees), and Cs are container blocks for their preceding reference block, // Container blocks only contain receipts of their preceding reference blocks. But they do not @@ -507,7 +515,7 @@ func withConsumers(t *testing.T, if authorized { // only authorized verification node has some chunks assigned to it. _, assignedChunkIDs = MockChunkAssignmentFixture(chunkAssigner, - flow.IdentityList{verID}, + flow.IdentityList{verID.Identity()}, completeERs, EvenChunkIndexAssigner) } @@ -527,7 +535,7 @@ func withConsumers(t *testing.T, // execution node exeNode, exeEngine, exeWG := SetupChunkDataPackProvider(t, hub, - exeID, + bootstrapNodesInfo[exeIndex], participants, chainID, completeERs, @@ -538,8 +546,8 @@ func withConsumers(t *testing.T, conNode, conEngine, conWG := SetupMockConsensusNode(t, unittest.Logger(), hub, - conID, - flow.IdentityList{verID}, + bootstrapNodesInfo[conIndex], + flow.IdentityList{verID.Identity()}, participants, completeERs, chainID, @@ -613,13 +621,22 @@ func bootstrapSystem( authorized bool, ) ( *enginemock.StateFixture, - *flow.Identity, - flow.IdentityList, + bootstrap.NodeInfo, + []bootstrap.NodeInfo, ) { - // creates identities to bootstrap system with - verID := unittest.IdentityFixture(unittest.WithRole(flow.RoleVerification)) - identities := unittest.CompleteIdentitySet(verID) - identities = append(identities, unittest.IdentityFixture(unittest.WithRole(flow.RoleExecution))) // adds extra execution node + // creates bootstrapNodesInfo to bootstrap system with + bootstrapNodesInfo := make([]bootstrap.NodeInfo, 0) + var verID bootstrap.NodeInfo + for _, missingRole := range unittest.CompleteIdentitySet() { + nodeInfo := unittest.PrivateNodeInfosFixture(1, unittest.WithRole(missingRole.Role))[0] + if nodeInfo.Role == flow.RoleVerification { + verID = nodeInfo + } + bootstrapNodesInfo = append(bootstrapNodesInfo, nodeInfo) + } + bootstrapNodesInfo = append(bootstrapNodesInfo, unittest.PrivateNodeInfosFixture(1, unittest.WithRole(flow.RoleExecution))...) // adds extra execution node + + identities := bootstrap.ToIdentityList(bootstrapNodesInfo) collector := &metrics.NoopCollector{} rootSnapshot := unittest.RootSnapshotFixture(identities) @@ -628,8 +645,9 @@ func bootstrapSystem( if !authorized { // creates a new verification node identity that is unauthorized for this epoch - verID = unittest.IdentityFixture(unittest.WithRole(flow.RoleVerification)) - identities = identities.Union(flow.IdentityList{verID}) + verID = unittest.PrivateNodeInfosFixture(1, unittest.WithRole(flow.RoleVerification))[0] + bootstrapNodesInfo = append(bootstrapNodesInfo, verID) + identities = append(identities, verID.Identity()) epochBuilder := unittest.NewEpochBuilder(t, stateFixture.State) epochBuilder. @@ -637,5 +655,5 @@ func bootstrapSystem( BuildEpoch() } - return stateFixture, verID, identities + return stateFixture, verID, bootstrapNodesInfo } diff --git a/insecure/corruptnet/network_test_helper.go b/insecure/corruptnet/network_test_helper.go index 11b45734575..f8948c9b65b 100644 --- a/insecure/corruptnet/network_test_helper.go +++ b/insecure/corruptnet/network_test_helper.go @@ -5,6 +5,7 @@ package corruptnet import ( "context" "fmt" + "github.com/onflow/flow-go/module/local" "net" "testing" "time" @@ -21,7 +22,6 @@ import ( "github.com/stretchr/testify/require" - "github.com/onflow/flow-go/engine/testutil" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/network/mocknetwork" "github.com/onflow/flow-go/utils/unittest" @@ -35,10 +35,10 @@ func corruptNetworkFixture(t *testing.T, logger zerolog.Logger, corruptedID ...* // create corruptible network with no attacker registered codec := unittest.NetworkCodec() - corruptedIdentity := unittest.IdentityFixture(unittest.WithAddress(insecure.DefaultAddress)) + corruptedIdentity := unittest.PrivateNodeInfosFixture(1, unittest.WithAddress(insecure.DefaultAddress))[0] // some tests will want to create corruptible network with a specific ID if len(corruptedID) > 0 { - corruptedIdentity = corruptedID[0] + corruptedIdentity.NodeID = corruptedID[0].NodeID } flowNetwork := mocknetwork.NewNetwork(t) @@ -65,11 +65,15 @@ func corruptNetworkFixture(t *testing.T, logger zerolog.Logger, corruptedID ...* err := ccf.RegisterAdapter(adapter) require.NoError(t, err) + private, err := corruptedIdentity.PrivateKeys() + require.NoError(t, err) + me, err := local.New(corruptedIdentity.Identity().IdentitySkeleton, private.StakingKey) + require.NoError(t, err) corruptibleNetwork, err := NewCorruptNetwork( logger, flow.BftTestnet, insecure.DefaultAddress, - testutil.LocalFixture(t, corruptedIdentity), + me, codec, flowNetwork, ccf) diff --git a/integration/dkg/dkg_emulator_suite.go b/integration/dkg/dkg_emulator_suite.go index 45e09d9c558..69738ecd90f 100644 --- a/integration/dkg/dkg_emulator_suite.go +++ b/integration/dkg/dkg_emulator_suite.go @@ -64,9 +64,10 @@ func (s *EmulatorSuite) SetupTest() { s.deployDKGContract() s.setupDKGAdmin() - s.netIDs = unittest.IdentityListFixture(numberOfNodes, unittest.WithRole(flow.RoleConsensus)) - for _, id := range s.netIDs { + boostrapNodesInfo := unittest.PrivateNodeInfosFixture(numberOfNodes, unittest.WithRole(flow.RoleConsensus)) + for _, id := range boostrapNodesInfo { s.nodeAccounts = append(s.nodeAccounts, s.createAndFundAccount(id)) + s.netIDs = append(s.netIDs, id.Identity()) } for _, acc := range s.nodeAccounts { @@ -88,6 +89,7 @@ func (s *EmulatorSuite) BeforeTest(_, testName string) { } // We need to initialise the nodes with a list of identities that contain // all roles, otherwise there would be an error initialising the first epoch + identities := unittest.CompleteIdentitySet(s.netIDs...) for _, node := range s.nodes { s.initEngines(node, identities) @@ -175,7 +177,7 @@ func (s *EmulatorSuite) setupDKGAdmin() { } // createAndFundAccount creates a nodeAccount and funds it in the emulator -func (s *EmulatorSuite) createAndFundAccount(netID *flow.Identity) *nodeAccount { +func (s *EmulatorSuite) createAndFundAccount(netID bootstrap.NodeInfo) *nodeAccount { accountPrivateKey := lib.RandomPrivateKey() accountKey := sdk.NewAccountKey(). FromPrivateKey(accountPrivateKey). diff --git a/integration/dkg/dkg_whiteboard_test.go b/integration/dkg/dkg_whiteboard_test.go index 6de192160a9..380c96ce6c9 100644 --- a/integration/dkg/dkg_whiteboard_test.go +++ b/integration/dkg/dkg_whiteboard_test.go @@ -1,6 +1,7 @@ package dkg import ( + "github.com/onflow/flow-go/model/bootstrap" "math/rand" "os" "testing" @@ -35,14 +36,19 @@ func createNodes( hub *stub.Hub, chainID flow.ChainID, whiteboard *whiteboard, - conIdentities flow.IdentityList, + conIdentities []bootstrap.NodeInfo, currentEpochSetup flow.EpochSetup, nextEpochSetup flow.EpochSetup, - firstBlock *flow.Header) ([]*node, flow.IdentityList) { + firstBlock *flow.Header) []*node { + + identities := make(flow.IdentityList, 0, len(conIdentities)) + for _, identity := range conIdentities { + identities = append(identities, identity.Identity()) + } // We need to initialise the nodes with a list of identities that contain // all roles, otherwise there would be an error initialising the first epoch - identities := unittest.CompleteIdentitySet(conIdentities...) + identities = unittest.CompleteIdentitySet(identities...) nodes := []*node{} for _, id := range conIdentities { @@ -57,14 +63,14 @@ func createNodes( firstBlock)) } - return nodes, conIdentities + return nodes } // createNode instantiates a node with a network hub, a whiteboard reference, // and a pre-set EpochSetup that will be used to trigger the next DKG run. func createNode( t *testing.T, - id *flow.Identity, + id bootstrap.NodeInfo, ids []*flow.Identity, hub *stub.Hub, chainID flow.ChainID, @@ -194,7 +200,11 @@ func TestWithWhiteboard(t *testing.T) { // we run the DKG protocol with N consensus nodes N := 10 - conIdentities := unittest.IdentityListFixture(N, unittest.WithRole(flow.RoleConsensus)) + bootstrapNodesInfo := unittest.PrivateNodeInfosFixture(N, unittest.WithRole(flow.RoleConsensus)) + conIdentities := make(flow.IdentitySkeletonList, 0, len(bootstrapNodesInfo)) + for _, identity := range bootstrapNodesInfo { + conIdentities = append(conIdentities, &identity.Identity().IdentitySkeleton) + } // The EpochSetup event is received at view 100. The phase transitions are // at views 150, 200, and 250. In between phase transitions, the controller @@ -239,12 +249,12 @@ func TestWithWhiteboard(t *testing.T) { RandomSource: []byte("random bytes for seed"), } - nodes, _ := createNodes( + nodes := createNodes( t, hub, chainID, whiteboard, - conIdentities, + bootstrapNodesInfo, currentEpochSetup, nextEpochSetup, firstBlock) diff --git a/integration/dkg/node.go b/integration/dkg/node.go index acd288e53dd..9ac40c6c38f 100644 --- a/integration/dkg/node.go +++ b/integration/dkg/node.go @@ -19,7 +19,7 @@ import ( ) type nodeAccount struct { - netID *flow.Identity + netID bootstrap.NodeInfo privKey crypto.PrivateKey accountKey *sdk.AccountKey accountID string diff --git a/module/dkg/broker.go b/module/dkg/broker.go index f4b319dbdf8..9e30f7816e7 100644 --- a/module/dkg/broker.go +++ b/module/dkg/broker.go @@ -345,7 +345,7 @@ func (b *Broker) Poll(referenceBlock flow.Identifier) error { continue } if !ok { - b.log.Error().Msg("invalid signature on broadcast dkg message") + b.log.Error().Err(err).Msg("invalid signature on broadcast dkg message") continue } b.log.Debug().Msgf("forwarding broadcast message to controller") @@ -470,7 +470,7 @@ func (b *Broker) prepareBroadcastMessage(data []byte) (messages.BroadcastDKGMess func (b *Broker) verifyBroadcastMessage(bcastMsg messages.BroadcastDKGMessage) (bool, error) { err := b.hasValidDKGInstanceID(bcastMsg.DKGMessage) if err != nil { - return false, err + return false, fmt.Errorf("invalid dkg instance: %w", err) } origin := b.committee[bcastMsg.CommitteeMemberIndex] signData := fingerprint.Fingerprint(bcastMsg.DKGMessage) From e4885ebd6d428c5096102e46c6b4f7b160843800 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Thu, 5 Oct 2023 14:25:53 +0300 Subject: [PATCH 26/39] Linted --- engine/verification/utils/unittest/helper.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/engine/verification/utils/unittest/helper.go b/engine/verification/utils/unittest/helper.go index 92be40d3774..d963fdb82bd 100644 --- a/engine/verification/utils/unittest/helper.go +++ b/engine/verification/utils/unittest/helper.go @@ -3,8 +3,6 @@ package verificationtest import ( "context" "fmt" - "github.com/onflow/flow-go/model/bootstrap" - "golang.org/x/exp/slices" "sync" "testing" "time" @@ -14,12 +12,14 @@ import ( "github.com/stretchr/testify/assert" testifymock "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" + "golang.org/x/exp/slices" "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/crypto" "github.com/onflow/flow-go/engine/testutil" enginemock "github.com/onflow/flow-go/engine/testutil/mock" "github.com/onflow/flow-go/engine/verification/assigner/blockconsumer" + "github.com/onflow/flow-go/model/bootstrap" "github.com/onflow/flow-go/model/chunks" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/flow/filter" From 7e5d3c90e86f971c887670f7d68cfbf317f93a9f Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Fri, 6 Oct 2023 09:11:49 +0300 Subject: [PATCH 27/39] Fixed leftover compilation issues --- insecure/corruptnet/network_egress_test.go | 10 +++++++--- insecure/integration/tests/composability_test.go | 13 +++++++++---- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/insecure/corruptnet/network_egress_test.go b/insecure/corruptnet/network_egress_test.go index c2b807990fa..81facb551db 100644 --- a/insecure/corruptnet/network_egress_test.go +++ b/insecure/corruptnet/network_egress_test.go @@ -13,11 +13,11 @@ import ( "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" - "github.com/onflow/flow-go/engine/testutil" "github.com/onflow/flow-go/insecure" mockinsecure "github.com/onflow/flow-go/insecure/mock" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/libp2p/message" + "github.com/onflow/flow-go/module/local" "github.com/onflow/flow-go/network/mocknetwork" "github.com/onflow/flow-go/utils/unittest" ) @@ -26,16 +26,20 @@ import ( // The attacker is mocked out in this test. func TestHandleOutgoingEvent_AttackerRegistered(t *testing.T) { codec := unittest.NetworkCodec() - corruptedIdentity := unittest.IdentityFixture(unittest.WithAddress(insecure.DefaultAddress)) + corruptedIdentity := unittest.PrivateNodeInfosFixture(1, unittest.WithAddress(insecure.DefaultAddress))[0] flowNetwork := mocknetwork.NewNetwork(t) ccf := mockinsecure.NewCorruptConduitFactory(t) ccf.On("RegisterEgressController", mock.Anything).Return(nil) + privateKeys, err := corruptedIdentity.PrivateKeys() + require.NoError(t, err) + me, err := local.New(corruptedIdentity.Identity().IdentitySkeleton, privateKeys.StakingKey) + require.NoError(t, err) corruptNetwork, err := NewCorruptNetwork( unittest.Logger(), flow.BftTestnet, insecure.DefaultAddress, - testutil.LocalFixture(t, corruptedIdentity), + me, codec, flowNetwork, ccf) diff --git a/insecure/integration/tests/composability_test.go b/insecure/integration/tests/composability_test.go index 4bac2aeb0c5..3d8190a68ca 100644 --- a/insecure/integration/tests/composability_test.go +++ b/insecure/integration/tests/composability_test.go @@ -13,13 +13,13 @@ import ( "github.com/stretchr/testify/require" - "github.com/onflow/flow-go/engine/testutil" "github.com/onflow/flow-go/insecure" "github.com/onflow/flow-go/insecure/corruptnet" "github.com/onflow/flow-go/insecure/orchestrator" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/libp2p/message" "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/module/local" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/stub" "github.com/onflow/flow-go/utils/unittest" @@ -122,7 +122,7 @@ func TestCorruptNetworkFrameworkHappyPath(t *testing.T) { // withCorruptNetwork creates a real corrupt network, starts it, runs the "run" function, and then stops it. func withCorruptNetwork(t *testing.T, run func(*testing.T, flow.Identity, *corruptnet.Network, *stub.Hub)) { codec := unittest.NetworkCodec() - corruptedIdentity := unittest.IdentityFixture(unittest.WithAddress(insecure.DefaultAddress)) + corruptedIdentity := unittest.PrivateNodeInfosFixture(1, unittest.WithAddress(insecure.DefaultAddress))[0] // life-cycle management of orchestratorNetwork. ctx, cancel := context.WithCancel(context.Background()) @@ -138,11 +138,16 @@ func withCorruptNetwork(t *testing.T, run func(*testing.T, flow.Identity, *corru hub := stub.NewNetworkHub() ccf := corruptnet.NewCorruptConduitFactory(unittest.Logger(), flow.BftTestnet) flowNetwork := stub.NewNetwork(t, corruptedIdentity.NodeID, hub, stub.WithConduitFactory(ccf)) + + privateKeys, err := corruptedIdentity.PrivateKeys() + require.NoError(t, err) + me, err := local.New(corruptedIdentity.Identity().IdentitySkeleton, privateKeys.StakingKey) + require.NoError(t, err) corruptNetwork, err := corruptnet.NewCorruptNetwork( unittest.Logger(), flow.BftTestnet, insecure.DefaultAddress, - testutil.LocalFixture(t, corruptedIdentity), + me, codec, flowNetwork, ccf) @@ -161,7 +166,7 @@ func withCorruptNetwork(t *testing.T, run func(*testing.T, flow.Identity, *corru flowNetwork.StartConDev(100*time.Millisecond, true) }, 100*time.Millisecond, "failed to start corrupted node network") - run(t, *corruptedIdentity, corruptNetwork, hub) + run(t, *corruptedIdentity.Identity(), corruptNetwork, hub) // terminates orchestratorNetwork cancel() From 67075373ef5bf441c9e2b5426ba109e8a0760ec4 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Fri, 6 Oct 2023 14:07:39 +0300 Subject: [PATCH 28/39] Fixed insecure tests --- insecure/corruptnet/network_test_helper.go | 26 +++++++++------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/insecure/corruptnet/network_test_helper.go b/insecure/corruptnet/network_test_helper.go index f8948c9b65b..76d8e2545ac 100644 --- a/insecure/corruptnet/network_test_helper.go +++ b/insecure/corruptnet/network_test_helper.go @@ -5,24 +5,21 @@ package corruptnet import ( "context" "fmt" - "github.com/onflow/flow-go/module/local" "net" "testing" "time" - "github.com/stretchr/testify/mock" - "github.com/rs/zerolog" - + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" "google.golang.org/grpc" grpcinsecure "google.golang.org/grpc/credentials/insecure" "github.com/onflow/flow-go/insecure" - "github.com/onflow/flow-go/module/irrecoverable" - - "github.com/stretchr/testify/require" - + "github.com/onflow/flow-go/model/bootstrap" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/module/local" "github.com/onflow/flow-go/network/mocknetwork" "github.com/onflow/flow-go/utils/unittest" ) @@ -31,14 +28,14 @@ import ( // By default, no attacker is registered on this corruptible network. // This function is not meant to be used by tests directly because it expects the corrupt network to be properly started and stopped. // Otherwise, it will throw mock expectations errors. -func corruptNetworkFixture(t *testing.T, logger zerolog.Logger, corruptedID ...*flow.Identity) (*Network, *mocknetwork.Adapter) { +func corruptNetworkFixture(t *testing.T, logger zerolog.Logger, corruptedID ...flow.Identifier) (*Network, *mocknetwork.Adapter, bootstrap.NodeInfo) { // create corruptible network with no attacker registered codec := unittest.NetworkCodec() corruptedIdentity := unittest.PrivateNodeInfosFixture(1, unittest.WithAddress(insecure.DefaultAddress))[0] // some tests will want to create corruptible network with a specific ID if len(corruptedID) > 0 { - corruptedIdentity.NodeID = corruptedID[0].NodeID + corruptedIdentity.NodeID = corruptedID[0] } flowNetwork := mocknetwork.NewNetwork(t) @@ -80,7 +77,7 @@ func corruptNetworkFixture(t *testing.T, logger zerolog.Logger, corruptedID ...* require.NoError(t, err) // return adapter so callers can set up test specific expectations - return corruptibleNetwork, adapter + return corruptibleNetwork, adapter, corruptedIdentity } // runCorruptNetworkTest creates and starts a corruptible network, runs the "run" function of a simulated attacker and then @@ -93,8 +90,6 @@ func runCorruptNetworkTest(t *testing.T, logger zerolog.Logger, insecure.CorruptNetwork_ProcessAttackerMessageClient, // gRPC interface that orchestrator network uses to send messages to this ccf. )) { - corruptedIdentity := unittest.IdentityFixture(unittest.WithAddress(insecure.DefaultAddress)) - // life-cycle management of corruptible network ctx, cancel := context.WithCancel(context.Background()) ccfCtx, errChan := irrecoverable.WithSignaler(ctx) @@ -107,7 +102,8 @@ func runCorruptNetworkTest(t *testing.T, logger zerolog.Logger, } }() - corruptibleNetwork, adapter := corruptNetworkFixture(t, logger, corruptedIdentity) + corruptedIdentifier := unittest.IdentifierFixture() + corruptibleNetwork, adapter, corruptedIdentity := corruptNetworkFixture(t, logger, corruptedIdentifier) // start corruptible network corruptibleNetwork.Start(ccfCtx) @@ -128,7 +124,7 @@ func runCorruptNetworkTest(t *testing.T, logger zerolog.Logger, stream, err := client.ProcessAttackerMessage(context.Background()) require.NoError(t, err) - run(*corruptedIdentity, corruptibleNetwork, adapter, stream) + run(*corruptedIdentity.Identity(), corruptibleNetwork, adapter, stream) // terminates orchestratorNetwork cancel() From 99aff8911606ca188e28e623a9357f009bfcf8d4 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Fri, 6 Oct 2023 15:05:47 +0300 Subject: [PATCH 29/39] Linted --- integration/dkg/dkg_whiteboard_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/dkg/dkg_whiteboard_test.go b/integration/dkg/dkg_whiteboard_test.go index 380c96ce6c9..87d13c623f0 100644 --- a/integration/dkg/dkg_whiteboard_test.go +++ b/integration/dkg/dkg_whiteboard_test.go @@ -1,7 +1,6 @@ package dkg import ( - "github.com/onflow/flow-go/model/bootstrap" "math/rand" "os" "testing" @@ -17,6 +16,7 @@ import ( "github.com/onflow/flow-go/crypto" dkgeng "github.com/onflow/flow-go/engine/consensus/dkg" "github.com/onflow/flow-go/engine/testutil" + "github.com/onflow/flow-go/model/bootstrap" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/dkg" msig "github.com/onflow/flow-go/module/signature" From 32dd1d2e53547555d6f92191b3f5ae19f6352972 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Wed, 11 Oct 2023 12:55:43 +0300 Subject: [PATCH 30/39] Apply suggestions from code review Co-authored-by: Alexander Hentschel --- model/flow/identity_list.go | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/model/flow/identity_list.go b/model/flow/identity_list.go index fc76a2cbcc8..3356c9ab373 100644 --- a/model/flow/identity_list.go +++ b/model/flow/identity_list.go @@ -11,7 +11,7 @@ import ( "github.com/onflow/flow-go/utils/rand" ) -// Notes on using generic types: +// Notes on runtime EFFICIENCY of GENERIC TYPES: // DO NOT pass an interface to a generic function (100x runtime cost as of go 1.20). // For example, consider the function // @@ -60,7 +60,11 @@ func (iy IdentitySkeleton) GetSkeleton() IdentitySkeleton { return iy } -// IdentityFilter is a filter on identities. +// IdentityFilter is a filter on identities. Mathematically, an IdentityFilter F +// can be described as a function F: 𝓘 → 𝐼, where 𝓘 denotes the set of all identities +// and 𝐼 ⊆ 𝓘. For an input identity i, F(i) returns true if and only if i passed the +// filter, i.e. i ∈ 𝐼. Returning false means that some necessary criterion was violated +// and identity i should be dropped, i.e. i ∉ 𝐼. type IdentityFilter[T GenericIdentity] func(*T) bool // IdentityOrder is a sort for identities. @@ -118,16 +122,12 @@ func (il GenericIdentityList[T]) Map(f IdentityMapFunc[T]) GenericIdentityList[T // // CAUTION: // All Identity fields are deep-copied, _except_ for their keys, which -// are copied by reference. +// are copied by reference as they are treated as immutable by convention. func (il GenericIdentityList[T]) Copy() GenericIdentityList[T] { dup := make(GenericIdentityList[T], 0, len(il)) - lenList := len(il) - - // performance tests show this is faster than 'range' - for i := 0; i < lenList; i++ { - // copy the object - next := *(il[i]) + for i := 0; i < lenList; i++ { // performance tests show this is faster than 'range' + next := *(il[i]) // copy the object dup = append(dup, &next) } return dup @@ -143,6 +143,10 @@ func (il GenericIdentityList[T]) Selector() IdentityFilter[T] { } } +// Lookup converts the identity slice to a map using the NodeIDs as keys. This +// is useful when _repeatedly_ querying identities by their NodeIDs. The +// conversation from slice to map incurs cost O(n), for `n` the slice length. +// For a _single_ lookup, use method `ByNodeID(Identifier)` (avoiding conversion). func (il GenericIdentityList[T]) Lookup() map[Identifier]*T { lookup := make(map[Identifier]*T, len(il)) for _, identity := range il { @@ -165,7 +169,7 @@ func (il GenericIdentityList[T]) Sorted(less IdentityOrder[T]) bool { return slices.IsSortedFunc(il, less) } -// NodeIDs returns the NodeIDs of the nodes in the list. +// NodeIDs returns the NodeIDs of the nodes in the list (order preserving). func (il GenericIdentityList[T]) NodeIDs() IdentifierList { nodeIDs := make([]Identifier, 0, len(il)) for _, id := range il { @@ -274,7 +278,7 @@ func (il GenericIdentityList[T]) Shuffle() (GenericIdentityList[T], error) { // sample contains `pct` percentage of the list. The sample is rounded up // if `pct>0`, so this will always select at least one identity. // -// NOTE: The input must be between 0-1. +// NOTE: The input must be between in the interval [0, 1.0] func (il GenericIdentityList[T]) SamplePct(pct float64) (GenericIdentityList[T], error) { if pct <= 0 { return GenericIdentityList[T]{}, nil @@ -314,7 +318,6 @@ func (il GenericIdentityList[T]) Union(other GenericIdentityList[T]) GenericIden lhs, rhs := (*a).GetNodeID(), (*b).GetNodeID() return bytes.Compare(lhs[:], rhs[:]) < 0 }) - return union } @@ -336,9 +339,8 @@ func IdentitySkeletonListEqualTo(lhs, rhs IdentitySkeletonList) bool { }) } -// Exists takes a previously sorted Identity list and searches it for the target value -// This code is optimized, so the coding style will be different -// target: value to search for +// Exists takes a previously sorted Identity list and searches it for the target +// identity by its NodeID. Caution: other identity fields are not compared. // CAUTION: The identity list MUST be sorted prior to calling this method func (il GenericIdentityList[T]) Exists(target *T) bool { return il.IdentifierExists((*target).GetNodeID()) From 497bec2c0ff8e31ba137f39732231c1a8fee75ba Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Wed, 11 Oct 2023 13:49:59 +0300 Subject: [PATCH 31/39] Apply suggestions from PR review --- model/flow/identity.go | 30 ++++++++++++++++++++++++++++++ model/flow/identity_list.go | 33 ++++----------------------------- 2 files changed, 34 insertions(+), 29 deletions(-) diff --git a/model/flow/identity.go b/model/flow/identity.go index c2940f5bec2..ee42bc47014 100644 --- a/model/flow/identity.go +++ b/model/flow/identity.go @@ -126,6 +126,36 @@ func (iy Identity) Checksum() Identifier { return MakeID(iy) } +// GetNodeID returns node ID for the identity. It is needed to satisfy GenericIdentity constraint. +func (iy IdentitySkeleton) GetNodeID() Identifier { + return iy.NodeID +} + +// GetRole returns a node role for the identity. It is needed to satisfy GenericIdentity constraint. +func (iy IdentitySkeleton) GetRole() Role { + return iy.Role +} + +// GetStakingPubKey returns staking public key for the identity. It is needed to satisfy GenericIdentity constraint. +func (iy IdentitySkeleton) GetStakingPubKey() crypto.PublicKey { + return iy.StakingPubKey +} + +// GetNetworkPubKey returns network public key for the identity. It is needed to satisfy GenericIdentity constraint. +func (iy IdentitySkeleton) GetNetworkPubKey() crypto.PublicKey { + return iy.NetworkPubKey +} + +// GetInitialWeight returns initial weight for the identity. It is needed to satisfy GenericIdentity constraint. +func (iy IdentitySkeleton) GetInitialWeight() uint64 { + return iy.InitialWeight +} + +// GetSkeleton returns the skeleton part for the identity. It is needed to satisfy GenericIdentity constraint. +func (iy IdentitySkeleton) GetSkeleton() IdentitySkeleton { + return iy +} + type encodableIdentitySkeleton struct { NodeID Identifier Address string `json:",omitempty"` diff --git a/model/flow/identity_list.go b/model/flow/identity_list.go index 3356c9ab373..90b9da1ed27 100644 --- a/model/flow/identity_list.go +++ b/model/flow/identity_list.go @@ -36,30 +36,6 @@ type GenericIdentity interface { GetSkeleton() IdentitySkeleton } -func (iy IdentitySkeleton) GetNodeID() Identifier { - return iy.NodeID -} - -func (iy IdentitySkeleton) GetRole() Role { - return iy.Role -} - -func (iy IdentitySkeleton) GetStakingPubKey() crypto.PublicKey { - return iy.StakingPubKey -} - -func (iy IdentitySkeleton) GetNetworkPubKey() crypto.PublicKey { - return iy.NetworkPubKey -} - -func (iy IdentitySkeleton) GetInitialWeight() uint64 { - return iy.InitialWeight -} - -func (iy IdentitySkeleton) GetSkeleton() IdentitySkeleton { - return iy -} - // IdentityFilter is a filter on identities. Mathematically, an IdentityFilter F // can be described as a function F: 𝓘 → 𝐼, where 𝓘 denotes the set of all identities // and 𝐼 ⊆ 𝓘. For an input identity i, F(i) returns true if and only if i passed the @@ -85,14 +61,13 @@ type IdentityList = GenericIdentityList[Identity] type GenericIdentityList[T GenericIdentity] []*T // Filter will apply a filter to the identity list. +// The resulting list will only contain entries that match the filtering criteria. func (il GenericIdentityList[T]) Filter(filter IdentityFilter[T]) GenericIdentityList[T] { var dup GenericIdentityList[T] -IDLoop: for _, identity := range il { - if !filter(identity) { - continue IDLoop + if filter(identity) { + dup = append(dup, identity) } - dup = append(dup, identity) } return dup } @@ -146,7 +121,7 @@ func (il GenericIdentityList[T]) Selector() IdentityFilter[T] { // Lookup converts the identity slice to a map using the NodeIDs as keys. This // is useful when _repeatedly_ querying identities by their NodeIDs. The // conversation from slice to map incurs cost O(n), for `n` the slice length. -// For a _single_ lookup, use method `ByNodeID(Identifier)` (avoiding conversion). +// For a _single_ lookup, use method `ByNodeID(Identifier)` (avoiding conversion). func (il GenericIdentityList[T]) Lookup() map[Identifier]*T { lookup := make(map[Identifier]*T, len(il)) for _, identity := range il { From 2a0b64bea54786b6d751ef3e67da2f4c97fc965d Mon Sep 17 00:00:00 2001 From: Alexander Hentschel Date: Mon, 16 Oct 2023 14:37:13 -0700 Subject: [PATCH 32/39] removing dapper copyright notice. --- model/flow/order/identity.go | 2 -- model/flow/order/identity_test.go | 2 -- 2 files changed, 4 deletions(-) diff --git a/model/flow/order/identity.go b/model/flow/order/identity.go index 9912aedfb23..0ed5b725668 100644 --- a/model/flow/order/identity.go +++ b/model/flow/order/identity.go @@ -1,5 +1,3 @@ -// (c) 2019 Dapper Labs - ALL RIGHTS RESERVED - package order import ( diff --git a/model/flow/order/identity_test.go b/model/flow/order/identity_test.go index 05c5704f515..8b7cecde78e 100644 --- a/model/flow/order/identity_test.go +++ b/model/flow/order/identity_test.go @@ -1,5 +1,3 @@ -// (c) 2019 Dapper Labs - ALL RIGHTS RESERVED - package order_test import ( From 4122b045b04e79072d3d09f5db4c1de8c6cae014 Mon Sep 17 00:00:00 2001 From: Alexander Hentschel Date: Mon, 16 Oct 2023 15:16:38 -0700 Subject: [PATCH 33/39] fixed typo --- model/flow/protocol_state.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model/flow/protocol_state.go b/model/flow/protocol_state.go index 10c98fda619..99949cf3cf5 100644 --- a/model/flow/protocol_state.go +++ b/model/flow/protocol_state.go @@ -49,7 +49,7 @@ type EpochStateContainer struct { // ActiveIdentities contains the dynamic identity properties for the nodes that // are active in this epoch. Active means that these nodes are authorized to contribute to // extending the chain. Nodes are listed in `Identities` if and only if - // they are part of the EpochSetup even for the respective epoch. + // they are part of the EpochSetup event for the respective epoch. // The dynamic identity properties can change from block to block. Each non-deferred // identity-mutating operation is applied independently to the `ActiveIdentities` // of the relevant epoch's EpochStateContainer separately. From 9feb09e146788555eff2af9c4470c6e7c1b2781a Mon Sep 17 00:00:00 2001 From: Alexander Hentschel Date: Mon, 16 Oct 2023 16:19:58 -0700 Subject: [PATCH 34/39] extended documentation of GenericIdentityList[T].Union --- model/flow/identity_list.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/model/flow/identity_list.go b/model/flow/identity_list.go index 90b9da1ed27..0aa05fcd0fb 100644 --- a/model/flow/identity_list.go +++ b/model/flow/identity_list.go @@ -271,7 +271,9 @@ func (il GenericIdentityList[T]) SamplePct(pct float64) (GenericIdentityList[T], // Union returns a new identity list containing every identity that occurs in // either `il`, or `other`, or both. There are no duplicates in the output, -// where duplicates are identities with the same node ID. +// where duplicates are identities with the same node ID. In case an entry +// with the same NodeID exists in the receiver `il` as well as in `other`, +// the identity from `il` is included in the output. // Receiver `il` and/or method input `other` can be nil or empty. // The returned IdentityList is sorted in canonical order. func (il GenericIdentityList[T]) Union(other GenericIdentityList[T]) GenericIdentityList[T] { From cf2469d7ca88fc596565d69f1696f22f5f472e52 Mon Sep 17 00:00:00 2001 From: Alexander Hentschel Date: Mon, 16 Oct 2023 23:53:30 -0700 Subject: [PATCH 35/39] marginal goDoc update --- model/flow/protocol_state.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model/flow/protocol_state.go b/model/flow/protocol_state.go index 99949cf3cf5..443c285a279 100644 --- a/model/flow/protocol_state.go +++ b/model/flow/protocol_state.go @@ -48,7 +48,7 @@ type EpochStateContainer struct { CommitID Identifier // ActiveIdentities contains the dynamic identity properties for the nodes that // are active in this epoch. Active means that these nodes are authorized to contribute to - // extending the chain. Nodes are listed in `Identities` if and only if + // extending the chain. Nodes are listed in `ActiveIdentities` if and only if // they are part of the EpochSetup event for the respective epoch. // The dynamic identity properties can change from block to block. Each non-deferred // identity-mutating operation is applied independently to the `ActiveIdentities` From c99c1ac090284528c91f633e4c6ee829369f7f31 Mon Sep 17 00:00:00 2001 From: Alexander Hentschel Date: Tue, 17 Oct 2023 16:57:40 -0700 Subject: [PATCH 36/39] =?UTF-8?q?=E2=80=A2=20extending=20documentation=20a?= =?UTF-8?q?round=20cluster=20committee=20=E2=80=A2=20compactifying=20code?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../hotstuff/committees/cluster_committee.go | 95 ++++++++++--------- state/protocol/cluster.go | 4 +- 2 files changed, 55 insertions(+), 44 deletions(-) diff --git a/consensus/hotstuff/committees/cluster_committee.go b/consensus/hotstuff/committees/cluster_committee.go index f1d388af2df..99e9e895a75 100644 --- a/consensus/hotstuff/committees/cluster_committee.go +++ b/consensus/hotstuff/committees/cluster_committee.go @@ -20,18 +20,18 @@ import ( // implementation reference blocks on the cluster chain, which in turn reference // blocks on the main chain - this implementation manages that translation. type Cluster struct { - state protocol.State - payloads storage.ClusterPayloads - me flow.Identifier - // pre-computed leader selection for the full lifecycle of the cluster - selection *leader.LeaderSelection - // a filter that returns all members of the cluster committee allowed to vote - clusterMemberFilter flow.IdentityFilter[flow.Identity] - // initial set of cluster members, WITHOUT dynamic weight changes - initialClusterMembers flow.IdentitySkeletonList + state protocol.State + payloads storage.ClusterPayloads + me flow.Identifier + selection *leader.LeaderSelection // pre-computed leader selection for the full lifecycle of the cluster + + clusterMembers flow.IdentitySkeletonList // cluster members in canonical order as specified by the epoch smart contract + clusterMemberFilter flow.IdentityFilter[flow.Identity] // filter that returns true for all members of the cluster committee allowed to vote + weightThresholdForQC uint64 // computed based on initial cluster committee weights + weightThresholdForTO uint64 // computed based on initial cluster committee weights + + // initialClusterIdentities lists full Identities for cluster members (in canonical order) at time of cluster initialization by Epoch smart contract initialClusterIdentities flow.IdentityList - weightThresholdForQC uint64 // computed based on initial cluster committee weights - weightThresholdForTO uint64 // computed based on initial cluster committee weights } var _ hotstuff.Replicas = (*Cluster)(nil) @@ -44,7 +44,6 @@ func NewClusterCommittee( epoch protocol.Epoch, me flow.Identifier, ) (*Cluster, error) { - selection, err := leader.SelectionForCluster(cluster, epoch) if err != nil { return nil, fmt.Errorf("could not compute leader selection for cluster: %w", err) @@ -53,18 +52,8 @@ func NewClusterCommittee( initialClusterMembers := cluster.Members() totalWeight := initialClusterMembers.TotalWeight() initialClusterMembersSelector := initialClusterMembers.Selector() - // the next section is not very nice, but there are no dynamic identities for root block, - // and we need them to specificially handle querying of identities for root block - initialClusterIdentities := make(flow.IdentityList, 0, len(cluster.Members())) - for _, skeleton := range initialClusterMembers { - initialClusterIdentities = append(initialClusterIdentities, &flow.Identity{ - IdentitySkeleton: *skeleton, - DynamicIdentity: flow.DynamicIdentity{ - Weight: skeleton.InitialWeight, - Ejected: false, - }, - }) - } + initialClusterIdentities := constructInitialClusterIdentities(initialClusterMembers) + com := &Cluster{ state: state, payloads: payloads, @@ -76,7 +65,7 @@ func NewClusterCommittee( filter.Not(filter.Ejected), filter.HasWeight(true), ), - initialClusterMembers: initialClusterMembers, + clusterMembers: initialClusterMembers, initialClusterIdentities: initialClusterIdentities, weightThresholdForQC: WeightThresholdToBuildQC(totalWeight), weightThresholdForTO: WeightThresholdToTimeout(totalWeight), @@ -90,17 +79,14 @@ func (c *Cluster) IdentitiesByBlock(blockID flow.Identifier) (flow.IdentityList, // blockID is a collection block not a block produced by consensus, // to query the identities from protocol state, we need to use the reference block id from the payload // - // first retrieve the cluster block payload + // first retrieve the cluster block's payload payload, err := c.payloads.ByBlockID(blockID) if err != nil { return nil, fmt.Errorf("could not get cluster payload: %w", err) } - // an empty reference block ID indicates a root block - isRootBlock := payload.ReferenceBlockID == flow.ZeroID - - // use the initial cluster members for root block - if isRootBlock { + // An empty reference block ID indicates a root block. In this case, use the initial cluster members for root block + if isRootBlock := payload.ReferenceBlockID == flow.ZeroID; isRootBlock { return c.initialClusterIdentities, nil } @@ -110,18 +96,14 @@ func (c *Cluster) IdentitiesByBlock(blockID flow.Identifier) (flow.IdentityList, } func (c *Cluster) IdentityByBlock(blockID flow.Identifier, nodeID flow.Identifier) (*flow.Identity, error) { - - // first retrieve the cluster block payload + // first retrieve the cluster block's payload payload, err := c.payloads.ByBlockID(blockID) if err != nil { return nil, fmt.Errorf("could not get cluster payload: %w", err) } - // an empty reference block ID indicates a root block - isRootBlock := payload.ReferenceBlockID == flow.ZeroID - - // use the initial cluster members for root block - if isRootBlock { + // An empty reference block ID indicates a root block. In this case, use the initial cluster members for root block + if isRootBlock := payload.ReferenceBlockID == flow.ZeroID; isRootBlock { identity, ok := c.initialClusterIdentities.ByNodeID(nodeID) if !ok { return nil, model.NewInvalidSignerErrorf("node %v is not an authorized hotstuff participant", nodeID) @@ -143,11 +125,12 @@ func (c *Cluster) IdentityByBlock(blockID flow.Identifier, nodeID flow.Identifie return identity, nil } -// IdentitiesByEpoch returns the initial cluster members for this epoch. The view -// parameter is the view in the cluster consensus. Since clusters only exist for -// one epoch, we don't need to check the view. +// IdentitiesByEpoch returns the IdentitySkeletons of the cluster members in canonical order. +// This represents the cluster composition at the time the cluster was specified by the epoch smart +// contract (hence, we return IdentitySkeletons as opposed to full identities). Since clusters only +// exist for one epoch, we don't need to check the view. func (c *Cluster) IdentitiesByEpoch(_ uint64) (flow.IdentitySkeletonList, error) { - return c.initialClusterMembers, nil + return c.clusterMembers, nil } // IdentityByEpoch returns the node from the initial cluster members for this epoch. @@ -158,7 +141,7 @@ func (c *Cluster) IdentitiesByEpoch(_ uint64) (flow.IdentitySkeletonList, error) // - model.InvalidSignerError if nodeID was not listed by the Epoch Setup event as an // authorized participant in this cluster func (c *Cluster) IdentityByEpoch(view uint64, participantID flow.Identifier) (*flow.IdentitySkeleton, error) { - identity, ok := c.initialClusterMembers.ByNodeID(participantID) + identity, ok := c.clusterMembers.ByNodeID(participantID) if !ok { return nil, model.NewInvalidSignerErrorf("node %v is not an authorized hotstuff participant", participantID) } @@ -196,3 +179,29 @@ func (c *Cluster) Self() flow.Identifier { func (c *Cluster) DKG(_ uint64) (hotstuff.DKG, error) { panic("queried DKG of cluster committee") } + +// constructInitialClusterIdentities extends the IdentitySkeletons of the cluster members to their full Identities +// (in canonical order). at time of cluster initialization by Epoch smart contract. This represents the cluster +// composition at the time the cluster was specified by the epoch smart contract. +// +// CONTEXT: The EpochSetup event contains the IdentitySkeletons for each cluster, thereby specifying cluster membership. +// While ejection status and dynamic weight are not part of the EpochSetup event, we can supplement this information as follows: +// - Per convention, service events are delivered (asynchronously) in an *order-preserving* manner. Furthermore, weight changes or +// node ejection is also mediated by system smart contracts and delivered via service events. +// - Therefore, the EpochSetup event contains the up-to-date snapshot of the cluster members. Any weight changes or node ejection +// that happened before should be reflected in the EpochSetup event. Specifically, the initial weight should be reduced and ejected +// nodes should be no longer listed in the EpochSetup event. Hence, when the EpochSetup event is emitted / processed, the weight of +// all cluster members equals their InitialWeight and the Ejected flag is false. +func constructInitialClusterIdentities(clusterMembers flow.IdentitySkeletonList) flow.IdentityList { + initialClusterIdentities := make(flow.IdentityList, 0, len(clusterMembers)) + for _, skeleton := range clusterMembers { + initialClusterIdentities = append(initialClusterIdentities, &flow.Identity{ + IdentitySkeleton: *skeleton, + DynamicIdentity: flow.DynamicIdentity{ + Weight: skeleton.InitialWeight, + Ejected: false, + }, + }) + } + return initialClusterIdentities +} diff --git a/state/protocol/cluster.go b/state/protocol/cluster.go index 6a62aa7c050..3001d026542 100644 --- a/state/protocol/cluster.go +++ b/state/protocol/cluster.go @@ -20,7 +20,9 @@ type Cluster interface { // EpochCounter returns the epoch counter for this cluster. EpochCounter() uint64 - // Members returns the initial set of collector nodes in this cluster. + // Members returns the IdentitySkeletons of the cluster members in canonical order. + // This represents the cluster composition at the time the cluster was specified by the epoch smart + // contract (hence, we return IdentitySkeletons as opposed to full identities). Members() flow.IdentitySkeletonList // RootBlock returns the root block for this cluster. From b2c97b5696506c80236821e1856c453e947f634f Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Thu, 19 Oct 2023 12:18:23 +0300 Subject: [PATCH 37/39] Apply suggestions from PR review --- cmd/bootstrap/run/cluster_qc.go | 4 +- .../committees/consensus_committee.go | 2 +- .../hotstuff/committees/leader/consensus.go | 2 +- consensus/integration/nodes_test.go | 1 - .../test/cluster_switchover_test.go | 2 +- engine/consensus/dkg/reactor_engine.go | 2 +- engine/execution/execution_test.go | 60 ++++++++----------- engine/verification/utils/unittest/helper.go | 6 +- insecure/corruptnet/network_egress_test.go | 2 +- insecure/corruptnet/network_test_helper.go | 2 +- .../integration/tests/composability_test.go | 2 +- model/flow/factory/cluster_list.go | 5 +- model/flow/filter/identity.go | 24 +++++--- model/flow/identity_list.go | 6 +- model/flow/mapfunc/identity.go | 6 ++ model/flow/protocol_state.go | 19 ++++-- utils/unittest/fixtures.go | 4 ++ 17 files changed, 83 insertions(+), 66 deletions(-) diff --git a/cmd/bootstrap/run/cluster_qc.go b/cmd/bootstrap/run/cluster_qc.go index 689a65a339c..39f06fd81f6 100644 --- a/cmd/bootstrap/run/cluster_qc.go +++ b/cmd/bootstrap/run/cluster_qc.go @@ -32,8 +32,8 @@ func GenerateClusterRootQC(signers []bootstrap.NodeInfo, allCommitteeMembers flo } // STEP 1.5: patch committee to include dynamic identities. This is a temporary measure until bootstrapping is refactored. - // We need to do this since the committee is used to create the QC uses dynamic identities, but clustering for root block contain only - // static identities since there no state transitions haven't happened yet. + // We need a Committee for creating the cluster's root QC and the Committee requires dynamic identities to be instantiated. + // The clustering for root block contain only static identities, since there no state transitions have happened yet. dynamicCommitteeMembers := make(flow.IdentityList, 0, len(allCommitteeMembers)) for _, participant := range allCommitteeMembers { dynamicCommitteeMembers = append(dynamicCommitteeMembers, &flow.Identity{ diff --git a/consensus/hotstuff/committees/consensus_committee.go b/consensus/hotstuff/committees/consensus_committee.go index 172f67cd4e8..f4dd5548670 100644 --- a/consensus/hotstuff/committees/consensus_committee.go +++ b/consensus/hotstuff/committees/consensus_committee.go @@ -55,7 +55,7 @@ func newStaticEpochInfo(epoch protocol.Epoch) (*staticEpochInfo, error) { if err != nil { return nil, fmt.Errorf("could not initial identities: %w", err) } - initialCommittee := initialIdentities.Filter(filter.IsAllowedConsensusCommitteeMember).ToSkeleton() + initialCommittee := initialIdentities.Filter(filter.IsConsensusCommitteeMember).ToSkeleton() dkg, err := epoch.DKG() if err != nil { return nil, fmt.Errorf("could not get dkg: %w", err) diff --git a/consensus/hotstuff/committees/leader/consensus.go b/consensus/hotstuff/committees/leader/consensus.go index a2c1400b8e0..f278e690f76 100644 --- a/consensus/hotstuff/committees/leader/consensus.go +++ b/consensus/hotstuff/committees/leader/consensus.go @@ -43,7 +43,7 @@ func SelectionForConsensus(epoch protocol.Epoch) (*LeaderSelection, error) { firstView, rng, int(finalView-firstView+1), // add 1 because both first/final view are inclusive - identities.Filter(filter.IsAllowedConsensusCommitteeMember), + identities.Filter(filter.IsConsensusCommitteeMember), ) return leaders, err } diff --git a/consensus/integration/nodes_test.go b/consensus/integration/nodes_test.go index 940556bea75..dbb6a9cd350 100644 --- a/consensus/integration/nodes_test.go +++ b/consensus/integration/nodes_test.go @@ -472,7 +472,6 @@ func createNode( rootQC, err := rootSnapshot.QuorumCertificate() require.NoError(t, err) - // selector := filter.HasRole[flow.Identity](flow.RoleConsensus) committee, err := committees.NewConsensusCommittee(state, localID) require.NoError(t, err) protocolStateEvents.AddConsumer(committee) diff --git a/engine/collection/test/cluster_switchover_test.go b/engine/collection/test/cluster_switchover_test.go index 1e72d754f79..d9797b89ab8 100644 --- a/engine/collection/test/cluster_switchover_test.go +++ b/engine/collection/test/cluster_switchover_test.go @@ -60,7 +60,7 @@ func NewClusterSwitchoverTestCase(t *testing.T, conf ClusterSwitchoverTestConf) identityRoles := unittest.CompleteIdentitySet(unittest.IdentityListFixture(int(conf.collectors), unittest.WithRole(flow.RoleCollection))...) identities := flow.IdentityList{} for _, missingRole := range identityRoles { - nodeInfo := unittest.PrivateNodeInfosFixture(1, unittest.WithRole(missingRole.Role))[0] + nodeInfo := unittest.PrivateNodeInfoFixture(unittest.WithRole(missingRole.Role)) tc.nodeInfos = append(tc.nodeInfos, nodeInfo) identities = append(identities, nodeInfo.Identity()) } diff --git a/engine/consensus/dkg/reactor_engine.go b/engine/consensus/dkg/reactor_engine.go index e9c0669cf9b..b1055b9ff89 100644 --- a/engine/consensus/dkg/reactor_engine.go +++ b/engine/consensus/dkg/reactor_engine.go @@ -181,7 +181,7 @@ func (e *ReactorEngine) startDKGForEpoch(currentEpochCounter uint64, first *flow log.Fatal().Err(err).Msg("could not retrieve epoch info") } - committee := curDKGInfo.identities.Filter(filter.IsAllowedConsensusCommitteeMember) + committee := curDKGInfo.identities.Filter(filter.IsConsensusCommitteeMember) log.Info(). Uint64("phase1", curDKGInfo.phase1FinalView). diff --git a/engine/execution/execution_test.go b/engine/execution/execution_test.go index cb4228842a7..0ac4d14e23a 100644 --- a/engine/execution/execution_test.go +++ b/engine/execution/execution_test.go @@ -43,26 +43,22 @@ func TestExecutionFlow(t *testing.T) { chainID := flow.Testnet - colID := unittest.PrivateNodeInfosFixture( - 1, + colID := unittest.PrivateNodeInfoFixture( unittest.WithRole(flow.RoleCollection), unittest.WithKeys, - )[0] - conID := unittest.PrivateNodeInfosFixture( - 1, + ) + conID := unittest.PrivateNodeInfoFixture( unittest.WithRole(flow.RoleConsensus), unittest.WithKeys, - )[0] - exeID := unittest.PrivateNodeInfosFixture( - 1, + ) + exeID := unittest.PrivateNodeInfoFixture( unittest.WithRole(flow.RoleExecution), unittest.WithKeys, - )[0] - verID := unittest.PrivateNodeInfosFixture( - 1, + ) + verID := unittest.PrivateNodeInfoFixture( unittest.WithRole(flow.RoleVerification), unittest.WithKeys, - )[0] + ) identities := unittest.CompleteIdentitySet(colID.Identity(), conID.Identity(), exeID.Identity(), verID.Identity()). Sort(order.Canonical[flow.Identity]) @@ -357,21 +353,18 @@ func TestFailedTxWillNotChangeStateCommitment(t *testing.T) { chainID := flow.Emulator - colNodeInfo := unittest.PrivateNodeInfosFixture( - 1, + colNodeInfo := unittest.PrivateNodeInfoFixture( unittest.WithRole(flow.RoleCollection), unittest.WithKeys, - )[0] - conNodeInfo := unittest.PrivateNodeInfosFixture( - 1, + ) + conNodeInfo := unittest.PrivateNodeInfoFixture( unittest.WithRole(flow.RoleConsensus), unittest.WithKeys, - )[0] - exe1NodeInfo := unittest.PrivateNodeInfosFixture( - 1, + ) + exe1NodeInfo := unittest.PrivateNodeInfoFixture( unittest.WithRole(flow.RoleExecution), unittest.WithKeys, - )[0] + ) colID := colNodeInfo.Identity() conID := conNodeInfo.Identity() @@ -516,31 +509,26 @@ func TestBroadcastToMultipleVerificationNodes(t *testing.T) { chainID := flow.Emulator - colID := unittest.PrivateNodeInfosFixture( - 1, + colID := unittest.PrivateNodeInfoFixture( unittest.WithRole(flow.RoleCollection), unittest.WithKeys, - )[0] - conID := unittest.PrivateNodeInfosFixture( - 1, + ) + conID := unittest.PrivateNodeInfoFixture( unittest.WithRole(flow.RoleConsensus), unittest.WithKeys, - )[0] - exeID := unittest.PrivateNodeInfosFixture( - 1, + ) + exeID := unittest.PrivateNodeInfoFixture( unittest.WithRole(flow.RoleExecution), unittest.WithKeys, - )[0] - ver1ID := unittest.PrivateNodeInfosFixture( - 1, + ) + ver1ID := unittest.PrivateNodeInfoFixture( unittest.WithRole(flow.RoleVerification), unittest.WithKeys, - )[0] - ver2ID := unittest.PrivateNodeInfosFixture( - 1, + ) + ver2ID := unittest.PrivateNodeInfoFixture( unittest.WithRole(flow.RoleVerification), unittest.WithKeys, - )[0] + ) identities := unittest.CompleteIdentitySet(colID.Identity(), conID.Identity(), diff --git a/engine/verification/utils/unittest/helper.go b/engine/verification/utils/unittest/helper.go index d963fdb82bd..d3ce766ec5b 100644 --- a/engine/verification/utils/unittest/helper.go +++ b/engine/verification/utils/unittest/helper.go @@ -628,13 +628,13 @@ func bootstrapSystem( bootstrapNodesInfo := make([]bootstrap.NodeInfo, 0) var verID bootstrap.NodeInfo for _, missingRole := range unittest.CompleteIdentitySet() { - nodeInfo := unittest.PrivateNodeInfosFixture(1, unittest.WithRole(missingRole.Role))[0] + nodeInfo := unittest.PrivateNodeInfoFixture(unittest.WithRole(missingRole.Role)) if nodeInfo.Role == flow.RoleVerification { verID = nodeInfo } bootstrapNodesInfo = append(bootstrapNodesInfo, nodeInfo) } - bootstrapNodesInfo = append(bootstrapNodesInfo, unittest.PrivateNodeInfosFixture(1, unittest.WithRole(flow.RoleExecution))...) // adds extra execution node + bootstrapNodesInfo = append(bootstrapNodesInfo, unittest.PrivateNodeInfoFixture(unittest.WithRole(flow.RoleExecution))) // adds extra execution node identities := bootstrap.ToIdentityList(bootstrapNodesInfo) @@ -645,7 +645,7 @@ func bootstrapSystem( if !authorized { // creates a new verification node identity that is unauthorized for this epoch - verID = unittest.PrivateNodeInfosFixture(1, unittest.WithRole(flow.RoleVerification))[0] + verID = unittest.PrivateNodeInfoFixture(unittest.WithRole(flow.RoleVerification)) bootstrapNodesInfo = append(bootstrapNodesInfo, verID) identities = append(identities, verID.Identity()) diff --git a/insecure/corruptnet/network_egress_test.go b/insecure/corruptnet/network_egress_test.go index 81facb551db..072f4394a9f 100644 --- a/insecure/corruptnet/network_egress_test.go +++ b/insecure/corruptnet/network_egress_test.go @@ -26,7 +26,7 @@ import ( // The attacker is mocked out in this test. func TestHandleOutgoingEvent_AttackerRegistered(t *testing.T) { codec := unittest.NetworkCodec() - corruptedIdentity := unittest.PrivateNodeInfosFixture(1, unittest.WithAddress(insecure.DefaultAddress))[0] + corruptedIdentity := unittest.PrivateNodeInfoFixture(unittest.WithAddress(insecure.DefaultAddress)) flowNetwork := mocknetwork.NewNetwork(t) ccf := mockinsecure.NewCorruptConduitFactory(t) ccf.On("RegisterEgressController", mock.Anything).Return(nil) diff --git a/insecure/corruptnet/network_test_helper.go b/insecure/corruptnet/network_test_helper.go index 76d8e2545ac..1f7ff8b1cf4 100644 --- a/insecure/corruptnet/network_test_helper.go +++ b/insecure/corruptnet/network_test_helper.go @@ -32,7 +32,7 @@ func corruptNetworkFixture(t *testing.T, logger zerolog.Logger, corruptedID ...f // create corruptible network with no attacker registered codec := unittest.NetworkCodec() - corruptedIdentity := unittest.PrivateNodeInfosFixture(1, unittest.WithAddress(insecure.DefaultAddress))[0] + corruptedIdentity := unittest.PrivateNodeInfoFixture(unittest.WithAddress(insecure.DefaultAddress)) // some tests will want to create corruptible network with a specific ID if len(corruptedID) > 0 { corruptedIdentity.NodeID = corruptedID[0] diff --git a/insecure/integration/tests/composability_test.go b/insecure/integration/tests/composability_test.go index 3d8190a68ca..9e996697b7e 100644 --- a/insecure/integration/tests/composability_test.go +++ b/insecure/integration/tests/composability_test.go @@ -122,7 +122,7 @@ func TestCorruptNetworkFrameworkHappyPath(t *testing.T) { // withCorruptNetwork creates a real corrupt network, starts it, runs the "run" function, and then stops it. func withCorruptNetwork(t *testing.T, run func(*testing.T, flow.Identity, *corruptnet.Network, *stub.Hub)) { codec := unittest.NetworkCodec() - corruptedIdentity := unittest.PrivateNodeInfosFixture(1, unittest.WithAddress(insecure.DefaultAddress))[0] + corruptedIdentity := unittest.PrivateNodeInfoFixture(unittest.WithAddress(insecure.DefaultAddress)) // life-cycle management of orchestratorNetwork. ctx, cancel := context.WithCancel(context.Background()) diff --git a/model/flow/factory/cluster_list.go b/model/flow/factory/cluster_list.go index aa86dcc7146..49ffae74447 100644 --- a/model/flow/factory/cluster_list.go +++ b/model/flow/factory/cluster_list.go @@ -15,7 +15,7 @@ import ( func NewClusterList(assignments flow.AssignmentList, collectors flow.IdentitySkeletonList) (flow.ClusterList, error) { // build a lookup for all the identities by node identifier - lookup := make(map[flow.Identifier]*flow.IdentitySkeleton) + lookup := collectors.Lookup() for _, collector := range collectors { lookup[collector.NodeID] = collector } @@ -23,7 +23,8 @@ func NewClusterList(assignments flow.AssignmentList, collectors flow.IdentitySke return nil, fmt.Errorf("duplicate collector in list") } - // replicate the identifier list but use identities instead + // assignments only contains the NodeIDs for each cluster. In the following, we + // substitute them with the respective IdentitySkeletons. clusters := make(flow.ClusterList, 0, len(assignments)) for i, participants := range assignments { cluster := make(flow.IdentitySkeletonList, 0, len(participants)) diff --git a/model/flow/filter/identity.go b/model/flow/filter/identity.go index c8b3bac7a96..99ac738f644 100644 --- a/model/flow/filter/identity.go +++ b/model/flow/filter/identity.go @@ -52,8 +52,11 @@ func Not[T flow.GenericIdentity](filter flow.IdentityFilter[T]) flow.IdentityFil } } -// In returns a filter for identities within the input list. This is equivalent -// to HasNodeID, but for list-typed inputs. +// In returns a filter for identities within the input list. For an input identity i, +// the filter returns true if and only if i ∈ list. +// Caution: The filter solely operates on NodeIDs. Other identity fields are not compared. +// This function is just a compact representation of `HasNodeID[T](list.NodeIDs()...)` +// which behaves algorithmically the same way. func In[T flow.GenericIdentity](list flow.GenericIdentityList[T]) flow.IdentityFilter[T] { return HasNodeID[T](list.NodeIDs()...) } @@ -91,7 +94,12 @@ func HasInitialWeight[T flow.GenericIdentity](hasWeight bool) flow.IdentityFilte } } -// HasWeight returns a filter for nodes with non-zero weight. +// HasWeight filters Identities by their weight: +// When `hasWeight == true`: +// - for an input identity i, the filter returns true if and only if i's weight is greater than zero +// +// When `hasWeight == false`: +// - for an input identity i, the filter returns true if and only if i's weight is zero func HasWeight(hasWeight bool) flow.IdentityFilter[flow.Identity] { return func(identity *flow.Identity) bool { return (identity.Weight > 0) == hasWeight @@ -122,9 +130,11 @@ var IsValidCurrentEpochParticipant = And( Not(Ejected), // ejection will change signer index ) -// IsAllowedConsensusCommitteeMember is a identity filter for all members of -// the consensus committee allowed to participate. -var IsAllowedConsensusCommitteeMember = And( +// IsConsensusCommitteeMember is an identity filter for all members of the consensus committee. +// Formally, a Node X is a Consensus Committee Member if and only if X is a consensus node with +// positive initial stake. This is specified by the EpochSetup Event and remains static +// throughout the epoch. +var IsConsensusCommitteeMember = And( HasRole[flow.IdentitySkeleton](flow.RoleConsensus), HasInitialWeight[flow.IdentitySkeleton](true), ) @@ -139,4 +149,4 @@ var IsVotingConsensusCommitteeMember = And[flow.Identity]( // IsValidDKGParticipant is an identity filter for all DKG participants. It is // equivalent to the filter for consensus committee members, as these are // the same group for now. -var IsValidDKGParticipant = IsAllowedConsensusCommitteeMember +var IsValidDKGParticipant = IsConsensusCommitteeMember diff --git a/model/flow/identity_list.go b/model/flow/identity_list.go index 0aa05fcd0fb..8e2f842dadc 100644 --- a/model/flow/identity_list.go +++ b/model/flow/identity_list.go @@ -317,8 +317,10 @@ func IdentitySkeletonListEqualTo(lhs, rhs IdentitySkeletonList) bool { } // Exists takes a previously sorted Identity list and searches it for the target -// identity by its NodeID. Caution: other identity fields are not compared. -// CAUTION: The identity list MUST be sorted prior to calling this method +// identity by its NodeID. +// CAUTION: +// - Other identity fields are not compared. +// - The identity list MUST be sorted prior to calling this method. func (il GenericIdentityList[T]) Exists(target *T) bool { return il.IdentifierExists((*target).GetNodeID()) } diff --git a/model/flow/mapfunc/identity.go b/model/flow/mapfunc/identity.go index 681e21d1088..a58bc7d0844 100644 --- a/model/flow/mapfunc/identity.go +++ b/model/flow/mapfunc/identity.go @@ -4,6 +4,9 @@ import ( "github.com/onflow/flow-go/model/flow" ) +// WithInitialWeight returns an anonymous function that assigns the given weight value +// to `Identity.InitialWeight`. This function is primarily intended for testing, as +// Identity structs should be immutable by convention. func WithInitialWeight(weight uint64) flow.IdentityMapFunc[flow.Identity] { return func(identity flow.Identity) flow.Identity { identity.InitialWeight = weight @@ -11,6 +14,9 @@ func WithInitialWeight(weight uint64) flow.IdentityMapFunc[flow.Identity] { } } +// WithWeight returns an anonymous function that assigns the given weight value +// to `Identity.Weight`. This function is primarily intended for testing, as +// Identity structs should be immutable by convention. func WithWeight(weight uint64) flow.IdentityMapFunc[flow.Identity] { return func(identity flow.Identity) flow.Identity { identity.Weight = weight diff --git a/model/flow/protocol_state.go b/model/flow/protocol_state.go index 443c285a279..cf3444a77ac 100644 --- a/model/flow/protocol_state.go +++ b/model/flow/protocol_state.go @@ -209,14 +209,14 @@ func NewRichProtocolStateEntry( } else { // if next epoch is not yet created, it means that we are in staking phase, // so we need to build the identity table using previous and current epoch setup events. - var otherIdentities IdentitySkeletonList + var previousEpochIdentities IdentitySkeletonList if previousEpochSetup != nil { - otherIdentities = previousEpochSetup.Participants + previousEpochIdentities = previousEpochSetup.Participants } result.CurrentEpochIdentityTable, err = BuildIdentityTable( protocolState.CurrentEpoch.ActiveIdentities, currentEpochSetup.Participants, - otherIdentities, + previousEpochIdentities, ) if err != nil { return nil, fmt.Errorf("could not build identity table for staking phase: %w", err) @@ -354,8 +354,9 @@ func (ll DynamicIdentityEntryList) Sort(less IdentifierOrder) DynamicIdentityEnt } // BuildIdentityTable constructs the full identity table for the target epoch by combining data from: -// 1. The target epoch's Dynamic Identities. -// 2. The target epoch's IdentitySkeletons +// 1. The Dynamic Identities for the nodes that are _active_ in the target epoch (i.e. the dynamic identity +// fields for the IdentitySkeletons contained in the EpochSetup event for the respective epoch). +// 2. The IdentitySkeletons for the nodes that are _active_ in the target epoch // (recorded in EpochSetup event and immutable throughout the epoch). // 3. [optional] An adjacent epoch's IdentitySkeletons (can be empty or nil), as recorded in the // adjacent epoch's setup event. For a target epoch N, the epochs N-1 and N+1 are defined to be @@ -369,7 +370,13 @@ func BuildIdentityTable( targetEpochIdentitySkeletons IdentitySkeletonList, adjacentEpochIdentitySkeletons IdentitySkeletonList, ) (IdentityList, error) { - // produce a unique set for current and previous epoch participants + // Combine the participants of the current and adjacent epoch. The method `GenericIdentityList.Union` + // already implements the following required conventions: + // 1. Preference for IdentitySkeleton of the target epoch: + // In case an IdentitySkeleton with the same NodeID exists in the target epoch as well as + // in the adjacent epoch, we use the IdentitySkeleton for the target epoch (for example, + // to account for changes of keys, address, initial weight, etc). + // 2. Canonical ordering allEpochParticipants := targetEpochIdentitySkeletons.Union(adjacentEpochIdentitySkeletons) // sanity check: size of identities should be equal to previous and current epoch participants combined if len(allEpochParticipants) != len(targetEpochDynamicIdentities) { diff --git a/utils/unittest/fixtures.go b/utils/unittest/fixtures.go index 2ad4a4609dc..8ba2c2f87b2 100644 --- a/utils/unittest/fixtures.go +++ b/utils/unittest/fixtures.go @@ -1094,6 +1094,10 @@ func NodeInfosFixture(n int, opts ...func(*flow.Identity)) []bootstrap.NodeInfo return nodeInfos } +func PrivateNodeInfoFixture(opts ...func(*flow.Identity)) bootstrap.NodeInfo { + return PrivateNodeInfosFixture(1, opts...)[0] +} + func PrivateNodeInfosFixture(n int, opts ...func(*flow.Identity)) []bootstrap.NodeInfo { il := IdentityListFixture(n, opts...) nodeInfos := make([]bootstrap.NodeInfo, 0, n) From 244af7b94621f4183609ee73684cd0aa66dc815c Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Thu, 19 Oct 2023 13:52:21 +0300 Subject: [PATCH 38/39] Simplified bootstrap in cluster switchover test --- engine/collection/test/cluster_switchover_test.go | 13 +++++-------- engine/verification/utils/unittest/helper.go | 1 - model/flow/filter/identity.go | 2 +- utils/unittest/fixtures.go | 7 +++++-- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/engine/collection/test/cluster_switchover_test.go b/engine/collection/test/cluster_switchover_test.go index d9797b89ab8..4785969fd28 100644 --- a/engine/collection/test/cluster_switchover_test.go +++ b/engine/collection/test/cluster_switchover_test.go @@ -56,14 +56,11 @@ func NewClusterSwitchoverTestCase(t *testing.T, conf ClusterSwitchoverTestConf) t: t, conf: conf, } - - identityRoles := unittest.CompleteIdentitySet(unittest.IdentityListFixture(int(conf.collectors), unittest.WithRole(flow.RoleCollection))...) - identities := flow.IdentityList{} - for _, missingRole := range identityRoles { - nodeInfo := unittest.PrivateNodeInfoFixture(unittest.WithRole(missingRole.Role)) - tc.nodeInfos = append(tc.nodeInfos, nodeInfo) - identities = append(identities, nodeInfo.Identity()) - } + tc.nodeInfos = unittest.PrivateNodeInfosFromIdentityList( + unittest.CompleteIdentitySet( + unittest.IdentityListFixture(int(conf.collectors), unittest.WithRole(flow.RoleCollection))...), + ) + identities := model.ToIdentityList(tc.nodeInfos) collectors := identities.Filter(filter.HasRole[flow.Identity](flow.RoleCollection)).ToSkeleton() assignment := unittest.ClusterAssignment(tc.conf.clusters, collectors) clusters, err := factory.NewClusterList(assignment, collectors) diff --git a/engine/verification/utils/unittest/helper.go b/engine/verification/utils/unittest/helper.go index d3ce766ec5b..2fa9e00a936 100644 --- a/engine/verification/utils/unittest/helper.go +++ b/engine/verification/utils/unittest/helper.go @@ -635,7 +635,6 @@ func bootstrapSystem( bootstrapNodesInfo = append(bootstrapNodesInfo, nodeInfo) } bootstrapNodesInfo = append(bootstrapNodesInfo, unittest.PrivateNodeInfoFixture(unittest.WithRole(flow.RoleExecution))) // adds extra execution node - identities := bootstrap.ToIdentityList(bootstrapNodesInfo) collector := &metrics.NoopCollector{} diff --git a/model/flow/filter/identity.go b/model/flow/filter/identity.go index 99ac738f644..a4121bcda4d 100644 --- a/model/flow/filter/identity.go +++ b/model/flow/filter/identity.go @@ -139,7 +139,7 @@ var IsConsensusCommitteeMember = And( HasInitialWeight[flow.IdentitySkeleton](true), ) -// IsVotingConsensusCommitteeMember is a identity filter for all members of +// IsVotingConsensusCommitteeMember is an identity filter for all members of // the consensus committee allowed to vote. var IsVotingConsensusCommitteeMember = And[flow.Identity]( HasRole[flow.Identity](flow.RoleConsensus), diff --git a/utils/unittest/fixtures.go b/utils/unittest/fixtures.go index 8ba2c2f87b2..2d479d98dcd 100644 --- a/utils/unittest/fixtures.go +++ b/utils/unittest/fixtures.go @@ -1099,8 +1099,11 @@ func PrivateNodeInfoFixture(opts ...func(*flow.Identity)) bootstrap.NodeInfo { } func PrivateNodeInfosFixture(n int, opts ...func(*flow.Identity)) []bootstrap.NodeInfo { - il := IdentityListFixture(n, opts...) - nodeInfos := make([]bootstrap.NodeInfo, 0, n) + return PrivateNodeInfosFromIdentityList(IdentityListFixture(n, opts...)) +} + +func PrivateNodeInfosFromIdentityList(il flow.IdentityList) []bootstrap.NodeInfo { + nodeInfos := make([]bootstrap.NodeInfo, 0, len(il)) for _, identity := range il { nodeInfo := bootstrap.PrivateNodeInfoFromIdentity(identity, KeyFixture(crypto.ECDSAP256), KeyFixture(crypto.BLSBLS12381)) nodeInfos = append(nodeInfos, nodeInfo) From ad075bc5ecb49fc298efe02fc8d23d72a1a04998 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Thu, 19 Oct 2023 14:12:16 +0300 Subject: [PATCH 39/39] Apply suggestions from PR review --- model/flow/filter/identity.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/model/flow/filter/identity.go b/model/flow/filter/identity.go index a4121bcda4d..8721f773767 100644 --- a/model/flow/filter/identity.go +++ b/model/flow/filter/identity.go @@ -7,9 +7,9 @@ import ( "github.com/onflow/flow-go/model/flow" ) -// Adapt adapts a filter for a specific identity type. -// -// Converts flow.IdentityFilter[flow.IdentitySkeleton] to flow.IdentityFilter[flow.Identity]. +// Adapt takes an IdentityFilter on the domain of IdentitySkeletons +// and adapts the filter to the domain of full Identities. In other words, it converts +// flow.IdentityFilter[flow.IdentitySkeleton] to flow.IdentityFilter[flow.Identity]. func Adapt(f flow.IdentityFilter[flow.IdentitySkeleton]) flow.IdentityFilter[flow.Identity] { return func(i *flow.Identity) bool { return f(&i.IdentitySkeleton)