diff --git a/Makefile b/Makefile index b67dcaff165..b31a373d815 100644 --- a/Makefile +++ b/Makefile @@ -114,6 +114,7 @@ generate-mocks: GO111MODULE=on mockery -name '.*' -dir=fvm -case=underscore -output="./fvm/mock" -outpkg="mock" GO111MODULE=on mockery -name '.*' -dir=network/gossip/libp2p/middleware -case=underscore -output="./network/gossip/libp2p/mock" -outpkg="mock" GO111MODULE=on mockery -name 'Connector' -dir=network/gossip/libp2p -case=underscore -output="./network/gossip/libp2p/mock" -outpkg="mock" + GO111MODULE=on mockery -name 'SubscriptionManager' -dir=network/gossip/libp2p/channel -case=underscore -output="./network/gossip/libp2p/mock" -outpkg="mock" GO111MODULE=on mockery -name 'Vertex' -dir="./consensus/hotstuff/forks/finalizer/forest" -case=underscore -output="./consensus/hotstuff/forks/finalizer/forest/mock" -outpkg="mock" GO111MODULE=on mockery -name '.*' -dir="./consensus/hotstuff" -case=underscore -output="./consensus/hotstuff/mocks" -outpkg="mocks" GO111MODULE=on mockery -name '.*' -dir="./engine/access/wrapper" -case=underscore -output="./engine/access/mock" -outpkg="mock" diff --git a/cmd/scaffold.go b/cmd/scaffold.go index 30f90d210dd..8ae1b0fbb66 100644 --- a/cmd/scaffold.go +++ b/cmd/scaffold.go @@ -176,28 +176,31 @@ func (fnb *FlowNodeBuilder) enqueueNetworkInit() { } fnb.Middleware = mw - nodeID, err := fnb.State.Final().Identity(fnb.Me.NodeID()) - if err != nil { - return nil, fmt.Errorf("could not get node id: %w", err) - } - nodeRole := nodeID.Role - participants, err := fnb.State.Final().Identities(libp2p.NetworkingSetFilter) if err != nil { return nil, fmt.Errorf("could not get network identities: %w", err) } - var nodeTopology topology.Topology - if nodeRole == flow.RoleCollection { - nodeTopology, err = topology.NewCollectionTopology(nodeID.NodeID, fnb.State) - } else { - nodeTopology, err = topology.NewRandPermTopology(nodeRole, nodeID.NodeID) - } + // creates topology, topology manager, and subscription managers + // + // topology + // subscription manager + subscriptionManager := libp2p.NewChannelSubscriptionManager(fnb.Middleware) + top, err := topology.NewTopicBasedTopology(fnb.NodeID, fnb.Logger, fnb.State, subscriptionManager) if err != nil { return nil, fmt.Errorf("could not create topology: %w", err) } - net, err := libp2p.NewNetwork(fnb.Logger, codec, participants, fnb.Me, fnb.Middleware, 10e6, nodeTopology, fnb.Metrics.Network) + // creates network instance + net, err := libp2p.NewNetwork(fnb.Logger, + codec, + participants, + fnb.Me, + fnb.Middleware, + 10e6, + top, + subscriptionManager, + fnb.Metrics.Network) if err != nil { return nil, fmt.Errorf("could not initialize network: %w", err) } diff --git a/engine/channels.go b/engine/channels.go index aff0c34058f..10c457f4857 100644 --- a/engine/channels.go +++ b/engine/channels.go @@ -9,6 +9,47 @@ import ( "github.com/onflow/flow-go/model/flow" ) +// init is called first time this package is imported. +// It creates and initializes the channel ID map. +func init() { + initializeChannelIdMap() +} + +// channelIdMap keeps a map between channel IDs and list of flow roles involved in that channel ID. +var channelIdMap map[string]flow.RoleList + +// RolesByChannelID returns list of flow roles involved in the channelID. +func RolesByChannelID(channelID string) (flow.RoleList, bool) { + if clusterChannelID, isCluster := IsClusterChannelID(channelID); isCluster { + // replaces channelID with the stripped-off channel prefix + channelID = clusterChannelID + } + roles, ok := channelIdMap[channelID] + return roles, ok +} + +// ChannelIDsByRole returns a list of all channel IDs the role subscribes to. +func ChannelIDsByRole(role flow.Role) []string { + channels := make([]string, 0) + for channelID, roles := range channelIdMap { + if roles.Contains(role) { + channels = append(channels, channelID) + } + } + + return channels +} + +// ChannelIDs returns all channelIDs nodes of any role have subscribed to. +func ChannelIDs() []string { + channelIDs := make([]string, 0) + for channelID := range channelIdMap { + channelIDs = append(channelIDs, channelID) + } + + return channelIDs +} + // channel IDs const ( @@ -49,6 +90,70 @@ const ( ProvideReceiptsByBlockID = RequestReceiptsByBlockID ) +// initializeChannelIdMap initializes an instance of channelIdMap and populates it with the channel IDs and their +// Note: Please update this map, if a new channel is defined or a the roles subscribing to a channel have changed +// corresponding list of roles. +func initializeChannelIdMap() { + channelIdMap = make(map[string]flow.RoleList) + + // Channels for test + channelIdMap[TestNetwork] = flow.RoleList{flow.RoleCollection, flow.RoleConsensus, flow.RoleExecution, + flow.RoleVerification, flow.RoleAccess} + channelIdMap[TestMetrics] = flow.RoleList{flow.RoleCollection, flow.RoleConsensus, flow.RoleExecution, + flow.RoleVerification, flow.RoleAccess} + + // Channels for consensus protocols + channelIdMap[ConsensusCommittee] = flow.RoleList{flow.RoleConsensus} + + // Channels for protocols actively synchronizing state across nodes + channelIdMap[SyncCommittee] = flow.RoleList{flow.RoleConsensus} + channelIdMap[SyncExecution] = flow.RoleList{flow.RoleExecution} + + // Channels for actively pushing entities to subscribers + channelIdMap[PushTransactions] = flow.RoleList{flow.RoleCollection} + channelIdMap[PushGuarantees] = flow.RoleList{flow.RoleCollection, flow.RoleConsensus} + channelIdMap[PushBlocks] = flow.RoleList{flow.RoleCollection, flow.RoleConsensus, flow.RoleExecution, + flow.RoleVerification, flow.RoleAccess} + channelIdMap[PushReceipts] = flow.RoleList{flow.RoleConsensus, flow.RoleExecution, flow.RoleVerification, + flow.RoleAccess} + channelIdMap[PushApprovals] = flow.RoleList{flow.RoleConsensus, flow.RoleVerification} + + // Channels for actively requesting missing entities + channelIdMap[RequestCollections] = flow.RoleList{flow.RoleCollection, flow.RoleExecution} + channelIdMap[RequestChunks] = flow.RoleList{flow.RoleExecution, flow.RoleVerification} + channelIdMap[RequestReceiptsByBlockID] = flow.RoleList{flow.RoleConsensus, flow.RoleExecution} + + // Channel aliases to make the code more readable / more robust to errors + channelIdMap[ReceiveGuarantees] = flow.RoleList{flow.RoleCollection, flow.RoleConsensus} + channelIdMap[ReceiveBlocks] = flow.RoleList{flow.RoleCollection, flow.RoleConsensus, flow.RoleExecution, + flow.RoleVerification, flow.RoleAccess} + channelIdMap[ReceiveReceipts] = flow.RoleList{flow.RoleConsensus, flow.RoleExecution, flow.RoleVerification, + flow.RoleAccess} + channelIdMap[ReceiveApprovals] = flow.RoleList{flow.RoleConsensus, flow.RoleVerification} + + channelIdMap[ProvideCollections] = flow.RoleList{flow.RoleCollection, flow.RoleExecution} + channelIdMap[ProvideChunks] = flow.RoleList{flow.RoleExecution, flow.RoleVerification} + channelIdMap[ProvideReceiptsByBlockID] = flow.RoleList{flow.RoleConsensus, flow.RoleExecution} + + channelIdMap[syncClusterPrefix] = flow.RoleList{flow.RoleCollection} + channelIdMap[consensusClusterPrefix] = flow.RoleList{flow.RoleCollection} +} + +// IsClusterChannelID returns true if channel ID is a cluster-related channel ID. +// At the current implementation, only collection nodes are involved in a cluster-related channel ID. +// If the channel ID is a cluster-related one, this method also strips off the channel prefix and returns it. +func IsClusterChannelID(channelID string) (string, bool) { + if strings.HasPrefix(channelID, syncClusterPrefix) { + return syncClusterPrefix, true + } + + if strings.HasPrefix(channelID, consensusClusterPrefix) { + return consensusClusterPrefix, true + } + + return "", false +} + // FullyQualifiedChannelName returns the unique channel name made up of channel name string suffixed with root block id // The root block id is used to prevent cross talks between nodes on different sporks func FullyQualifiedChannelName(channelID string, rootBlockID string) string { diff --git a/engine/channels_test.go b/engine/channels_test.go new file mode 100644 index 00000000000..f60eaac83ee --- /dev/null +++ b/engine/channels_test.go @@ -0,0 +1,90 @@ +package engine + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/model/flow" +) + +// TestGetRolesByChannelID_NonClusterChannelID evaluates correctness of GetRoleByChannelID function against +// inclusion and exclusion of roles. Essentially, the test evaluates that RolesByChannelID +// operates on top of channelIdMap. +func TestGetRolesByChannelID_NonClusterChannelID(t *testing.T) { + // asserts existing topic with its role + // the roles list should contain collection and consensus roles + roles, ok := RolesByChannelID(PushGuarantees) + assert.True(t, ok) + assert.Len(t, roles, 2) + assert.Contains(t, roles, flow.RoleConsensus) + assert.Contains(t, roles, flow.RoleCollection) + assert.NotContains(t, roles, flow.RoleExecution) + assert.NotContains(t, roles, flow.RoleVerification) + assert.NotContains(t, roles, flow.RoleAccess) + + // asserts a non-existing topic + roles, ok = RolesByChannelID("non-existing-topic") + assert.False(t, ok) + assert.Nil(t, roles) +} + +// TestGetRolesByChannelID_ClusterChannelID evaluates correctness of GetRoleByChannelID function against +// cluster channel ids. Essentially, the test evaluates that RolesByChannelID +// operates on top of channelIdMap, and correctly identifies and strips of the cluster channel ids. +func TestGetRolesByChannelID_ClusterChannelID(t *testing.T) { + // creates a cluster channel id + conClusterChannel := ChannelConsensusCluster("some-consensus-cluster-id") + + // the roles list should contain collection + roles, ok := RolesByChannelID(conClusterChannel) + assert.True(t, ok) + assert.Len(t, roles, 1) + assert.Contains(t, roles, flow.RoleCollection) +} + +// TestGetChannelIDByRole evaluates retrieving channel IDs associated with a role from the +// channel IDs map using ChannelIDsByRole. Essentially it evaluates that ChannelIDsByRole +// operates on top of channelIDMap. +func TestGetChannelIDByRole(t *testing.T) { + // asserts topics by the role for verification node + // it should have the topics of + // - PushBlocks + // - PushReceipts + // - PushApprovals + // - ProvideChunks + // - TestNetwork + // - TestMetric + // the roles list should contain collection and consensus roles + topics := ChannelIDsByRole(flow.RoleVerification) + assert.Len(t, topics, 6) + assert.Contains(t, topics, PushBlocks) + assert.Contains(t, topics, PushReceipts) + assert.Contains(t, topics, PushApprovals) + assert.Contains(t, topics, RequestChunks) + assert.Contains(t, topics, TestMetrics) + assert.Contains(t, topics, TestNetwork) +} + +// TestIsClusterChannelID verifies the correctness of IsClusterChannelID method +// against cluster and non-cluster channel ids. +func TestIsClusterChannelID(t *testing.T) { + // creates a consensus cluster channel and verifies it + conClusterChannel := ChannelConsensusCluster("some-consensus-cluster-id") + clusterChannelID, ok := IsClusterChannelID(conClusterChannel) + require.True(t, ok) + require.Equal(t, clusterChannelID, consensusClusterPrefix) + + // creates a sync cluster channel and verifies it + syncClusterID := ChannelSyncCluster("some-sync-cluster-id") + clusterChannelID, ok = IsClusterChannelID(syncClusterID) + require.True(t, ok) + require.Equal(t, clusterChannelID, syncClusterPrefix) + + // non-cluster channel should not be verified + clusterChannelID, ok = IsClusterChannelID("non-cluster-channel-id") + require.False(t, ok) + require.Empty(t, clusterChannelID) + +} diff --git a/go.mod b/go.mod index 84b1f908ecc..680ff42b7fc 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.13 require ( cloud.google.com/go/storage v1.10.0 github.com/HdrHistogram/hdrhistogram-go v0.9.0 // indirect + github.com/bsipos/thist v1.0.0 github.com/btcsuite/btcd v0.20.1-beta github.com/codahale/hdrhistogram v0.9.0 // indirect github.com/davecgh/go-spew v1.1.1 diff --git a/go.sum b/go.sum index b1133bf80f0..eeab1630101 100644 --- a/go.sum +++ b/go.sum @@ -71,6 +71,7 @@ github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrU github.com/VictoriaMetrics/fastcache v1.5.3 h1:2odJnXLbFZcoV9KYtQ+7TH1UOq3dn3AssMgieaezkR4= github.com/VictoriaMetrics/fastcache v1.5.3/go.mod h1:+jv9Ckb+za/P1ZRg/sulP5Ni1v49daAVERr0H3CuscE= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= +github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af h1:wVe6/Ea46ZMeNkQjjBW6xcqyQA/j5e0D6GytH95g0gQ= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -93,6 +94,8 @@ github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bsipos/thist v1.0.0 h1:vZ3W5/ZnT54s4LHeonTCbnzCb20ERlJUnhiwXoGpsbY= +github.com/bsipos/thist v1.0.0/go.mod h1:7i0xwRua1/bmUxcxi2xAxaFL895rLtOpKUwnw3NrT8I= github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ= github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= github.com/btcsuite/btcd v0.0.0-20190523000118-16327141da8c/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= @@ -181,6 +184,7 @@ github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6 h1:u/UEqS66A5ckRmS4yNpjmVH56sVtS/RfclBAYocb4as= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ= +github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90 h1:WXb3TSNmHp2vHoCroCIB1foO/yQ36swABL8aOVeDpgg= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -208,6 +212,7 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -377,6 +382,7 @@ github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfE github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.1.1-0.20170430222011-975b5c4c7c21/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5 h1:PJr+ZMXIecYc1Ey2zucXdR73SMBtgjPgwa31099IMv0= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= @@ -930,6 +936,7 @@ golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 h1:QE6XYQK6naiK1EPAe1g/ILLxN golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b h1:+qEpEAPhDZ1o0x3tHzZTQDArnOixOzGD9HUJfcg0mb4= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -1035,6 +1042,7 @@ golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606050223-4d9ae51c2468/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= @@ -1088,6 +1096,7 @@ gonum.org/v1/gonum v0.6.1 h1:/LSrTrgZtpbXyAR6+0e152SROCkJJSh7goYWVmdPFGc= gonum.org/v1/gonum v0.6.1/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0 h1:OE9mWmgKkjJyEmDAAtGMPjXu+YNeGvK9VTSHY6+Qihc= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b h1:Qh4dB5D/WpoUUp3lSod7qgoyEHbDGPUWjIbnqdqqe1k= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= @@ -1219,6 +1228,7 @@ honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 honnef.co/go/tools v0.0.1-2020.1.4 h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK8= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/pdf v0.1.1 h1:k1MczvYDUvJBe93bYd7wrZLLUEcLZAuF824/I4e5Xr4= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/integration/go.sum b/integration/go.sum index 1048512a895..28e351da80f 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -91,6 +91,7 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/bsipos/thist v1.0.0/go.mod h1:7i0xwRua1/bmUxcxi2xAxaFL895rLtOpKUwnw3NrT8I= github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ= github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= github.com/btcsuite/btcd v0.0.0-20190523000118-16327141da8c/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= @@ -1143,6 +1144,7 @@ golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606050223-4d9ae51c2468/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= diff --git a/model/flow/role.go b/model/flow/role.go index 142a3199f9d..936d7790dbc 100644 --- a/model/flow/role.go +++ b/model/flow/role.go @@ -69,3 +69,37 @@ func (r *Role) UnmarshalText(text []byte) error { func Roles() []Role { return []Role{RoleCollection, RoleConsensus, RoleExecution, RoleVerification, RoleAccess} } + +// RoleList defines a slice of roles in flow system. +type RoleList []Role + +// Contains returns true if RoleList contains the role, otherwise false. +func (r RoleList) Contains(role Role) bool { + for _, each := range r { + if each == role { + return true + } + } + return false +} + +// Union returns a new role list containing every role that occurs in +// either `r`, or `other`, or both. There are no duplicate roles in the output, +func (r RoleList) Union(other RoleList) RoleList { + // stores the output, the union of the two lists + union := make(RoleList, 0, len(r)+len(other)) + + // efficient lookup to avoid duplicates + added := make(map[Role]struct{}) + + // adds all roles, skips duplicates + for _, role := range append(r, other...) { + if _, exists := added[role]; exists { + continue + } + union = append(union, role) + added[role] = struct{}{} + } + + return union +} diff --git a/model/flow/role_test.go b/model/flow/role_test.go index 25d6d22a972..a95f7cd2084 100644 --- a/model/flow/role_test.go +++ b/model/flow/role_test.go @@ -20,3 +20,44 @@ func TestRoleJSON(t *testing.T) { assert.NoError(t, err) assert.Equal(t, r, actual) } + +// TestRoleList_Contains evaluates correctness of Contains method of RoleList. +func TestRoleList_Contains(t *testing.T) { + roleList := flow.RoleList{flow.RoleConsensus, flow.RoleVerification} + + // asserts Contains returns true for roles in the list + assert.True(t, roleList.Contains(flow.RoleConsensus)) + assert.True(t, roleList.Contains(flow.RoleVerification)) + + // asserts Contains returns false for roles not in the list + assert.False(t, roleList.Contains(flow.RoleAccess)) + assert.False(t, roleList.Contains(flow.RoleExecution)) + assert.False(t, roleList.Contains(flow.RoleCollection)) + +} + +// TestRoleList_Union evaluates correctness of Union method of RoleList. +func TestRoleList_Union(t *testing.T) { + this := flow.RoleList{flow.RoleConsensus, flow.RoleVerification} + other := flow.RoleList{flow.RoleConsensus, flow.RoleExecution} + + union := this.Union(other) + + // asserts length of role lists + assert.Len(t, union, 3) + assert.Len(t, this, 2) + assert.Len(t, other, 2) + + // asserts content of role lists + // this + assert.Contains(t, this, flow.RoleConsensus) + assert.Contains(t, this, flow.RoleVerification) + // other + assert.Contains(t, other, flow.RoleConsensus) + assert.Contains(t, other, flow.RoleExecution) + // union + assert.Contains(t, union, flow.RoleConsensus) + assert.Contains(t, union, flow.RoleVerification) + assert.Contains(t, union, flow.RoleExecution) + +} diff --git a/network/gossip/libp2p/channel/subscriptionManager.go b/network/gossip/libp2p/channel/subscriptionManager.go new file mode 100644 index 00000000000..e3c58f9e9ba --- /dev/null +++ b/network/gossip/libp2p/channel/subscriptionManager.go @@ -0,0 +1,19 @@ +package channel + +import ( + "github.com/onflow/flow-go/network" +) + +type SubscriptionManager interface { + // Register registers an engine on the channel ID into the subscription manager. + Register(channelID string, engine network.Engine) error + + // Unregister removes the engine associated with a channel ID + Unregister(channelID string) error + + // GetEngine returns engine associated with a channel ID. + GetEngine(channelID string) (network.Engine, error) + + // GetChannelIDs returns all the channel IDs registered in this subscription manager. + GetChannelIDs() []string +} diff --git a/network/gossip/libp2p/mock/subscription_manager.go b/network/gossip/libp2p/mock/subscription_manager.go new file mode 100644 index 00000000000..fa04c0c44b7 --- /dev/null +++ b/network/gossip/libp2p/mock/subscription_manager.go @@ -0,0 +1,80 @@ +// Code generated by mockery v1.0.0. DO NOT EDIT. + +package mock + +import ( + network "github.com/onflow/flow-go/network" + mock "github.com/stretchr/testify/mock" +) + +// SubscriptionManager is an autogenerated mock type for the SubscriptionManager type +type SubscriptionManager struct { + mock.Mock +} + +// GetChannelIDs provides a mock function with given fields: +func (_m *SubscriptionManager) GetChannelIDs() []string { + ret := _m.Called() + + var r0 []string + if rf, ok := ret.Get(0).(func() []string); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]string) + } + } + + return r0 +} + +// GetEngine provides a mock function with given fields: channelID +func (_m *SubscriptionManager) GetEngine(channelID string) (network.Engine, error) { + ret := _m.Called(channelID) + + var r0 network.Engine + if rf, ok := ret.Get(0).(func(string) network.Engine); ok { + r0 = rf(channelID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(network.Engine) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(channelID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Register provides a mock function with given fields: channelID, engine +func (_m *SubscriptionManager) Register(channelID string, engine network.Engine) error { + ret := _m.Called(channelID, engine) + + var r0 error + if rf, ok := ret.Get(0).(func(string, network.Engine) error); ok { + r0 = rf(channelID, engine) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Unregister provides a mock function with given fields: channelID +func (_m *SubscriptionManager) Unregister(channelID string) error { + ret := _m.Called(channelID) + + var r0 error + if rf, ok := ret.Get(0).(func(string) error); ok { + r0 = rf(channelID) + } else { + r0 = ret.Error(0) + } + + return r0 +} diff --git a/network/gossip/libp2p/network.go b/network/gossip/libp2p/network.go index d18e216893c..05d828598ca 100644 --- a/network/gossip/libp2p/network.go +++ b/network/gossip/libp2p/network.go @@ -9,10 +9,12 @@ import ( "github.com/rs/zerolog" "github.com/onflow/flow-go/crypto/hash" + channels "github.com/onflow/flow-go/engine" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/gossip/libp2p/cache" + "github.com/onflow/flow-go/network/gossip/libp2p/channel" "github.com/onflow/flow-go/network/gossip/libp2p/message" "github.com/onflow/flow-go/network/gossip/libp2p/middleware" "github.com/onflow/flow-go/network/gossip/libp2p/queue" @@ -25,18 +27,19 @@ type identifierFilter func(ids ...flow.Identifier) ([]flow.Identifier, error) // the protocols for handshakes, authentication, gossiping and heartbeats. type Network struct { sync.RWMutex - logger zerolog.Logger - codec network.Codec - ids flow.IdentityList - me module.Local - mw middleware.Middleware - top topology.Topology - metrics module.NetworkMetrics - rcache *cache.RcvCache // used to deduplicate incoming messages - queue queue.MessageQueue - ctx context.Context - cancel context.CancelFunc - subscriptionMgr *subscriptionManager + logger zerolog.Logger + codec network.Codec + ids flow.IdentityList + me module.Local + mw middleware.Middleware + top topology.Topology // used to determine fanout connections + metrics module.NetworkMetrics + rcache *cache.RcvCache // used to deduplicate incoming messages + queue queue.MessageQueue + ctx context.Context + cancel context.CancelFunc + subMngr channel.SubscriptionManager // used to keep track of subscribed channels + } // NewNetwork creates a new naive overlay network, using the given middleware to @@ -51,6 +54,7 @@ func NewNetwork( mw middleware.Middleware, csize int, top topology.Topology, + sm channel.SubscriptionManager, metrics module.NetworkMetrics, ) (*Network, error) { @@ -60,14 +64,14 @@ func NewNetwork( } o := &Network{ - logger: log, - codec: codec, - me: me, - mw: mw, - rcache: rcache, - top: top, - metrics: metrics, - subscriptionMgr: newSubscriptionManager(mw), + logger: log, + codec: codec, + me: me, + mw: mw, + rcache: rcache, + top: top, + metrics: metrics, + subMngr: sm, } o.ctx, o.cancel = context.WithCancel(context.Background()) o.ids = ids @@ -110,12 +114,19 @@ func (n *Network) Done() <-chan struct{} { // returning a conduit to directly submit messages to the message bus of the // engine. func (n *Network) Register(channelID string, engine network.Engine) (network.Conduit, error) { + if _, ok := channels.RolesByChannelID(channelID); !ok { + return nil, fmt.Errorf("unknown channel id: %s, should be registered in topic map", channelID) + } - err := n.subscriptionMgr.register(channelID, engine) + err := n.subMngr.Register(channelID, engine) if err != nil { return nil, fmt.Errorf("failed to register engine for channel %s: %w", channelID, err) } + n.logger.Info(). + Str("channel_id", channelID). + Msg("channel successfully registered") + // create a cancellable child context ctx, cancel := context.WithCancel(n.ctx) @@ -137,7 +148,7 @@ func (n *Network) Register(channelID string, engine network.Engine) (network.Con // unregister unregisters the engine for the specified channel. The engine will no longer be able to send or // receive messages from that channelID func (n *Network) unregister(channelID string) error { - err := n.subscriptionMgr.unregister(channelID) + err := n.subMngr.Unregister(channelID) if err != nil { return fmt.Errorf("failed to unregister engine for channelID %s: %w", channelID, err) } @@ -155,17 +166,16 @@ func (n *Network) Identity() (map[flow.Identifier]flow.Identity, error) { return identifierToID, nil } -// Topology returns the identities of a uniform subset of nodes in protocol state using the topology provided earlier +// Topology returns the identities of a uniform subset of nodes in protocol state using the topology provided earlier. +// Independent invocations of Topology on different nodes collectively constructs a connected network graph. func (n *Network) Topology() (flow.IdentityList, error) { - n.RLock() - defer n.RUnlock() - // fanout is currently set to half of the system size for connectivity assurance - fanout := uint(len(n.ids)+1) / 2 - subset, err := n.top.Subset(n.ids, fanout) + n.Lock() + defer n.Unlock() + top, err := n.top.GenerateFanout(n.ids) if err != nil { - return nil, fmt.Errorf("failed to derive list of peer nodes to connect to: %w", err) + return nil, fmt.Errorf("could not generate topology: %w", err) } - return subset, nil + return top, nil } func (n *Network) Receive(nodeID flow.Identifier, msg *message.Message) error { @@ -419,7 +429,7 @@ func (n *Network) sendOnChannel(channelID string, message interface{}, targetIDs // when it gets a message from the queue func (n *Network) queueSubmitFunc(message interface{}) { qm := message.(queue.QueueMessage) - eng, err := n.subscriptionMgr.getEngine(qm.ChannelID) + eng, err := n.subMngr.GetEngine(qm.ChannelID) if err != nil { n.logger.Error(). Err(err). diff --git a/network/gossip/libp2p/peerManager_test.go b/network/gossip/libp2p/peerManager_test.go index d70fb82b31d..e172e4aca67 100644 --- a/network/gossip/libp2p/peerManager_test.go +++ b/network/gossip/libp2p/peerManager_test.go @@ -146,7 +146,7 @@ func (suite *PeerManagerTestSuite) TestPeriodicPeerUpdate() { connector.On("DisconnectPeers", suite.ctx, testifymock.Anything).Return(nil) pm := NewPeerManager(suite.ctx, suite.log, idProvider, connector) PeerUpdateInterval = 5 * time.Millisecond - unittest.RequireClosesBefore(suite.T(), pm.Ready(), 2*time.Second) + unittest.RequireCloseBefore(suite.T(), pm.Ready(), 2*time.Second, "could not start peer manager") unittest.RequireReturnsBefore(suite.T(), wg.Wait, 2*PeerUpdateInterval, "ConnectPeers is not running on UpdateIntervals") @@ -182,7 +182,7 @@ func (suite *PeerManagerTestSuite) TestOnDemandPeerUpdate() { connector.On("DisconnectPeers", suite.ctx, testifymock.Anything).Return(nil) pm := NewPeerManager(suite.ctx, suite.log, idProvider, connector) - unittest.RequireClosesBefore(suite.T(), pm.Ready(), 2*time.Second) + unittest.RequireCloseBefore(suite.T(), pm.Ready(), 2*time.Second, "could not start peer manager") unittest.RequireReturnsBefore(suite.T(), wg.Wait, 1*time.Second, "ConnectPeers is not running on startup") @@ -221,7 +221,7 @@ func (suite *PeerManagerTestSuite) TestConcurrentOnDemandPeerUpdate() { // start the peer manager // this should trigger the first update and which will block on the ConnectPeers to return - unittest.RequireClosesBefore(suite.T(), pm.Ready(), 2*time.Second) + unittest.RequireCloseBefore(suite.T(), pm.Ready(), 2*time.Second, "could not start peer manager") // assert that the first update started assert.Eventually(suite.T(), func() bool { @@ -238,7 +238,7 @@ func (suite *PeerManagerTestSuite) TestConcurrentOnDemandPeerUpdate() { // assert that only two calls to ConnectPeers were made (one by the periodic update and one by the on-demand update) assert.Eventually(suite.T(), func() bool { return connector.AssertNumberOfCalls(suite.T(), "ConnectPeers", 2) - }, 3*time.Second, 100*time.Millisecond) + }, 10*time.Second, 100*time.Millisecond) } // assertListsEqual asserts that two identity list are equal ignoring the order diff --git a/network/gossip/libp2p/subscriptionManager.go b/network/gossip/libp2p/subscriptionManager.go index 32fc32bbec4..f261729bd05 100644 --- a/network/gossip/libp2p/subscriptionManager.go +++ b/network/gossip/libp2p/subscriptionManager.go @@ -8,23 +8,23 @@ import ( "github.com/onflow/flow-go/network/gossip/libp2p/middleware" ) -// subscriptionManager manages the engine to channelID subscription -type subscriptionManager struct { - sync.RWMutex +// ChannelSubscriptionManager manages the engine to channelID subscription +type ChannelSubscriptionManager struct { + mu sync.RWMutex engines map[string]network.Engine mw middleware.Middleware } -func newSubscriptionManager(mw middleware.Middleware) *subscriptionManager { - return &subscriptionManager{ +func NewChannelSubscriptionManager(mw middleware.Middleware) *ChannelSubscriptionManager { + return &ChannelSubscriptionManager{ engines: make(map[string]network.Engine), mw: mw, } } -func (sm *subscriptionManager) register(channelID string, engine network.Engine) error { - sm.Lock() - defer sm.Unlock() +func (sm *ChannelSubscriptionManager) Register(channelID string, engine network.Engine) error { + sm.mu.Lock() + defer sm.mu.Unlock() // check if the engine engineID is already taken _, ok := sm.engines[channelID] @@ -44,9 +44,9 @@ func (sm *subscriptionManager) register(channelID string, engine network.Engine) return nil } -func (sm *subscriptionManager) unregister(channelID string) error { - sm.Lock() - defer sm.Unlock() +func (sm *ChannelSubscriptionManager) Unregister(channelID string) error { + sm.mu.Lock() + defer sm.mu.Unlock() // check if there is a registered engine for the given channelID _, ok := sm.engines[channelID] @@ -65,12 +65,26 @@ func (sm *subscriptionManager) unregister(channelID string) error { return nil } -func (sm *subscriptionManager) getEngine(channelID string) (network.Engine, error) { - sm.RLock() - defer sm.RUnlock() +func (sm *ChannelSubscriptionManager) GetEngine(channelID string) (network.Engine, error) { + sm.mu.RLock() + defer sm.mu.RUnlock() + eng, found := sm.engines[channelID] if !found { return nil, fmt.Errorf("subscriptionManager: engine for channelID %s not found", channelID) } return eng, nil } + +// GetChannelIDs returns list of topics this subscription manager has an engine registered for. +func (sm *ChannelSubscriptionManager) GetChannelIDs() []string { + sm.mu.RLock() + defer sm.mu.RUnlock() + + topics := make([]string, 0) + for topic := range sm.engines { + topics = append(topics, topic) + } + + return topics +} diff --git a/network/gossip/libp2p/test/echoengine_test.go b/network/gossip/libp2p/test/echoengine_test.go index 2fb3617c7fa..15d02739e98 100644 --- a/network/gossip/libp2p/test/echoengine_test.go +++ b/network/gossip/libp2p/test/echoengine_test.go @@ -25,10 +25,9 @@ import ( // single message from one engine to the other one through different scenarios. type EchoEngineTestSuite struct { suite.Suite - ConduitWrapper // used as a wrapper around conduit methods - nets []*libp2p.Network // used to keep track of the networks - mws []*libp2p.Middleware // used to keep track of the middlewares associated with networks - ids flow.IdentityList // used to keep track of the identifiers associated with networks + ConduitWrapper // used as a wrapper around conduit methods + nets []*libp2p.Network // used to keep track of the networks + ids flow.IdentityList // used to keep track of the identifiers associated with networks } // Some tests are skipped to speedup the build. @@ -39,346 +38,368 @@ func TestStubEngineTestSuite(t *testing.T) { suite.Run(t, new(EchoEngineTestSuite)) } -func (s *EchoEngineTestSuite) SetupTest() { +func (suite *EchoEngineTestSuite) SetupTest() { const count = 2 - logger := zerolog.New(os.Stderr).Level(zerolog.ErrorLevel) golog.SetAllLoggers(golog.LevelError) - s.ids, s.mws, s.nets = generateIDsMiddlewaresNetworks(s.T(), count, logger, 100, nil, false) + // both nodes should be of the same role to get connected on epidemic dissemination + suite.ids, _, suite.nets = GenerateIDsMiddlewaresNetworks(suite.T(), count, logger, 100, nil, !DryRun) } // TearDownTest closes the networks within a specified timeout -func (s *EchoEngineTestSuite) TearDownTest() { - for _, net := range s.nets { - select { - // closes the network - case <-net.Done(): - continue - case <-time.After(3 * time.Second): - s.Suite.Fail("could not stop the network") - } - } +func (suite *EchoEngineTestSuite) TearDownTest() { + stopNetworks(suite.T(), suite.nets, 3*time.Second) +} + +// TestUnknownChannelID evaluates that registering an engine with an unknown channel ID returns an error. +// All channel IDs should be registered as topics in engine.topicMap. +func (suite *EchoEngineTestSuite) TestUnknownChannelID() { + e := NewEchoEngine(suite.T(), suite.nets[0], 1, engine.TestNetwork, false, suite.Unicast) + _, err := suite.nets[0].Register("unknown-channel-id", e) + require.Error(suite.T(), err) +} + +// TestClusterChannelID evaluates that registering a cluster channel ID is done without any error. +func (suite *EchoEngineTestSuite) TestClusterChannelID() { + e := NewEchoEngine(suite.T(), suite.nets[0], 1, engine.TestNetwork, false, suite.Unicast) + // creates a cluster channel ID + clusterChannelID := engine.ChannelSyncCluster(flow.Testnet) + // registers engine with cluster channel ID + _, err := suite.nets[0].Register(clusterChannelID, e) + // registering cluster channel ID should not cause an error + require.NoError(suite.T(), err) +} + +// TestDuplicateChannelID evaluates that registering an engine with duplicate channel ID returns an error. +func (suite *EchoEngineTestSuite) TestDuplicateChannelID() { + // creates an echo engine, which registers it on test network channel + e := NewEchoEngine(suite.T(), suite.nets[0], 1, engine.TestNetwork, false, suite.Unicast) + + // attempts to register the same engine again on test network channel which + // should cause an error + _, err := suite.nets[0].Register(engine.TestNetwork, e) + require.Error(suite.T(), err) } // TestSingleMessage_Submit tests sending a single message from sender to receiver using // the Submit method of Conduit. -func (s *EchoEngineTestSuite) TestSingleMessage_Submit() { - s.skipTest("covered by TestEchoMultiMsgAsync_Submit") +func (suite *EchoEngineTestSuite) TestSingleMessage_Submit() { + suite.skipTest("covered by TestEchoMultiMsgAsync_Submit") // set to false for no echo expectation - s.singleMessage(false, s.Submit) + suite.singleMessage(false, suite.Submit) } // TestSingleMessage_Publish tests sending a single message from sender to receiver using // the Publish method of Conduit. -func (s *EchoEngineTestSuite) TestSingleMessage_Publish() { - s.skipTest("covered by TestEchoMultiMsgAsync_Publish") +func (suite *EchoEngineTestSuite) TestSingleMessage_Publish() { + suite.skipTest("covered by TestEchoMultiMsgAsync_Publish") // set to false for no echo expectation - s.singleMessage(false, s.Publish) + suite.singleMessage(false, suite.Publish) } // TestSingleMessage_Unicast tests sending a single message from sender to receiver using // the Unicast method of Conduit. -func (s *EchoEngineTestSuite) TestSingleMessage_Unicast() { - s.skipTest("covered by TestEchoMultiMsgAsync_Unicast") +func (suite *EchoEngineTestSuite) TestSingleMessage_Unicast() { + suite.skipTest("covered by TestEchoMultiMsgAsync_Unicast") // set to false for no echo expectation - s.singleMessage(false, s.Unicast) + suite.singleMessage(false, suite.Unicast) } // TestSingleMessage_Multicast tests sending a single message from sender to receiver using // the Multicast method of Conduit. -func (s *EchoEngineTestSuite) TestSingleMessage_Multicast() { - s.skipTest("covered by TestEchoMultiMsgAsync_Multicast") +func (suite *EchoEngineTestSuite) TestSingleMessage_Multicast() { + suite.skipTest("covered by TestEchoMultiMsgAsync_Multicast") // set to false for no echo expectation - s.singleMessage(false, s.Multicast) + suite.singleMessage(false, suite.Multicast) } // TestSingleEcho_Submit tests sending a single message from sender to receiver using // the Submit method of its Conduit. // It also evaluates the correct reception of an echo message back. -func (s *EchoEngineTestSuite) TestSingleEcho_Submit() { - s.skipTest("covered by TestEchoMultiMsgAsync_Submit") +func (suite *EchoEngineTestSuite) TestSingleEcho_Submit() { + suite.skipTest("covered by TestEchoMultiMsgAsync_Submit") // set to true for an echo expectation - s.singleMessage(true, s.Submit) + suite.singleMessage(true, suite.Submit) } // TestSingleEcho_Publish tests sending a single message from sender to receiver using // the Publish method of its Conduit. // It also evaluates the correct reception of an echo message back. -func (s *EchoEngineTestSuite) TestSingleEcho_Publish() { - s.skipTest("covered by TestEchoMultiMsgAsync_Publish") +func (suite *EchoEngineTestSuite) TestSingleEcho_Publish() { + suite.skipTest("covered by TestEchoMultiMsgAsync_Publish") // set to true for an echo expectation - s.singleMessage(true, s.Publish) + suite.singleMessage(true, suite.Publish) } // TestSingleEcho_Unicast tests sending a single message from sender to receiver using // the Unicast method of its Conduit. // It also evaluates the correct reception of an echo message back. -func (s *EchoEngineTestSuite) TestSingleEcho_Unicast() { - s.skipTest("covered by TestEchoMultiMsgAsync_Unicast") +func (suite *EchoEngineTestSuite) TestSingleEcho_Unicast() { + suite.skipTest("covered by TestEchoMultiMsgAsync_Unicast") // set to true for an echo expectation - s.singleMessage(true, s.Unicast) + suite.singleMessage(true, suite.Unicast) } // TestSingleEcho_Multicast tests sending a single message from sender to receiver using // the Multicast method of its Conduit. // It also evaluates the correct reception of an echo message back. -func (s *EchoEngineTestSuite) TestSingleEcho_Multicast() { - s.skipTest("covered by TestEchoMultiMsgAsync_Multicast") +func (suite *EchoEngineTestSuite) TestSingleEcho_Multicast() { + suite.skipTest("covered by TestEchoMultiMsgAsync_Multicast") // set to true for an echo expectation - s.singleMessage(true, s.Multicast) + suite.singleMessage(true, suite.Multicast) } // TestMultiMsgSync_Submit tests sending multiple messages from sender to receiver // using the Submit method of its Conduit. // Sender and receiver are synced over reception. -func (s *EchoEngineTestSuite) TestMultiMsgSync_Submit() { - s.skipTest("covered by TestEchoMultiMsgAsync_Submit") +func (suite *EchoEngineTestSuite) TestMultiMsgSync_Submit() { + suite.skipTest("covered by TestEchoMultiMsgAsync_Submit") // set to false for no echo expectation - s.multiMessageSync(false, 10, s.Submit) + suite.multiMessageSync(false, 10, suite.Submit) } // TestMultiMsgSync_Publish tests sending multiple messages from sender to receiver // using the Publish method of its Conduit. // Sender and receiver are synced over reception. -func (s *EchoEngineTestSuite) TestMultiMsgSync_Publish() { - s.skipTest("covered by TestEchoMultiMsgAsync_Publish") +func (suite *EchoEngineTestSuite) TestMultiMsgSync_Publish() { + suite.skipTest("covered by TestEchoMultiMsgAsync_Publish") // set to false for no echo expectation - s.multiMessageSync(false, 10, s.Publish) + suite.multiMessageSync(false, 10, suite.Publish) } // TestMultiMsgSync_Unicast tests sending multiple messages from sender to receiver // using the Unicast method of its Conduit. // Sender and receiver are synced over reception. -func (s *EchoEngineTestSuite) TestMultiMsgSync_Unicast() { - s.skipTest("covered by TestEchoMultiMsgAsync_Unicast") +func (suite *EchoEngineTestSuite) TestMultiMsgSync_Unicast() { + suite.skipTest("covered by TestEchoMultiMsgAsync_Unicast") // set to false for no echo expectation - s.multiMessageSync(false, 10, s.Unicast) + suite.multiMessageSync(false, 10, suite.Unicast) } // TestMultiMsgSync_Multicast tests sending multiple messages from sender to receiver // using the Multicast method of its Conduit. // Sender and receiver are synced over reception. -func (s *EchoEngineTestSuite) TestMultiMsgSync_Multicast() { - s.skipTest("covered by TestEchoMultiMsgAsync_Multicast") +func (suite *EchoEngineTestSuite) TestMultiMsgSync_Multicast() { + suite.skipTest("covered by TestEchoMultiMsgAsync_Multicast") // set to false for no echo expectation - s.multiMessageSync(false, 10, s.Multicast) + suite.multiMessageSync(false, 10, suite.Multicast) } // TestEchoMultiMsgSync_Submit tests sending multiple messages from sender to receiver // using the Submit method of its Conduit. // It also evaluates the correct reception of an echo message back for each send // sender and receiver are synced over reception. -func (s *EchoEngineTestSuite) TestEchoMultiMsgSync_Submit() { - s.skipTest("covered by TestEchoMultiMsgAsync_Submit") +func (suite *EchoEngineTestSuite) TestEchoMultiMsgSync_Submit() { + suite.skipTest("covered by TestEchoMultiMsgAsync_Submit") // set to true for an echo expectation - s.multiMessageSync(true, 10, s.Submit) + suite.multiMessageSync(true, 10, suite.Submit) } // TestEchoMultiMsgSync_Publish tests sending multiple messages from sender to receiver // using the Publish method of its Conduit. // It also evaluates the correct reception of an echo message back for each send // sender and receiver are synced over reception. -func (s *EchoEngineTestSuite) TestEchoMultiMsgSync_Publish() { - s.skipTest("covered by TestEchoMultiMsgAsync_Publish") +func (suite *EchoEngineTestSuite) TestEchoMultiMsgSync_Publish() { + suite.skipTest("covered by TestEchoMultiMsgAsync_Publish") // set to true for an echo expectation - s.multiMessageSync(true, 10, s.Publish) + suite.multiMessageSync(true, 10, suite.Publish) } // TestEchoMultiMsgSync_Unicast tests sending multiple messages from sender to receiver // using the Unicast method of its Conduit. // It also evaluates the correct reception of an echo message back for each send // sender and receiver are synced over reception. -func (s *EchoEngineTestSuite) TestEchoMultiMsgSync_Unicast() { - s.skipTest("covered by TestEchoMultiMsgAsync_Unicast") +func (suite *EchoEngineTestSuite) TestEchoMultiMsgSync_Unicast() { + suite.skipTest("covered by TestEchoMultiMsgAsync_Unicast") // set to true for an echo expectation - s.multiMessageSync(true, 10, s.Submit) + suite.multiMessageSync(true, 10, suite.Submit) } // TestEchoMultiMsgSync_Multicast tests sending multiple messages from sender to receiver // using the Multicast method of its Conduit. // It also evaluates the correct reception of an echo message back for each send // sender and receiver are synced over reception. -func (s *EchoEngineTestSuite) TestEchoMultiMsgSync_Multicast() { - s.skipTest("covered by TestEchoMultiMsgAsync_Multicast") +func (suite *EchoEngineTestSuite) TestEchoMultiMsgSync_Multicast() { + suite.skipTest("covered by TestEchoMultiMsgAsync_Multicast") // set to true for an echo expectation - s.multiMessageSync(true, 10, s.Multicast) + suite.multiMessageSync(true, 10, suite.Multicast) } // TestMultiMsgAsync_Submit tests sending multiple messages from sender to receiver // using the Submit method of their Conduit. // Sender and receiver are not synchronized. -func (s *EchoEngineTestSuite) TestMultiMsgAsync_Submit() { - s.skipTest("covered by TestEchoMultiMsgAsync_Submit") +func (suite *EchoEngineTestSuite) TestMultiMsgAsync_Submit() { + suite.skipTest("covered by TestEchoMultiMsgAsync_Submit") // set to false for no echo expectation - s.multiMessageAsync(false, 10, s.Submit) + suite.multiMessageAsync(false, 10, suite.Submit) } // TestMultiMsgAsync_Publish tests sending multiple messages from sender to receiver // using the Publish method of their Conduit. // Sender and receiver are not synchronized -func (s *EchoEngineTestSuite) TestMultiMsgAsync_Publish() { - s.skipTest("covered by TestEchoMultiMsgAsync_Publish") +func (suite *EchoEngineTestSuite) TestMultiMsgAsync_Publish() { + suite.skipTest("covered by TestEchoMultiMsgAsync_Publish") // set to false for no echo expectation - s.multiMessageAsync(false, 10, s.Publish) + suite.multiMessageAsync(false, 10, suite.Publish) } // TestMultiMsgAsync_Unicast tests sending multiple messages from sender to receiver // using the Unicast method of their Conduit. // Sender and receiver are not synchronized -func (s *EchoEngineTestSuite) TestMultiMsgAsync_Unicast() { - s.skipTest("covered by TestEchoMultiMsgAsync_Unicast") +func (suite *EchoEngineTestSuite) TestMultiMsgAsync_Unicast() { + suite.skipTest("covered by TestEchoMultiMsgAsync_Unicast") // set to false for no echo expectation - s.multiMessageAsync(false, 10, s.Unicast) + suite.multiMessageAsync(false, 10, suite.Unicast) } // TestMultiMsgAsync_Multicast tests sending multiple messages from sender to receiver // using the Multicast method of their Conduit. // Sender and receiver are not synchronized. -func (s *EchoEngineTestSuite) TestMultiMsgAsync_Multicast() { - s.skipTest("covered by TestEchoMultiMsgAsync_Multicast") +func (suite *EchoEngineTestSuite) TestMultiMsgAsync_Multicast() { + suite.skipTest("covered by TestEchoMultiMsgAsync_Multicast") // set to false for no echo expectation - s.multiMessageAsync(false, 10, s.Multicast) + suite.multiMessageAsync(false, 10, suite.Multicast) } // TestEchoMultiMsgAsync_Submit tests sending multiple messages from sender to receiver // using the Submit method of their Conduit. // It also evaluates the correct reception of an echo message back for each send. // Sender and receiver are not synchronized -func (s *EchoEngineTestSuite) TestEchoMultiMsgAsync_Submit() { +func (suite *EchoEngineTestSuite) TestEchoMultiMsgAsync_Submit() { // set to true for an echo expectation - s.multiMessageAsync(true, 10, s.Submit) + suite.multiMessageAsync(true, 10, suite.Submit) } // TestEchoMultiMsgAsync_Publish tests sending multiple messages from sender to receiver // using the Publish method of their Conduit. // It also evaluates the correct reception of an echo message back for each send. // Sender and receiver are not synchronized -func (s *EchoEngineTestSuite) TestEchoMultiMsgAsync_Publish() { +func (suite *EchoEngineTestSuite) TestEchoMultiMsgAsync_Publish() { // set to true for an echo expectation - s.multiMessageAsync(true, 10, s.Publish) + suite.multiMessageAsync(true, 10, suite.Publish) } // TestEchoMultiMsgAsync_Unicast tests sending multiple messages from sender to receiver // using the Unicast method of their Conduit. // It also evaluates the correct reception of an echo message back for each send. // Sender and receiver are not synchronized -func (s *EchoEngineTestSuite) TestEchoMultiMsgAsync_Unicast() { +func (suite *EchoEngineTestSuite) TestEchoMultiMsgAsync_Unicast() { // set to true for an echo expectation - s.multiMessageAsync(true, 10, s.Unicast) + suite.multiMessageAsync(true, 10, suite.Unicast) } // TestEchoMultiMsgAsync_Multicast tests sending multiple messages from sender to receiver // using the Multicast method of their Conduit. // It also evaluates the correct reception of an echo message back for each send. // Sender and receiver are not synchronized -func (s *EchoEngineTestSuite) TestEchoMultiMsgAsync_Multicast() { +func (suite *EchoEngineTestSuite) TestEchoMultiMsgAsync_Multicast() { // set to true for an echo expectation - s.multiMessageAsync(true, 10, s.Multicast) + suite.multiMessageAsync(true, 10, suite.Multicast) } // TestDuplicateMessageSequential_Submit evaluates the correctness of network layer on deduplicating // the received messages over Submit method of nodes' Conduits. // Messages are delivered to the receiver in a sequential manner. -func (s *EchoEngineTestSuite) TestDuplicateMessageSequential_Submit() { - s.skipTest("covered by TestDuplicateMessageParallel_Submit") - s.duplicateMessageSequential(s.Submit) +func (suite *EchoEngineTestSuite) TestDuplicateMessageSequential_Submit() { + suite.skipTest("covered by TestDuplicateMessageParallel_Submit") + suite.duplicateMessageSequential(suite.Submit) } // TestDuplicateMessageSequential_Publish evaluates the correctness of network layer on deduplicating // the received messages over Publish method of nodes' Conduits. // Messages are delivered to the receiver in a sequential manner. -func (s *EchoEngineTestSuite) TestDuplicateMessageSequential_Publish() { - s.skipTest("covered by TestDuplicateMessageParallel_Publish") - s.duplicateMessageSequential(s.Publish) +func (suite *EchoEngineTestSuite) TestDuplicateMessageSequential_Publish() { + suite.skipTest("covered by TestDuplicateMessageParallel_Publish") + suite.duplicateMessageSequential(suite.Publish) } // TestDuplicateMessageSequential_Unicast evaluates the correctness of network layer on deduplicating // the received messages over Unicast method of nodes' Conduits. // Messages are delivered to the receiver in a sequential manner. -func (s *EchoEngineTestSuite) TestDuplicateMessageSequential_Unicast() { - s.skipTest("covered by TestDuplicateMessageParallel_Unicast") - s.duplicateMessageSequential(s.Unicast) +func (suite *EchoEngineTestSuite) TestDuplicateMessageSequential_Unicast() { + suite.skipTest("covered by TestDuplicateMessageParallel_Unicast") + suite.duplicateMessageSequential(suite.Unicast) } // TestDuplicateMessageSequential_Multicast evaluates the correctness of network layer on deduplicating // the received messages over Multicast method of nodes' Conduits. // Messages are delivered to the receiver in a sequential manner. -func (s *EchoEngineTestSuite) TestDuplicateMessageSequential_Multicast() { - s.skipTest("covered by TestDuplicateMessageParallel_Multicast") - s.duplicateMessageSequential(s.Multicast) +func (suite *EchoEngineTestSuite) TestDuplicateMessageSequential_Multicast() { + suite.skipTest("covered by TestDuplicateMessageParallel_Multicast") + suite.duplicateMessageSequential(suite.Multicast) } // TestDuplicateMessageParallel_Submit evaluates the correctness of network layer // on deduplicating the received messages via Submit method of nodes' Conduits. // Messages are delivered to the receiver in parallel via the Submit method of Conduits. -func (s *EchoEngineTestSuite) TestDuplicateMessageParallel_Submit() { - s.duplicateMessageParallel(s.Submit) +func (suite *EchoEngineTestSuite) TestDuplicateMessageParallel_Submit() { + suite.duplicateMessageParallel(suite.Submit) } // TestDuplicateMessageParallel_Publish evaluates the correctness of network layer // on deduplicating the received messages via Publish method of nodes' Conduits. // Messages are delivered to the receiver in parallel via the Publish method of Conduits. -func (s *EchoEngineTestSuite) TestDuplicateMessageParallel_Publish() { - s.duplicateMessageParallel(s.Publish) +func (suite *EchoEngineTestSuite) TestDuplicateMessageParallel_Publish() { + suite.duplicateMessageParallel(suite.Publish) } // TestDuplicateMessageParallel_Unicast evaluates the correctness of network layer // on deduplicating the received messages via Unicast method of nodes' Conduits. // Messages are delivered to the receiver in parallel via the Unicast method of Conduits. -func (s *EchoEngineTestSuite) TestDuplicateMessageParallel_Unicast() { - s.duplicateMessageParallel(s.Unicast) +func (suite *EchoEngineTestSuite) TestDuplicateMessageParallel_Unicast() { + suite.duplicateMessageParallel(suite.Unicast) } // TestDuplicateMessageParallel_Multicast evaluates the correctness of network layer // on deduplicating the received messages via Multicast method of nodes' Conduits. // Messages are delivered to the receiver in parallel via the Multicast method of Conduits. -func (s *EchoEngineTestSuite) TestDuplicateMessageParallel_Multicast() { - s.duplicateMessageParallel(s.Multicast) +func (suite *EchoEngineTestSuite) TestDuplicateMessageParallel_Multicast() { + suite.duplicateMessageParallel(suite.Multicast) } // TestDuplicateMessageDifferentChan_Submit evaluates the correctness of network layer // on deduplicating the received messages via Submit method of Conduits against different engine ids. In specific, the // desire behavior is that the deduplication should happen based on both eventID and channelID. // Messages are sent via the Submit method of the Conduits. -func (s *EchoEngineTestSuite) TestDuplicateMessageDifferentChan_Submit() { - s.duplicateMessageDifferentChan(s.Submit) +func (suite *EchoEngineTestSuite) TestDuplicateMessageDifferentChan_Submit() { + suite.duplicateMessageDifferentChan(suite.Submit) } // TestDuplicateMessageDifferentChan_Publish evaluates the correctness of network layer // on deduplicating the received messages against different engine ids. In specific, the // desire behavior is that the deduplication should happen based on both eventID and channelID. // Messages are sent via the Publish methods of the Conduits. -func (s *EchoEngineTestSuite) TestDuplicateMessageDifferentChan_Publish() { - s.duplicateMessageDifferentChan(s.Publish) +func (suite *EchoEngineTestSuite) TestDuplicateMessageDifferentChan_Publish() { + suite.duplicateMessageDifferentChan(suite.Publish) } // TestDuplicateMessageDifferentChan_Unicast evaluates the correctness of network layer // on deduplicating the received messages against different engine ids. In specific, the // desire behavior is that the deduplication should happen based on both eventID and channelID. // Messages are sent via the Unicast methods of the Conduits. -func (s *EchoEngineTestSuite) TestDuplicateMessageDifferentChan_Unicast() { - s.duplicateMessageDifferentChan(s.Unicast) +func (suite *EchoEngineTestSuite) TestDuplicateMessageDifferentChan_Unicast() { + suite.duplicateMessageDifferentChan(suite.Unicast) } // TestDuplicateMessageDifferentChan_Multicast evaluates the correctness of network layer // on deduplicating the received messages against different engine ids. In specific, the // desire behavior is that the deduplication should happen based on both eventID and channelID. // Messages are sent via the Multicast methods of the Conduits. -func (s *EchoEngineTestSuite) TestDuplicateMessageDifferentChan_Multicast() { - s.duplicateMessageDifferentChan(s.Multicast) +func (suite *EchoEngineTestSuite) TestDuplicateMessageDifferentChan_Multicast() { + suite.duplicateMessageDifferentChan(suite.Multicast) } // duplicateMessageSequential is a helper function that sends duplicate messages sequentially // from a receiver to the sender via the injected send wrapper function of conduit. -func (s *EchoEngineTestSuite) duplicateMessageSequential(send ConduitSendWrapperFunc) { +func (suite *EchoEngineTestSuite) duplicateMessageSequential(send ConduitSendWrapperFunc) { sndID := 0 rcvID := 1 // registers engines in the network // sender's engine - sender := NewEchoEngine(s.Suite.T(), s.nets[sndID], 10, engine.TestNetwork, false, send) + sender := NewEchoEngine(suite.Suite.T(), suite.nets[sndID], 10, engine.TestNetwork, false, send) // receiver's engine - receiver := NewEchoEngine(s.Suite.T(), s.nets[rcvID], 10, engine.TestNetwork, false, send) + receiver := NewEchoEngine(suite.Suite.T(), suite.nets[rcvID], 10, engine.TestNetwork, false, send) // allow nodes to heartbeat and discover each other if using PubSub optionalSleep(send) @@ -390,28 +411,28 @@ func (s *EchoEngineTestSuite) duplicateMessageSequential(send ConduitSendWrapper // sends the same message 10 times for i := 0; i < 10; i++ { - require.NoError(s.Suite.T(), send(event, sender.con, s.ids[rcvID].NodeID)) + require.NoError(suite.Suite.T(), send(event, sender.con, suite.ids[rcvID].NodeID)) } time.Sleep(1 * time.Second) // receiver should only see the message once, and the rest should be dropped due to // duplication - require.Equal(s.Suite.T(), 1, receiver.seen[event.Text]) - require.Len(s.Suite.T(), receiver.seen, 1) + require.Equal(suite.Suite.T(), 1, receiver.seen[event.Text]) + require.Len(suite.Suite.T(), receiver.seen, 1) } // duplicateMessageParallel is a helper function that sends duplicate messages concurrent;u // from a receiver to the sender via the injected send wrapper function of conduit. -func (s *EchoEngineTestSuite) duplicateMessageParallel(send ConduitSendWrapperFunc) { +func (suite *EchoEngineTestSuite) duplicateMessageParallel(send ConduitSendWrapperFunc) { sndID := 0 rcvID := 1 // registers engines in the network // sender's engine - sender := NewEchoEngine(s.Suite.T(), s.nets[sndID], 10, engine.TestNetwork, false, send) + sender := NewEchoEngine(suite.Suite.T(), suite.nets[sndID], 10, engine.TestNetwork, false, send) // receiver's engine - receiver := NewEchoEngine(s.Suite.T(), s.nets[rcvID], 10, engine.TestNetwork, false, send) + receiver := NewEchoEngine(suite.Suite.T(), suite.nets[rcvID], 10, engine.TestNetwork, false, send) // allow nodes to heartbeat and discover each other optionalSleep(send) @@ -427,7 +448,7 @@ func (s *EchoEngineTestSuite) duplicateMessageParallel(send ConduitSendWrapperFu wg.Add(1) go func() { defer wg.Done() - require.NoError(s.Suite.T(), send(event, sender.con, s.ids[rcvID].NodeID)) + require.NoError(suite.Suite.T(), send(event, sender.con, suite.ids[rcvID].NodeID)) }() } wg.Wait() @@ -435,13 +456,13 @@ func (s *EchoEngineTestSuite) duplicateMessageParallel(send ConduitSendWrapperFu // receiver should only see the message once, and the rest should be dropped due to // duplication - require.Equal(s.Suite.T(), 1, receiver.seen[event.Text]) - require.Len(s.Suite.T(), receiver.seen, 1) + require.Equal(suite.Suite.T(), 1, receiver.seen[event.Text]) + require.Len(suite.Suite.T(), receiver.seen, 1) } // duplicateMessageDifferentChan is a helper function that sends the same message from two distinct // sender engines to the two distinct receiver engines via the send wrapper function of Conduits. -func (s *EchoEngineTestSuite) duplicateMessageDifferentChan(send ConduitSendWrapperFunc) { +func (suite *EchoEngineTestSuite) duplicateMessageDifferentChan(send ConduitSendWrapperFunc) { const ( sndNode = iota rcvNode @@ -452,19 +473,19 @@ func (s *EchoEngineTestSuite) duplicateMessageDifferentChan(send ConduitSendWrap ) // registers engines in the network // first type - // sender's engine - sender1 := NewEchoEngine(s.Suite.T(), s.nets[sndNode], 10, channel1, false, send) + // sender'suite engine + sender1 := NewEchoEngine(suite.Suite.T(), suite.nets[sndNode], 10, channel1, false, send) // receiver's engine - receiver1 := NewEchoEngine(s.Suite.T(), s.nets[rcvNode], 10, channel1, false, send) + receiver1 := NewEchoEngine(suite.Suite.T(), suite.nets[rcvNode], 10, channel1, false, send) // second type // registers engines in the network - // sender's engine - sender2 := NewEchoEngine(s.Suite.T(), s.nets[sndNode], 10, channel2, false, send) + // sender'suite engine + sender2 := NewEchoEngine(suite.Suite.T(), suite.nets[sndNode], 10, channel2, false, send) // receiver's engine - receiver2 := NewEchoEngine(s.Suite.T(), s.nets[rcvNode], 10, channel2, false, send) + receiver2 := NewEchoEngine(suite.Suite.T(), suite.nets[rcvNode], 10, channel2, false, send) // allow nodes to heartbeat and discover each other optionalSleep(send) @@ -481,10 +502,10 @@ func (s *EchoEngineTestSuite) duplicateMessageDifferentChan(send ConduitSendWrap go func() { defer wg.Done() // sender1 to receiver1 on channel1 - require.NoError(s.Suite.T(), send(event, sender1.con, s.ids[rcvNode].NodeID)) + require.NoError(suite.Suite.T(), send(event, sender1.con, suite.ids[rcvNode].NodeID)) // sender2 to receiver2 on channel2 - require.NoError(s.Suite.T(), send(event, sender2.con, s.ids[rcvNode].NodeID)) + require.NoError(suite.Suite.T(), send(event, sender2.con, suite.ids[rcvNode].NodeID)) }() } wg.Wait() @@ -492,26 +513,26 @@ func (s *EchoEngineTestSuite) duplicateMessageDifferentChan(send ConduitSendWrap // each receiver should only see the message once, and the rest should be dropped due to // duplication - require.Equal(s.Suite.T(), 1, receiver1.seen[event.Text]) - require.Equal(s.Suite.T(), 1, receiver2.seen[event.Text]) + require.Equal(suite.Suite.T(), 1, receiver1.seen[event.Text]) + require.Equal(suite.Suite.T(), 1, receiver2.seen[event.Text]) - require.Len(s.Suite.T(), receiver1.seen, 1) - require.Len(s.Suite.T(), receiver2.seen, 1) + require.Len(suite.Suite.T(), receiver1.seen, 1) + require.Len(suite.Suite.T(), receiver2.seen, 1) } // singleMessage sends a single message from one network instance to the other one // it evaluates the correctness of implementation against correct delivery of the message. // in case echo is true, it also evaluates correct reception of the echo message from the receiver side -func (s *EchoEngineTestSuite) singleMessage(echo bool, send ConduitSendWrapperFunc) { +func (suite *EchoEngineTestSuite) singleMessage(echo bool, send ConduitSendWrapperFunc) { sndID := 0 rcvID := 1 // registers engines in the network // sender's engine - sender := NewEchoEngine(s.Suite.T(), s.nets[sndID], 10, engine.TestNetwork, echo, send) + sender := NewEchoEngine(suite.Suite.T(), suite.nets[sndID], 10, engine.TestNetwork, echo, send) // receiver's engine - receiver := NewEchoEngine(s.Suite.T(), s.nets[rcvID], 10, engine.TestNetwork, echo, send) + receiver := NewEchoEngine(suite.Suite.T(), suite.nets[rcvID], 10, engine.TestNetwork, echo, send) // allow nodes to heartbeat and discover each other optionalSleep(send) @@ -520,27 +541,27 @@ func (s *EchoEngineTestSuite) singleMessage(echo bool, send ConduitSendWrapperFu event := &message.TestMessage{ Text: "hello", } - require.NoError(s.Suite.T(), send(event, sender.con, s.ids[rcvID].NodeID)) + require.NoError(suite.Suite.T(), send(event, sender.con, suite.ids[rcvID].NodeID)) // evaluates reception of echo request select { case <-receiver.received: // evaluates reception of message at the other side // does not evaluate the content - require.NotNil(s.Suite.T(), receiver.originID) - require.NotNil(s.Suite.T(), receiver.event) - assert.Equal(s.Suite.T(), s.ids[sndID].NodeID, receiver.originID) + require.NotNil(suite.Suite.T(), receiver.originID) + require.NotNil(suite.Suite.T(), receiver.event) + assert.Equal(suite.Suite.T(), suite.ids[sndID].NodeID, receiver.originID) // evaluates proper reception of event // casts the received event at the receiver side rcvEvent, ok := (<-receiver.event).(*message.TestMessage) // evaluates correctness of casting - require.True(s.Suite.T(), ok) + require.True(suite.Suite.T(), ok) // evaluates content of received message - assert.Equal(s.Suite.T(), event, rcvEvent) + assert.Equal(suite.Suite.T(), event, rcvEvent) case <-time.After(10 * time.Second): - assert.Fail(s.Suite.T(), "sender failed to send a message to receiver") + assert.Fail(suite.Suite.T(), "sender failed to send a message to receiver") } // evaluates echo back @@ -550,23 +571,23 @@ func (s *EchoEngineTestSuite) singleMessage(echo bool, send ConduitSendWrapperFu case <-sender.received: // evaluates reception of message at the other side // does not evaluate the content - require.NotNil(s.Suite.T(), sender.originID) - require.NotNil(s.Suite.T(), sender.event) - assert.Equal(s.Suite.T(), s.ids[rcvID].NodeID, sender.originID) + require.NotNil(suite.Suite.T(), sender.originID) + require.NotNil(suite.Suite.T(), sender.event) + assert.Equal(suite.Suite.T(), suite.ids[rcvID].NodeID, sender.originID) // evaluates proper reception of event // casts the received event at the receiver side rcvEvent, ok := (<-sender.event).(*message.TestMessage) // evaluates correctness of casting - require.True(s.Suite.T(), ok) + require.True(suite.Suite.T(), ok) // evaluates content of received message echoEvent := &message.TestMessage{ Text: fmt.Sprintf("%s: %s", receiver.echomsg, event.Text), } - assert.Equal(s.Suite.T(), echoEvent, rcvEvent) + assert.Equal(suite.Suite.T(), echoEvent, rcvEvent) case <-time.After(10 * time.Second): - assert.Fail(s.Suite.T(), "receiver failed to send an echo message back to sender") + assert.Fail(suite.Suite.T(), "receiver failed to send an echo message back to sender") } } } @@ -576,15 +597,15 @@ func (s *EchoEngineTestSuite) singleMessage(echo bool, send ConduitSendWrapperFu // sender and receiver are sync over reception, i.e., sender sends one message at a time and // waits for its reception // count defines number of messages -func (s *EchoEngineTestSuite) multiMessageSync(echo bool, count int, send ConduitSendWrapperFunc) { +func (suite *EchoEngineTestSuite) multiMessageSync(echo bool, count int, send ConduitSendWrapperFunc) { sndID := 0 rcvID := 1 // registers engines in the network // sender's engine - sender := NewEchoEngine(s.Suite.T(), s.nets[sndID], 10, engine.TestNetwork, echo, send) + sender := NewEchoEngine(suite.Suite.T(), suite.nets[sndID], 10, engine.TestNetwork, echo, send) // receiver's engine - receiver := NewEchoEngine(s.Suite.T(), s.nets[rcvID], 10, engine.TestNetwork, echo, send) + receiver := NewEchoEngine(suite.Suite.T(), suite.nets[rcvID], 10, engine.TestNetwork, echo, send) // allow nodes to heartbeat and discover each other optionalSleep(send) @@ -595,26 +616,26 @@ func (s *EchoEngineTestSuite) multiMessageSync(echo bool, count int, send Condui Text: fmt.Sprintf("hello%d", i), } // sends a message from sender to receiver using send wrapper function - require.NoError(s.Suite.T(), send(event, sender.con, s.ids[rcvID].NodeID)) + require.NoError(suite.Suite.T(), send(event, sender.con, suite.ids[rcvID].NodeID)) select { case <-receiver.received: // evaluates reception of message at the other side // does not evaluate the content - require.NotNil(s.Suite.T(), receiver.originID) - require.NotNil(s.Suite.T(), receiver.event) - assert.Equal(s.Suite.T(), s.ids[sndID].NodeID, receiver.originID) + require.NotNil(suite.Suite.T(), receiver.originID) + require.NotNil(suite.Suite.T(), receiver.event) + assert.Equal(suite.Suite.T(), suite.ids[sndID].NodeID, receiver.originID) // evaluates proper reception of event // casts the received event at the receiver side rcvEvent, ok := (<-receiver.event).(*message.TestMessage) // evaluates correctness of casting - require.True(s.Suite.T(), ok) + require.True(suite.Suite.T(), ok) // evaluates content of received message - assert.Equal(s.Suite.T(), event, rcvEvent) + assert.Equal(suite.Suite.T(), event, rcvEvent) case <-time.After(2 * time.Second): - assert.Fail(s.Suite.T(), "sender failed to send a message to receiver") + assert.Fail(suite.Suite.T(), "sender failed to send a message to receiver") } // evaluates echo back @@ -624,23 +645,23 @@ func (s *EchoEngineTestSuite) multiMessageSync(echo bool, count int, send Condui case <-sender.received: // evaluates reception of message at the other side // does not evaluate the content - require.NotNil(s.Suite.T(), sender.originID) - require.NotNil(s.Suite.T(), sender.event) - assert.Equal(s.Suite.T(), s.ids[rcvID].NodeID, sender.originID) + require.NotNil(suite.Suite.T(), sender.originID) + require.NotNil(suite.Suite.T(), sender.event) + assert.Equal(suite.Suite.T(), suite.ids[rcvID].NodeID, sender.originID) // evaluates proper reception of event // casts the received event at the receiver side rcvEvent, ok := (<-sender.event).(*message.TestMessage) // evaluates correctness of casting - require.True(s.Suite.T(), ok) + require.True(suite.Suite.T(), ok) // evaluates content of received message echoEvent := &message.TestMessage{ Text: fmt.Sprintf("%s: %s", receiver.echomsg, event.Text), } - assert.Equal(s.Suite.T(), echoEvent, rcvEvent) + assert.Equal(suite.Suite.T(), echoEvent, rcvEvent) case <-time.After(10 * time.Second): - assert.Fail(s.Suite.T(), "receiver failed to send an echo message back to sender") + assert.Fail(suite.Suite.T(), "receiver failed to send an echo message back to sender") } } @@ -652,16 +673,16 @@ func (s *EchoEngineTestSuite) multiMessageSync(echo bool, count int, send Condui // it evaluates the correctness of implementation against correct delivery of the messages. // sender and receiver are async, i.e., sender sends all its message at blast // count defines number of messages -func (s *EchoEngineTestSuite) multiMessageAsync(echo bool, count int, send ConduitSendWrapperFunc) { +func (suite *EchoEngineTestSuite) multiMessageAsync(echo bool, count int, send ConduitSendWrapperFunc) { sndID := 0 rcvID := 1 // registers engines in the network // sender's engine - sender := NewEchoEngine(s.Suite.T(), s.nets[sndID], 10, engine.TestNetwork, echo, send) + sender := NewEchoEngine(suite.Suite.T(), suite.nets[sndID], 10, engine.TestNetwork, echo, send) // receiver's engine - receiver := NewEchoEngine(s.Suite.T(), s.nets[rcvID], 10, engine.TestNetwork, echo, send) + receiver := NewEchoEngine(suite.Suite.T(), suite.nets[rcvID], 10, engine.TestNetwork, echo, send) // allow nodes to heartbeat and discover each other optionalSleep(send) @@ -677,7 +698,7 @@ func (s *EchoEngineTestSuite) multiMessageAsync(echo bool, count int, send Condu event := &message.TestMessage{ Text: fmt.Sprintf("hello%d", i), } - require.NoError(s.Suite.T(), send(event, sender.con, s.ids[1].NodeID)) + require.NoError(suite.Suite.T(), send(event, sender.con, suite.ids[1].NodeID)) } for i := 0; i < count; i++ { @@ -685,25 +706,25 @@ func (s *EchoEngineTestSuite) multiMessageAsync(echo bool, count int, send Condu case <-receiver.received: // evaluates reception of message at the other side // does not evaluate the content - require.NotNil(s.Suite.T(), receiver.originID) - require.NotNil(s.Suite.T(), receiver.event) - assert.Equal(s.Suite.T(), s.ids[0].NodeID, receiver.originID) + require.NotNil(suite.Suite.T(), receiver.originID) + require.NotNil(suite.Suite.T(), receiver.event) + assert.Equal(suite.Suite.T(), suite.ids[0].NodeID, receiver.originID) // evaluates proper reception of event // casts the received event at the receiver side rcvEvent, ok := (<-receiver.event).(*message.TestMessage) // evaluates correctness of casting - require.True(s.Suite.T(), ok) + require.True(suite.Suite.T(), ok) // evaluates content of received message // the content should not yet received and be unique _, rcv := received[rcvEvent.Text] - assert.False(s.Suite.T(), rcv) + assert.False(suite.Suite.T(), rcv) // marking event as received received[rcvEvent.Text] = struct{}{} case <-time.After(2 * time.Second): - assert.Fail(s.Suite.T(), "sender failed to send a message to receiver") + assert.Fail(suite.Suite.T(), "sender failed to send a message to receiver") } } @@ -715,33 +736,33 @@ func (s *EchoEngineTestSuite) multiMessageAsync(echo bool, count int, send Condu case <-sender.received: // evaluates reception of message at the other side // does not evaluate the content - require.NotNil(s.Suite.T(), sender.originID) - require.NotNil(s.Suite.T(), sender.event) - assert.Equal(s.Suite.T(), s.ids[rcvID].NodeID, sender.originID) + require.NotNil(suite.Suite.T(), sender.originID) + require.NotNil(suite.Suite.T(), sender.event) + assert.Equal(suite.Suite.T(), suite.ids[rcvID].NodeID, sender.originID) // evaluates proper reception of event // casts the received event at the receiver side rcvEvent, ok := (<-sender.event).(*message.TestMessage) // evaluates correctness of casting - require.True(s.Suite.T(), ok) + require.True(suite.Suite.T(), ok) // evaluates content of received echo message // the content should not yet received and be unique _, rcv := received[rcvEvent.Text] - assert.False(s.Suite.T(), rcv) + assert.False(suite.Suite.T(), rcv) // echo messages should start with prefix msg of receiver that echos back - assert.True(s.Suite.T(), strings.HasPrefix(rcvEvent.Text, receiver.echomsg)) + assert.True(suite.Suite.T(), strings.HasPrefix(rcvEvent.Text, receiver.echomsg)) // marking echo event as received received[rcvEvent.Text] = struct{}{} case <-time.After(10 * time.Second): - assert.Fail(s.Suite.T(), "receiver failed to send an echo message back to sender") + assert.Fail(suite.Suite.T(), "receiver failed to send an echo message back to sender") } } } } -func (s *EchoEngineTestSuite) skipTest(reason string) { +func (suite *EchoEngineTestSuite) skipTest(reason string) { if _, found := os.LookupEnv("AllNetworkTest"); !found { - s.T().Skip(reason) + suite.T().Skip(reason) } } diff --git a/network/gossip/libp2p/test/epochtransition_test.go b/network/gossip/libp2p/test/epochtransition_test.go index c60b88951bc..1f2fcd4beab 100644 --- a/network/gossip/libp2p/test/epochtransition_test.go +++ b/network/gossip/libp2p/test/epochtransition_test.go @@ -45,72 +45,71 @@ func TestEpochTransitionTestSuite(t *testing.T) { suite.Run(t, new(MutableIdentityTableSuite)) } -func (ts *MutableIdentityTableSuite) SetupTest() { +func (suite *MutableIdentityTableSuite) SetupTest() { rand.Seed(time.Now().UnixNano()) nodeCount := 10 - ts.logger = zerolog.New(os.Stderr).Level(zerolog.ErrorLevel) + suite.logger = zerolog.New(os.Stderr).Level(zerolog.ErrorLevel) golog.SetAllLoggers(golog.LevelError) // create ids - ids, mws := generateIDsAndMiddlewares(ts.T(), nodeCount, ts.logger) - ts.ids = ids - ts.mws = mws + ids, mws := GenerateIDsAndMiddlewares(suite.T(), nodeCount, !DryRun, suite.logger) + suite.ids = ids + suite.mws = mws // setup state related mocks final := unittest.BlockHeaderFixture() - ts.state = new(protocol.ReadOnlyState) - ts.snapshot = new(protocol.Snapshot) - ts.snapshot.On("Head").Return(&final, nil) - ts.snapshot.On("Phase").Return(flow.EpochPhaseCommitted, nil) - ts.snapshot.On("Identities", testifymock.Anything).Return( - func(flow.IdentityFilter) flow.IdentityList { return ts.ids }, + suite.state = new(protocol.ReadOnlyState) + suite.snapshot = new(protocol.Snapshot) + suite.snapshot.On("Head").Return(&final, nil) + suite.snapshot.On("Phase").Return(flow.EpochPhaseCommitted, nil) + suite.snapshot.On("Identities", testifymock.Anything).Return( + func(flow.IdentityFilter) flow.IdentityList { return suite.ids }, func(flow.IdentityFilter) error { return nil }) - ts.state.On("Final").Return(ts.snapshot, nil) + suite.state.On("Final").Return(suite.snapshot, nil) + + // all nodes use the same state mock + states := make([]*protocol.ReadOnlyState, nodeCount) + for i := 0; i < nodeCount; i++ { + states[i] = suite.state + } // create networks using the mocked state and default topology - nets := generateNetworks(ts.T(), ts.logger, ids, mws, 100, nil, false) - ts.nets = nets + sms := GenerateSubscriptionManagers(suite.T(), mws) + nets := GenerateNetworks(suite.T(), suite.logger, ids, mws, 100, nil, sms, !DryRun) + suite.nets = nets // generate the refreshers - ts.idRefreshers = ts.generateNodeIDRefreshers(nets) + suite.idRefreshers = suite.generateNodeIDRefreshers(nets) // generate the engines - ts.engines = generateEngines(ts.T(), nets) + suite.engines = GenerateEngines(suite.T(), nets) } // TearDownTest closes the networks within a specified timeout -func (ts *MutableIdentityTableSuite) TearDownTest() { - for _, net := range ts.nets { - select { - // closes the network - case <-net.Done(): - continue - case <-time.After(3 * time.Second): - ts.Suite.Fail("could not stop the network") - } - } +func (suite *MutableIdentityTableSuite) TearDownTest() { + stopNetworks(suite.T(), suite.nets, 3*time.Second) } // TestNewNodeAdded tests that when a new node is added to the identity list // (ie. as a result of a EpochSetup event) that it can connect to the network. -func (ts *MutableIdentityTableSuite) TestNewNodeAdded() { +func (suite *MutableIdentityTableSuite) TestNewNodeAdded() { // create the id, middleware and network for a new node - ids, mws, nets := generateIDsMiddlewaresNetworks(ts.T(), 1, ts.logger, 100, nil, false) + ids, mws, nets := GenerateIDsMiddlewaresNetworks(suite.T(), 1, suite.logger, 100, nil, !DryRun) newID := ids[0] - ts.nets = append(ts.nets, nets[0]) + suite.nets = append(suite.nets, nets[0]) newMiddleware := mws[0] - newIDs := append(ts.ids, ids...) - ts.ids = newIDs + newIDs := append(suite.ids, ids...) + suite.ids = newIDs // create a new refresher - newIDRefresher := ts.generateNodeIDRefreshers(nets) - newIDRefreshers := append(ts.idRefreshers, newIDRefresher...) + newIDRefresher := suite.generateNodeIDRefreshers(nets) + newIDRefreshers := append(suite.idRefreshers, newIDRefresher...) // create the engine for the new node - newEngine := generateEngines(ts.T(), nets) - newEngines := append(ts.engines, newEngine...) + newEngine := GenerateEngines(suite.T(), nets) + newEngines := append(suite.engines, newEngine...) // trigger the identity table change event for _, n := range newIDRefreshers { @@ -119,28 +118,28 @@ func (ts *MutableIdentityTableSuite) TestNewNodeAdded() { // check if the new node has sufficient connections with the existing nodes // if it does, then it has been inducted successfully in the network - checkConnectivity(ts.T(), newMiddleware, newIDs.Filter(filter.Not(filter.HasNodeID(newID.NodeID)))) + checkConnectivity(suite.T(), newMiddleware, newIDs.Filter(filter.Not(filter.HasNodeID(newID.NodeID)))) // check that all the engines on this new epoch can talk to each other - sendMessagesAndVerify(ts.T(), newIDs, newEngines, ts.Publish) + sendMessagesAndVerify(suite.T(), newIDs, newEngines, suite.Publish) } // TestNodeRemoved tests that when an existing node is removed from the identity // list (ie. as a result of an ejection or transition into an epoch where that node // has un-staked) that it cannot connect to the network. -func (ts *MutableIdentityTableSuite) TestNodeRemoved() { +func (suite *MutableIdentityTableSuite) TestNodeRemoved() { // choose a random node to remove - removeIndex := rand.Intn(len(ts.ids)) - removedID := ts.ids[removeIndex] + removeIndex := rand.Intn(len(suite.ids)) + removedID := suite.ids[removeIndex] // remove the identity at that index from the ids - newIDs := ts.ids.Filter(filter.Not(filter.HasNodeID(removedID.NodeID))) - ts.ids = newIDs + newIDs := suite.ids.Filter(filter.Not(filter.HasNodeID(removedID.NodeID))) + suite.ids = newIDs // create a list of engines except for the removed node var newEngines []*MeshEngine - for i, eng := range ts.engines { + for i, eng := range suite.engines { if i == removeIndex { continue } @@ -149,12 +148,12 @@ func (ts *MutableIdentityTableSuite) TestNodeRemoved() { // trigger an epoch phase change for all nodes // from flow.EpochPhaseStaking to flow.EpochPhaseSetup - for _, n := range ts.idRefreshers { + for _, n := range suite.idRefreshers { n.OnIdentityTableChanged() } // check that all remaining engines can still talk to each other - sendMessagesAndVerify(ts.T(), newIDs, newEngines, ts.Publish) + sendMessagesAndVerify(suite.T(), newIDs, newEngines, suite.Publish) // TODO check that messages to/from evicted node are not delivered } @@ -210,10 +209,10 @@ func sendMessagesAndVerify(t *testing.T, ids flow.IdentityList, engs []*MeshEngi unittest.AssertReturnsBefore(t, wg.Wait, 5*time.Second) } -func (ts *MutableIdentityTableSuite) generateNodeIDRefreshers(nets []*libp2p.Network) []*libp2p.NodeIDRefresher { +func (suite *MutableIdentityTableSuite) generateNodeIDRefreshers(nets []*libp2p.Network) []*libp2p.NodeIDRefresher { refreshers := make([]*libp2p.NodeIDRefresher, len(nets)) for i, net := range nets { - refreshers[i] = libp2p.NewNodeIDRefresher(ts.logger, ts.state, net.SetIDs) + refreshers[i] = libp2p.NewNodeIDRefresher(suite.logger, suite.state, net.SetIDs) } return refreshers } diff --git a/network/gossip/libp2p/test/meshengine_test.go b/network/gossip/libp2p/test/meshengine_test.go index bf5bdfe4461..9619f37ff1c 100644 --- a/network/gossip/libp2p/test/meshengine_test.go +++ b/network/gossip/libp2p/test/meshengine_test.go @@ -28,10 +28,9 @@ import ( // of engines over a complete graph type MeshEngineTestSuite struct { suite.Suite - ConduitWrapper // used as a wrapper around conduit methods - nets []*libp2p.Network // used to keep track of the networks - mws []*libp2p.Middleware // used to keep track of the middlewares associated with networks - ids flow.IdentityList // used to keep track of the identifiers associated with networks + ConduitWrapper // used as a wrapper around conduit methods + nets []*libp2p.Network // used to keep track of the networks + ids flow.IdentityList // used to keep track of the identifiers associated with networks } // TestMeshNetTestSuite runs all tests in this test suit @@ -41,142 +40,132 @@ func TestMeshNetTestSuite(t *testing.T) { // SetupTest is executed prior to each test in this test suit // it creates and initializes a set of network instances -func (m *MeshEngineTestSuite) SetupTest() { +func (suite *MeshEngineTestSuite) SetupTest() { // defines total number of nodes in our network (minimum 3 needed to use 1-k messaging) const count = 10 logger := zerolog.New(os.Stderr).Level(zerolog.ErrorLevel) golog.SetAllLoggers(golog.LevelError) - var err error - m.ids, m.mws, m.nets = generateIDsMiddlewaresNetworks(m.T(), count, logger, 100, nil, false) - require.NoError(m.Suite.T(), err) + suite.ids, _, suite.nets = GenerateIDsMiddlewaresNetworks(suite.T(), count, logger, 100, nil, !DryRun) } // TearDownTest closes the networks within a specified timeout -func (m *MeshEngineTestSuite) TearDownTest() { - for _, net := range m.nets { - select { - // closes the network - case <-net.Done(): - continue - case <-time.After(3 * time.Second): - m.Suite.Fail("could not stop the network") - } - } +func (suite *MeshEngineTestSuite) TearDownTest() { + stopNetworks(suite.T(), suite.nets, 3*time.Second) } // TestAllToAll_Submit evaluates the network of mesh engines against allToAllScenario scenario. // Network instances during this test use their Submit method to disseminate messages. -func (m *MeshEngineTestSuite) TestAllToAll_Submit() { - m.allToAllScenario(m.Submit) +func (suite *MeshEngineTestSuite) TestAllToAll_Submit() { + suite.allToAllScenario(suite.Submit) } // TestAllToAll_Publish evaluates the network of mesh engines against allToAllScenario scenario. // Network instances during this test use their Publish method to disseminate messages. -func (m *MeshEngineTestSuite) TestAllToAll_Publish() { - m.allToAllScenario(m.Publish) +func (suite *MeshEngineTestSuite) TestAllToAll_Publish() { + suite.allToAllScenario(suite.Publish) } // TestAllToAll_Multicast evaluates the network of mesh engines against allToAllScenario scenario. // Network instances during this test use their Multicast method to disseminate messages. -func (m *MeshEngineTestSuite) TestAllToAll_Multicast() { - m.allToAllScenario(m.Multicast) +func (suite *MeshEngineTestSuite) TestAllToAll_Multicast() { + suite.allToAllScenario(suite.Multicast) } // TestAllToAll_Unicast evaluates the network of mesh engines against allToAllScenario scenario. // Network instances during this test use their Unicast method to disseminate messages. -func (m *MeshEngineTestSuite) TestAllToAll_Unicast() { - m.allToAllScenario(m.Unicast) +func (suite *MeshEngineTestSuite) TestAllToAll_Unicast() { + suite.allToAllScenario(suite.Unicast) } // TestTargetedValidators_Submit tests if only the intended recipients in a 1-k messaging actually receive the message. // The messages are disseminated through the Submit method of conduits. -func (m *MeshEngineTestSuite) TestTargetedValidators_Submit() { - m.targetValidatorScenario(m.Submit) +func (suite *MeshEngineTestSuite) TestTargetedValidators_Submit() { + suite.targetValidatorScenario(suite.Submit) } // TestTargetedValidators_Unicast tests if only the intended recipients in a 1-k messaging actually receive the message. // The messages are disseminated through the Unicast method of conduits. -func (m *MeshEngineTestSuite) TestTargetedValidators_Unicast() { - m.targetValidatorScenario(m.Unicast) +func (suite *MeshEngineTestSuite) TestTargetedValidators_Unicast() { + suite.targetValidatorScenario(suite.Unicast) } // TestTargetedValidators_Multicast tests if only the intended recipients in a 1-k messaging actually receive the //message. // The messages are disseminated through the Multicast method of conduits. -func (m *MeshEngineTestSuite) TestTargetedValidators_Multicast() { - m.targetValidatorScenario(m.Multicast) +func (suite *MeshEngineTestSuite) TestTargetedValidators_Multicast() { + suite.targetValidatorScenario(suite.Multicast) } // TestTargetedValidators_Publish tests if only the intended recipients in a 1-k messaging actually receive the message. // The messages are disseminated through the Multicast method of conduits. -func (m *MeshEngineTestSuite) TestTargetedValidators_Publish() { - m.targetValidatorScenario(m.Publish) +func (suite *MeshEngineTestSuite) TestTargetedValidators_Publish() { + suite.targetValidatorScenario(suite.Publish) } // TestMaxMessageSize_Submit evaluates the messageSizeScenario scenario using // the Submit method of conduits. -func (m *MeshEngineTestSuite) TestMaxMessageSize_Submit() { - m.messageSizeScenario(m.Submit, libp2p.DefaultMaxPubSubMsgSize) +func (suite *MeshEngineTestSuite) TestMaxMessageSize_Submit() { + suite.messageSizeScenario(suite.Submit, libp2p.DefaultMaxPubSubMsgSize) } // TestMaxMessageSize_Unicast evaluates the messageSizeScenario scenario using // the Unicast method of conduits. -func (m *MeshEngineTestSuite) TestMaxMessageSize_Unicast() { - m.messageSizeScenario(m.Unicast, libp2p.DefaultMaxUnicastMsgSize) +func (suite *MeshEngineTestSuite) TestMaxMessageSize_Unicast() { + suite.messageSizeScenario(suite.Unicast, libp2p.DefaultMaxUnicastMsgSize) } // TestMaxMessageSize_Multicast evaluates the messageSizeScenario scenario using // the Multicast method of conduits. -func (m *MeshEngineTestSuite) TestMaxMessageSize_Multicast() { - m.messageSizeScenario(m.Multicast, libp2p.DefaultMaxPubSubMsgSize) +func (suite *MeshEngineTestSuite) TestMaxMessageSize_Multicast() { + suite.messageSizeScenario(suite.Multicast, libp2p.DefaultMaxPubSubMsgSize) } // TestMaxMessageSize_Publish evaluates the messageSizeScenario scenario using the // Publish method of conduits. -func (m *MeshEngineTestSuite) TestMaxMessageSize_Publish() { - m.messageSizeScenario(m.Publish, libp2p.DefaultMaxPubSubMsgSize) +func (suite *MeshEngineTestSuite) TestMaxMessageSize_Publish() { + suite.messageSizeScenario(suite.Publish, libp2p.DefaultMaxPubSubMsgSize) } // TestUnregister_Publish tests that an engine cannot send any message using Publish // or receive any messages after the conduit is closed -func (m *MeshEngineTestSuite) TestUnregister_Publish() { - m.conduitCloseScenario(m.Publish) +func (suite *MeshEngineTestSuite) TestUnregister_Publish() { + suite.conduitCloseScenario(suite.Publish) } // TestUnregister_Publish tests that an engine cannot send any message using Multicast // or receive any messages after the conduit is closed -func (m *MeshEngineTestSuite) TestUnregister_Multicast() { - m.conduitCloseScenario(m.Multicast) +func (suite *MeshEngineTestSuite) TestUnregister_Multicast() { + suite.conduitCloseScenario(suite.Multicast) } // TestUnregister_Publish tests that an engine cannot send any message using Submit // or receive any messages after the conduit is closed -func (m *MeshEngineTestSuite) TestUnregister_Submit() { - m.conduitCloseScenario(m.Submit) +func (suite *MeshEngineTestSuite) TestUnregister_Submit() { + suite.conduitCloseScenario(suite.Submit) } // TestUnregister_Publish tests that an engine cannot send any message using Unicast // or receive any messages after the conduit is closed -func (m *MeshEngineTestSuite) TestUnregister_Unicast() { - m.conduitCloseScenario(m.Unicast) +func (suite *MeshEngineTestSuite) TestUnregister_Unicast() { + suite.conduitCloseScenario(suite.Unicast) } // allToAllScenario creates a complete mesh of the engines // each engine x then sends a "hello from node x" to other engines // it evaluates the correctness of message delivery as well as content of the message -func (m *MeshEngineTestSuite) allToAllScenario(send ConduitSendWrapperFunc) { +func (suite *MeshEngineTestSuite) allToAllScenario(send ConduitSendWrapperFunc) { // allows nodes to find each other in case of Mulitcast and Publish optionalSleep(send) // creating engines - count := len(m.nets) + count := len(suite.nets) engs := make([]*MeshEngine, 0) wg := sync.WaitGroup{} // logs[i][j] keeps the message that node i sends to node j logs := make(map[int][]string) - for i := range m.nets { - eng := NewMeshEngine(m.Suite.T(), m.nets[i], count-1, engine.TestNetwork) + for i := range suite.nets { + eng := NewMeshEngine(suite.Suite.T(), suite.nets[i], count-1, engine.TestNetwork) engs = append(engs, eng) logs[i] = make([]string, 0) } @@ -185,19 +174,19 @@ func (m *MeshEngineTestSuite) allToAllScenario(send ConduitSendWrapperFunc) { time.Sleep(2 * time.Second) // Each node broadcasting a message to all others - for i := range m.nets { + for i := range suite.nets { event := &message.TestMessage{ Text: fmt.Sprintf("hello from node %v", i), } // others keeps the identifier of all nodes except ith node - others := m.ids.Filter(filter.Not(filter.HasNodeID(m.ids[i].NodeID))).NodeIDs() - require.NoError(m.Suite.T(), send(event, engs[i].con, others...)) + others := suite.ids.Filter(filter.Not(filter.HasNodeID(suite.ids[i].NodeID))).NodeIDs() + require.NoError(suite.Suite.T(), send(event, engs[i].con, others...)) wg.Add(count - 1) } // fires a goroutine for each engine that listens to incoming messages - for i := range m.nets { + for i := range suite.nets { go func(e *MeshEngine) { for x := 0; x < count-1; x++ { <-e.received @@ -206,28 +195,28 @@ func (m *MeshEngineTestSuite) allToAllScenario(send ConduitSendWrapperFunc) { }(engs[i]) } - unittest.AssertReturnsBefore(m.Suite.T(), wg.Wait, 30*time.Second) + unittest.AssertReturnsBefore(suite.Suite.T(), wg.Wait, 30*time.Second) // evaluates that all messages are received for index, e := range engs { // confirms the number of received messages at each node if len(e.event) != (count - 1) { - assert.Fail(m.Suite.T(), + assert.Fail(suite.Suite.T(), fmt.Sprintf("Message reception mismatch at node %v. Expected: %v, Got: %v", index, count-1, len(e.event))) } // extracts failed messages receivedIndices, err := extractSenderID(count, e.event, "hello from node") - require.NoError(m.Suite.T(), err) + require.NoError(suite.Suite.T(), err) for j := 0; j < count; j++ { // evaluates self-gossip if j == index { - assert.False(m.Suite.T(), (receivedIndices)[index], fmt.Sprintf("self gossiped for node %v detected", index)) + assert.False(suite.Suite.T(), (receivedIndices)[index], fmt.Sprintf("self gossiped for node %v detected", index)) } // evaluates content if !(receivedIndices)[j] { - assert.False(m.Suite.T(), (receivedIndices)[index], + assert.False(suite.Suite.T(), (receivedIndices)[index], fmt.Sprintf("Message not found in node #%v's messages. Expected: Message from node %v. Got: No message", index, j)) } } @@ -238,14 +227,14 @@ func (m *MeshEngineTestSuite) allToAllScenario(send ConduitSendWrapperFunc) { // based on identifiers list. // It then verifies that only the intended recipients receive the message. // Message dissemination is done using the send wrapper of conduit. -func (m *MeshEngineTestSuite) targetValidatorScenario(send ConduitSendWrapperFunc) { +func (suite *MeshEngineTestSuite) targetValidatorScenario(send ConduitSendWrapperFunc) { // creating engines - count := len(m.nets) + count := len(suite.nets) engs := make([]*MeshEngine, 0) wg := sync.WaitGroup{} - for i := range m.nets { - eng := NewMeshEngine(m.Suite.T(), m.nets[i], count-1, engine.TestNetwork) + for i := range suite.nets { + eng := NewMeshEngine(suite.Suite.T(), suite.nets[i], count-1, engine.TestNetwork) engs = append(engs, eng) } @@ -253,7 +242,7 @@ func (m *MeshEngineTestSuite) targetValidatorScenario(send ConduitSendWrapperFun time.Sleep(5 * time.Second) // choose half of the nodes as target - allIds := m.ids.NodeIDs() + allIds := suite.ids.NodeIDs() var targets []flow.Identifier // create a target list of half of the nodes for i := 0; i < len(allIds)/2; i++ { @@ -264,7 +253,7 @@ func (m *MeshEngineTestSuite) targetValidatorScenario(send ConduitSendWrapperFun event := &message.TestMessage{ Text: "hello from node 0", } - require.NoError(m.Suite.T(), send(event, engs[len(engs)-1].con, targets...)) + require.NoError(suite.Suite.T(), send(event, engs[len(engs)-1].con, targets...)) // fires a goroutine for all engines to listens for the incoming message for i := 0; i < len(allIds)/2; i++ { @@ -275,14 +264,14 @@ func (m *MeshEngineTestSuite) targetValidatorScenario(send ConduitSendWrapperFun }(engs[i]) } - unittest.AssertReturnsBefore(m.T(), wg.Wait, 10*time.Second) + unittest.AssertReturnsBefore(suite.T(), wg.Wait, 10*time.Second) // evaluates that all messages are received for index, e := range engs { if index < len(engs)/2 { - assert.Len(m.Suite.T(), e.event, 1, fmt.Sprintf("message not received %v", index)) + assert.Len(suite.Suite.T(), e.event, 1, fmt.Sprintf("message not received %v", index)) } else { - assert.Len(m.Suite.T(), e.event, 0, fmt.Sprintf("message received when none was expected %v", index)) + assert.Len(suite.Suite.T(), e.event, 0, fmt.Sprintf("message received when none was expected %v", index)) } } } @@ -290,14 +279,14 @@ func (m *MeshEngineTestSuite) targetValidatorScenario(send ConduitSendWrapperFun // messageSizeScenario provides a scenario to check if a message of maximum permissible size can be sent //successfully. // It broadcasts a message from the first node to all the nodes in the identifiers list using send wrapper function. -func (m *MeshEngineTestSuite) messageSizeScenario(send ConduitSendWrapperFunc, size uint) { +func (suite *MeshEngineTestSuite) messageSizeScenario(send ConduitSendWrapperFunc, size uint) { // creating engines - count := len(m.nets) + count := len(suite.nets) engs := make([]*MeshEngine, 0) wg := sync.WaitGroup{} - for i := range m.nets { - eng := NewMeshEngine(m.Suite.T(), m.nets[i], count-1, engine.TestNetwork) + for i := range suite.nets { + eng := NewMeshEngine(suite.Suite.T(), suite.nets[i], count-1, engine.TestNetwork) engs = append(engs, eng) } @@ -305,15 +294,15 @@ func (m *MeshEngineTestSuite) messageSizeScenario(send ConduitSendWrapperFunc, s time.Sleep(2 * time.Second) // others keeps the identifier of all nodes except node that is sender. - others := m.ids.Filter(filter.Not(filter.HasNodeID(m.ids[0].NodeID))).NodeIDs() + others := suite.ids.Filter(filter.Not(filter.HasNodeID(suite.ids[0].NodeID))).NodeIDs() // generates and sends an event of custom size to the network - payload := libp2p.NetworkPayloadFixture(m.T(), size) + payload := libp2p.NetworkPayloadFixture(suite.T(), size) event := &message.TestMessage{ Text: string(payload), } - require.NoError(m.T(), send(event, engs[0].con, others...)) + require.NoError(suite.T(), send(event, engs[0].con, others...)) // fires a goroutine for all engines (except sender) to listen for the incoming message for _, eng := range engs[1:] { @@ -324,26 +313,26 @@ func (m *MeshEngineTestSuite) messageSizeScenario(send ConduitSendWrapperFunc, s }(eng) } - unittest.AssertReturnsBefore(m.Suite.T(), wg.Wait, 30*time.Second) + unittest.AssertReturnsBefore(suite.Suite.T(), wg.Wait, 30*time.Second) // evaluates that all messages are received for index, e := range engs[1:] { - assert.Len(m.Suite.T(), e.event, 1, "message not received by engine %d", index+1) + assert.Len(suite.Suite.T(), e.event, 1, "message not received by engine %d", index+1) } } // conduitCloseScenario tests after a Conduit is closed, an engine cannot send or receive a message for that channel ID -func (m *MeshEngineTestSuite) conduitCloseScenario(send ConduitSendWrapperFunc) { +func (suite *MeshEngineTestSuite) conduitCloseScenario(send ConduitSendWrapperFunc) { optionalSleep(send) // creating engines - count := len(m.nets) + count := len(suite.nets) engs := make([]*MeshEngine, 0) wg := sync.WaitGroup{} - for i := range m.nets { - eng := NewMeshEngine(m.Suite.T(), m.nets[i], count-1, engine.TestNetwork) + for i := range suite.nets { + eng := NewMeshEngine(suite.Suite.T(), suite.nets[i], count-1, engine.TestNetwork) engs = append(engs, eng) } @@ -353,28 +342,28 @@ func (m *MeshEngineTestSuite) conduitCloseScenario(send ConduitSendWrapperFunc) // unregister a random engine from the test topic by calling close on it's conduit unregisterIndex := rand.Intn(count) err := engs[unregisterIndex].con.Close() - assert.NoError(m.T(), err) + assert.NoError(suite.T(), err) // each node attempts to broadcast a message to all others - for i := range m.nets { + for i := range suite.nets { event := &message.TestMessage{ Text: fmt.Sprintf("hello from node %v", i), } // others keeps the identifier of all nodes except ith node - others := m.ids.Filter(filter.Not(filter.HasNodeID(m.ids[i].NodeID))).NodeIDs() + others := suite.ids.Filter(filter.Not(filter.HasNodeID(suite.ids[i].NodeID))).NodeIDs() if i == unregisterIndex { // assert that unsubscribed engine cannot publish on that topic - require.Error(m.Suite.T(), send(event, engs[i].con, others...)) + require.Error(suite.Suite.T(), send(event, engs[i].con, others...)) continue } - require.NoError(m.Suite.T(), send(event, engs[i].con, others...)) + require.NoError(suite.Suite.T(), send(event, engs[i].con, others...)) } // fire a goroutine to listen for incoming messages for each engine except for the one which unregistered - for i := range m.nets { + for i := range suite.nets { if i == unregisterIndex { continue } @@ -389,11 +378,11 @@ func (m *MeshEngineTestSuite) conduitCloseScenario(send ConduitSendWrapperFunc) } // assert every one except the unsubscribed engine received the message - unittest.AssertReturnsBefore(m.Suite.T(), wg.Wait, 2*time.Second) + unittest.AssertReturnsBefore(suite.Suite.T(), wg.Wait, 2*time.Second) // assert that the unregistered engine did not receive the message unregisteredEng := engs[unregisterIndex] - assert.Emptyf(m.T(), unregisteredEng.received, "unregistered engine received the topic message") + assert.Emptyf(suite.T(), unregisteredEng.received, "unregistered engine received the topic message") } // extractSenderID returns a bool array with the index i true if there is a message from node i in the provided messages. diff --git a/network/gossip/libp2p/test/middleware_test.go b/network/gossip/libp2p/test/middleware_test.go index 7f9dc35ce6b..a51d74cd951 100644 --- a/network/gossip/libp2p/test/middleware_test.go +++ b/network/gossip/libp2p/test/middleware_test.go @@ -45,13 +45,10 @@ func (m *MiddlewareTestSuite) SetupTest() { golog.SetAllLoggers(golog.LevelError) m.size = 2 // operates on two middlewares - m.metrics = metrics.NewNoopCollector() - // create and start the middlewares - var err error - m.ids, m.mws = generateIDsAndMiddlewares(m.T(), m.size, logger) - require.NoError(m.T(), err) + m.ids, m.mws = GenerateIDsAndMiddlewares(m.T(), m.size, !DryRun, logger) + require.Len(m.Suite.T(), m.ids, m.size) require.Len(m.Suite.T(), m.mws, m.size) diff --git a/network/gossip/libp2p/test/testUtil.go b/network/gossip/libp2p/test/testUtil.go index 3abc911a623..99b944dfcb4 100644 --- a/network/gossip/libp2p/test/testUtil.go +++ b/network/gossip/libp2p/test/testUtil.go @@ -15,16 +15,22 @@ import ( "github.com/onflow/flow-go/engine" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/flow/filter" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/lifecycle" "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/module/mock" "github.com/onflow/flow-go/network/codec/json" "github.com/onflow/flow-go/network/gossip/libp2p" + "github.com/onflow/flow-go/network/gossip/libp2p/channel" "github.com/onflow/flow-go/network/gossip/libp2p/topology" + "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/utils/unittest" ) var rootBlockID string +const DryRun = true + var allocator *portAllocator // init is a built-in golang function getting called first time this @@ -34,38 +40,39 @@ func init() { rootBlockID = unittest.IdentifierFixture().String() } -// generateIDs generate flow Identities with a valid port and networking key -func generateIDs(t *testing.T, n int) (flow.IdentityList, []crypto.PrivateKey) { - identities := make([]*flow.Identity, n) +// GenerateIDs generate flow Identities with a valid port and networking key +func GenerateIDs(t *testing.T, n int, dryRunMode bool, opts ...func(*flow.Identity)) (flow.IdentityList, + []crypto.PrivateKey) { privateKeys := make([]crypto.PrivateKey, n) - freePorts := allocator.getFreePorts(t, n) + var freePorts []int - for i := 0; i < n; i++ { + if !dryRunMode { + // get free ports + freePorts = allocator.getFreePorts(t, n) + } - identifier := unittest.IdentifierFixture() + identities := unittest.IdentityListFixture(n, opts...) + // generates keys and address for the node + for i, id := range identities { // generate key - key, err := GenerateNetworkingKey(identifier) + key, err := GenerateNetworkingKey(id.NodeID) require.NoError(t, err) privateKeys[i] = key + port := 0 - port := freePorts[i] - - opt := []func(id *flow.Identity){ - func(id *flow.Identity) { - id.NodeID = identifier - id.Address = fmt.Sprintf("0.0.0.0:%d", port) - id.NetworkPubKey = key.PublicKey() - }, + if !dryRunMode { + port = freePorts[i] } - identities[i] = unittest.IdentityFixture(opt...) + identities[i].Address = fmt.Sprintf("0.0.0.0:%d", port) + identities[i].NetworkPubKey = key.PublicKey() } return identities, privateKeys } -// generateMiddlewares creates and initializes middleware instances for all the identities -func generateMiddlewares(t *testing.T, log zerolog.Logger, identities flow.IdentityList, keys []crypto.PrivateKey) []*libp2p.Middleware { +// GenerateMiddlewares creates and initializes middleware instances for all the identities +func GenerateMiddlewares(t *testing.T, log zerolog.Logger, identities flow.IdentityList, keys []crypto.PrivateKey) []*libp2p.Middleware { metrics := metrics.NewNoopCollector() mws := make([]*libp2p.Middleware, len(identities)) for i, id := range identities { @@ -85,20 +92,32 @@ func generateMiddlewares(t *testing.T, log zerolog.Logger, identities flow.Ident return mws } -// generateNetworks generates the network for the given middlewares -func generateNetworks(t *testing.T, log zerolog.Logger, ids flow.IdentityList, mws []*libp2p.Middleware, csize int, tops []topology.Topology, dryrun bool) []*libp2p.Network { +// GenerateNetworks generates the network for the given middlewares +func GenerateNetworks(t *testing.T, + log zerolog.Logger, + ids flow.IdentityList, + mws []*libp2p.Middleware, + csize int, + tops []topology.Topology, + sms []channel.SubscriptionManager, + dryRunMode bool) []*libp2p.Network { count := len(ids) nets := make([]*libp2p.Network, 0) metrics := metrics.NewNoopCollector() - // if no topology is passed in, use the default topology for all networks + // checks if necessary to generate topology managers if tops == nil { - tops = make([]topology.Topology, count) - for i, id := range ids { - rpt, err := topology.NewRandPermTopology(id.Role, id.NodeID) - require.NoError(t, err) - tops[i] = rpt - } + // nil topology managers means generating default ones + + // creates default topology + // + // mocks state for collector nodes topology + // considers only a single cluster as higher cluster numbers are tested + // in collectionTopology_test + state, _ := topology.CreateMockStateForCollectionNodes(t, + ids.Filter(filter.HasRole(flow.RoleCollection)), 1) + // creates topology instances for the nodes based on their roles + tops = GenerateTopologies(t, state, ids, sms, log) } for i := 0; i < count; i++ { @@ -110,14 +129,14 @@ func generateNetworks(t *testing.T, log zerolog.Logger, ids flow.IdentityList, m me.On("Address").Return(ids[i].Address) // create the network - net, err := libp2p.NewNetwork(log, json.NewCodec(), ids, me, mws[i], csize, tops[i], metrics) + net, err := libp2p.NewNetwork(log, json.NewCodec(), ids, me, mws[i], csize, tops[i], sms[i], metrics) require.NoError(t, err) nets = append(nets, net) } // if dryrun then don't actually start the network - if !dryrun { + if !dryRunMode { for _, net := range nets { <-net.Ready() } @@ -125,20 +144,31 @@ func generateNetworks(t *testing.T, log zerolog.Logger, ids flow.IdentityList, m return nets } -func generateIDsAndMiddlewares(t *testing.T, n int, log zerolog.Logger) (flow.IdentityList, []*libp2p.Middleware) { - ids, keys := generateIDs(t, n) - mws := generateMiddlewares(t, log, ids, keys) +func GenerateIDsAndMiddlewares(t *testing.T, + n int, + dryRunMode bool, + log zerolog.Logger) (flow.IdentityList, + []*libp2p.Middleware) { + + ids, keys := GenerateIDs(t, n, dryRunMode) + mws := GenerateMiddlewares(t, log, ids, keys) return ids, mws } -func generateIDsMiddlewaresNetworks(t *testing.T, n int, log zerolog.Logger, csize int, tops []topology.Topology, dryrun bool) (flow.IdentityList, []*libp2p.Middleware, []*libp2p.Network) { - ids, mws := generateIDsAndMiddlewares(t, n, log) - networks := generateNetworks(t, log, ids, mws, csize, tops, dryrun) +func GenerateIDsMiddlewaresNetworks(t *testing.T, + n int, + log zerolog.Logger, + csize int, + tops []topology.Topology, + dryRun bool) (flow.IdentityList, []*libp2p.Middleware, []*libp2p.Network) { + ids, mws := GenerateIDsAndMiddlewares(t, n, dryRun, log) + sms := GenerateSubscriptionManagers(t, mws) + networks := GenerateNetworks(t, log, ids, mws, csize, tops, sms, dryRun) return ids, mws, networks } -// generateEngines generates MeshEngines for the given networks -func generateEngines(t *testing.T, nets []*libp2p.Network) []*MeshEngine { +// GenerateEngines generates MeshEngines for the given networks +func GenerateEngines(t *testing.T, nets []*libp2p.Network) []*MeshEngine { count := len(nets) engs := make([]*MeshEngine, count) for i, n := range nets { @@ -162,3 +192,44 @@ func GenerateNetworkingKey(s flow.Identifier) (crypto.PrivateKey, error) { copy(seed, s[:]) return crypto.GeneratePrivateKey(crypto.ECDSASecp256k1, seed) } + +// CreateTopologies is a test helper on receiving an identity list, creates a topology per identity +// and returns the slice of topologies. +func GenerateTopologies(t *testing.T, state protocol.State, identities flow.IdentityList, + subMngrs []channel.SubscriptionManager, logger zerolog.Logger) []topology.Topology { + tops := make([]topology.Topology, 0) + for i, id := range identities { + var top topology.Topology + var err error + + top, err = topology.NewTopicBasedTopology(id.NodeID, logger, state, subMngrs[i]) + require.NoError(t, err) + + tops = append(tops, top) + } + return tops +} + +// GenerateSubscriptionManagers creates and returns a ChannelSubscriptionManager for each middleware object. +func GenerateSubscriptionManagers(t *testing.T, mws []*libp2p.Middleware) []channel.SubscriptionManager { + require.NotEmpty(t, mws) + + sms := make([]channel.SubscriptionManager, len(mws)) + for i, mw := range mws { + sms[i] = libp2p.NewChannelSubscriptionManager(mw) + } + return sms +} + +// stopNetworks stops network instances in parallel and fails the test if they could not be stopped within the +// duration. +func stopNetworks(t *testing.T, nets []*libp2p.Network, duration time.Duration) { + // casts nets instances into ReadyDoneAware components + comps := make([]module.ReadyDoneAware, 0, len(nets)) + for _, net := range nets { + comps = append(comps, net) + } + + unittest.RequireCloseBefore(t, lifecycle.AllDone(comps...), duration, + "could not stop the networks") +} diff --git a/network/gossip/libp2p/test/topology_test.go b/network/gossip/libp2p/test/topology_test.go deleted file mode 100644 index 02f4608d05c..00000000000 --- a/network/gossip/libp2p/test/topology_test.go +++ /dev/null @@ -1,135 +0,0 @@ -package test - -import ( - "math" - "os" - "sort" - "testing" - - golog "github.com/ipfs/go-log" - "github.com/rs/zerolog" - "github.com/rs/zerolog/log" - "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" - - "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/model/flow/filter" - "github.com/onflow/flow-go/network/gossip/libp2p/topology" - "github.com/onflow/flow-go/utils/unittest" -) - -// TopologyTestSuite tests the bare minimum requirements of a randomized -// topology that is needed for our network. It should not replace the information -// theory assumptions behind the schemes, e.g., random oracle model of hashes -type TopologyTestSuite struct { - suite.Suite -} - -// TestNetworkTestSuit starts all the tests in this test suite -func TestNetworkTestSuit(t *testing.T) { - suite.Run(t, new(TopologyTestSuite)) -} - -func (n *TopologyTestSuite) TestTopologySize() { - totalNodes := 100 - logger := zerolog.New(os.Stderr).Level(zerolog.ErrorLevel) - golog.SetAllLoggers(golog.LevelError) - - // create totalNodes number of networks - _, _, nets := generateIDsMiddlewaresNetworks(n.T(), totalNodes, logger, 100, nil, true) - - // determine the expected size of the id list that should be returned by RandPermTopology - rndSubsetSize := int(math.Ceil(float64(totalNodes+1) / 2)) - oneOfEachNodetype := 0 // there is only one node type in this test - remaining := totalNodes - rndSubsetSize - oneOfEachNodetype - halfOfRemainingNodes := int(math.Ceil(float64(remaining+1) / 2)) - expectedSize := rndSubsetSize + oneOfEachNodetype + halfOfRemainingNodes - - top, err := nets[0].Topology() - require.NoError(n.T(), err) - // assert id list returned is of expected size - require.Len(n.T(), top, expectedSize) -} - -// TestMembership evaluates every id in topology to be a protocol id -func (n *TopologyTestSuite) TestMembership() { - logger := log.Output(zerolog.ConsoleWriter{Out: os.Stderr}).With().Caller().Logger() - ids, _, nets := generateIDsMiddlewaresNetworks(n.T(), 100, logger, 100, nil, true) - - // get the subset from the first network - subset, err := nets[0].Topology() - require.NoError(n.T(), err) - - // every id in topology should be an id of the protocol - require.Empty(n.T(), subset.Filter(filter.Not(filter.In(ids)))) -} - -// TestDeteministicity verifies that the same seed generates the same topology -func (n *TopologyTestSuite) TestDeteministicity() { - top, err := topology.NewRandPermTopology(flow.RoleCollection, unittest.IdentifierFixture()) - require.NoError(n.T(), err) - - totalIDs := 100 - ids := unittest.IdentityListFixture(totalIDs) - - // topology of size count/2 - topSize := uint(totalIDs / 2) - var previous, current []string - - // call the topology.Subset function 100 times and assert that output is always the same - for i := 0; i < 100; i++ { - previous = current - current = nil - // generate a new topology with the same ids, size and seed - idMap, err := top.Subset(ids, topSize) - require.NoError(n.T(), err) - - for _, v := range idMap { - current = append(current, v.NodeID.String()) - } - // no guarantees about order is made by Topology.Subset(), hence sort the return values before comparision - sort.Strings(current) - - if previous == nil { - continue - } - - // assert that a different seed generates a different topology - require.Equal(n.T(), previous, current) - } -} - -// TestUniqueness verifies that different seeds generates different topologies -func (n *TopologyTestSuite) TestUniqueness() { - - totalIDs := 100 - ids := unittest.IdentityListFixture(totalIDs) - - // topology of size count/2 - topSize := uint(totalIDs / 2) - var previous, current []string - - // call the topology.Subset function 100 times and assert that output is always different - for i := 0; i < 100; i++ { - previous = current - current = nil - // generate a new topology with a the same ids, size but a different seed for each iteration - identity, _ := ids.ByIndex(uint(i)) - top, err := topology.NewRandPermTopology(flow.RoleCollection, identity.NodeID) - require.NoError(n.T(), err) - idMap, err := top.Subset(ids, topSize) - require.NoError(n.T(), err) - - for _, v := range idMap { - current = append(current, v.NodeID.String()) - } - sort.Strings(current) - - if previous == nil { - continue - } - - // assert that a different seed generates a different topology - require.NotEqual(n.T(), previous, current) - } -} diff --git a/network/gossip/libp2p/topology/collectionTopology.go b/network/gossip/libp2p/topology/collectionTopology.go deleted file mode 100644 index 3ce5bb0468c..00000000000 --- a/network/gossip/libp2p/topology/collectionTopology.go +++ /dev/null @@ -1,83 +0,0 @@ -package topology - -import ( - "fmt" - - "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/model/flow/filter" - "github.com/onflow/flow-go/state/protocol" -) - -// CollectionTopology builds on top of RandPermTopology and generates a deterministic random topology for collection node -// such that nodes withing the same collection cluster form a connected graph. -type CollectionTopology struct { - RandPermTopology - nodeID flow.Identifier - state protocol.ReadOnlyState - seed int64 -} - -func NewCollectionTopology(nodeID flow.Identifier, state protocol.ReadOnlyState) (CollectionTopology, error) { - rpt, err := NewRandPermTopology(flow.RoleCollection, nodeID) - if err != nil { - return CollectionTopology{}, err - } - return CollectionTopology{ - RandPermTopology: rpt, - nodeID: nodeID, - state: state, - }, nil -} - -// Subset samples the idList and returns a list of nodes to connect with such that: -// a. this node is directly or indirectly connected to all other nodes in the same cluster -// b. to all other nodes -// c. to at least one node of each type and -///d. to all other nodes of the same type. -// The collection nodes within a collection cluster need to form a connected graph among themselves independent of any -// other nodes to ensure reliable dissemination of cluster specific topic messages. e.g ClusterBlockProposal -// Similarly, all nodes of network need to form a connected graph, to ensure reliable dissemination of messages for -// topics subscribed by all node types e.g. BlockProposals -// Each node should be connected to at least one node of each type to ensure nodes don't form an island of a specific -// role, specially since some node types are represented by a very small number of nodes (e.g. few access nodes compared -//to tens or hundreds of collection nodes) -// Finally, all nodes of the same type should form a connected graph for exchanging messages for role specific topics -// e.g. Transaction -func (c CollectionTopology) Subset(idList flow.IdentityList, fanout uint) (flow.IdentityList, error) { - - randPermSample, err := c.RandPermTopology.Subset(idList, fanout) - if err != nil { - return nil, err - } - - clusterPeers, err := c.clusterPeers() - if err != nil { - return nil, fmt.Errorf("failed to find cluster peers for node %s", c.nodeID.String()) - } - clusterSample, _ := connectedGraphSample(clusterPeers, c.seed) - - // include only those cluster peers which have not already been chosen by RandPermTopology - uniqueClusterSample := clusterSample.Filter(filter.Not(filter.In(randPermSample))) - - // add those to the earlier sample from randPerm - randPermSample = append(randPermSample, uniqueClusterSample...) - - // return the aggregated set - return randPermSample, nil -} - -// clusterPeers returns the list of other nodes within the same cluster as this node -func (c CollectionTopology) clusterPeers() (flow.IdentityList, error) { - currentEpoch := c.state.Final().Epochs().Current() - clusterList, err := currentEpoch.Clustering() - if err != nil { - return nil, err - } - - myCluster, _, found := clusterList.ByNodeID(c.nodeID) - if !found { - return nil, fmt.Errorf("failed to find the cluster for node ID %s", c.nodeID.String()) - } - - return myCluster, nil -} diff --git a/network/gossip/libp2p/topology/collectionTopology_test.go b/network/gossip/libp2p/topology/collectionTopology_test.go deleted file mode 100644 index bf96d23d706..00000000000 --- a/network/gossip/libp2p/topology/collectionTopology_test.go +++ /dev/null @@ -1,74 +0,0 @@ -package topology_test - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" - - "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/model/flow/filter" - "github.com/onflow/flow-go/network/gossip/libp2p/topology" - protocol "github.com/onflow/flow-go/state/protocol/mock" - "github.com/onflow/flow-go/utils/unittest" -) - -type CollectionTopologyTestSuite struct { - suite.Suite - state *protocol.State - snapshot *protocol.Snapshot - epochQuery *protocol.EpochQuery - clusterList flow.ClusterList - ids flow.IdentityList - collectors flow.IdentityList -} - -func TestCollectionTopologyTestSuite(t *testing.T) { - suite.Run(t, new(CollectionTopologyTestSuite)) -} - -func (suite *CollectionTopologyTestSuite) SetupTest() { - suite.state = new(protocol.State) - suite.snapshot = new(protocol.Snapshot) - suite.epochQuery = new(protocol.EpochQuery) - nClusters := 3 - nCollectors := 7 - suite.collectors = unittest.IdentityListFixture(nCollectors, unittest.WithRole(flow.RoleCollection)) - suite.ids = append(unittest.IdentityListFixture(1000, unittest.WithAllRolesExcept(flow.RoleCollection)), suite.collectors...) - assignments := unittest.ClusterAssignment(uint(nClusters), suite.collectors) - clusters, err := flow.NewClusterList(assignments, suite.collectors) - require.NoError(suite.T(), err) - suite.clusterList = clusters - epoch := new(protocol.Epoch) - epoch.On("Clustering").Return(clusters, nil).Times(nCollectors) - suite.epochQuery.On("Current").Return(epoch).Times(nCollectors) - suite.snapshot.On("Epochs").Return(suite.epochQuery).Times(nCollectors) - suite.state.On("Final").Return(suite.snapshot, nil).Times(nCollectors) -} - -// TestSubset tests that the collection nodes using CollectionTopology form a connected graph and nodes within the same -// collection clusters also form a connected graph -func (suite *CollectionTopologyTestSuite) TestSubset() { - var adjencyMap = make(map[flow.Identifier]flow.IdentityList, len(suite.collectors)) - // for each of the collector node, find a subset of nodes it should connect to using the CollectionTopology - for _, c := range suite.collectors { - collectionTopology, err := topology.NewCollectionTopology(c.NodeID, suite.state) - assert.NoError(suite.T(), err) - subset, err := collectionTopology.Subset(suite.ids, uint(len(suite.ids))) - assert.NoError(suite.T(), err) - adjencyMap[c.NodeID] = subset - } - - // check that all collection nodes are either directly connected or indirectly connected via other collection nodes - checkConnectednessByRole(suite.T(), adjencyMap, suite.collectors, flow.RoleCollection) - - // check that each of the collection clusters forms a connected graph - for _, cluster := range suite.clusterList { - checkConnectednessByCluster(suite.T(), adjencyMap, cluster, suite.collectors) - } -} - -func checkConnectednessByCluster(t *testing.T, adjMap map[flow.Identifier]flow.IdentityList, cluster flow.IdentityList, ids flow.IdentityList) { - checkGraphConnected(t, adjMap, ids, filter.In(cluster)) -} diff --git a/network/gossip/libp2p/topology/fanout.go b/network/gossip/libp2p/topology/fanout.go new file mode 100644 index 00000000000..09d709c5a1a --- /dev/null +++ b/network/gossip/libp2p/topology/fanout.go @@ -0,0 +1,16 @@ +package topology + +import ( + "math" +) + +// FanoutFunc represents a function type that receiving total number of nodes +// in flow system, returns fanout of individual nodes. +type FanoutFunc func(size int) int + +// LinearFanoutFunc guarantees full network connectivity in a deterministic way. +// Given system of `size` nodes, it returns `size+1/2`. +func LinearFanout(size int) int { + fanout := math.Ceil(float64(size+1) / 2) + return int(fanout) +} diff --git a/network/gossip/libp2p/topology/helper.go b/network/gossip/libp2p/topology/helper.go new file mode 100644 index 00000000000..02e679fac26 --- /dev/null +++ b/network/gossip/libp2p/topology/helper.go @@ -0,0 +1,139 @@ +package topology + +import ( + "fmt" + "math" + "testing" + + "github.com/stretchr/testify/assert" + testifymock "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/engine" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/model/flow/filter" + "github.com/onflow/flow-go/network/gossip/libp2p/channel" + "github.com/onflow/flow-go/network/gossip/libp2p/mock" + "github.com/onflow/flow-go/state/protocol" + protocolmock "github.com/onflow/flow-go/state/protocol/mock" + "github.com/onflow/flow-go/utils/unittest" +) + +// CreateMockStateForCollectionNodes is a test helper function that generate a mock state +// clustering collection nodes into `clusterNum` clusters. +func CreateMockStateForCollectionNodes(t *testing.T, collectorIds flow.IdentityList, + clusterNum uint) (protocol.State, flow.ClusterList) { + state := new(protocolmock.State) + snapshot := new(protocolmock.Snapshot) + epochQuery := new(protocolmock.EpochQuery) + epoch := new(protocolmock.Epoch) + assignments := unittest.ClusterAssignment(clusterNum, collectorIds) + clusters, err := flow.NewClusterList(assignments, collectorIds) + require.NoError(t, err) + + epoch.On("Clustering").Return(clusters, nil) + epochQuery.On("Current").Return(epoch) + snapshot.On("Epochs").Return(epochQuery) + state.On("Final").Return(snapshot, nil) + + return state, clusters +} + +// CheckConnectedness verifies graph as a whole is connected. +func CheckConnectedness(t *testing.T, adjMap map[flow.Identifier]flow.IdentityList, ids flow.IdentityList) { + CheckGraphConnected(t, adjMap, ids, filter.Any) +} + +// CheckConnectednessByChannelID verifies that the subgraph of nodes subscribed to a channelID is connected. +func CheckConnectednessByChannelID(t *testing.T, adjMap map[flow.Identifier]flow.IdentityList, ids flow.IdentityList, + channelID string) { + roles, ok := engine.RolesByChannelID(channelID) + require.True(t, ok) + CheckGraphConnected(t, adjMap, ids, filter.HasRole(roles...)) +} + +// CheckGraphConnected checks if the graph represented by the adjacency matrix is connected. +// It traverses the adjacency map starting from an arbitrary node and checks if all nodes that satisfy the filter +// were visited. +func CheckGraphConnected(t *testing.T, adjMap map[flow.Identifier]flow.IdentityList, ids flow.IdentityList, f flow.IdentityFilter) { + + // filter the ids and find the expected node count + expectedIDs := ids.Filter(f) + expectedCount := len(expectedIDs) + + // start with an arbitrary node which satisfies the filter + startID := expectedIDs.Sample(1)[0].NodeID + + visited := make(map[flow.Identifier]bool) + dfs(startID, adjMap, visited, f) + + // assert that expected number of nodes were visited by DFS + assert.Equal(t, expectedCount, len(visited)) +} + +// MockSubscriptionManager returns a list of mocked subscription manages for the input +// identities. It only mocks the GetChannelIDs method of the subscription manager. Other methods +// return an error, as they are not supposed to be invoked. +func MockSubscriptionManager(t *testing.T, ids flow.IdentityList) []channel.SubscriptionManager { + require.NotEmpty(t, ids) + + sms := make([]channel.SubscriptionManager, len(ids)) + for i, id := range ids { + sm := &mock.SubscriptionManager{} + err := fmt.Errorf("this method should not be called on mock subscription manager") + sm.On("Register", testifymock.Anything, testifymock.Anything).Return(err) + sm.On("Unregister", testifymock.Anything).Return(err) + sm.On("GetEngine", testifymock.Anything).Return(err) + sm.On("GetChannelIDs").Return(engine.ChannelIDsByRole(id.Role)) + sms[i] = sm + } + + return sms +} + +// CheckMembership checks each identity in a top list belongs to all identity list. +func CheckMembership(t *testing.T, top flow.IdentityList, all flow.IdentityList) { + for _, id := range top { + require.Contains(t, all, id) + } +} + +// TODO: fix this test after we have fanout optimized. +// CheckTopologySize evaluates that overall topology size of a node is bound by the fanout of system. +func CheckTopologySize(t *testing.T, total int, top flow.IdentityList) { + t.Skip("this test requires optimizing the fanout per topic") + fanout := (total + 1) / 2 + require.True(t, len(top) <= fanout) +} + +// ClusterNum is a test helper determines the number of clusters of specific `size`. +func ClusterNum(t *testing.T, ids flow.IdentityList, size int) int { + collectors := ids.Filter(filter.HasRole(flow.RoleCollection)) + + // we need at least two collector nodes to generate a cluster + // and check the connectedness + require.True(t, len(collectors) >= 2) + require.True(t, size > 0) + + clusterNum := len(collectors) / size + return int(math.Max(float64(clusterNum), 1)) +} + +// DFS is a test helper function checking graph connectedness. It fails if +// graph represented by `adjMap` is not connected, i.e., there is more than a single +// connected component. +func dfs(currentID flow.Identifier, + adjMap map[flow.Identifier]flow.IdentityList, + visited map[flow.Identifier]bool, + filter flow.IdentityFilter) { + + if visited[currentID] { + return + } + + visited[currentID] = true + + for _, id := range adjMap[currentID].Filter(filter) { + dfs(id.NodeID, adjMap, visited, filter) + } +} diff --git a/network/gossip/libp2p/topology/randPermTopology.go b/network/gossip/libp2p/topology/randPermTopology.go deleted file mode 100644 index f2675570587..00000000000 --- a/network/gossip/libp2p/topology/randPermTopology.go +++ /dev/null @@ -1,67 +0,0 @@ -package topology - -import ( - "fmt" - - "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/model/flow/filter" -) - -var _ Topology = &RandPermTopology{} - -// RandPermTopology generates a random topology from a given set of nodes and for a given role -// The topology generated is a union of three sets: -// 1. a random subset of the size (n+1)/2 to make sure the nodes form a connected graph with no islands -// 2. one node of each of the flow role from the remaining ids to make a node can talk to any other type of node -// 3. (n+1)/2 of the nodes of the same role as this node from the remaining ids to make sure that nodes of the same type -// form a connected graph with no islands. -type RandPermTopology struct { - myRole flow.Role - seed int64 -} - -func NewRandPermTopology(role flow.Role, id flow.Identifier) (RandPermTopology, error) { - seed, err := seedFromID(id) - if err != nil { - return RandPermTopology{}, fmt.Errorf("failed to seed topology: %w", err) - } - return RandPermTopology{ - myRole: role, - seed: seed, - }, nil -} - -func (r RandPermTopology) Subset(idList flow.IdentityList, fanout uint) (flow.IdentityList, error) { - - if uint(len(idList)) < fanout { - return nil, fmt.Errorf("cannot sample topology idList %d smaller than desired fanout %d", len(idList), fanout) - } - - // connect to (n+1)/2 other nodes to ensure graph is connected (no islands) - result, remainder := connectedGraphSample(idList, r.seed) - - // find one id for each role from the remaining list, if it hasn't already been chosen - for _, role := range flow.Roles() { - - if len(result.Filter(filter.HasRole(role))) > 0 { - // we already have a node with this role - continue - } - - var selectedIDs flow.IdentityList - - // connect to one of each type (ignore remainder) - selectedIDs, _ = oneOfEachRoleSample(remainder, r.seed, role) - - // add it to result - result = append(result, selectedIDs...) - } - - // connect to (k+1)/2 other nodes of the same type to ensure all nodes of the same type are fully connected, - // where k is the number of nodes of each type - selfRoleIDs, _ := connectedGraphByRoleSample(remainder, r.seed, r.myRole) // ignore the remaining ids - - result = append(result, selfRoleIDs...) - - return result, nil -} diff --git a/network/gossip/libp2p/topology/randPermTopology_test.go b/network/gossip/libp2p/topology/randPermTopology_test.go deleted file mode 100644 index 64952b15f61..00000000000 --- a/network/gossip/libp2p/topology/randPermTopology_test.go +++ /dev/null @@ -1,186 +0,0 @@ -package topology_test - -import ( - "math" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/suite" - - "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/model/flow/filter" - "github.com/onflow/flow-go/network/gossip/libp2p/topology" - "github.com/onflow/flow-go/utils/unittest" -) - -type RandPermTopologyTestSuite struct { - suite.Suite -} - -func TestRandPermTopologyTestSuite(t *testing.T) { - suite.Run(t, new(RandPermTopologyTestSuite)) -} - -// TestNodesConnected tests overall node connectedness and connectedness by role by keeping nodes of one role type in -// minority (~2%) -func (r *RandPermTopologyTestSuite) TestNodesConnected() { - - // test vectors for different network sizes - testVector := []struct { - total int - minorityRole flow.Role - nodeRole flow.Role - }{ - // integration tests - order of 10s - { - total: 12, - minorityRole: flow.RoleCollection, - nodeRole: flow.RoleCollection, - }, - { - total: 12, - minorityRole: flow.RoleCollection, - nodeRole: flow.RoleConsensus, - }, - // alpha main net order of 100s - { - total: 100, - minorityRole: flow.RoleCollection, - nodeRole: flow.RoleCollection, - }, - { - total: 100, - minorityRole: flow.RoleCollection, - nodeRole: flow.RoleConsensus, - }, - // mature flow order of 1000s - { - total: 1000, - minorityRole: flow.RoleCollection, - nodeRole: flow.RoleCollection, - }, - { - total: 1000, - minorityRole: flow.RoleCollection, - nodeRole: flow.RoleConsensus, - }, - } - - for _, v := range testVector { - r.testTopology(v.total, v.minorityRole, v.nodeRole) - } -} - -func (r *RandPermTopologyTestSuite) testTopology(total int, minorityRole flow.Role, nodeRole flow.Role) { - - distribution := createDistribution(total, minorityRole) - - ids := make(flow.IdentityList, 0) - for role, count := range distribution { - roleIDs := unittest.IdentityListFixture(count, unittest.WithRole(role)) - ids = append(ids, roleIDs...) - } - - n := len(ids) - adjencyMap := make(map[flow.Identifier]flow.IdentityList, n) - - for _, id := range ids { - rpt, err := topology.NewRandPermTopology(id.Role, id.NodeID) - r.NoError(err) - top, err := rpt.Subset(ids, uint(n)) - r.NoError(err) - adjencyMap[id.NodeID] = top - } - - // check that nodes of the same role form a connected graph - checkConnectednessByRole(r.T(), adjencyMap, ids, minorityRole) - - // check that nodes form a connected graph - checkConnectedness(r.T(), adjencyMap, ids) -} - -// TestSubsetDeterminism tests that if the id list remains the same, the Topology.Subset call always yields the same -// list of nodes -func (r *RandPermTopologyTestSuite) TestSubsetDeterminism() { - ids := unittest.IdentityListFixture(100, unittest.WithAllRoles()) - for _, id := range ids { - rpt, err := topology.NewRandPermTopology(flow.RoleConsensus, id.NodeID) - r.NoError(err) - var prev flow.IdentityList - for i := 0; i < 10; i++ { - current, err := rpt.Subset(ids, uint(100)) - r.NoError(err) - if prev != nil { - assert.EqualValues(r.T(), prev, current) - } - } - } -} - -// createDistribution creates a count distribution of ~total number of nodes with 2% minority node count -func createDistribution(total int, minority flow.Role) map[flow.Role]int { - - minorityPercentage := 0.02 - count := func(per float64) int { - nodes := int(math.Ceil(per * float64(total))) // assume atleast one node of the minority role - return nodes - } - minorityCount, majorityCount := count(minorityPercentage), count(1-minorityPercentage) - roles := flow.Roles() - totalRoles := len(roles) - 1 - majorityCountPerRole := int(math.Ceil(float64(majorityCount) / float64(totalRoles))) - - countMap := make(map[flow.Role]int, totalRoles) // map of role to the number of nodes for that role - for _, r := range roles { - if r == minority { - countMap[r] = minorityCount - } else { - countMap[r] = majorityCountPerRole - } - } - return countMap -} - -func checkConnectednessByRole(t *testing.T, adjMap map[flow.Identifier]flow.IdentityList, ids flow.IdentityList, role flow.Role) { - checkGraphConnected(t, adjMap, ids, filter.HasRole(role)) -} - -func checkConnectedness(t *testing.T, adjMap map[flow.Identifier]flow.IdentityList, ids flow.IdentityList) { - checkGraphConnected(t, adjMap, ids, filter.Any) -} - -// checkGraphConnected checks if the graph represented by the adjacency matrix is connected. -// It traverses the adjacency map starting from an arbitrary node and checks if all nodes that satisfy the filter -// were visited. -func checkGraphConnected(t *testing.T, adjMap map[flow.Identifier]flow.IdentityList, ids flow.IdentityList, f flow.IdentityFilter) { - - // filter the ids and find the expected node count - expectedIDs := ids.Filter(f) - expectedCount := len(expectedIDs) - - // start with an arbitrary node which satisfies the filter - startID := expectedIDs.Sample(1)[0].NodeID - - visited := make(map[flow.Identifier]bool) - dfs(startID, adjMap, visited, f) - - // assert that expected number of nodes were visited by DFS - assert.Equal(t, expectedCount, len(visited)) -} - -// dfs to check graph connectedness -func dfs(currentID flow.Identifier, - adjMap map[flow.Identifier]flow.IdentityList, - visited map[flow.Identifier]bool, - filter flow.IdentityFilter) { - - if visited[currentID] { - return - } - - visited[currentID] = true - - for _, id := range adjMap[currentID].Filter(filter) { - dfs(id.NodeID, adjMap, visited, filter) - } -} diff --git a/network/gossip/libp2p/topology/topicBasedTopology.go b/network/gossip/libp2p/topology/topicBasedTopology.go new file mode 100644 index 00000000000..d222445256a --- /dev/null +++ b/network/gossip/libp2p/topology/topicBasedTopology.go @@ -0,0 +1,229 @@ +package topology + +import ( + "fmt" + + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/engine" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/model/flow/filter" + "github.com/onflow/flow-go/network/gossip/libp2p/channel" + "github.com/onflow/flow-go/state/protocol" +) + +// TopicBasedTopology is a deterministic topology mapping that creates a connected graph component among the nodes +// involved in each topic. +type TopicBasedTopology struct { + me flow.Identifier // used to keep identifier of the node + state protocol.ReadOnlyState // used to keep a read only protocol state + subMngr channel.SubscriptionManager // used to keep track topics the node subscribed to + logger zerolog.Logger + seed int64 +} + +// NewTopicBasedTopology returns an instance of the TopicBasedTopology. +func NewTopicBasedTopology(nodeID flow.Identifier, + logger zerolog.Logger, + state protocol.ReadOnlyState, + subMngr channel.SubscriptionManager) (*TopicBasedTopology, error) { + seed, err := seedFromID(nodeID) + if err != nil { + return nil, fmt.Errorf("could not generate seed from id:%w", err) + } + + t := &TopicBasedTopology{ + me: nodeID, + state: state, + seed: seed, + subMngr: subMngr, + logger: logger.With().Str("component:", "topic-based-topology").Logger(), + } + + return t, nil +} + +// GenerateFanout receives IdentityList of entire network and constructs the fanout IdentityList +// of this instance. A node directly communicates with its fanout IdentityList on epidemic dissemination +// of the messages (i.e., publish and multicast). +// Independent invocations of GenerateFanout on different nodes collaboratively must construct a cohesive +// connected graph of nodes that enables them talking to each other. +func (t TopicBasedTopology) GenerateFanout(ids flow.IdentityList) (flow.IdentityList, error) { + myChannelIDs := t.subMngr.GetChannelIDs() + if len(myChannelIDs) == 0 { + // no subscribed channel id, hence skip topology creation + // we do not return an error at this state as invocation of MakeTopology may happen before + // node subscribing to all its channels. + t.logger.Warn().Msg("skips generating fanout with no subscribed channels") + return flow.IdentityList{}, nil + } + + // finds all interacting roles with this node + myInteractingRoles := flow.RoleList{} + for _, myChannel := range myChannelIDs { + roles, ok := engine.RolesByChannelID(myChannel) + if !ok { + return nil, fmt.Errorf("could not extract roles for channel: %s", myChannel) + } + myInteractingRoles = myInteractingRoles.Union(roles) + } + + // builds a connected component per role this node interact with, + var myFanout flow.IdentityList + for _, role := range myInteractingRoles { + if role == flow.RoleCollection { + // we do not build connected component for collection nodes based on their role + // rather we build it based on their cluster identity in the next step. + continue + } + roleFanout, err := t.subsetRole(ids, nil, flow.RoleList{role}) + if err != nil { + return nil, fmt.Errorf("failed to derive list of peer nodes to connect for role %s: %w", role, err) + } + myFanout = myFanout.Union(roleFanout) + } + + // stitches the role-based components that subscribed to the same channel id together. + for _, myChannel := range myChannelIDs { + shouldHave := myFanout.Copy() + + topicFanout, err := t.subsetChannel(ids, shouldHave, myChannel) + if err != nil { + return nil, fmt.Errorf("could not generate fanout for topic %s: %w", myChannel, err) + } + myFanout = myFanout.Union(topicFanout) + } + + if len(myFanout) == 0 { + return nil, fmt.Errorf("topology size reached zero") + } + t.logger.Debug(). + Int("fanout", len(myFanout)). + Msg("fanout successfully generated") + return myFanout, nil +} + +// subsetChannel returns a random subset of the identity list that is passed. `shouldHave` represents set of +// identities that should be included in the returned subset. +// Returned identities should all subscribed to the specified `channel`. +// Note: this method should not include identity of its executor. +func (t *TopicBasedTopology) subsetChannel(ids flow.IdentityList, shouldHave flow.IdentityList, + channel string) (flow.IdentityList, error) { + if _, ok := engine.IsClusterChannelID(channel); ok { + return t.clusterChannelHandler(ids, shouldHave) + } + return t.nonClusterChannelHandler(ids, shouldHave, channel) +} + +// subsetRole returns a random subset of the identity list that is passed. `shouldHave` represents set of +// identities that should be included in the returned subset. +// Returned identities should all be of one of the specified `roles`. +// Note: this method should not include identity of its executor. +func (t TopicBasedTopology) subsetRole(ids flow.IdentityList, shouldHave flow.IdentityList, roles flow.RoleList) (flow.IdentityList, error) { + // excludes irrelevant roles and the node itself from both should have and ids set + shouldHave = shouldHave.Filter(filter.And( + filter.HasRole(roles...), + filter.Not(filter.HasNodeID(t.me)), + )) + + ids = ids.Filter(filter.And( + filter.HasRole(roles...), + filter.Not(filter.HasNodeID(t.me)), + )) + + sample, err := t.sampleConnectedGraph(ids, shouldHave) + if err != nil { + return nil, fmt.Errorf("could not sample a connected graph: %w", err) + } + + return sample, nil +} + +// sampleConnectedGraph receives two lists: all and shouldHave. It then samples a connected fanout +// for the caller that includes the shouldHave set. Independent invocations of this method over +// different nodes, should create a connected graph. +// Fanout is the set of nodes that this instance should get connected to in order to create a +// connected graph. +func (t TopicBasedTopology) sampleConnectedGraph(all flow.IdentityList, shouldHave flow.IdentityList) (flow.IdentityList, error) { + + if len(all) == 0 { + t.logger.Debug().Msg("skips sampling connected graph with zero nodes") + return flow.IdentityList{}, nil + } + + if len(shouldHave) == 0 { + // choose (n+1)/2 random nodes so that each node in the graph will have a degree >= (n+1) / 2, + // guaranteeing a connected graph. + size := uint(LinearFanout(len(all))) + return all.DeterministicSample(size, t.seed), nil + + } + // checks `shouldHave` be a subset of `all` + nonMembers := shouldHave.Filter(filter.Not(filter.In(all))) + if len(nonMembers) != 0 { + return nil, fmt.Errorf("should have identities is not a subset of all: %v", nonMembers) + } + + // total sample size + totalSize := LinearFanout(len(all)) + + if totalSize < len(shouldHave) { + // total fanout size needed is already satisfied by shouldHave set. + return shouldHave, nil + } + + // subset size excluding should have ones + subsetSize := totalSize - len(shouldHave) + + // others are all excluding should have ones + others := all.Filter(filter.Not(filter.In(shouldHave))) + others = others.DeterministicSample(uint(subsetSize), t.seed) + + return others.Union(shouldHave), nil + +} + +// clusterPeers returns the list of other nodes within the same cluster as this node. +func (t TopicBasedTopology) clusterPeers() (flow.IdentityList, error) { + currentEpoch := t.state.Final().Epochs().Current() + clusterList, err := currentEpoch.Clustering() + if err != nil { + return nil, fmt.Errorf("failed to extract cluster list %w", err) + } + + myCluster, _, found := clusterList.ByNodeID(t.me) + if !found { + return nil, fmt.Errorf("failed to find the cluster for node ID %s", t.me.String()) + } + + return myCluster, nil +} + +// clusterChannelHandler returns a connected graph fanout of peers in the same cluster as executor of this instance. +func (t TopicBasedTopology) clusterChannelHandler(ids, shouldHave flow.IdentityList) (flow.IdentityList, error) { + // extracts cluster peer ids to which the node belongs to. + clusterPeers, err := t.clusterPeers() + if err != nil { + return nil, fmt.Errorf("failed to find cluster peers for node %s: %w", t.me.String(), err) + } + + // samples a connected graph topology from the cluster peers + return t.subsetRole(clusterPeers, shouldHave.Filter(filter.HasNodeID(clusterPeers.NodeIDs()...)), flow.RoleList{flow.RoleCollection}) +} + +// nonClusterChannelHandler returns a connected graph fanout of peers from `ids` that subscribed to `channel`. +// The returned sample contains `shouldHave` ones that also subscribed to `channel`. +func (t TopicBasedTopology) nonClusterChannelHandler(ids, shouldHave flow.IdentityList, channel string) (flow.IdentityList, error) { + if _, ok := engine.IsClusterChannelID(channel); ok { + return nil, fmt.Errorf("could not handle cluster channel: %s", channel) + } + + // extracts flow roles subscribed to topic. + roles, ok := engine.RolesByChannelID(channel) + if !ok { + return nil, fmt.Errorf("unknown topic with no subscribed roles: %s", channel) + } + + // samples a connected graph topology + return t.subsetRole(ids.Filter(filter.HasRole(roles...)), shouldHave, roles) +} diff --git a/network/gossip/libp2p/topology/topicBasedTopology_test.go b/network/gossip/libp2p/topology/topicBasedTopology_test.go new file mode 100644 index 00000000000..dbe81c4f2fb --- /dev/null +++ b/network/gossip/libp2p/topology/topicBasedTopology_test.go @@ -0,0 +1,465 @@ +package topology + +import ( + "os" + "sort" + "testing" + + "github.com/rs/zerolog" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/onflow/flow-go/engine" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/model/flow/filter" + "github.com/onflow/flow-go/network/gossip/libp2p/channel" + protocol2 "github.com/onflow/flow-go/state/protocol" + "github.com/onflow/flow-go/utils/unittest" +) + +// TopicAwareTopologyTestSuite tests the bare minimum requirements of a +// topology that is needed for our network. It should not replace the information +// theory assumptions behind the schemes, e.g., random oracle model of hashes +type TopicAwareTopologyTestSuite struct { + suite.Suite + state protocol2.State // represents a mocked protocol state + all flow.IdentityList // represents the identity list of all nodes in the system + clusters flow.ClusterList // represents list of cluster ids of collection nodes + subMngr []channel.SubscriptionManager + logger zerolog.Logger + fanout uint // represents maximum number of connections this peer allows to have +} + +// TestTopicAwareTopologyTestSuite starts all the tests in this test suite +func TestTopicAwareTopologyTestSuite(t *testing.T) { + suite.Run(t, new(TopicAwareTopologyTestSuite)) +} + +// SetupTest initiates the test setups prior to each test +func (suite *TopicAwareTopologyTestSuite) SetupTest() { + // we consider fanout as maximum number of connections the node allows to have + // TODO: optimize value of fanout. + suite.fanout = 100 + + nClusters := 3 + nCollectors := 100 + nTotal := 1000 + + suite.logger = zerolog.New(os.Stderr).Level(zerolog.DebugLevel) + + collectors := unittest.IdentityListFixture(nCollectors, unittest.WithRole(flow.RoleCollection)) + others := unittest.IdentityListFixture(nTotal, unittest.WithAllRolesExcept(flow.RoleCollection)) + suite.all = append(others, collectors...) + + // mocks state for collector nodes topology + suite.state, suite.clusters = CreateMockStateForCollectionNodes(suite.T(), + suite.all.Filter(filter.HasRole(flow.RoleCollection)), uint(nClusters)) + + suite.subMngr = MockSubscriptionManager(suite.T(), suite.all) +} + +// TestTopologySize_Topic verifies that size of each topology fanout per topic is greater than +// `(k+1)/2` where `k` is number of nodes subscribed to a topic. It does that over 100 random iterations. +func (suite *TopicAwareTopologyTestSuite) TestTopologySize_Topic() { + for i := 0; i < 100; i++ { + top, err := NewTopicBasedTopology(suite.all[0].NodeID, suite.logger, suite.state, suite.subMngr[0]) + require.NoError(suite.T(), err) + + topics := engine.ChannelIDsByRole(suite.all[0].Role) + require.Greater(suite.T(), len(topics), 1) + + for _, topic := range topics { + // extracts total number of nodes subscribed to topic + roles, ok := engine.RolesByChannelID(topic) + require.True(suite.T(), ok) + + ids, err := top.subsetChannel(suite.all, nil, topic) + require.NoError(suite.T(), err) + + // counts total number of nodes that has the roles and are not `suite.me` (node of interest). + total := len(suite.all.Filter(filter.And(filter.HasRole(roles...), + filter.Not(filter.HasNodeID(suite.all[0].NodeID))))) + require.True(suite.T(), float64(len(ids)) >= (float64)(total+1)/2) + } + } +} + +// TestDeteministicity is a weak test that verifies the same seed generates the same topology for a topic. +// +// It also checks the topology against non-inclusion of the node itself in its own topology. +func (suite *TopicAwareTopologyTestSuite) TestDeteministicity() { + // creates a topology using the graph sampler + top, err := NewTopicBasedTopology(suite.all[0].NodeID, suite.logger, suite.state, suite.subMngr[0]) + require.NoError(suite.T(), err) + + topics := engine.ChannelIDsByRole(suite.all[0].Role) + require.Greater(suite.T(), len(topics), 1) + + // for each topic samples 100 topologies + // all topologies for a topic should be the same + for _, topic := range topics { + var previous, current []string + for i := 0; i < 100; i++ { + previous = current + current = nil + + // generate a new topology with a the same all, size and seed + ids, err := top.subsetChannel(suite.all, nil, topic) + require.NoError(suite.T(), err) + + // topology should not contain the node itself + require.Empty(suite.T(), ids.Filter(filter.HasNodeID(suite.all[0].NodeID))) + + for _, v := range ids { + current = append(current, v.NodeID.String()) + } + // no guarantees about order is made by Topology.subsetChannel(), hence sort the return values before comparision + sort.Strings(current) + + if previous == nil { + continue + } + + // assert that a different seed generates a different topology + require.Equal(suite.T(), previous, current) + } + } +} + +// TestUniqueness generates a topology for the first topic of consensus nodes. +// Since topologies are seeded with the node ids, it evaluates that every two consecutive +// topologies of the same topic for distinct nodes are distinct. +// +// It also checks the topology against non-inclusion of the node itself in its own topology. +// +// Note: currently we are using a linear fanout for guaranteed delivery, hence there are +// C(n, (n+1)/2) many unique topologies for the same topic across different nodes. Even for small numbers +// like n = 300, the potential outcomes are large enough (i.e., 10e88) so that the uniqueness is guaranteed. +// This test however, performs a very weak uniqueness test by checking the uniqueness among consecutive topologies. +func (suite *TopicAwareTopologyTestSuite) TestUniqueness() { + var previous, current []string + + // for each topic samples 100 topologies + // all topologies for a topic should be the same + topics := engine.ChannelIDsByRole(flow.RoleConsensus) + require.Greater(suite.T(), len(topics), 1) + + for i, identity := range suite.all { + // extracts all topics node (i) subscribed to + if identity.Role != flow.RoleConsensus { + continue + } + + previous = current + current = nil + + // creates and samples a new topic aware topology for the first topic of consensus nodes + top, err := NewTopicBasedTopology(identity.NodeID, suite.logger, suite.state, suite.subMngr[i]) + require.NoError(suite.T(), err) + ids, err := top.subsetChannel(suite.all, nil, topics[0]) + require.NoError(suite.T(), err) + + // topology should not contain the node itself + require.Empty(suite.T(), ids.Filter(filter.HasNodeID(identity.NodeID))) + + for _, v := range ids { + current = append(current, v.NodeID.String()) + } + sort.Strings(current) + + if previous == nil { + continue + } + + // assert that a different seed generates a different topology + require.NotEqual(suite.T(), previous, current) + } +} + +// TestConnectedness_NonClusterTopics checks whether graph components corresponding to a +// non-cluster channel ID are individually connected. +func (suite *TopicAwareTopologyTestSuite) TestConnectedness_NonClusterChannelID() { + channelID := engine.TestNetwork + // adjacency map keeps graph component of a single channel ID + channelIDAdjMap := make(map[flow.Identifier]flow.IdentityList) + + for i, id := range suite.all { + // creates a topic-based topology for node + top, err := NewTopicBasedTopology(id.NodeID, suite.logger, suite.state, suite.subMngr[i]) + require.NoError(suite.T(), err) + + // samples subset of topology + subset, err := top.subsetChannel(suite.all, nil, channelID) + require.NoError(suite.T(), err) + + channelIDAdjMap[id.NodeID] = subset + } + + CheckConnectednessByChannelID(suite.T(), channelIDAdjMap, suite.all, channelID) +} + +// TestConnectedness_NonClusterChannelID checks whether graph components corresponding to a +// cluster channel ID are individually connected. +func (suite *TopicAwareTopologyTestSuite) TestConnectedness_ClusterChannelID() { + // picks one cluster channel ID as sample + channelID := clusterChannelIDs(suite.T())[0] + + // adjacency map keeps graph component of a single channel ID + channelIDAdjMap := make(map[flow.Identifier]flow.IdentityList) + + // iterates over collection nodes + for i, id := range suite.all.Filter(filter.HasRole(flow.RoleCollection)) { + // creates a channelID-based topology for node + top, err := NewTopicBasedTopology(id.NodeID, suite.logger, suite.state, suite.subMngr[i]) + require.NoError(suite.T(), err) + + // samples subset of topology + subset, err := top.subsetChannel(suite.all, nil, channelID) + require.NoError(suite.T(), err) + + channelIDAdjMap[id.NodeID] = subset + } + + // check that each of the collection clusters forms a connected graph + for _, cluster := range suite.clusters { + suite.checkConnectednessByCluster(suite.T(), channelIDAdjMap, cluster) + } +} + +// TestLinearFanout_UnconditionalSampling evaluates that sampling a connected graph fanout +// with an empty `shouldHave` list follows the LinearFanoutFunc, +// and it also does not contain duplicate element. +func (suite *TopicAwareTopologyTestSuite) TestLinearFanout_UnconditionalSampling() { + // samples with no `shouldHave` set. + top, err := NewTopicBasedTopology(suite.all[0].NodeID, suite.logger, suite.state, suite.subMngr[0]) + require.NoError(suite.T(), err) + + sample, err := top.sampleConnectedGraph(suite.all, nil) + require.NoError(suite.T(), err) + + // the LinearFanoutGraphSampler utilizes the LinearFanoutFunc. Hence any sample it makes should have + // the size equal to applying LinearFanoutFunc over the original set. + expectedFanout := LinearFanout(len(suite.all)) + require.Equal(suite.T(), len(sample), expectedFanout) + + // checks sample does not include any duplicate + suite.uniquenessCheck(sample) +} + +// TestLinearFanout_ConditionalSampling evaluates that sampling a connected graph fanout with a shouldHave set +// follows the LinearFanoutFunc, and it also does not contain duplicate element. +func (suite *TopicAwareTopologyTestSuite) TestLinearFanout_ConditionalSampling() { + // samples 10 all into `shouldHave` set. + shouldHave := suite.all.Sample(10) + + // creates a topology for the node + top, err := NewTopicBasedTopology(suite.all[0].NodeID, suite.logger, suite.state, suite.subMngr[0]) + require.NoError(suite.T(), err) + + // samples a connected graph of `all` that includes `shouldHave` set. + sample, err := top.sampleConnectedGraph(suite.all, shouldHave) + require.NoError(suite.T(), err) + + // the LinearFanoutGraphSampler utilizes the LinearFanoutFunc. Hence any sample it makes should have + // the size equal to applying LinearFanoutFunc over the original set. + expectedFanout := LinearFanout(len(suite.all)) + require.Equal(suite.T(), len(sample), expectedFanout) + + // checks sample does not include any duplicate + suite.uniquenessCheck(sample) + + // checks inclusion of all shouldHave ones into sample + for _, id := range shouldHave { + require.Contains(suite.T(), sample, id) + } +} + +// TestLinearFanout_SmallerAll evaluates that sampling a connected graph fanout with a shouldHave set +// that is greater than required fanout, returns the `shouldHave` set instead. +func (suite *TopicAwareTopologyTestSuite) TestLinearFanout_SmallerAll() { + // samples 10 all into 'shouldHave'. + shouldHave := suite.all.Sample(10) + // samples a smaller component of all with 5 nodes and combines with `shouldHave` + smallerAll := suite.all.Filter(filter.Not(filter.In(shouldHave))).Sample(5).Union(shouldHave) + + // creates a topology for the node + top, err := NewTopicBasedTopology(suite.all[0].NodeID, suite.logger, suite.state, suite.subMngr[0]) + require.NoError(suite.T(), err) + + // total size of smallerAll is 15, and it requires a linear fanout of 8 which is less than + // size of `shouldHave` set, so the shouldHave itself should return + sample, err := top.sampleConnectedGraph(smallerAll, shouldHave) + require.NoError(suite.T(), err) + require.Equal(suite.T(), len(sample), len(shouldHave)) + require.ElementsMatch(suite.T(), sample, shouldHave) +} + +// TestLinearFanout_SubsetViolence evaluates that trying to sample a connected graph when `shouldHave` +// is not a subset of `all` returns an error. +func (suite *TopicAwareTopologyTestSuite) TestLinearFanout_SubsetViolation() { + // samples 10 all into 'shouldHave', + shouldHave := suite.all.Sample(10) + // excludes one of the `shouldHave` all from all, hence it is no longer a subset + excludedAll := suite.all.Filter(filter.Not(filter.HasNodeID(shouldHave[0].NodeID))) + + // creates a topology for the node + top, err := NewTopicBasedTopology(suite.all[0].NodeID, suite.logger, suite.state, suite.subMngr[0]) + require.NoError(suite.T(), err) + + // since `shouldHave` is not a subset of `excludedAll` it should return an error + _, err = top.sampleConnectedGraph(excludedAll, shouldHave) + require.Error(suite.T(), err) +} + +// TestLinearFanout_EmptyAllSet evaluates that trying to sample a connected graph when `all` +// is empty does not return an error. +func (suite *TopicAwareTopologyTestSuite) TestLinearFanout_EmptyAllSet() { + // samples 10 all into 'shouldHave'. + shouldHave := suite.all.Sample(10) + + // creates a topology for the node + top, err := NewTopicBasedTopology(suite.all[0].NodeID, suite.logger, suite.state, suite.subMngr[0]) + require.NoError(suite.T(), err) + + // sampling with empty `all` and non-empty `shouldHave` + _, err = top.sampleConnectedGraph(flow.IdentityList{}, shouldHave) + require.NoError(suite.T(), err) + + // sampling with empty all and nil `shouldHave` + _, err = top.sampleConnectedGraph(flow.IdentityList{}, nil) + require.NoError(suite.T(), err) + + // sampling with nil all and nil `shouldHave` + _, err = top.sampleConnectedGraph(nil, nil) + require.NoError(suite.T(), err) +} + +// TestConnectedness_Unconditionally evaluates that samples returned by the sampleConnectedGraph with +// empty `shouldHave` constitute a connected graph. +func (suite *TopicAwareTopologyTestSuite) TestConnectedness_Unconditionally() { + adjMap := make(map[flow.Identifier]flow.IdentityList) + for i, id := range suite.all { + // creates a topology for the node + top, err := NewTopicBasedTopology(id.NodeID, suite.logger, suite.state, suite.subMngr[i]) + require.NoError(suite.T(), err) + + // samples a graph and stores it in adjacency map + sample, err := top.sampleConnectedGraph(suite.all, nil) + require.NoError(suite.T(), err) + adjMap[id.NodeID] = sample + } + + CheckGraphConnected(suite.T(), adjMap, suite.all, filter.In(suite.all)) +} + +// TestConnectedness_Conditionally evaluates that samples returned by the sampleConnectedGraph with +// some `shouldHave` constitute a connected graph. +func (suite *TopicAwareTopologyTestSuite) TestConnectedness_Conditionally() { + adjMap := make(map[flow.Identifier]flow.IdentityList) + for i, id := range suite.all { + // creates a topology for the node + top, err := NewTopicBasedTopology(id.NodeID, suite.logger, suite.state, suite.subMngr[i]) + require.NoError(suite.T(), err) + + // samples a graph and stores it in adjacency map + // sampling is done with a non-empty `shouldHave` subset of 10 randomly chosen all + shouldHave := suite.all.Sample(10) + sample, err := top.sampleConnectedGraph(suite.all, shouldHave) + require.NoError(suite.T(), err) + + // evaluates inclusion of should haves in sample + for _, shouldHaveID := range shouldHave { + require.Contains(suite.T(), sample, shouldHaveID) + } + adjMap[id.NodeID] = sample + } + + CheckGraphConnected(suite.T(), adjMap, suite.all, filter.In(suite.all)) +} + +// TestSubsetRoleConnectedness_Conditionally evaluates that subset returned by subsetRole with a non-empty `shouldHave` set +// is a connected graph among specified roles. +func (suite *TopicAwareTopologyTestSuite) TestSubsetRoleConnectedness_Conditionally() { + adjMap := make(map[flow.Identifier]flow.IdentityList) + for i, id := range suite.all { + // creates a topology for the node + top, err := NewTopicBasedTopology(id.NodeID, suite.logger, suite.state, suite.subMngr[i]) + require.NoError(suite.T(), err) + + // samples a graph among consensus nodes and stores it in adjacency map + // sampling is done with a non-empty `shouldHave` subset of 10 randomly + // chosen all excluding the node itself. + shouldHave := suite.all.Filter(filter.Not(filter.HasNodeID(id.NodeID))).Sample(10) + sample, err := top.subsetRole(suite.all, shouldHave, flow.RoleList{flow.RoleConsensus}) + require.NoError(suite.T(), err) + + // evaluates inclusion of should haves consensus nodes in sample + for _, shouldHaveID := range shouldHave { + if shouldHaveID.Role != flow.RoleConsensus { + continue + } + require.Contains(suite.T(), sample, shouldHaveID) + } + adjMap[id.NodeID] = sample + } + + // evaluates connectedness of consensus nodes graph. + CheckGraphConnected(suite.T(), adjMap, suite.all, filter.HasRole(flow.RoleConsensus)) +} + +// TestSubsetRoleConnectedness_Unconditionally evaluates that subset returned by subsetRole with an `shouldHave` set +// is a connected graph among specified roles. +func (suite *TopicAwareTopologyTestSuite) TestSubsetRoleConnectedness_Unconditionally() { + adjMap := make(map[flow.Identifier]flow.IdentityList) + for i, id := range suite.all { + // creates a topology for the node + top, err := NewTopicBasedTopology(id.NodeID, suite.logger, suite.state, suite.subMngr[i]) + require.NoError(suite.T(), err) + + // samples a graph among consensus nodes and stores it in adjacency map + // sampling is done with an non-empty set. + sample, err := top.subsetRole(suite.all, nil, flow.RoleList{flow.RoleConsensus}) + require.NoError(suite.T(), err) + adjMap[id.NodeID] = sample + } + + // evaluates connectedness of consensus nodes graph. + CheckGraphConnected(suite.T(), adjMap, suite.all, filter.HasRole(flow.RoleConsensus)) +} + +// uniquenessCheck is a test helper method that fails the test if all include any duplicate identity. +func (suite *TopicAwareTopologyTestSuite) uniquenessCheck(ids flow.IdentityList) { + seen := make(map[flow.Identifier]struct{}) + for _, id := range ids { + // checks if id is duplicate in ids list + _, ok := seen[id.NodeID] + require.False(suite.T(), ok) + + // marks id as seen + seen[id.NodeID] = struct{}{} + } +} + +// clusterChannelIDs is a test helper method that returns all cluster-based channel ids. +func clusterChannelIDs(t *testing.T) []string { + ccids := make([]string, 0) + for _, channelID := range engine.ChannelIDs() { + if _, ok := engine.IsClusterChannelID(channelID); !ok { + continue + } + ccids = append(ccids, channelID) + } + + require.NotEmpty(t, ccids) + return ccids +} + +// checkConnectednessByCluster is a test helper that checks all nodes belong to a cluster are connected. +func (suite *TopicAwareTopologyTestSuite) checkConnectednessByCluster(t *testing.T, + adjMap map[flow.Identifier]flow.IdentityList, + cluster flow.IdentityList) { + CheckGraphConnected(t, + adjMap, + suite.all, + filter.In(cluster)) +} diff --git a/network/gossip/libp2p/topology/topology.go b/network/gossip/libp2p/topology/topology.go index b361cfd95a5..2992496e30b 100644 --- a/network/gossip/libp2p/topology/topology.go +++ b/network/gossip/libp2p/topology/topology.go @@ -4,6 +4,10 @@ import "github.com/onflow/flow-go/model/flow" // Topology provides a subset of nodes which a given node should directly connect to for 1-k messaging type Topology interface { - // Subset returns a random subset of the identity list that is passed - Subset(idList flow.IdentityList, fanout uint) (flow.IdentityList, error) + // GenerateFanout receives IdentityList of entire network and constructs the fanout IdentityList + // of this instance. A node directly communicates with its fanout IdentityList on epidemic dissemination + // of the messages (i.e., publish and multicast). + // Independent invocations of GenerateFanout on different nodes collaboratively must construct a cohesive + // connected graph of nodes that enables them talking to each other. + GenerateFanout(ids flow.IdentityList) (flow.IdentityList, error) } diff --git a/network/gossip/libp2p/topology/topology_test.go b/network/gossip/libp2p/topology/topology_test.go new file mode 100644 index 00000000000..25c2963d2f2 --- /dev/null +++ b/network/gossip/libp2p/topology/topology_test.go @@ -0,0 +1,176 @@ +package topology_test + +import ( + "fmt" + "os" + "testing" + + "github.com/bsipos/thist" + "github.com/rs/zerolog" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/model/flow/filter" + "github.com/onflow/flow-go/network/gossip/libp2p/channel" + "github.com/onflow/flow-go/network/gossip/libp2p/test" + "github.com/onflow/flow-go/network/gossip/libp2p/topology" + "github.com/onflow/flow-go/state/protocol" + "github.com/onflow/flow-go/utils/unittest" +) + +// TopologyTestSuite tests the end-to-end connectedness of topology +type TopologyTestSuite struct { + suite.Suite +} + +// TestTopologyTestSuite runs all tests in this test suite +func TestTopologyTestSuite(t *testing.T) { + suite.Run(t, new(TopologyTestSuite)) +} + +// TestLowScale creates systems with +// 10 access nodes +// 100 collection nodes in 4 clusters +// 120 consensus nodes +// 5 execution nodes +// 100 verification nodes +// and builds a stateful topology for the systems. +// For each system, it then checks the end-to-end connectedness of the topology graph. +func (suite *TopologyTestSuite) TestLowScale() { + suite.multiSystemEndToEndConnectedness(1, 10, 100, 120, 5, 100, 4) +} + +// TestModerateScale creates systems with +// 20 access nodes +// 200 collection nodes in 8 clusters +// 240 consensus nodes +// 10 execution nodes +// 100 verification nodes +// and builds a stateful topology for the systems. +// For each system, it then checks the end-to-end connectedness of the topology graph. +func (suite *TopologyTestSuite) TestModerateScale() { + suite.multiSystemEndToEndConnectedness(1, 20, 200, 240, 10, 200, 8) +} + +// TestHighScale creates systems with +// 40 access nodes +// 400 collection nodes in 16 clusters +// 480 consensus nodes +// 20 execution nodes +// 400 verification nodes +// and builds a stateful topology for the systems. +// For each system, it then checks the end-to-end connectedness of the topology graph. +func (suite *TopologyTestSuite) TestHighScale() { + suite.multiSystemEndToEndConnectedness(1, 40, 400, 480, 20, 400, 16) +} + +// generateSystem is a test helper that given number of nodes per role as well as desire number of clusters +// generates the protocol state, identity list and subscription managers for the nodes. +// - acc: number of access nodes +// - col: number of collection nodes +// - exe: number of execution nodes +// - ver: number of verification nodes +// - cluster: number of clusters of collection nodes +func (suite *TopologyTestSuite) generateSystem(acc, col, con, exe, ver, cluster int) (protocol.State, + flow.IdentityList, + []channel.SubscriptionManager) { + + collector, _ := test.GenerateIDs(suite.T(), col, test.DryRun, unittest.WithRole(flow.RoleCollection)) + access, _ := test.GenerateIDs(suite.T(), acc, test.DryRun, unittest.WithRole(flow.RoleAccess)) + consensus, _ := test.GenerateIDs(suite.T(), con, test.DryRun, unittest.WithRole(flow.RoleConsensus)) + verification, _ := test.GenerateIDs(suite.T(), ver, test.DryRun, unittest.WithRole(flow.RoleVerification)) + execution, _ := test.GenerateIDs(suite.T(), exe, test.DryRun, unittest.WithRole(flow.RoleExecution)) + + ids := flow.IdentityList{} + ids = ids.Union(collector) + ids = ids.Union(access) + ids = ids.Union(consensus) + ids = ids.Union(verification) + ids = ids.Union(execution) + + // mocks state for collector nodes topology + state, _ := topology.CreateMockStateForCollectionNodes(suite.T(), + ids.Filter(filter.HasRole(flow.RoleCollection)), uint(cluster)) + + subMngrs := topology.MockSubscriptionManager(suite.T(), ids) + + return state, ids, subMngrs +} + +// multiSystemEndToEndConnectedness is a test helper evaluates end-to-end connectedness of the system graph +// over several number of systems each with specified number of nodes on each role. +func (suite *TopologyTestSuite) multiSystemEndToEndConnectedness(system, acc, col, con, exe, ver, cluster int) { + // creates a histogram to keep average fanout of nodes in systems + var aveHist *thist.Hist + if suite.trace() { + aveHist = thist.NewHist(nil, fmt.Sprintf("Average fanout for %d systems", system), "fit", 10, false) + } + + for j := 0; j < system; j++ { + // adjacency map keeps graph component of a single channel ID + adjMap := make(map[flow.Identifier]flow.IdentityList) + + // creates a flow system + state, ids, subMngrs := suite.generateSystem(acc, col, con, exe, ver, cluster) + + var systemHist *thist.Hist + if suite.trace() { + // creates a fanout histogram for this system + systemHist = thist.NewHist(nil, fmt.Sprintf("System #%d fanout", j), "auto", -1, false) + } + + totalFanout := 0 // keeps summation of nodes' fanout for statistical reason + + // creates topology of the nodes + for i, id := range ids { + fanout := suite.topologyScenario(id.NodeID, subMngrs[i], ids, state) + adjMap[id.NodeID] = fanout + + if suite.trace() { + systemHist.Update(float64(len(fanout))) + } + totalFanout += len(fanout) + } + + if suite.trace() { + // prints fanout histogram of this system + fmt.Println(systemHist.Draw()) + // keeps track of average fanout per node + aveHist.Update(float64(totalFanout) / float64(len(ids))) + } + + // checks end-to-end connectedness of the topology + topology.CheckConnectedness(suite.T(), adjMap, ids) + } + + if suite.trace() { + fmt.Println(aveHist.Draw()) + } +} + +// topologyScenario is a test helper that creates a StatefulTopologyManager with the LinearFanoutFunc, +// it creates a TopicBasedTopology for the node and returns its fanout. +func (suite *TopologyTestSuite) topologyScenario(me flow.Identifier, + subMngr channel.SubscriptionManager, + ids flow.IdentityList, + state protocol.ReadOnlyState) flow.IdentityList { + + logger := zerolog.New(os.Stderr).Level(zerolog.DebugLevel) + + // creates topology of the node + top, err := topology.NewTopicBasedTopology(me, logger, state, subMngr) + require.NoError(suite.T(), err) + + // generates topology of node + myFanout, err := top.GenerateFanout(ids) + require.NoError(suite.T(), err) + + return myFanout +} + +// trace returns true if local environment variable Trace is found. +func (suite *TopologyTestSuite) trace() bool { + _, found := os.LookupEnv("Trace") + return found +} diff --git a/network/gossip/libp2p/topology/topology_utils.go b/network/gossip/libp2p/topology/topology_utils.go index 510b28aff02..50c3c26019a 100644 --- a/network/gossip/libp2p/topology/topology_utils.go +++ b/network/gossip/libp2p/topology/topology_utils.go @@ -3,67 +3,17 @@ package topology import ( "bytes" "encoding/binary" - "math" + "fmt" "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/model/flow/filter" ) -// connectedGraph returns a random subset of length (n+1)/2. -// If each node connects to the nodes returned by connectedGraph, the graph of such nodes is connected. -func connectedGraph(ids flow.IdentityList, seed int64) flow.IdentityList { - // choose (n+1)/2 random nodes so that each node in the graph will have a degree >= (n+1) / 2, - // guaranteeing a connected graph. - size := uint(math.Ceil(float64(len(ids)+1) / 2)) - return ids.DeterministicSample(size, seed) -} - -// connectedGraph returns a random subset of length (n+1)/2 of the specified role -func connectedGraphByRole(ids flow.IdentityList, seed int64, role flow.Role) flow.IdentityList { - filteredIds := ids.Filter(filter.HasRole(role)) - if len(filteredIds) == 0 { - // there are no more nodes of this role to choose from - return flow.IdentityList{} - } - return connectedGraph(filteredIds, seed) -} - -// oneOfRole returns one random id of the given role -func oneOfRole(ids flow.IdentityList, seed int64, role flow.Role) flow.IdentityList { - filteredIds := ids.Filter(filter.HasRole(role)) - if len(filteredIds) == 0 { - // there are no more nodes of this role to choose from - return flow.IdentityList{} - } - - // choose 1 out of all the remaining nodes of this role - selectedID := filteredIds.DeterministicSample(1, seed) - - return selectedID -} - -func connectedGraphSample(ids flow.IdentityList, seed int64) (flow.IdentityList, flow.IdentityList) { - result := connectedGraph(ids, seed) - remainder := ids.Filter(filter.Not(filter.In(result))) - return result, remainder -} - -func connectedGraphByRoleSample(ids flow.IdentityList, seed int64, role flow.Role) (flow.IdentityList, flow.IdentityList) { - result := connectedGraphByRole(ids, seed, role) - remainder := ids.Filter(filter.Not(filter.In(result))) - return result, remainder -} - -func oneOfEachRoleSample(ids flow.IdentityList, seed int64, role flow.Role) (flow.IdentityList, flow.IdentityList) { - result := oneOfRole(ids, seed, role) - remainder := ids.Filter(filter.Not(filter.In(result))) - return result, remainder -} - // seedFromID generates a int64 seed from a flow.Identifier func seedFromID(id flow.Identifier) (int64, error) { var seed int64 buf := bytes.NewBuffer(id[:]) - err := binary.Read(buf, binary.LittleEndian, &seed) - return seed, err + if err := binary.Read(buf, binary.LittleEndian, &seed); err != nil { + return -1, fmt.Errorf("could not read random bytes: %w", err) + } + return seed, nil } diff --git a/utils/unittest/unittest.go b/utils/unittest/unittest.go index 6f403c361cb..79b148d9da3 100644 --- a/utils/unittest/unittest.go +++ b/utils/unittest/unittest.go @@ -54,17 +54,6 @@ func AssertClosesBefore(t *testing.T, done <-chan struct{}, duration time.Durati } } -// RequireClosesBefore requires that the given channel closes before the -// duration expires. -func RequireClosesBefore(t *testing.T, done <-chan struct{}, duration time.Duration) { - select { - case <-time.After(duration): - require.Fail(t, "channel did not return in time") - case <-done: - return - } -} - // RequireReturnBefore requires that the given function returns before the // duration expires. func RequireReturnsBefore(t testing.TB, f func(), duration time.Duration, message string) { @@ -75,10 +64,16 @@ func RequireReturnsBefore(t testing.TB, f func(), duration time.Duration, messag close(done) }() + RequireCloseBefore(t, done, duration, "could not close done channel on time") +} + +// RequireCloseBefore requires that the given channel returns before the +// duration expires. +func RequireCloseBefore(t testing.TB, c <-chan struct{}, duration time.Duration, message string) { select { case <-time.After(duration): - require.Fail(t, "function did not return on time: "+message) - case <-done: + require.Fail(t, "function did not return in time: "+message) + case <-c: return } }