test(p2p): regression test for kademlia

ChainSafe · Aug 6, 2024 · 8680331 · 8680331
1 parent 55c8beb
commit 8680331
Show file tree

Hide file tree

Showing 10 changed files with 1,255 additions and 119 deletions.
diff --git a/Makefile b/Makefile
@@ -115,7 +115,8 @@ test-release: go-mod
 test-all: test test-release
 
 go-mod:
-	(cd $(PWD)/src/libp2p_bitswap/tests/go-app && go mod vendor && go build -o /tmp/forest-go-compat-test) || \
+	(cd $(PWD)/src/libp2p_bitswap/tests/go-app && go mod vendor && go build -o /tmp/forest-go-bitswap-compat-test) && \
+	(cd $(PWD)/src/libp2p/tests/go-kad && go mod vendor && go build -o /tmp/forest-go-kad-compat-test) || \
 	(echo "Some tests require Go 1.21.x to be installed, follow instructions at https://go.dev/dl/" && exit 1)
 
 # Checks if all headers are present and adds if not

diff --git a/src/libp2p/discovery.rs b/src/libp2p/discovery.rs
@@ -26,7 +26,7 @@ use libp2p::{
     upnp, StreamProtocol,
 };
 use tokio::time::Interval;
-use tracing::{debug, info, trace, warn};
+use tracing::{debug, info, trace};
 
 use crate::utils::version::FOREST_VERSION_STRING;
 
@@ -133,35 +133,17 @@ impl<'a> DiscoveryConfig<'a> {
         } = self;
 
         let mut peers = HashSet::new();
-
         let kademlia_opt = if enable_kademlia {
-            // This is a workaround for <https://github.com/ChainSafe/forest/issues/4576>
-            // `go-libp2p-kad-dht` sends the remote peer id as query key during bootstrap,
-            // however, `rust-libp2p` returns empty peer list when the key it receives
-            // matches the local one. It's fine to use a random peer id as a workaround
-            // since Kademlia not used as value providers in filecoin p2p network.
-            let kad_random_peer_id = PeerId::random();
-            let store = MemoryStore::new(kad_random_peer_id);
-            let kad_config = {
-                let mut cfg = kad::Config::default();
-                cfg.set_protocol_names(vec![StreamProtocol::try_from_owned(format!(
+            let mut kademlia = new_kademlia(
+                local_peer_id,
+                vec![StreamProtocol::try_from_owned(format!(
                     "/fil/kad/{network_name}/kad/1.0.0"
-                ))?]);
-                cfg
-            };
-            let mut kademlia = kad::Behaviour::with_config(kad_random_peer_id, store, kad_config);
-            // `set_mode(Server)` fixes https://github.com/ChainSafe/forest/issues/3620
-            // but it should not be required as the behaviour should automatically switch to server mode
-            // according to the doc. It might be a bug in `libp2p`.
-            // We should fix the bug or report with a minimal reproduction.
-            kademlia.set_mode(Some(kad::Mode::Server));
+                ))?],
+            );
             for (peer_id, addr) in &user_defined {
                 kademlia.add_address(peer_id, addr.clone());
                 peers.insert(*peer_id);
             }
-            if let Err(e) = kademlia.bootstrap() {
-                warn!("Kademlia bootstrap failed: {}", e);
-            }
             Some(kademlia)
         } else {
             None
@@ -198,6 +180,31 @@ impl<'a> DiscoveryConfig<'a> {
     }
 }
 
+pub fn new_kademlia(
+    _peer_id: PeerId,
+    protocols: Vec<StreamProtocol>,
+) -> kad::Behaviour<kad::store::MemoryStore> {
+    // This is a workaround for <https://github.com/ChainSafe/forest/issues/4576>
+    // `go-libp2p-kad-dht` sends the remote peer id as query key during bootstrap,
+    // however, `rust-libp2p` returns empty peer list when the key it receives
+    // matches the local one. It's fine to use a random peer id as a workaround
+    // since Kademlia not used as value providers in filecoin p2p network.
+    let peer_id = PeerId::random();
+    let store = MemoryStore::new(peer_id);
+    let kad_config = {
+        let mut cfg = kad::Config::default();
+        cfg.set_protocol_names(protocols);
+        cfg
+    };
+    let mut kademlia = kad::Behaviour::with_config(peer_id, store, kad_config);
+    // `set_mode(Server)` fixes https://github.com/ChainSafe/forest/issues/3620
+    // but it should not be required as the behaviour should automatically switch to server mode
+    // according to the doc. It might be a bug in `libp2p`.
+    // We should fix the bug or report with a minimal reproduction.
+    kademlia.set_mode(Some(kad::Mode::Server));
+    kademlia
+}
+
 /// Implementation of `NetworkBehaviour` that discovers the nodes on the
 /// network.
 // Behaviours that manage connections should come first, to get rid of some panics in debug build.

diff --git a/src/libp2p/mod.rs b/src/libp2p/mod.rs
@@ -26,4 +26,5 @@ pub use self::{config::*, peer_manager::*, service::*};
 #[cfg(test)]
 mod tests {
     mod decode_test;
+    mod kad_go_compat;
 }
diff --git a/src/libp2p/tests/go-kad/.gitignore b/src/libp2p/tests/go-kad/.gitignore
@@ -0,0 +1,27 @@
+# Created by https://www.toptal.com/developers/gitignore/api/go
+# Edit at https://www.toptal.com/developers/gitignore?templates=go
+
+### Go ###
+# If you prefer the allow list template instead of the deny list, see community template:
+# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
+#
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+
+# Test binary, built with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Dependency directories (remove the comment below to include it)
+vendor/
+
+# Go workspace file
+go.work
+
+# End of https://www.toptal.com/developers/gitignore/api/go
diff --git a/src/libp2p/tests/go-kad/go.mod b/src/libp2p/tests/go-kad/go.mod
@@ -0,0 +1,134 @@
+module test/v2
+
+go 1.21
+
+require (
+	github.com/libp2p/go-libp2p v0.36.0
+	github.com/libp2p/go-libp2p-kad-dht v0.25.2
+	github.com/multiformats/go-multiaddr v0.13.0
+)
+
+require (
+	github.com/benbjohnson/clock v1.3.5 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/containerd/cgroups v1.1.0 // indirect
+	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
+	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
+	github.com/elastic/gosigar v0.14.3 // indirect
+	github.com/flynn/noise v1.1.0 // indirect
+	github.com/francoispqt/gojay v1.2.13 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
+	github.com/godbus/dbus/v5 v5.1.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/google/gopacket v1.1.19 // indirect
+	github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/gorilla/websocket v1.5.3 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/hashicorp/golang-lru v1.0.2 // indirect
+	github.com/huin/goupnp v1.3.0 // indirect
+	github.com/ipfs/boxo v0.21.0 // indirect
+	github.com/ipfs/go-cid v0.4.1 // indirect
+	github.com/ipfs/go-datastore v0.6.0 // indirect
+	github.com/ipfs/go-log v1.0.5 // indirect
+	github.com/ipfs/go-log/v2 v2.5.1 // indirect
+	github.com/ipld/go-ipld-prime v0.21.0 // indirect
+	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
+	github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
+	github.com/jbenet/goprocess v0.1.4 // indirect
+	github.com/klauspost/compress v1.17.9 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.8 // indirect
+	github.com/koron/go-ssdp v0.0.4 // indirect
+	github.com/libp2p/go-buffer-pool v0.1.0 // indirect
+	github.com/libp2p/go-cidranger v1.1.0 // indirect
+	github.com/libp2p/go-flow-metrics v0.1.0 // indirect
+	github.com/libp2p/go-libp2p-asn-util v0.4.1 // indirect
+	github.com/libp2p/go-libp2p-kbucket v0.6.3 // indirect
+	github.com/libp2p/go-libp2p-record v0.2.0 // indirect
+	github.com/libp2p/go-libp2p-routing-helpers v0.7.4 // indirect
+	github.com/libp2p/go-msgio v0.3.0 // indirect
+	github.com/libp2p/go-nat v0.2.0 // indirect
+	github.com/libp2p/go-netroute v0.2.1 // indirect
+	github.com/libp2p/go-reuseport v0.4.0 // indirect
+	github.com/libp2p/go-yamux/v4 v4.0.1 // indirect
+	github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/miekg/dns v1.1.61 // indirect
+	github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect
+	github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect
+	github.com/minio/sha256-simd v1.0.1 // indirect
+	github.com/mr-tron/base58 v1.2.0 // indirect
+	github.com/multiformats/go-base32 v0.1.0 // indirect
+	github.com/multiformats/go-base36 v0.2.0 // indirect
+	github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect
+	github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect
+	github.com/multiformats/go-multibase v0.2.0 // indirect
+	github.com/multiformats/go-multicodec v0.9.0 // indirect
+	github.com/multiformats/go-multihash v0.2.3 // indirect
+	github.com/multiformats/go-multistream v0.5.0 // indirect
+	github.com/multiformats/go-varint v0.0.7 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/onsi/ginkgo/v2 v2.19.1 // indirect
+	github.com/opencontainers/runtime-spec v1.2.0 // indirect
+	github.com/opentracing/opentracing-go v1.2.0 // indirect
+	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
+	github.com/pion/datachannel v1.5.8 // indirect
+	github.com/pion/dtls/v2 v2.2.12 // indirect
+	github.com/pion/ice/v2 v2.3.33 // indirect
+	github.com/pion/interceptor v0.1.29 // indirect
+	github.com/pion/logging v0.2.2 // indirect
+	github.com/pion/mdns v0.0.12 // indirect
+	github.com/pion/randutil v0.1.0 // indirect
+	github.com/pion/rtcp v1.2.14 // indirect
+	github.com/pion/rtp v1.8.8 // indirect
+	github.com/pion/sctp v1.8.20 // indirect
+	github.com/pion/sdp/v3 v3.0.9 // indirect
+	github.com/pion/srtp/v2 v2.0.20 // indirect
+	github.com/pion/stun v0.6.1 // indirect
+	github.com/pion/transport/v2 v2.2.10 // indirect
+	github.com/pion/turn/v2 v2.1.6 // indirect
+	github.com/pion/webrtc/v3 v3.2.50 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/polydawn/refmt v0.89.0 // indirect
+	github.com/prometheus/client_golang v1.19.1 // indirect
+	github.com/prometheus/client_model v0.6.1 // indirect
+	github.com/prometheus/common v0.55.0 // indirect
+	github.com/prometheus/procfs v0.15.1 // indirect
+	github.com/quic-go/qpack v0.4.0 // indirect
+	github.com/quic-go/quic-go v0.45.2 // indirect
+	github.com/quic-go/webtransport-go v0.8.0 // indirect
+	github.com/raulk/go-watchdog v1.3.0 // indirect
+	github.com/spaolacci/murmur3 v1.1.0 // indirect
+	github.com/stretchr/testify v1.9.0 // indirect
+	github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect
+	github.com/wlynxg/anet v0.0.3 // indirect
+	go.opencensus.io v0.24.0 // indirect
+	go.opentelemetry.io/otel v1.28.0 // indirect
+	go.opentelemetry.io/otel/metric v1.28.0 // indirect
+	go.opentelemetry.io/otel/trace v1.28.0 // indirect
+	go.uber.org/dig v1.17.1 // indirect
+	go.uber.org/fx v1.22.1 // indirect
+	go.uber.org/mock v0.4.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	go.uber.org/zap v1.27.0 // indirect
+	golang.org/x/crypto v0.25.0 // indirect
+	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
+	golang.org/x/mod v0.20.0 // indirect
+	golang.org/x/net v0.27.0 // indirect
+	golang.org/x/sync v0.8.0 // indirect
+	golang.org/x/sys v0.23.0 // indirect
+	golang.org/x/text v0.16.0 // indirect
+	golang.org/x/tools v0.23.0 // indirect
+	gonum.org/v1/gonum v0.15.0 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	lukechampine.com/blake3 v1.3.0 // indirect
+)