Skip to content

Commit

Permalink
wip
Browse files Browse the repository at this point in the history
  • Loading branch information
@dennis-tra
dennis-tra committed Oct 16, 2024
1 parent 0b271be commit 0ab7c1b
Show file tree
Hide file tree
Showing 7 changed files with 53 additions and 510 deletions.
47 changes: 0 additions & 47 deletions discv4/README.md
View file

This file was deleted.

88 changes: 48 additions & 40 deletions discv4/crawler_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,23 +5,24 @@ import (
"testing"

mapset "github.com/deckarep/golang-set/v2"
"github.com/libp2p/go-libp2p/core/peer"
"github.com/stretchr/testify/assert"
)

func Test_determineStrategy(t *testing.T) {
tests := []struct {
name string
sets []mapset.Set[string]
sets [][]string
errs []error
want CrawlStrategy
}{
{
// simulates, we received the same response three times (success case)
name: "all same (3)",
sets: []mapset.Set[string]{
mapset.NewThreadUnsafeSet("0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15"),
mapset.NewThreadUnsafeSet("0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15"),
mapset.NewThreadUnsafeSet("0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15"),
sets: [][]string{
{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15"},
{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15"},
{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15"},
},
want: crawlStrategySingleProbe,
},
Expand All @@ -30,9 +31,9 @@ func Test_determineStrategy(t *testing.T) {
// error. This indicates a flaky connection. Just issue one probe
// for each bucket but also retry if failed.
name: "all same with error (2)",
sets: []mapset.Set[string]{
mapset.NewThreadUnsafeSet("0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15"),
mapset.NewThreadUnsafeSet("0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15"),
sets: [][]string{
{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15"},
{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15"},
},
errs: []error{
fmt.Errorf("some error"),
Expand All @@ -42,83 +43,90 @@ func Test_determineStrategy(t *testing.T) {
{
// simulates: remote peer replaced a node in its RT during probing
name: "single diff full responses (3)",
sets: []mapset.Set[string]{
mapset.NewThreadUnsafeSet("A", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15"),
mapset.NewThreadUnsafeSet("B", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15"),
mapset.NewThreadUnsafeSet("B", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15"),
sets: [][]string{
{"A", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15"},
{"B", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15"},
{"B", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15"},
},
want: crawlStrategySingleProbe,
},
{
name: "single diff full responses (2)",
sets: []mapset.Set[string]{
mapset.NewThreadUnsafeSet("A", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15"),
mapset.NewThreadUnsafeSet("B", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15"),
sets: [][]string{
{"A", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15"},
{"B", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15"},
},
want: crawlStrategyMultiProbe,
},
{
name: "partial response, full bucket",
sets: []mapset.Set[string]{
mapset.NewThreadUnsafeSet( /* missing */ "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15"),
mapset.NewThreadUnsafeSet("0", "1", "2" /* missing */, "6", "7", "8", "9", "10", "11", "12", "13", "14", "15"),
mapset.NewThreadUnsafeSet("0", "1", "2", "3", "4", "5" /* missing */, "9", "10", "11", "12", "13", "14", "15"),
sets: [][]string{
{ /* missing */ "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15"},
{"0", "1", "2" /* missing */, "6", "7", "8", "9", "10", "11", "12", "13", "14", "15"},
{"0", "1", "2", "3", "4", "5" /* missing */, "9", "10", "11", "12", "13", "14", "15"},
},
want: crawlStrategyMultiProbe,
},
{
// simulates: the weird node that only returns three peers for each
// request and all of them are different
name: "weird node (all different responses)",
sets: []mapset.Set[string]{
mapset.NewThreadUnsafeSet("0", "1", "2"),
mapset.NewThreadUnsafeSet("3", "4", "5"),
mapset.NewThreadUnsafeSet("6", "7", "8"),
sets: [][]string{
{"0", "1", "2"},
{"3", "4", "5"},
{"6", "7", "8"},
},
want: crawlStrategyRandomProbe,
},
{
// simulates: the weird node that only returns three peers for each
// request and all of them are different
name: "weird node (single overlap responses)",
sets: []mapset.Set[string]{
mapset.NewThreadUnsafeSet("0", "1", "2"),
mapset.NewThreadUnsafeSet("3", "4", "0"),
mapset.NewThreadUnsafeSet("6", "4", "8"),
sets: [][]string{
{"0", "1", "2"},
{"3", "4", "0"},
{"6", "4", "8"},
},
want: crawlStrategyRandomProbe,
},
{
name: "more than 16 peers in each bucket",
sets: []mapset.Set[string]{
mapset.NewThreadUnsafeSet("0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15"),
mapset.NewThreadUnsafeSet("16", "17", "18", "19", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15"),
mapset.NewThreadUnsafeSet("16", "17", "20", "21", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15"),
sets: [][]string{
{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15"},
{"16", "17", "18", "19", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15"},
{"16", "17", "20", "21", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15"},
},
want: crawlStrategyMultiProbe,
},
{
name: "partially filled bucket",
sets: []mapset.Set[string]{
mapset.NewThreadUnsafeSet("0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13"),
mapset.NewThreadUnsafeSet("0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13"),
mapset.NewThreadUnsafeSet("0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13"),
sets: [][]string{
{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13"},
{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13"},
{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13"},
},
want: crawlStrategySingleProbe,
},
{
name: "received v4wire.MaxNeighbors responses, full bucket",
sets: []mapset.Set[string]{
mapset.NewThreadUnsafeSet("0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11"),
mapset.NewThreadUnsafeSet("2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13"),
mapset.NewThreadUnsafeSet("5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15"),
sets: [][]string{
{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11"},
{"2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13"},
{"5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15"},
},
want: crawlStrategyMultiProbe,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := determineStrategy(tt.sets, tt.errs)
var sets []mapset.Set[peer.ID]
for i, s := range tt.sets {
sets = append(sets, mapset.NewThreadUnsafeSet[peer.ID]())
for _, item := range s {
sets[i].Add(peer.ID(item))
}
}
got := determineStrategy(sets)
assert.Equal(t, tt.want, got)
})
}
Expand Down
6 changes: 2 additions & 4 deletions discv4/driver_crawler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,6 @@ import (
"net/netip"
"time"

"golang.org/x/net/context"

"github.com/dennis-tra/nebula-crawler/tele"

ethcrypto "github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/crypto/secp256k1"
"github.com/ethereum/go-ethereum/p2p/discover"
Expand All @@ -25,11 +21,13 @@ import (
log "github.com/sirupsen/logrus"
"go.opentelemetry.io/otel/metric"
"go.opentelemetry.io/otel/trace"
"golang.org/x/net/context"

"github.com/dennis-tra/nebula-crawler/config"
"github.com/dennis-tra/nebula-crawler/core"
"github.com/dennis-tra/nebula-crawler/db"
"github.com/dennis-tra/nebula-crawler/db/models"
"github.com/dennis-tra/nebula-crawler/tele"
"github.com/dennis-tra/nebula-crawler/utils"
)

Expand Down
Loading

0 comments on commit 0ab7c1b

Please sign in to comment.