Merge pull request #927 from iotaledger/fix/autopeering+inx

Autopeering and INX fixes

components/inx/server_utxo.go

@@ -37,8 +37,7 @@ func NewLedgerOutput(o *utxoledger.Output, slotIncluded ...iotago.SlotIndex) (*i
 		},
 	}
 
-	if includedSlot > 0 &&
-		includedSlot <= latestCommitment.Slot() &&
+	if includedSlot <= latestCommitment.Slot() &&
 		includedSlot >= deps.Protocol.CommittedAPI().ProtocolParameters().GenesisSlot() {
 		includedCommitment, err := deps.Protocol.Engines.Main.Get().Storage.Commitments().Load(includedSlot)
 		if err != nil {
@@ -64,8 +63,7 @@ func NewLedgerSpent(s *utxoledger.Spent) (*inx.LedgerSpent, error) {
 
 	latestCommitment := deps.Protocol.Engines.Main.Get().SyncManager.LatestCommitment()
 	spentSlot := s.SlotSpent()
-	if spentSlot > 0 &&
-		spentSlot <= latestCommitment.Slot() &&
+	if spentSlot <= latestCommitment.Slot() &&
 		spentSlot >= deps.Protocol.CommittedAPI().ProtocolParameters().GenesisSlot() {
 		spentCommitment, err := deps.Protocol.Engines.Main.Get().Storage.Commitments().Load(spentSlot)
 		if err != nil {

components/p2p/component.go

@@ -215,23 +215,7 @@ func provide(c *dig.Container) error {
 			libp2p.NATPortMap(),
 			libp2p.DisableRelay(),
 			// Define a custom address factory to inject external addresses to the DHT advertisements.
-			libp2p.AddrsFactory(func() func(addrs []multiaddr.Multiaddr) []multiaddr.Multiaddr {
-				var externalMultiAddrs []multiaddr.Multiaddr
-				if len(ParamsP2P.ExternalMultiAddresses) > 0 {
-					for _, externalMultiAddress := range ParamsP2P.ExternalMultiAddresses {
-						addr, err := multiaddr.NewMultiaddr(externalMultiAddress)
-						if err != nil {
-							Component.LogPanicf("unable to parse external multi address %s: %s", externalMultiAddress, err)
-						}
-
-						externalMultiAddrs = append(externalMultiAddrs, addr)
-					}
-				}
-
-				return func(addrs []multiaddr.Multiaddr) []multiaddr.Multiaddr {
-					return append(addrs, externalMultiAddrs...)
-				}
-			}()),
+			libp2p.AddrsFactory(externalAddresses(ParamsP2P.Autopeering.ExternalMultiAddresses, ParamsP2P.Autopeering.AllowLocalIPs)),
 		)
 		if err != nil {
 			Component.LogFatalf("unable to initialize libp2p host: %s", err)
@@ -259,27 +243,11 @@ func provide(c *dig.Container) error {
 	}
 
 	return c.Provide(func(inDeps p2pManagerDeps) network.Manager {
-		peersMultiAddresses, err := getMultiAddrsFromString(ParamsPeers.BootstrapPeers)
-		if err != nil {
-			Component.LogFatalf("Failed to parse bootstrapPeers param: %s", err)
-		}
-
-		for _, multiAddr := range peersMultiAddresses {
-			bootstrapPeer, err := network.NewPeerFromMultiAddr(multiAddr)
-			if err != nil {
-				Component.LogFatalf("Failed to parse bootstrap peer multiaddress: %s", err)
-			}
-
-			if err := inDeps.PeerDB.UpdatePeer(bootstrapPeer); err != nil {
-				Component.LogErrorf("Failed to update bootstrap peer: %s", err)
-			}
-		}
-
 		onBlockSentCallback := func() {
 			inDeps.P2PMetrics.OutgoingBlocks.Add(1)
 		}
 
-		return p2p.NewManager(Component.Logger, inDeps.Host, inDeps.PeerDB, ParamsP2P.Autopeering.MaxPeers, onBlockSentCallback)
+		return p2p.NewManager(Component.Logger, inDeps.Host, inDeps.PeerDB, ParamsP2P.Autopeering.MaxPeers, ParamsP2P.Autopeering.AllowLocalIPs, onBlockSentCallback)
 	})
 }
 
@@ -320,7 +288,7 @@ func run() error {
 	if err := Component.Daemon().BackgroundWorker(Component.Name, func(ctx context.Context) {
 		defer deps.NetworkManager.Shutdown()
 
-		if err := deps.NetworkManager.Start(ctx, deps.Protocol.LatestAPI().ProtocolParameters().NetworkName()); err != nil {
+		if err := deps.NetworkManager.Start(ctx, deps.Protocol.LatestAPI().ProtocolParameters().NetworkName(), bootstrapPeers()); err != nil {
 			Component.LogFatalf("Failed to start p2p manager: %s", err)
 		}
 
@@ -335,6 +303,24 @@ func run() error {
 	return nil
 }
 
+func bootstrapPeers() []peer.AddrInfo {
+	peersMultiAddresses, err := getMultiAddrsFromString(ParamsP2P.Autopeering.BootstrapPeers)
+	if err != nil {
+		Component.LogFatalf("Failed to parse bootstrapPeers param: %s", err)
+	}
+
+	addrInfos := make([]peer.AddrInfo, 0, len(peersMultiAddresses))
+	for _, multiAddr := range peersMultiAddresses {
+		addrInfo, err := peer.AddrInfoFromP2pAddr(multiAddr)
+		if err != nil {
+			Component.LogFatalf("Failed to parse bootstrap peer multiaddress: %s", err)
+		}
+		addrInfos = append(addrInfos, *addrInfo)
+	}
+
+	return addrInfos
+}
+
 func getMultiAddrsFromString(peers []string) ([]multiaddr.Multiaddr, error) {
 	peersMultiAddresses := make([]multiaddr.Multiaddr, 0, len(peers))
 
@@ -368,3 +354,25 @@ func connectConfigKnownPeers() {
 		}
 	}
 }
+
+func externalAddresses(additionalMultiaddresses []string, allowLocalNetworks bool) network.AddressFilter {
+	var externalMultiAddrs []multiaddr.Multiaddr
+
+	// Add the external multi addresses to the list of addresses to be announced.
+	if len(additionalMultiaddresses) > 0 {
+		for _, externalMultiAddress := range additionalMultiaddresses {
+			addr, err := multiaddr.NewMultiaddr(externalMultiAddress)
+			if err != nil {
+				Component.LogPanicf("unable to parse external multi address %s: %s", externalMultiAddress, err)
+			}
+
+			externalMultiAddrs = append(externalMultiAddrs, addr)
+		}
+	}
+
+	publicFilter := network.PublicOnlyAddressesFilter(allowLocalNetworks)
+
+	return func(addresses []multiaddr.Multiaddr) []multiaddr.Multiaddr {
+		return publicFilter(append(addresses, externalMultiAddrs...))
+	}
+}

components/p2p/params.go

@@ -22,14 +22,21 @@ type ParametersP2P struct {
 		LowWatermark int `default:"5" usage:"the minimum connections count to hold after the high watermark was reached"`
 	}
 
-	// ExternalMultiAddress defines additional p2p multiaddresses to be advertised via DHT.
-	ExternalMultiAddresses []string `default:"" usage:"external reacheable multi addresses advertised to the network"`
-
 	// Defines the private key used to derive the node identity (optional).
 	IdentityPrivateKey string `default:"" usage:"private key used to derive the node identity (optional)"`
 
 	Autopeering struct {
-		MaxPeers int `default:"5" usage:"the max number of autopeer connections. Set to 0 to disable autopeering."`
+		// MaxPeers defines the max number of auto-peer connections. Set to 0 to disable auto-peering.
+		MaxPeers int `default:"5" usage:"the max number of auto-peer connections. Set to 0 to disable auto-peering."`
+
+		// Defines the peers to be used as discovery for other peers (CLI).
+		BootstrapPeers []string `default:"" usage:"peers to be used as discovery for other peers"`
+
+		// AllowLocalIPs defines if local IPs are allowed to be used for autopeering.
+		AllowLocalIPs bool `default:"false" usage:"allow local IPs to be used for autopeering"`
+
+		// ExternalMultiAddress defines additional p2p multiaddresses to be advertised via DHT.
+		ExternalMultiAddresses []string `default:"" usage:"external reacheable multi addresses advertised to the network"`
 	}
 
 	Database struct {
@@ -44,8 +51,6 @@ type ParametersPeers struct {
 	Peers []string `default:"" usage:"the static peers this node should retain a connection to (CLI)"`
 	// Defines the aliases of the static peers (must be the same length like CfgP2PPeers) (CLI).
 	PeerAliases []string `default:"" usage:"the aliases of the static peers (must be the same amount like \"p2p.peers\""`
-	// Defines the peers to be used as discovery for other peers (CLI).
-	BootstrapPeers []string `default:"" usage:"peers to be used as discovery for other peers (CLI)"`
 }
 
 var (

config_defaults.json

@@ -26,10 +26,12 @@
       "highWatermark": 10,
       "lowWatermark": 5
     },
-    "externalMultiAddresses": [],
     "identityPrivateKey": "",
     "autopeering": {
-      "maxPeers": 5
+      "maxPeers": 5,
+      "bootstrapPeers": [],
+      "allowLocalIPs": false,
+      "externalMultiAddresses": []
     },
     "db": {
       "path": "testnet/p2pstore"

documentation/configuration.md

@@ -93,14 +93,13 @@ Example:
 
 ## <a id="p2p"></a> 3. Peer to Peer
 
-| Name                                        | Description                                                   | Type   | Default value                                |
-| ------------------------------------------- | ------------------------------------------------------------- | ------ | -------------------------------------------- |
-| bindMultiAddresses                          | The bind multi addresses for p2p connections                  | array  | /ip4/0.0.0.0/tcp/15600<br/>/ip6/::/tcp/15600 |
-| [connectionManager](#p2p_connectionmanager) | Configuration for connectionManager                           | object |                                              |
-| externalMultiAddresses                      | External reacheable multi addresses advertised to the network | array  |                                              |
-| identityPrivateKey                          | Private key used to derive the node identity (optional)       | string | ""                                           |
-| [autopeering](#p2p_autopeering)             | Configuration for autopeering                                 | object |                                              |
-| [db](#p2p_db)                               | Configuration for Database                                    | object |                                              |
+| Name                                        | Description                                             | Type   | Default value                                |
+| ------------------------------------------- | ------------------------------------------------------- | ------ | -------------------------------------------- |
+| bindMultiAddresses                          | The bind multi addresses for p2p connections            | array  | /ip4/0.0.0.0/tcp/15600<br/>/ip6/::/tcp/15600 |
+| [connectionManager](#p2p_connectionmanager) | Configuration for connectionManager                     | object |                                              |
+| identityPrivateKey                          | Private key used to derive the node identity (optional) | string | ""                                           |
+| [autopeering](#p2p_autopeering)             | Configuration for autopeering                           | object |                                              |
+| [db](#p2p_db)                               | Configuration for Database                              | object |                                              |
 
 ### <a id="p2p_connectionmanager"></a> ConnectionManager
 
@@ -111,9 +110,12 @@ Example:
 
 ### <a id="p2p_autopeering"></a> Autopeering
 
-| Name     | Description                                                              | Type | Default value |
-| -------- | ------------------------------------------------------------------------ | ---- | ------------- |
-| maxPeers | The max number of autopeer connections. Set to 0 to disable autopeering. | int  | 5             |
+| Name                   | Description                                                                | Type    | Default value |
+| ---------------------- | -------------------------------------------------------------------------- | ------- | ------------- |
+| maxPeers               | The max number of auto-peer connections. Set to 0 to disable auto-peering. | int     | 5             |
+| bootstrapPeers         | Peers to be used as discovery for other peers                              | array   |               |
+| allowLocalIPs          | Allow local IPs to be used for autopeering                                 | boolean | false         |
+| externalMultiAddresses | External reacheable multi addresses advertised to the network              | array   |               |
 
 ### <a id="p2p_db"></a> Database
 
@@ -134,10 +136,12 @@ Example:
         "highWatermark": 10,
         "lowWatermark": 5
       },
-      "externalMultiAddresses": [],
       "identityPrivateKey": "",
       "autopeering": {
-        "maxPeers": 5
+        "maxPeers": 5,
+        "bootstrapPeers": [],
+        "allowLocalIPs": false,
+        "externalMultiAddresses": []
       },
       "db": {
         "path": "testnet/p2pstore"

go.mod

@@ -169,6 +169,7 @@ require (
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasttemplate v1.2.2 // indirect
 	github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect
+	github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/otel v1.25.0 // indirect
 	go.opentelemetry.io/otel/metric v1.25.0 // indirect

go.sum

@@ -695,6 +695,8 @@ github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSD
 github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
 github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k=
 github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc=
+github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 h1:E9S12nwJwEOXe2d6gT6qxdvqMnNq+VnSsKPgm2ZZNds=
+github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7/go.mod h1:X2c0RVCI1eSUFI8eLcY3c0423ykwiUdxLJtkDvruhjI=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=

pkg/network/filter.go

@@ -0,0 +1,70 @@
+package network
+
+import (
+	"fmt"
+
+	"github.com/multiformats/go-multiaddr"
+	mamask "github.com/whyrusleeping/multiaddr-filter"
+
+	"github.com/iotaledger/hive.go/lo"
+)
+
+// Based on https://github.com/ipfs/kubo/blob/master/config/profile.go
+// defaultServerFilters has is a list of IPv4 and IPv6 prefixes that are private, local only, or unrouteable.
+// according to https://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml
+// and https://www.iana.org/assignments/iana-ipv6-special-registry/iana-ipv6-special-registry.xhtml
+var reservedFilters = []string{
+	"/ip4/0.0.0.0/ipcidr/32",
+	"/ip4/100.64.0.0/ipcidr/10",
+	"/ip4/127.0.0.0/ipcidr/8",
+	"/ip4/169.254.0.0/ipcidr/16",
+	"/ip4/192.0.0.0/ipcidr/24",
+	"/ip4/192.0.2.0/ipcidr/24",
+	"/ip4/192.31.196.0/ipcidr/24",
+	"/ip4/192.52.193.0/ipcidr/24",
+	"/ip4/198.18.0.0/ipcidr/15",
+	"/ip4/198.51.100.0/ipcidr/24",
+	"/ip4/203.0.113.0/ipcidr/24",
+	"/ip4/240.0.0.0/ipcidr/4",
+
+	"/ip6/::/ipcidr/128",
+	"/ip6/::1/ipcidr/128",
+	"/ip6/100::/ipcidr/64",
+	"/ip6/2001:2::/ipcidr/48",
+	"/ip6/2001:db8::/ipcidr/32",
+}
+
+var localNetworks = []string{
+	"/ip4/10.0.0.0/ipcidr/8",
+	"/ip4/172.16.0.0/ipcidr/12",
+	"/ip4/192.168.0.0/ipcidr/16",
+
+	"/ip6/fc00::/ipcidr/7",
+	"/ip6/fe80::/ipcidr/10",
+}
+
+type AddressFilter = func([]multiaddr.Multiaddr) []multiaddr.Multiaddr
+
+func PublicOnlyAddressesFilter(allowLocalNetworks bool) AddressFilter {
+	// Create a filter that blocks localhost and reserved addresses.
+	filters := multiaddr.NewFilters()
+
+	filtersToApply := reservedFilters
+	if !allowLocalNetworks {
+		filtersToApply = append(filtersToApply, localNetworks...)
+	}
+
+	for _, addr := range filtersToApply {
+		f, err := mamask.NewMask(addr)
+		if err != nil {
+			panic(fmt.Sprintf("unable to parse ip mask filter %s: %s", addr, err))
+		}
+		filters.AddFilter(*f, multiaddr.ActionDeny)
+	}
+
+	return func(addresses []multiaddr.Multiaddr) []multiaddr.Multiaddr {
+		return lo.Filter(addresses, func(m multiaddr.Multiaddr) bool {
+			return !filters.AddrBlocked(m)
+		})
+	}
+}

pkg/network/manager.go

@@ -47,6 +47,6 @@ type Manager interface {
 
 	P2PHost() host.Host
 
-	Start(ctx context.Context, networkID string) error
+	Start(ctx context.Context, networkID string, bootstrapPeers []peer.AddrInfo) error
 	Shutdown()
 }