Improve node connection resiliency

cmd/node/main.go

@@ -127,6 +127,7 @@ func run() int {
 		host.WithDialBackWebsocketPort(cfg.Connectivity.WebsocketDialbackPort),
 		host.WithWebsocket(cfg.Connectivity.Websocket),
 		host.WithWebsocketPort(cfg.Connectivity.WebsocketPort),
+		host.WithMustReachBootNodes(cfg.Connectivity.MustReachBootNodes),
 	}
 
 	if !cfg.Connectivity.NoDialbackPeers {

config/config.go

@@ -59,15 +59,16 @@ type Log struct {
 
 // Connectivity describes the libp2p host that the node will use.
 type Connectivity struct {
-	Address               string `koanf:"address"           flag:"address,a"`
-	Port                  uint   `koanf:"port"              flag:"port,p"`
-	PrivateKey            string `koanf:"private-key"       flag:"private-key"`
-	DialbackAddress       string `koanf:"dialback-address"  flag:"dialback-address"`
-	DialbackPort          uint   `koanf:"dialback-port"     flag:"dialback-port"`
-	Websocket             bool   `koanf:"websocket"         flag:"websocket,w"`
-	WebsocketPort         uint   `koanf:"websocket-port"    flag:"websocket-port"`
+	Address               string `koanf:"address"                 flag:"address,a"`
+	Port                  uint   `koanf:"port"                    flag:"port,p"`
+	PrivateKey            string `koanf:"private-key"             flag:"private-key"`
+	DialbackAddress       string `koanf:"dialback-address"        flag:"dialback-address"`
+	DialbackPort          uint   `koanf:"dialback-port"           flag:"dialback-port"`
+	Websocket             bool   `koanf:"websocket"               flag:"websocket,w"`
+	WebsocketPort         uint   `koanf:"websocket-port"          flag:"websocket-port"`
 	WebsocketDialbackPort uint   `koanf:"websocket-dialback-port" flag:"websocket-dialback-port"`
-	NoDialbackPeers       bool   `koanf:"no-dialback-peers" flag:"no-dialback-peers"`
+	NoDialbackPeers       bool   `koanf:"no-dialback-peers"       flag:"no-dialback-peers"`
+	MustReachBootNodes    bool   `koanf:"must-reach-boot-nodes"   flag:"must-reach-boot-nodes"`
 }
 
 type Head struct {
@@ -139,6 +140,8 @@ func getFlagDescription(flag string) string {
 		return "memory limit (kB) for Blockless Functions"
 	case "no-dialback-peers":
 		return "start without dialing back peers from previous runs"
+	case "must-reach-boot-nodes":
+		return "halt node if we fail to reach boot nodes on start"
 	default:
 		return ""
 	}

host/config.go

@@ -10,11 +10,13 @@ import (
 
 // defaultConfig used to create Host.
 var defaultConfig = Config{
-	PrivateKey:          "",
-	ConnectionThreshold: 20,
-	DialBackPeersLimit:  100,
-	DiscoveryInterval:   10 * time.Second,
-	Websocket:           false,
+	PrivateKey:                         "",
+	ConnectionThreshold:                20,
+	DialBackPeersLimit:                 100,
+	DiscoveryInterval:                  10 * time.Second,
+	Websocket:                          false,
+	BootNodesReachabilityCheckInterval: 1 * time.Minute,
+	MustReachBootNodes:                 defaultMustReachBootNodes,
 }
 
 // Config represents the Host configuration.
@@ -33,6 +35,9 @@ type Config struct {
 	DialBackAddress       string
 	DialBackPort          uint
 	DialBackWebsocketPort uint
+
+	BootNodesReachabilityCheckInterval time.Duration
+	MustReachBootNodes                 bool
 }
 
 // WithPrivateKey specifies the private key for the Host.
@@ -108,3 +113,17 @@ func WithWebsocketPort(port uint) func(*Config) {
 		cfg.WebsocketPort = port
 	}
 }
+
+// WithMustReachBootNodes specifies if we should treat failure to reach boot nodes as a halting error.
+func WithMustReachBootNodes(b bool) func(*Config) {
+	return func(cfg *Config) {
+		cfg.MustReachBootNodes = b
+	}
+}
+
+// WithBootNodesReachabilityInterval specifies how often should we recheck and reconnect to boot nodes.
+func WithBootNodesReachabilityInterval(d time.Duration) func(cfg *Config) {
+	return func(cfg *Config) {
+		cfg.BootNodesReachabilityCheckInterval = d
+	}
+}