ir: retry mainnet RPC connection if the node is outdated

We can have a set of RPC nodes configured and we will switch them in runtime,
but if the first node is outdated then the IR node will _always_ fail to
start until RPC node catches up. We can have a better behaviour with node
switching to other RPCs in this phase.

Can't be done this easily for the sidechain since in most cases we have an
internal node and it needs to go through the chain completely.

Refs. #2426.

Signed-off-by: Roman Khimov <[email protected]>

CHANGELOG.md

@@ -16,6 +16,7 @@ Changelog for NeoFS Node
 - Double voting for validators on IR startup (#2365)
 - Skip unexpected notary events on notary request parsing step (#2315)
 - Session inactivity on object PUT request relay (#2460)
+- Missing connection retries on IR node startup whe the first configured mainner RPC node is not in sync (#XXX)
 
 ### Removed
 - Deprecated `morph.rpc_endpoint` SN and `morph.endpoint.client` IR config sections (#2400)

pkg/innerring/innerring.go

@@ -479,8 +479,13 @@ func New(ctx context.Context, log *zap.Logger, cfg *viper.Viper, errChan chan<-
 			return nil, err
 		}
 
-		// create mainnet listener
-		server.mainnetListener, err = createListener(ctx, server.mainnetClient, mainnetChain)
+		// create mainnet listener, retry with a different node if current one is not up to date
+		for {
+			server.mainnetListener, err = createListener(ctx, server.mainnetClient, mainnetChain)
+			if !errors.Is(err, subscriber.ErrStaleNode) || !server.mainnetClient.SwitchRPC() {
+				break
+			}
+		}
 		if err != nil {
 			return nil, err
 		}

pkg/morph/subscriber/subscriber.go

@@ -68,6 +68,7 @@ func (s *subscriber) NotificationChannels() NotificationChannels {
 }
 
 var (
+	ErrStaleNode = errors.New("RPC node is not yet up to date")
 	errNilParams = errors.New("chain/subscriber: config was not provided to the constructor")
 
 	errNilLogger = errors.New("chain/subscriber: logger was not provided to the constructor")
@@ -333,7 +334,7 @@ func awaitHeight(cli *client.Client, startFrom uint32) error {
 	}
 
 	if height < startFrom+1 {
-		return fmt.Errorf("RPC block counter %d didn't reach expected height %d", height, startFrom)
+		return fmt.Errorf("%w: expected %d height, got %d count", ErrStaleNode, startFrom, height)
 	}
 
 	return nil