From 60673aa29fabf0293b428608840511fef6c86a20 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 20 Aug 2024 13:55:18 -0400 Subject: [PATCH 001/174] MultiNode integration setup --- pkg/solana/chain.go | 7 +- pkg/solana/chain_multinode.go | 268 +++++++ pkg/solana/chain_test.go | 3 +- pkg/solana/client/client.go | 8 +- pkg/solana/client/client_test.go | 4 +- pkg/solana/client/multinode/models.go | 121 +++ pkg/solana/client/multinode/multi_node.go | 379 +++++++++ pkg/solana/client/multinode/node.go | 331 ++++++++ pkg/solana/client/multinode/node_fsm.go | 370 +++++++++ pkg/solana/client/multinode/node_lifecycle.go | 732 ++++++++++++++++++ pkg/solana/client/multinode/node_selector.go | 43 + pkg/solana/client/multinode/poller.go | 99 +++ pkg/solana/client/multinode/poller_test.go | 187 +++++ pkg/solana/client/multinode/send_only_node.go | 183 +++++ .../multinode/send_only_node_lifecycle.go | 67 ++ .../client/multinode/send_only_node_test.go | 139 ++++ .../client/multinode/transaction_sender.go | 277 +++++++ .../multinode/transaction_sender_test.go | 360 +++++++++ pkg/solana/client/multinode/types.go | 124 +++ pkg/solana/client/rpc_client.go | 318 ++++++++ pkg/solana/config/multinode.go | 86 ++ pkg/solana/config/toml.go | 8 +- 22 files changed, 4103 insertions(+), 11 deletions(-) create mode 100644 pkg/solana/chain_multinode.go create mode 100644 pkg/solana/client/multinode/models.go create mode 100644 pkg/solana/client/multinode/multi_node.go create mode 100644 pkg/solana/client/multinode/node.go create mode 100644 pkg/solana/client/multinode/node_fsm.go create mode 100644 pkg/solana/client/multinode/node_lifecycle.go create mode 100644 pkg/solana/client/multinode/node_selector.go create mode 100644 pkg/solana/client/multinode/poller.go create mode 100644 pkg/solana/client/multinode/poller_test.go create mode 100644 pkg/solana/client/multinode/send_only_node.go create mode 100644 pkg/solana/client/multinode/send_only_node_lifecycle.go create mode 100644 pkg/solana/client/multinode/send_only_node_test.go create mode 100644 pkg/solana/client/multinode/transaction_sender.go create mode 100644 pkg/solana/client/multinode/transaction_sender_test.go create mode 100644 pkg/solana/client/multinode/types.go create mode 100644 pkg/solana/client/rpc_client.go create mode 100644 pkg/solana/config/multinode.go diff --git a/pkg/solana/chain.go b/pkg/solana/chain.go index 4e03fd425..20f6322d5 100644 --- a/pkg/solana/chain.go +++ b/pkg/solana/chain.go @@ -114,7 +114,8 @@ func (v *verifiedCachedClient) verifyChainID() (bool, error) { v.chainIDVerifiedLock.Lock() defer v.chainIDVerifiedLock.Unlock() - v.chainID, err = v.ReaderWriter.ChainID() + strID, err := v.ReaderWriter.ChainID(context.Background()) + v.chainID = strID.String() if err != nil { v.chainIDVerified = false return v.chainIDVerified, fmt.Errorf("failed to fetch ChainID in verifiedCachedClient: %w", err) @@ -186,13 +187,13 @@ func (v *verifiedCachedClient) LatestBlockhash() (*rpc.GetLatestBlockhashResult, return v.ReaderWriter.LatestBlockhash() } -func (v *verifiedCachedClient) ChainID() (string, error) { +func (v *verifiedCachedClient) ChainID(ctx context.Context) (client.StringID, error) { verified, err := v.verifyChainID() if !verified { return "", err } - return v.chainID, nil + return client.StringID(v.chainID), nil } func (v *verifiedCachedClient) GetFeeForMessage(msg string) (uint64, error) { diff --git a/pkg/solana/chain_multinode.go b/pkg/solana/chain_multinode.go new file mode 100644 index 000000000..82fb5b23f --- /dev/null +++ b/pkg/solana/chain_multinode.go @@ -0,0 +1,268 @@ +package solana + +import ( + "context" + "errors" + "fmt" + "math/big" + "sync" + "time" + + solanago "github.com/gagliardetto/solana-go" + "github.com/gagliardetto/solana-go/programs/system" + "github.com/smartcontractkit/chainlink-common/pkg/chains" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/loop" + "github.com/smartcontractkit/chainlink-common/pkg/services" + relaytypes "github.com/smartcontractkit/chainlink-common/pkg/types" + "github.com/smartcontractkit/chainlink-solana/pkg/solana/client" + mn "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" + "github.com/smartcontractkit/chainlink-solana/pkg/solana/config" + "github.com/smartcontractkit/chainlink-solana/pkg/solana/monitor" + "github.com/smartcontractkit/chainlink-solana/pkg/solana/txm" +) + +func NewMultiNodeChain(cfg *config.TOMLConfig, opts ChainOpts) (Chain, error) { + if !cfg.IsEnabled() { + return nil, fmt.Errorf("cannot create new chain with ID %s: chain is disabled", *cfg.ChainID) + } + c, err := newMultiNodeChain(*cfg.ChainID, cfg, opts.KeyStore, opts.Logger) + if err != nil { + return nil, err + } + return c, nil +} + +var _ Chain = (*multiNodeChain)(nil) + +type multiNodeChain struct { + services.StateMachine + id string + cfg *config.TOMLConfig + multiNode *mn.MultiNode[client.StringID, *client.RpcClient] + txSender *mn.TransactionSender[*solanago.Transaction, client.StringID, *client.RpcClient] + txm *txm.Txm + balanceMonitor services.Service + lggr logger.Logger + + clientLock sync.RWMutex +} + +func newMultiNodeChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr logger.Logger) (*multiNodeChain, error) { + lggr = logger.With(lggr, "chainID", id, "chain", "solana") + + chainFamily := "solana" + + cfg.BlockHistoryPollPeriod() + + mnCfg := cfg.MultiNodeConfig() + + var nodes []mn.Node[client.StringID, *client.RpcClient] + + for i, nodeInfo := range cfg.ListNodes() { + // create client and check + rpcClient, err := client.NewRpcClient(nodeInfo.URL.String(), cfg, DefaultRequestTimeout, logger.Named(lggr, "Client."+*nodeInfo.Name)) + if err != nil { + lggr.Warnw("failed to create client", "name", *nodeInfo.Name, "solana-url", nodeInfo.URL.String(), "err", err.Error()) + continue + } + + newNode := mn.NewNode[client.StringID, *client.Head, *client.RpcClient]( + mnCfg, mnCfg, lggr, *nodeInfo.URL.URL(), nil, *nodeInfo.Name, + int32(i), client.StringID(id), 0, rpcClient, chainFamily) + + nodes = append(nodes, newNode) + } + + multiNode := mn.NewMultiNode[client.StringID, *client.RpcClient]( + lggr, + mn.NodeSelectionModeRoundRobin, + time.Duration(0), // TODO: set lease duration + nodes, + []mn.SendOnlyNode[client.StringID, *client.RpcClient]{}, // TODO: no send only nodes? + client.StringID(id), + chainFamily, + time.Duration(0), // TODO: set deathDeclarationDelay + ) + + classifySendError := func(tx *solanago.Transaction, err error) mn.SendTxReturnCode { + return 0 // TODO ClassifySendError(err, clientErrors, logger.Sugared(logger.Nop()), tx, common.Address{}, false) + } + + txSender := mn.NewTransactionSender[*solanago.Transaction, client.StringID, *client.RpcClient]( + lggr, + client.StringID(id), + chainFamily, + multiNode, + classifySendError, + 0, // use the default value provided by the implementation + ) + + var ch = multiNodeChain{ + id: id, + cfg: cfg, + multiNode: multiNode, + txSender: txSender, + lggr: logger.Named(lggr, "Chain"), + } + + tc := func() (client.ReaderWriter, error) { + return ch.multiNode.SelectRPC() + } + + ch.txm = txm.NewTxm(ch.id, tc, cfg, ks, lggr) + bc := func() (monitor.BalanceClient, error) { + return ch.multiNode.SelectRPC() + } + ch.balanceMonitor = monitor.NewBalanceMonitor(ch.id, cfg, lggr, ks, bc) + return &ch, nil +} + +// ChainService interface +func (c *multiNodeChain) GetChainStatus(ctx context.Context) (relaytypes.ChainStatus, error) { + toml, err := c.cfg.TOMLString() + if err != nil { + return relaytypes.ChainStatus{}, err + } + return relaytypes.ChainStatus{ + ID: c.id, + Enabled: c.cfg.IsEnabled(), + Config: toml, + }, nil +} + +func (c *multiNodeChain) ListNodeStatuses(ctx context.Context, pageSize int32, pageToken string) (stats []relaytypes.NodeStatus, nextPageToken string, total int, err error) { + return chains.ListNodeStatuses(int(pageSize), pageToken, c.listNodeStatuses) +} + +func (c *multiNodeChain) Transact(ctx context.Context, from, to string, amount *big.Int, balanceCheck bool) error { + return c.sendTx(ctx, from, to, amount, balanceCheck) +} + +func (c *multiNodeChain) listNodeStatuses(start, end int) ([]relaytypes.NodeStatus, int, error) { + stats := make([]relaytypes.NodeStatus, 0) + total := len(c.cfg.Nodes) + if start >= total { + return stats, total, chains.ErrOutOfRange + } + if end > total { + end = total + } + nodes := c.cfg.Nodes[start:end] + for _, node := range nodes { + stat, err := config.NodeStatus(node, c.ChainID()) + if err != nil { + return stats, total, err + } + stats = append(stats, stat) + } + return stats, total, nil +} + +func (c *multiNodeChain) Name() string { + return c.lggr.Name() +} + +func (c *multiNodeChain) ID() string { + return c.id +} + +func (c *multiNodeChain) Config() config.Config { + return c.cfg +} + +func (c *multiNodeChain) TxManager() TxManager { + return c.txm +} + +func (c *multiNodeChain) Reader() (client.Reader, error) { + return c.multiNode.SelectRPC() +} + +func (c *multiNodeChain) ChainID() string { + return c.id +} + +func (c *multiNodeChain) Start(ctx context.Context) error { + return c.StartOnce("Chain", func() error { + c.lggr.Debug("Starting") + c.lggr.Debug("Starting txm") + c.lggr.Debug("Starting balance monitor") + var ms services.MultiStart + return ms.Start(ctx, c.txm, c.balanceMonitor) + }) +} + +func (c *multiNodeChain) Close() error { + return c.StopOnce("Chain", func() error { + c.lggr.Debug("Stopping") + c.lggr.Debug("Stopping txm") + c.lggr.Debug("Stopping balance monitor") + return services.CloseAll(c.txm, c.balanceMonitor) + }) +} + +func (c *multiNodeChain) Ready() error { + return errors.Join( + c.StateMachine.Ready(), + c.txm.Ready(), + ) +} + +func (c *multiNodeChain) HealthReport() map[string]error { + report := map[string]error{c.Name(): c.Healthy()} + services.CopyHealth(report, c.txm.HealthReport()) + return report +} + +func (c *multiNodeChain) sendTx(ctx context.Context, from, to string, amount *big.Int, balanceCheck bool) error { + reader, err := c.Reader() + if err != nil { + return fmt.Errorf("chain unreachable: %w", err) + } + + fromKey, err := solanago.PublicKeyFromBase58(from) + if err != nil { + return fmt.Errorf("failed to parse from key: %w", err) + } + toKey, err := solanago.PublicKeyFromBase58(to) + if err != nil { + return fmt.Errorf("failed to parse to key: %w", err) + } + if !amount.IsUint64() { + return fmt.Errorf("amount %s overflows uint64", amount) + } + amountI := amount.Uint64() + + blockhash, err := reader.LatestBlockhash() + if err != nil { + return fmt.Errorf("failed to get latest block hash: %w", err) + } + tx, err := solanago.NewTransaction( + []solanago.Instruction{ + system.NewTransferInstruction( + amountI, + fromKey, + toKey, + ).Build(), + }, + blockhash.Value.Blockhash, + solanago.TransactionPayer(fromKey), + ) + if err != nil { + return fmt.Errorf("failed to create tx: %w", err) + } + + if balanceCheck { + if err = solanaValidateBalance(reader, fromKey, amountI, tx.Message.ToBase64()); err != nil { + return fmt.Errorf("failed to validate balance: %w", err) + } + } + + txm := c.TxManager() + err = txm.Enqueue("", tx) + if err != nil { + return fmt.Errorf("transaction failed: %w", err) + } + return nil +} diff --git a/pkg/solana/chain_test.go b/pkg/solana/chain_test.go index aa52b8b4d..3f1fdaf23 100644 --- a/pkg/solana/chain_test.go +++ b/pkg/solana/chain_test.go @@ -1,6 +1,7 @@ package solana import ( + "context" "errors" "fmt" "io" @@ -174,7 +175,7 @@ func TestSolanaChain_VerifiedClient(t *testing.T) { testChain.id = "incorrect" c, err = testChain.verifiedClient(node) assert.NoError(t, err) - _, err = c.ChainID() + _, err = c.ChainID(context.Background()) // expect error from id mismatch (even if using a cached client) when performing RPC calls assert.Error(t, err) assert.Equal(t, fmt.Sprintf("client returned mismatched chain id (expected: %s, got: %s): %s", "incorrect", "devnet", node.URL), err.Error()) diff --git a/pkg/solana/client/client.go b/pkg/solana/client/client.go index e51c93837..d007e3c4c 100644 --- a/pkg/solana/client/client.go +++ b/pkg/solana/client/client.go @@ -33,7 +33,7 @@ type Reader interface { Balance(addr solana.PublicKey) (uint64, error) SlotHeight() (uint64, error) LatestBlockhash() (*rpc.GetLatestBlockhashResult, error) - ChainID() (string, error) + ChainID(ctx context.Context) (StringID, error) GetFeeForMessage(msg string) (uint64, error) GetLatestBlock() (*rpc.GetBlockResult, error) } @@ -142,11 +142,11 @@ func (c *Client) LatestBlockhash() (*rpc.GetLatestBlockhashResult, error) { return v.(*rpc.GetLatestBlockhashResult), err } -func (c *Client) ChainID() (string, error) { +func (c *Client) ChainID(ctx context.Context) (StringID, error) { done := c.latency("chain_id") defer done() - ctx, cancel := context.WithTimeout(context.Background(), c.contextDuration) + ctx, cancel := context.WithTimeout(ctx, c.contextDuration) defer cancel() v, err, _ := c.requestGroup.Do("GetGenesisHash", func() (interface{}, error) { return c.rpc.GetGenesisHash(ctx) @@ -168,7 +168,7 @@ func (c *Client) ChainID() (string, error) { c.log.Warnf("unknown genesis hash - assuming solana chain is 'localnet'") network = "localnet" } - return network, nil + return StringID(network), nil } func (c *Client) GetFeeForMessage(msg string) (uint64, error) { diff --git a/pkg/solana/client/client_test.go b/pkg/solana/client/client_test.go index ab9dba263..6f2276bd3 100644 --- a/pkg/solana/client/client_test.go +++ b/pkg/solana/client/client_test.go @@ -76,7 +76,7 @@ func TestClient_Reader_Integration(t *testing.T) { assert.Equal(t, uint64(5000), fee) // get chain ID based on gensis hash - network, err := c.ChainID() + network, err := c.ChainID(context.Background()) assert.NoError(t, err) assert.Equal(t, "localnet", network) @@ -120,7 +120,7 @@ func TestClient_Reader_ChainID(t *testing.T) { // get chain ID based on gensis hash for _, n := range networks { - network, err := c.ChainID() + network, err := c.ChainID(context.Background()) assert.NoError(t, err) assert.Equal(t, n, network) } diff --git a/pkg/solana/client/multinode/models.go b/pkg/solana/client/multinode/models.go new file mode 100644 index 000000000..526bb25c8 --- /dev/null +++ b/pkg/solana/client/multinode/models.go @@ -0,0 +1,121 @@ +package client + +import ( + "bytes" + "fmt" +) + +type SendTxReturnCode int + +// SendTxReturnCode is a generalized client error that dictates what should be the next action, depending on the RPC error response. +const ( + Successful SendTxReturnCode = iota + 1 + Fatal // Unrecoverable error. Most likely the attempt should be thrown away. + Retryable // The error returned by the RPC indicates that if we retry with the same attempt, the tx will eventually go through. + Underpriced // Attempt was underpriced. New estimation is needed with bumped gas price. + Unknown // Tx failed with an error response that is not recognized by the client. + Unsupported // Attempt failed with an error response that is not supported by the client for the given chain. + TransactionAlreadyKnown // The transaction that was sent has already been received by the RPC. + InsufficientFunds // Tx was rejected due to insufficient funds. + ExceedsMaxFee // Attempt's fee was higher than the node's limit and got rejected. + FeeOutOfValidRange // This error is returned when we use a fee price suggested from an RPC, but the network rejects the attempt due to an invalid range(mostly used by L2 chains). Retry by requesting a new suggested fee price. + TerminallyStuck // The error returned when a transaction is or could get terminally stuck in the mempool without any chance of inclusion. + sendTxReturnCodeLen // tracks the number of errors. Must always be last +) + +// sendTxSevereErrors - error codes which signal that transaction would never be accepted in its current form by the node +var sendTxSevereErrors = []SendTxReturnCode{Fatal, Underpriced, Unsupported, ExceedsMaxFee, FeeOutOfValidRange, Unknown} + +// sendTxSuccessfulCodes - error codes which signal that transaction was accepted by the node +var sendTxSuccessfulCodes = []SendTxReturnCode{Successful, TransactionAlreadyKnown} + +func (c SendTxReturnCode) String() string { + switch c { + case Successful: + return "Successful" + case Fatal: + return "Fatal" + case Retryable: + return "Retryable" + case Underpriced: + return "Underpriced" + case Unknown: + return "Unknown" + case Unsupported: + return "Unsupported" + case TransactionAlreadyKnown: + return "TransactionAlreadyKnown" + case InsufficientFunds: + return "InsufficientFunds" + case ExceedsMaxFee: + return "ExceedsMaxFee" + case FeeOutOfValidRange: + return "FeeOutOfValidRange" + case TerminallyStuck: + return "TerminallyStuck" + default: + return fmt.Sprintf("SendTxReturnCode(%d)", c) + } +} + +type NodeTier int + +const ( + Primary = NodeTier(iota) + Secondary +) + +func (n NodeTier) String() string { + switch n { + case Primary: + return "primary" + case Secondary: + return "secondary" + default: + return fmt.Sprintf("NodeTier(%d)", n) + } +} + +// syncStatus - defines problems related to RPC's state synchronization. Can be used as a bitmask to define multiple issues +type syncStatus int + +const ( + // syncStatusSynced - RPC is fully synced + syncStatusSynced = 0 + // syncStatusNotInSyncWithPool - RPC is lagging behind the highest block observed within the pool of RPCs + syncStatusNotInSyncWithPool syncStatus = 1 << iota + // syncStatusNoNewHead - RPC failed to produce a new head for too long + syncStatusNoNewHead + // syncStatusNoNewFinalizedHead - RPC failed to produce a new finalized head for too long + syncStatusNoNewFinalizedHead + syncStatusLen +) + +func (s syncStatus) String() string { + if s == syncStatusSynced { + return "Synced" + } + var result bytes.Buffer + for i := syncStatusNotInSyncWithPool; i < syncStatusLen; i = i << 1 { + if i&s == 0 { + continue + } + result.WriteString(i.string()) + result.WriteString(",") + } + result.Truncate(result.Len() - 1) + return result.String() +} + +func (s syncStatus) string() string { + switch s { + case syncStatusNotInSyncWithPool: + return "NotInSyncWithRPCPool" + case syncStatusNoNewHead: + return "NoNewHead" + case syncStatusNoNewFinalizedHead: + return "NoNewFinalizedHead" + default: + return fmt.Sprintf("syncStatus(%d)", s) + } +} diff --git a/pkg/solana/client/multinode/multi_node.go b/pkg/solana/client/multinode/multi_node.go new file mode 100644 index 000000000..386e09554 --- /dev/null +++ b/pkg/solana/client/multinode/multi_node.go @@ -0,0 +1,379 @@ +package client + +import ( + "context" + "errors" + "fmt" + "math/big" + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/services" +) + +var ( + // PromMultiNodeRPCNodeStates reports current RPC node state + PromMultiNodeRPCNodeStates = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "multi_node_states", + Help: "The number of RPC nodes currently in the given state for the given chain", + }, []string{"network", "chainId", "state"}) + ErroringNodeError = fmt.Errorf("no live nodes available") +) + +// MultiNode is a generalized multi node client interface that includes methods to interact with different chains. +// It also handles multiple node RPC connections simultaneously. +type MultiNode[ + CHAIN_ID ID, + RPC any, +] struct { + services.StateMachine + primaryNodes []Node[CHAIN_ID, RPC] + sendOnlyNodes []SendOnlyNode[CHAIN_ID, RPC] + chainID CHAIN_ID + lggr logger.SugaredLogger + selectionMode string + nodeSelector NodeSelector[CHAIN_ID, RPC] + leaseDuration time.Duration + leaseTicker *time.Ticker + chainFamily string + reportInterval time.Duration + deathDeclarationDelay time.Duration + + activeMu sync.RWMutex + activeNode Node[CHAIN_ID, RPC] + + chStop services.StopChan + wg sync.WaitGroup +} + +func NewMultiNode[ + CHAIN_ID ID, + RPC any, +]( + lggr logger.Logger, + selectionMode string, // type of the "best" RPC selector (e.g HighestHead, RoundRobin, etc.) + leaseDuration time.Duration, // defines interval on which new "best" RPC should be selected + primaryNodes []Node[CHAIN_ID, RPC], + sendOnlyNodes []SendOnlyNode[CHAIN_ID, RPC], + chainID CHAIN_ID, // configured chain ID (used to verify that passed primaryNodes belong to the same chain) + chainFamily string, // name of the chain family - used in the metrics + deathDeclarationDelay time.Duration, +) *MultiNode[CHAIN_ID, RPC] { + nodeSelector := newNodeSelector(selectionMode, primaryNodes) + // Prometheus' default interval is 15s, set this to under 7.5s to avoid + // aliasing (see: https://en.wikipedia.org/wiki/Nyquist_frequency) + const reportInterval = 6500 * time.Millisecond + c := &MultiNode[CHAIN_ID, RPC]{ + primaryNodes: primaryNodes, + sendOnlyNodes: sendOnlyNodes, + chainID: chainID, + lggr: logger.Sugared(lggr).Named("MultiNode").With("chainID", chainID.String()), + selectionMode: selectionMode, + nodeSelector: nodeSelector, + chStop: make(services.StopChan), + leaseDuration: leaseDuration, + chainFamily: chainFamily, + reportInterval: reportInterval, + deathDeclarationDelay: deathDeclarationDelay, + } + + c.lggr.Debugf("The MultiNode is configured to use NodeSelectionMode: %s", selectionMode) + + return c +} + +func (c *MultiNode[CHAIN_ID, RPC]) ChainID() CHAIN_ID { + return c.chainID +} + +func (c *MultiNode[CHAIN_ID, RPC]) DoAll(ctx context.Context, do func(ctx context.Context, rpc RPC, isSendOnly bool)) error { + var err error + ok := c.IfNotStopped(func() { + ctx, _ = c.chStop.Ctx(ctx) + + callsCompleted := 0 + for _, n := range c.primaryNodes { + select { + case <-ctx.Done(): + err = ctx.Err() + return + default: + if n.State() != NodeStateAlive { + continue + } + do(ctx, n.RPC(), false) + callsCompleted++ + } + } + if callsCompleted == 0 { + err = ErroringNodeError + } + + for _, n := range c.sendOnlyNodes { + select { + case <-ctx.Done(): + err = ctx.Err() + return + default: + if n.State() != NodeStateAlive { + continue + } + do(ctx, n.RPC(), true) + } + } + }) + if !ok { + return errors.New("MultiNode is stopped") + } + return err +} + +func (c *MultiNode[CHAIN_ID, RPC]) NodeStates() map[string]NodeState { + states := map[string]NodeState{} + for _, n := range c.primaryNodes { + states[n.String()] = n.State() + } + for _, n := range c.sendOnlyNodes { + states[n.String()] = n.State() + } + return states +} + +// Start starts every node in the pool +// +// Nodes handle their own redialing and runloops, so this function does not +// return any error if the nodes aren't available +func (c *MultiNode[CHAIN_ID, RPC]) Start(ctx context.Context) error { + return c.StartOnce("MultiNode", func() (merr error) { + if len(c.primaryNodes) == 0 { + return fmt.Errorf("no available nodes for chain %s", c.chainID.String()) + } + var ms services.MultiStart + for _, n := range c.primaryNodes { + if n.ConfiguredChainID().String() != c.chainID.String() { + return ms.CloseBecause(fmt.Errorf("node %s has configured chain ID %s which does not match multinode configured chain ID of %s", n.String(), n.ConfiguredChainID().String(), c.chainID.String())) + } + n.SetPoolChainInfoProvider(c) + // node will handle its own redialing and automatic recovery + if err := ms.Start(ctx, n); err != nil { + return err + } + } + for _, s := range c.sendOnlyNodes { + if s.ConfiguredChainID().String() != c.chainID.String() { + return ms.CloseBecause(fmt.Errorf("sendonly node %s has configured chain ID %s which does not match multinode configured chain ID of %s", s.String(), s.ConfiguredChainID().String(), c.chainID.String())) + } + if err := ms.Start(ctx, s); err != nil { + return err + } + } + c.wg.Add(1) + go c.runLoop() + + if c.leaseDuration.Seconds() > 0 && c.selectionMode != NodeSelectionModeRoundRobin { + c.lggr.Infof("The MultiNode will switch to best node every %s", c.leaseDuration.String()) + c.wg.Add(1) + go c.checkLeaseLoop() + } else { + c.lggr.Info("Best node switching is disabled") + } + + return nil + }) +} + +// Close tears down the MultiNode and closes all nodes +func (c *MultiNode[CHAIN_ID, RPC]) Close() error { + return c.StopOnce("MultiNode", func() error { + close(c.chStop) + c.wg.Wait() + + return services.CloseAll(services.MultiCloser(c.primaryNodes), services.MultiCloser(c.sendOnlyNodes)) + }) +} + +// SelectRPC returns an RPC of an active node. If there are no active nodes it returns an error. +// Call this method from your chain-specific client implementation to access any chain-specific rpc calls. +func (c *MultiNode[CHAIN_ID, RPC]) SelectRPC() (rpc RPC, err error) { + n, err := c.selectNode() + if err != nil { + return rpc, err + } + return n.RPC(), nil +} + +// selectNode returns the active Node, if it is still NodeStateAlive, otherwise it selects a new one from the NodeSelector. +func (c *MultiNode[CHAIN_ID, RPC]) selectNode() (node Node[CHAIN_ID, RPC], err error) { + c.activeMu.RLock() + node = c.activeNode + c.activeMu.RUnlock() + if node != nil && node.State() == NodeStateAlive { + return // still alive + } + + // select a new one + c.activeMu.Lock() + defer c.activeMu.Unlock() + node = c.activeNode + if node != nil && node.State() == NodeStateAlive { + return // another goroutine beat us here + } + + if c.activeNode != nil { + c.activeNode.UnsubscribeAllExceptAliveLoop() + } + c.activeNode = c.nodeSelector.Select() + + if c.activeNode == nil { + c.lggr.Criticalw("No live RPC nodes available", "NodeSelectionMode", c.nodeSelector.Name()) + errmsg := fmt.Errorf("no live nodes available for chain %s", c.chainID.String()) + c.SvcErrBuffer.Append(errmsg) + err = ErroringNodeError + } + + return c.activeNode, err +} + +// LatestChainInfo - returns number of live nodes available in the pool, so we can prevent the last alive node in a pool from being marked as out-of-sync. +// Return highest ChainInfo most recently received by the alive nodes. +// E.g. If Node A's the most recent block is 10 and highest 15 and for Node B it's - 12 and 14. This method will return 12. +func (c *MultiNode[CHAIN_ID, RPC]) LatestChainInfo() (int, ChainInfo) { + var nLiveNodes int + ch := ChainInfo{ + TotalDifficulty: big.NewInt(0), + } + for _, n := range c.primaryNodes { + if s, nodeChainInfo := n.StateAndLatest(); s == NodeStateAlive { + nLiveNodes++ + ch.BlockNumber = max(ch.BlockNumber, nodeChainInfo.BlockNumber) + ch.FinalizedBlockNumber = max(ch.FinalizedBlockNumber, nodeChainInfo.FinalizedBlockNumber) + ch.TotalDifficulty = MaxTotalDifficulty(ch.TotalDifficulty, nodeChainInfo.TotalDifficulty) + } + } + return nLiveNodes, ch +} + +// HighestUserObservations - returns highest ChainInfo ever observed by any user of the MultiNode +func (c *MultiNode[CHAIN_ID, RPC]) HighestUserObservations() ChainInfo { + ch := ChainInfo{ + TotalDifficulty: big.NewInt(0), + } + for _, n := range c.primaryNodes { + nodeChainInfo := n.HighestUserObservations() + ch.BlockNumber = max(ch.BlockNumber, nodeChainInfo.BlockNumber) + ch.FinalizedBlockNumber = max(ch.FinalizedBlockNumber, nodeChainInfo.FinalizedBlockNumber) + ch.TotalDifficulty = MaxTotalDifficulty(ch.TotalDifficulty, nodeChainInfo.TotalDifficulty) + } + return ch +} + +func (c *MultiNode[CHAIN_ID, RPC]) checkLease() { + bestNode := c.nodeSelector.Select() + for _, n := range c.primaryNodes { + // Terminate client subscriptions. Services are responsible for reconnecting, which will be routed to the new + // best node. Only terminate connections with more than 1 subscription to account for the aliveLoop subscription + if n.State() == NodeStateAlive && n != bestNode { + c.lggr.Infof("Switching to best node from %q to %q", n.String(), bestNode.String()) + n.UnsubscribeAllExceptAliveLoop() + } + } + + c.activeMu.Lock() + defer c.activeMu.Unlock() + if bestNode != c.activeNode { + if c.activeNode != nil { + c.activeNode.UnsubscribeAllExceptAliveLoop() + } + c.activeNode = bestNode + } +} + +func (c *MultiNode[CHAIN_ID, RPC]) checkLeaseLoop() { + defer c.wg.Done() + c.leaseTicker = time.NewTicker(c.leaseDuration) + defer c.leaseTicker.Stop() + + for { + select { + case <-c.leaseTicker.C: + c.checkLease() + case <-c.chStop: + return + } + } +} + +func (c *MultiNode[CHAIN_ID, RPC]) runLoop() { + defer c.wg.Done() + + nodeStates := make([]nodeWithState, len(c.primaryNodes)) + for i, n := range c.primaryNodes { + nodeStates[i] = nodeWithState{ + Node: n.String(), + State: n.State().String(), + DeadSince: nil, + } + } + + c.report(nodeStates) + + monitor := services.NewTicker(c.reportInterval) + defer monitor.Stop() + + for { + select { + case <-monitor.C: + c.report(nodeStates) + case <-c.chStop: + return + } + } +} + +type nodeWithState struct { + Node string + State string + DeadSince *time.Time +} + +func (c *MultiNode[CHAIN_ID, RPC]) report(nodesStateInfo []nodeWithState) { + start := time.Now() + var dead int + counts := make(map[NodeState]int) + for i, n := range c.primaryNodes { + state := n.State() + counts[state]++ + nodesStateInfo[i].State = state.String() + if state == NodeStateAlive { + nodesStateInfo[i].DeadSince = nil + continue + } + + if nodesStateInfo[i].DeadSince == nil { + nodesStateInfo[i].DeadSince = &start + } + + if start.Sub(*nodesStateInfo[i].DeadSince) >= c.deathDeclarationDelay { + dead++ + } + } + for _, state := range allNodeStates { + count := counts[state] + PromMultiNodeRPCNodeStates.WithLabelValues(c.chainFamily, c.chainID.String(), state.String()).Set(float64(count)) + } + + total := len(c.primaryNodes) + live := total - dead + c.lggr.Tracew(fmt.Sprintf("MultiNode state: %d/%d nodes are alive", live, total), "nodeStates", nodesStateInfo) + if total == dead { + rerr := fmt.Errorf("no primary nodes available: 0/%d nodes are alive", total) + c.lggr.Criticalw(rerr.Error(), "nodeStates", nodesStateInfo) + c.SvcErrBuffer.Append(rerr) + } else if dead > 0 { + c.lggr.Errorw(fmt.Sprintf("At least one primary node is dead: %d/%d nodes are alive", live, total), "nodeStates", nodesStateInfo) + } +} diff --git a/pkg/solana/client/multinode/node.go b/pkg/solana/client/multinode/node.go new file mode 100644 index 000000000..c3532b1a1 --- /dev/null +++ b/pkg/solana/client/multinode/node.go @@ -0,0 +1,331 @@ +package client + +import ( + "context" + "errors" + "fmt" + "net/url" + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/services" +) + +const QueryTimeout = 10 * time.Second + +var errInvalidChainID = errors.New("invalid chain id") + +var ( + promPoolRPCNodeVerifies = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_verifies", + Help: "The total number of chain ID verifications for the given RPC node", + }, []string{"network", "chainID", "nodeName"}) + promPoolRPCNodeVerifiesFailed = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_verifies_failed", + Help: "The total number of failed chain ID verifications for the given RPC node", + }, []string{"network", "chainID", "nodeName"}) + promPoolRPCNodeVerifiesSuccess = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_verifies_success", + Help: "The total number of successful chain ID verifications for the given RPC node", + }, []string{"network", "chainID", "nodeName"}) +) + +type NodeConfig interface { + PollFailureThreshold() uint32 + PollInterval() time.Duration + SelectionMode() string + SyncThreshold() uint32 + NodeIsSyncingEnabled() bool + FinalizedBlockPollInterval() time.Duration + EnforceRepeatableRead() bool + DeathDeclarationDelay() time.Duration +} + +type ChainConfig interface { + NodeNoNewHeadsThreshold() time.Duration + NoNewFinalizedHeadsThreshold() time.Duration + FinalityDepth() uint32 + FinalityTagEnabled() bool + FinalizedBlockOffset() uint32 +} + +type Node[ + CHAIN_ID ID, + RPC any, +] interface { + // State returns most accurate state of the Node on the moment of call. + // While some of the checks may be performed in the background and State may return cached value, critical, like + // `FinalizedBlockOutOfSync`, must be executed upon every call. + State() NodeState + // StateAndLatest returns nodeState with the latest ChainInfo observed by Node during current lifecycle. + StateAndLatest() (NodeState, ChainInfo) + // HighestUserObservations - returns highest ChainInfo ever observed by underlying RPC excluding results of health check requests + HighestUserObservations() ChainInfo + SetPoolChainInfoProvider(PoolChainInfoProvider) + // Name is a unique identifier for this node. + Name() string + // String - returns string representation of the node, useful for debugging (name + URLS used to connect to the RPC) + String() string + RPC() RPC + // UnsubscribeAllExceptAliveLoop - closes all subscriptions except the aliveLoop subscription + UnsubscribeAllExceptAliveLoop() + ConfiguredChainID() CHAIN_ID + // Order - returns priority order configured for the RPC + Order() int32 + // Start - starts health checks + Start(context.Context) error + Close() error +} + +type node[ + CHAIN_ID ID, + HEAD Head, + RPC RPCClient[CHAIN_ID, HEAD], +] struct { + services.StateMachine + lfcLog logger.Logger + name string + id int32 + chainID CHAIN_ID + nodePoolCfg NodeConfig + chainCfg ChainConfig + order int32 + chainFamily string + + ws url.URL + http *url.URL + + rpc RPC + + stateMu sync.RWMutex // protects state* fields + state NodeState + + poolInfoProvider PoolChainInfoProvider + + stopCh services.StopChan + // wg waits for subsidiary goroutines + wg sync.WaitGroup + + aliveLoopSub Subscription + finalizedBlockSub Subscription +} + +func NewNode[ + CHAIN_ID ID, + HEAD Head, + RPC RPCClient[CHAIN_ID, HEAD], +]( + nodeCfg NodeConfig, + chainCfg ChainConfig, + lggr logger.Logger, + wsuri url.URL, + httpuri *url.URL, + name string, + id int32, + chainID CHAIN_ID, + nodeOrder int32, + rpc RPC, + chainFamily string, +) Node[CHAIN_ID, RPC] { + n := new(node[CHAIN_ID, HEAD, RPC]) + n.name = name + n.id = id + n.chainID = chainID + n.nodePoolCfg = nodeCfg + n.chainCfg = chainCfg + n.ws = wsuri + n.order = nodeOrder + if httpuri != nil { + n.http = httpuri + } + n.stopCh = make(services.StopChan) + lggr = logger.Named(lggr, "Node") + lggr = logger.With(lggr, + "nodeTier", Primary.String(), + "nodeName", name, + "node", n.String(), + "chainID", chainID, + "nodeOrder", n.order, + ) + n.lfcLog = logger.Named(lggr, "Lifecycle") + n.rpc = rpc + n.chainFamily = chainFamily + return n +} + +func (n *node[CHAIN_ID, HEAD, RPC]) String() string { + s := fmt.Sprintf("(%s)%s:%s", Primary.String(), n.name, n.ws.String()) + if n.http != nil { + s = s + fmt.Sprintf(":%s", n.http.String()) + } + return s +} + +func (n *node[CHAIN_ID, HEAD, RPC]) ConfiguredChainID() (chainID CHAIN_ID) { + return n.chainID +} + +func (n *node[CHAIN_ID, HEAD, RPC]) Name() string { + return n.name +} + +func (n *node[CHAIN_ID, HEAD, RPC]) RPC() RPC { + return n.rpc +} + +// unsubscribeAllExceptAliveLoop is not thread-safe; it should only be called +// while holding the stateMu lock. +func (n *node[CHAIN_ID, HEAD, RPC]) unsubscribeAllExceptAliveLoop() { + aliveLoopSub := n.aliveLoopSub + finalizedBlockSub := n.finalizedBlockSub + n.rpc.UnsubscribeAllExcept(aliveLoopSub, finalizedBlockSub) +} + +func (n *node[CHAIN_ID, HEAD, RPC]) UnsubscribeAllExceptAliveLoop() { + n.stateMu.Lock() + defer n.stateMu.Unlock() + n.unsubscribeAllExceptAliveLoop() +} + +func (n *node[CHAIN_ID, HEAD, RPC]) Close() error { + return n.StopOnce(n.name, n.close) +} + +func (n *node[CHAIN_ID, HEAD, RPC]) close() error { + defer func() { + n.wg.Wait() + n.rpc.Close() + }() + + n.stateMu.Lock() + defer n.stateMu.Unlock() + + close(n.stopCh) + n.state = NodeStateClosed + return nil +} + +// Start dials and verifies the node +// Should only be called once in a node's lifecycle +// Return value is necessary to conform to interface but this will never +// actually return an error. +func (n *node[CHAIN_ID, HEAD, RPC]) Start(startCtx context.Context) error { + return n.StartOnce(n.name, func() error { + n.start(startCtx) + return nil + }) +} + +// start initially dials the node and verifies chain ID +// This spins off lifecycle goroutines. +// Not thread-safe. +// Node lifecycle is synchronous: only one goroutine should be running at a +// time. +func (n *node[CHAIN_ID, HEAD, RPC]) start(startCtx context.Context) { + if n.state != NodeStateUndialed { + panic(fmt.Sprintf("cannot dial node with state %v", n.state)) + } + + if err := n.rpc.Dial(startCtx); err != nil { + n.lfcLog.Errorw("Dial failed: Node is unreachable", "err", err) + n.declareUnreachable() + return + } + n.setState(NodeStateDialed) + + state := n.verifyConn(startCtx, n.lfcLog) + n.declareState(state) +} + +// verifyChainID checks that connection to the node matches the given chain ID +// Not thread-safe +// Pure verifyChainID: does not mutate node "state" field. +func (n *node[CHAIN_ID, HEAD, RPC]) verifyChainID(callerCtx context.Context, lggr logger.Logger) NodeState { + promPoolRPCNodeVerifies.WithLabelValues(n.chainFamily, n.chainID.String(), n.name).Inc() + promFailed := func() { + promPoolRPCNodeVerifiesFailed.WithLabelValues(n.chainFamily, n.chainID.String(), n.name).Inc() + } + + st := n.getCachedState() + switch st { + case NodeStateClosed: + // The node is already closed, and any subsequent transition is invalid. + // To make spotting such transitions a bit easier, return the invalid node state. + return NodeStateLen + case NodeStateDialed, NodeStateOutOfSync, NodeStateInvalidChainID, NodeStateSyncing: + default: + panic(fmt.Sprintf("cannot verify node in state %v", st)) + } + + var chainID CHAIN_ID + var err error + if chainID, err = n.rpc.ChainID(callerCtx); err != nil { + promFailed() + lggr.Errorw("Failed to verify chain ID for node", "err", err, "nodeState", n.getCachedState()) + return NodeStateUnreachable + } else if chainID.String() != n.chainID.String() { + promFailed() + err = fmt.Errorf( + "rpc ChainID doesn't match local chain ID: RPC ID=%s, local ID=%s, node name=%s: %w", + chainID.String(), + n.chainID.String(), + n.name, + errInvalidChainID, + ) + lggr.Errorw("Failed to verify RPC node; remote endpoint returned the wrong chain ID", "err", err, "nodeState", n.getCachedState()) + return NodeStateInvalidChainID + } + + promPoolRPCNodeVerifiesSuccess.WithLabelValues(n.chainFamily, n.chainID.String(), n.name).Inc() + + return NodeStateAlive +} + +// createVerifiedConn - establishes new connection with the RPC and verifies that it's valid: chainID matches, and it's not syncing. +// Returns desired state if one of the verifications fails. Otherwise, returns NodeStateAlive. +func (n *node[CHAIN_ID, HEAD, RPC]) createVerifiedConn(ctx context.Context, lggr logger.Logger) NodeState { + if err := n.rpc.Dial(ctx); err != nil { + n.lfcLog.Errorw("Dial failed: Node is unreachable", "err", err, "nodeState", n.getCachedState()) + return NodeStateUnreachable + } + + return n.verifyConn(ctx, lggr) +} + +// verifyConn - verifies that current connection is valid: chainID matches, and it's not syncing. +// Returns desired state if one of the verifications fails. Otherwise, returns NodeStateAlive. +func (n *node[CHAIN_ID, HEAD, RPC]) verifyConn(ctx context.Context, lggr logger.Logger) NodeState { + state := n.verifyChainID(ctx, lggr) + if state != NodeStateAlive { + return state + } + + if n.nodePoolCfg.NodeIsSyncingEnabled() { + isSyncing, err := n.rpc.IsSyncing(ctx) + if err != nil { + lggr.Errorw("Unexpected error while verifying RPC node synchronization status", "err", err, "nodeState", n.getCachedState()) + return NodeStateUnreachable + } + + if isSyncing { + lggr.Errorw("Verification failed: Node is syncing", "nodeState", n.getCachedState()) + return NodeStateSyncing + } + } + + return NodeStateAlive +} + +func (n *node[CHAIN_ID, HEAD, RPC]) Order() int32 { + return n.order +} + +func (n *node[CHAIN_ID, HEAD, RPC]) newCtx() (context.Context, context.CancelFunc) { + ctx, cancel := n.stopCh.NewCtx() + ctx = CtxAddHealthCheckFlag(ctx) + return ctx, cancel +} diff --git a/pkg/solana/client/multinode/node_fsm.go b/pkg/solana/client/multinode/node_fsm.go new file mode 100644 index 000000000..1111210c4 --- /dev/null +++ b/pkg/solana/client/multinode/node_fsm.go @@ -0,0 +1,370 @@ +package client + +import ( + "fmt" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var ( + promPoolRPCNodeTransitionsToAlive = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_num_transitions_to_alive", + Help: transitionString(NodeStateAlive), + }, []string{"chainID", "nodeName"}) + promPoolRPCNodeTransitionsToInSync = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_num_transitions_to_in_sync", + Help: fmt.Sprintf("%s to %s", transitionString(NodeStateOutOfSync), NodeStateAlive), + }, []string{"chainID", "nodeName"}) + promPoolRPCNodeTransitionsToOutOfSync = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_num_transitions_to_out_of_sync", + Help: transitionString(NodeStateOutOfSync), + }, []string{"chainID", "nodeName"}) + promPoolRPCNodeTransitionsToUnreachable = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_num_transitions_to_unreachable", + Help: transitionString(NodeStateUnreachable), + }, []string{"chainID", "nodeName"}) + promPoolRPCNodeTransitionsToInvalidChainID = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_num_transitions_to_invalid_chain_id", + Help: transitionString(NodeStateInvalidChainID), + }, []string{"chainID", "nodeName"}) + promPoolRPCNodeTransitionsToUnusable = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_num_transitions_to_unusable", + Help: transitionString(NodeStateUnusable), + }, []string{"chainID", "nodeName"}) + promPoolRPCNodeTransitionsToSyncing = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_num_transitions_to_syncing", + Help: transitionString(NodeStateSyncing), + }, []string{"chainID", "nodeName"}) +) + +// NodeState represents the current state of the node +// Node is a FSM (finite state machine) +type NodeState int + +func (n NodeState) String() string { + switch n { + case NodeStateUndialed: + return "Undialed" + case NodeStateDialed: + return "Dialed" + case NodeStateInvalidChainID: + return "InvalidChainID" + case NodeStateAlive: + return "Alive" + case NodeStateUnreachable: + return "Unreachable" + case NodeStateUnusable: + return "Unusable" + case NodeStateOutOfSync: + return "OutOfSync" + case NodeStateClosed: + return "Closed" + case NodeStateSyncing: + return "Syncing" + case NodeStateFinalizedBlockOutOfSync: + return "FinalizedBlockOutOfSync" + default: + return fmt.Sprintf("NodeState(%d)", n) + } +} + +// GoString prints a prettier state +func (n NodeState) GoString() string { + return fmt.Sprintf("NodeState%s(%d)", n.String(), n) +} + +const ( + // NodeStateUndialed is the first state of a virgin node + NodeStateUndialed = NodeState(iota) + // NodeStateDialed is after a node has successfully dialed but before it has verified the correct chain ID + NodeStateDialed + // NodeStateInvalidChainID is after chain ID verification failed + NodeStateInvalidChainID + // NodeStateAlive is a healthy node after chain ID verification succeeded + NodeStateAlive + // NodeStateUnreachable is a node that cannot be dialed or has disconnected + NodeStateUnreachable + // NodeStateOutOfSync is a node that is accepting connections but exceeded + // the failure threshold without sending any new heads. It will be + // disconnected, then put into a revive loop and re-awakened after redial + // if a new head arrives + NodeStateOutOfSync + // NodeStateUnusable is a sendonly node that has an invalid URL that can never be reached + NodeStateUnusable + // NodeStateClosed is after the connection has been closed and the node is at the end of its lifecycle + NodeStateClosed + // NodeStateSyncing is a node that is actively back-filling blockchain. Usually, it's a newly set up node that is + // still syncing the chain. The main difference from `NodeStateOutOfSync` is that it represents state relative + // to other primary nodes configured in the MultiNode. In contrast, `NodeStateSyncing` represents the internal state of + // the node (RPC). + NodeStateSyncing + // nodeStateFinalizedBlockOutOfSync - node is lagging behind on latest finalized block + NodeStateFinalizedBlockOutOfSync + // nodeStateLen tracks the number of states + NodeStateLen +) + +// allNodeStates represents all possible states a node can be in +var allNodeStates []NodeState + +func init() { + for s := NodeState(0); s < NodeStateLen; s++ { + allNodeStates = append(allNodeStates, s) + } +} + +// FSM methods + +// State allows reading the current state of the node. +func (n *node[CHAIN_ID, HEAD, RPC]) State() NodeState { + n.stateMu.RLock() + defer n.stateMu.RUnlock() + return n.recalculateState() +} + +func (n *node[CHAIN_ID, HEAD, RPC]) getCachedState() NodeState { + n.stateMu.RLock() + defer n.stateMu.RUnlock() + return n.state +} + +func (n *node[CHAIN_ID, HEAD, RPC]) recalculateState() NodeState { + if n.state != NodeStateAlive { + return n.state + } + + // double check that node is not lagging on finalized block + if n.nodePoolCfg.EnforceRepeatableRead() && n.isFinalizedBlockOutOfSync() { + return NodeStateFinalizedBlockOutOfSync + } + + return NodeStateAlive +} + +func (n *node[CHAIN_ID, HEAD, RPC]) isFinalizedBlockOutOfSync() bool { + if n.poolInfoProvider == nil { + return false + } + + highestObservedByCaller := n.poolInfoProvider.HighestUserObservations() + latest, _ := n.rpc.GetInterceptedChainInfo() + if n.chainCfg.FinalityTagEnabled() { + return latest.FinalizedBlockNumber < highestObservedByCaller.FinalizedBlockNumber-int64(n.chainCfg.FinalizedBlockOffset()) + } + + return latest.BlockNumber < highestObservedByCaller.BlockNumber-int64(n.chainCfg.FinalizedBlockOffset()) +} + +// StateAndLatest returns nodeState with the latest ChainInfo observed by Node during current lifecycle. +func (n *node[CHAIN_ID, HEAD, RPC]) StateAndLatest() (NodeState, ChainInfo) { + n.stateMu.RLock() + defer n.stateMu.RUnlock() + latest, _ := n.rpc.GetInterceptedChainInfo() + return n.recalculateState(), latest +} + +// HighestUserObservations - returns highest ChainInfo ever observed by external user of the Node +func (n *node[CHAIN_ID, HEAD, RPC]) HighestUserObservations() ChainInfo { + _, highestUserObservations := n.rpc.GetInterceptedChainInfo() + return highestUserObservations +} +func (n *node[CHAIN_ID, HEAD, RPC]) SetPoolChainInfoProvider(poolInfoProvider PoolChainInfoProvider) { + n.poolInfoProvider = poolInfoProvider +} + +// setState is only used by internal state management methods. +// This is low-level; care should be taken by the caller to ensure the new state is a valid transition. +// State changes should always be synchronous: only one goroutine at a time should change state. +// n.stateMu should not be locked for long periods of time because external clients expect a timely response from n.State() +func (n *node[CHAIN_ID, HEAD, RPC]) setState(s NodeState) { + n.stateMu.Lock() + defer n.stateMu.Unlock() + n.state = s +} + +// declareXXX methods change the state and pass conrol off the new state +// management goroutine + +func (n *node[CHAIN_ID, HEAD, RPC]) declareAlive() { + n.transitionToAlive(func() { + n.lfcLog.Infow("RPC Node is online", "nodeState", n.state) + n.wg.Add(1) + go n.aliveLoop() + }) +} + +func (n *node[CHAIN_ID, HEAD, RPC]) transitionToAlive(fn func()) { + promPoolRPCNodeTransitionsToAlive.WithLabelValues(n.chainID.String(), n.name).Inc() + n.stateMu.Lock() + defer n.stateMu.Unlock() + if n.state == NodeStateClosed { + return + } + switch n.state { + case NodeStateDialed, NodeStateInvalidChainID, NodeStateSyncing: + n.state = NodeStateAlive + default: + panic(transitionFail(n.state, NodeStateAlive)) + } + fn() +} + +// declareInSync puts a node back into Alive state, allowing it to be used by +// pool consumers again +func (n *node[CHAIN_ID, HEAD, RPC]) declareInSync() { + n.transitionToInSync(func() { + n.lfcLog.Infow("RPC Node is back in sync", "nodeState", n.state) + n.wg.Add(1) + go n.aliveLoop() + }) +} + +func (n *node[CHAIN_ID, HEAD, RPC]) transitionToInSync(fn func()) { + promPoolRPCNodeTransitionsToAlive.WithLabelValues(n.chainID.String(), n.name).Inc() + promPoolRPCNodeTransitionsToInSync.WithLabelValues(n.chainID.String(), n.name).Inc() + n.stateMu.Lock() + defer n.stateMu.Unlock() + if n.state == NodeStateClosed { + return + } + switch n.state { + case NodeStateOutOfSync, NodeStateSyncing: + n.state = NodeStateAlive + default: + panic(transitionFail(n.state, NodeStateAlive)) + } + fn() +} + +// declareOutOfSync puts a node into OutOfSync state, disconnecting all current +// clients and making it unavailable for use until back in-sync. +func (n *node[CHAIN_ID, HEAD, RPC]) declareOutOfSync(syncIssues syncStatus) { + n.transitionToOutOfSync(func() { + n.lfcLog.Errorw("RPC Node is out of sync", "nodeState", n.state, "syncIssues", syncIssues) + n.wg.Add(1) + go n.outOfSyncLoop(syncIssues) + }) +} + +func (n *node[CHAIN_ID, HEAD, RPC]) transitionToOutOfSync(fn func()) { + promPoolRPCNodeTransitionsToOutOfSync.WithLabelValues(n.chainID.String(), n.name).Inc() + n.stateMu.Lock() + defer n.stateMu.Unlock() + if n.state == NodeStateClosed { + return + } + switch n.state { + case NodeStateAlive: + n.unsubscribeAllExceptAliveLoop() + n.state = NodeStateOutOfSync + default: + panic(transitionFail(n.state, NodeStateOutOfSync)) + } + fn() +} + +func (n *node[CHAIN_ID, HEAD, RPC]) declareUnreachable() { + n.transitionToUnreachable(func() { + n.lfcLog.Errorw("RPC Node is unreachable", "nodeState", n.state) + n.wg.Add(1) + go n.unreachableLoop() + }) +} + +func (n *node[CHAIN_ID, HEAD, RPC]) transitionToUnreachable(fn func()) { + promPoolRPCNodeTransitionsToUnreachable.WithLabelValues(n.chainID.String(), n.name).Inc() + n.stateMu.Lock() + defer n.stateMu.Unlock() + if n.state == NodeStateClosed { + return + } + switch n.state { + case NodeStateUndialed, NodeStateDialed, NodeStateAlive, NodeStateOutOfSync, NodeStateInvalidChainID, NodeStateSyncing: + n.unsubscribeAllExceptAliveLoop() + n.state = NodeStateUnreachable + default: + panic(transitionFail(n.state, NodeStateUnreachable)) + } + fn() +} + +func (n *node[CHAIN_ID, HEAD, RPC]) declareState(state NodeState) { + if n.getCachedState() == NodeStateClosed { + return + } + switch state { + case NodeStateInvalidChainID: + n.declareInvalidChainID() + case NodeStateUnreachable: + n.declareUnreachable() + case NodeStateSyncing: + n.declareSyncing() + case NodeStateAlive: + n.declareAlive() + default: + panic(fmt.Sprintf("%#v state declaration is not implemented", state)) + } +} + +func (n *node[CHAIN_ID, HEAD, RPC]) declareInvalidChainID() { + n.transitionToInvalidChainID(func() { + n.lfcLog.Errorw("RPC Node has the wrong chain ID", "nodeState", n.state) + n.wg.Add(1) + go n.invalidChainIDLoop() + }) +} + +func (n *node[CHAIN_ID, HEAD, RPC]) transitionToInvalidChainID(fn func()) { + promPoolRPCNodeTransitionsToInvalidChainID.WithLabelValues(n.chainID.String(), n.name).Inc() + n.stateMu.Lock() + defer n.stateMu.Unlock() + if n.state == NodeStateClosed { + return + } + switch n.state { + case NodeStateDialed, NodeStateOutOfSync, NodeStateSyncing: + n.unsubscribeAllExceptAliveLoop() + n.state = NodeStateInvalidChainID + default: + panic(transitionFail(n.state, NodeStateInvalidChainID)) + } + fn() +} + +func (n *node[CHAIN_ID, HEAD, RPC]) declareSyncing() { + n.transitionToSyncing(func() { + n.lfcLog.Errorw("RPC Node is syncing", "nodeState", n.state) + n.wg.Add(1) + go n.syncingLoop() + }) +} + +func (n *node[CHAIN_ID, HEAD, RPC]) transitionToSyncing(fn func()) { + promPoolRPCNodeTransitionsToSyncing.WithLabelValues(n.chainID.String(), n.name).Inc() + n.stateMu.Lock() + defer n.stateMu.Unlock() + if n.state == NodeStateClosed { + return + } + switch n.state { + case NodeStateDialed, NodeStateOutOfSync, NodeStateInvalidChainID: + n.unsubscribeAllExceptAliveLoop() + n.state = NodeStateSyncing + default: + panic(transitionFail(n.state, NodeStateSyncing)) + } + + if !n.nodePoolCfg.NodeIsSyncingEnabled() { + panic("unexpected transition to NodeStateSyncing, while it's disabled") + } + fn() +} + +func transitionString(state NodeState) string { + return fmt.Sprintf("Total number of times node has transitioned to %s", state) +} + +func transitionFail(from NodeState, to NodeState) string { + return fmt.Sprintf("cannot transition from %#v to %#v", from, to) +} diff --git a/pkg/solana/client/multinode/node_lifecycle.go b/pkg/solana/client/multinode/node_lifecycle.go new file mode 100644 index 000000000..823a1abc3 --- /dev/null +++ b/pkg/solana/client/multinode/node_lifecycle.go @@ -0,0 +1,732 @@ +package client + +import ( + "context" + "fmt" + "math" + "math/big" + "time" + + "github.com/smartcontractkit/chainlink/v2/common/types" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/utils" + bigmath "github.com/smartcontractkit/chainlink-common/pkg/utils/big_math" + + iutils "github.com/smartcontractkit/chainlink/v2/common/internal/utils" +) + +var ( + promPoolRPCNodeHighestSeenBlock = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "pool_rpc_node_highest_seen_block", + Help: "The highest seen block for the given RPC node", + }, []string{"chainID", "nodeName"}) + promPoolRPCNodeHighestFinalizedBlock = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "pool_rpc_node_highest_finalized_block", + Help: "The highest seen finalized block for the given RPC node", + }, []string{"chainID", "nodeName"}) + promPoolRPCNodeNumSeenBlocks = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_num_seen_blocks", + Help: "The total number of new blocks seen by the given RPC node", + }, []string{"chainID", "nodeName"}) + promPoolRPCNodePolls = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_polls_total", + Help: "The total number of poll checks for the given RPC node", + }, []string{"chainID", "nodeName"}) + promPoolRPCNodePollsFailed = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_polls_failed", + Help: "The total number of failed poll checks for the given RPC node", + }, []string{"chainID", "nodeName"}) + promPoolRPCNodePollsSuccess = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_polls_success", + Help: "The total number of successful poll checks for the given RPC node", + }, []string{"chainID", "nodeName"}) +) + +// zombieNodeCheckInterval controls how often to re-check to see if we need to +// state change in case we have to force a state transition due to no available +// nodes. +// NOTE: This only applies to out-of-sync nodes if they are the last available node +func zombieNodeCheckInterval(noNewHeadsThreshold time.Duration) time.Duration { + interval := noNewHeadsThreshold + if interval <= 0 || interval > QueryTimeout { + interval = QueryTimeout + } + return utils.WithJitter(interval) +} + +const ( + msgCannotDisable = "but cannot disable this connection because there are no other RPC endpoints, or all other RPC endpoints are dead." + msgDegradedState = "Chainlink is now operating in a degraded state and urgent action is required to resolve the issue" +) + +// Node is a FSM +// Each state has a loop that goes with it, which monitors the node and moves it into another state as necessary. +// Only one loop must run at a time. +// Each loop passes control onto the next loop as it exits, except when the node is Closed which terminates the loop permanently. + +// This handles node lifecycle for the ALIVE state +// Should only be run ONCE per node, after a successful Dial +func (n *node[CHAIN_ID, HEAD, RPC]) aliveLoop() { + defer n.wg.Done() + ctx, cancel := n.newCtx() + defer cancel() + + { + // sanity check + state := n.getCachedState() + switch state { + case NodeStateAlive: + case NodeStateClosed: + return + default: + panic(fmt.Sprintf("aliveLoop can only run for node in Alive state, got: %s", state)) + } + } + + noNewHeadsTimeoutThreshold := n.chainCfg.NodeNoNewHeadsThreshold() + noNewFinalizedBlocksTimeoutThreshold := n.chainCfg.NoNewFinalizedHeadsThreshold() + pollFailureThreshold := n.nodePoolCfg.PollFailureThreshold() + pollInterval := n.nodePoolCfg.PollInterval() + + lggr := logger.Sugared(n.lfcLog).Named("Alive").With("noNewHeadsTimeoutThreshold", noNewHeadsTimeoutThreshold, "pollInterval", pollInterval, "pollFailureThreshold", pollFailureThreshold) + lggr.Tracew("Alive loop starting", "nodeState", n.getCachedState()) + + headsSub, err := n.registerNewSubscription(ctx, lggr.With("subscriptionType", "heads"), + n.chainCfg.NodeNoNewHeadsThreshold(), n.rpc.SubscribeToHeads) + if err != nil { + lggr.Errorw("Initial subscribe for heads failed", "nodeState", n.getCachedState(), "err", err) + n.declareUnreachable() + return + } + + n.stateMu.Lock() + n.aliveLoopSub = headsSub.sub + n.stateMu.Unlock() + defer func() { + defer headsSub.sub.Unsubscribe() + n.stateMu.Lock() + n.aliveLoopSub = nil + n.stateMu.Unlock() + }() + + var pollCh <-chan time.Time + if pollInterval > 0 { + lggr.Debug("Polling enabled") + pollT := time.NewTicker(pollInterval) + defer pollT.Stop() + pollCh = pollT.C + if pollFailureThreshold > 0 { + // polling can be enabled with no threshold to enable polling but + // the node will not be marked offline regardless of the number of + // poll failures + lggr.Debug("Polling liveness checking enabled") + } + } else { + lggr.Debug("Polling disabled") + } + + var finalizedHeadsSub headSubscription[HEAD] + if n.chainCfg.FinalityTagEnabled() { + finalizedHeadsSub, err = n.registerNewSubscription(ctx, lggr.With("subscriptionType", "finalizedHeads"), + n.chainCfg.NoNewFinalizedHeadsThreshold(), n.rpc.SubscribeToFinalizedHeads) + if err != nil { + lggr.Errorw("Failed to subscribe to finalized heads", "err", err) + n.declareUnreachable() + return + } + + n.stateMu.Lock() + n.finalizedBlockSub = finalizedHeadsSub.sub + n.stateMu.Unlock() + defer func() { + finalizedHeadsSub.Unsubscribe() + n.stateMu.Lock() + n.finalizedBlockSub = nil + n.stateMu.Unlock() + }() + } + + localHighestChainInfo, _ := n.rpc.GetInterceptedChainInfo() + var pollFailures uint32 + + for { + select { + case <-ctx.Done(): + return + case <-pollCh: + promPoolRPCNodePolls.WithLabelValues(n.chainID.String(), n.name).Inc() + lggr.Tracew("Pinging RPC", "nodeState", n.State(), "pollFailures", pollFailures) + pollCtx, cancel := context.WithTimeout(ctx, pollInterval) + err = n.RPC().Ping(pollCtx) + cancel() + if err != nil { + // prevent overflow + if pollFailures < math.MaxUint32 { + promPoolRPCNodePollsFailed.WithLabelValues(n.chainID.String(), n.name).Inc() + pollFailures++ + } + lggr.Warnw(fmt.Sprintf("Poll failure, RPC endpoint %s failed to respond properly", n.String()), "err", err, "pollFailures", pollFailures, "nodeState", n.getCachedState()) + } else { + lggr.Debugw("Ping successful", "nodeState", n.State()) + promPoolRPCNodePollsSuccess.WithLabelValues(n.chainID.String(), n.name).Inc() + pollFailures = 0 + } + if pollFailureThreshold > 0 && pollFailures >= pollFailureThreshold { + lggr.Errorw(fmt.Sprintf("RPC endpoint failed to respond to %d consecutive polls", pollFailures), "pollFailures", pollFailures, "nodeState", n.getCachedState()) + if n.poolInfoProvider != nil { + if l, _ := n.poolInfoProvider.LatestChainInfo(); l < 2 { + lggr.Criticalf("RPC endpoint failed to respond to polls; %s %s", msgCannotDisable, msgDegradedState) + continue + } + } + n.declareUnreachable() + return + } + _, latestChainInfo := n.StateAndLatest() + if outOfSync, liveNodes := n.syncStatus(latestChainInfo.BlockNumber, latestChainInfo.TotalDifficulty); outOfSync { + // note: there must be another live node for us to be out of sync + lggr.Errorw("RPC endpoint has fallen behind", "blockNumber", latestChainInfo.BlockNumber, "totalDifficulty", latestChainInfo.TotalDifficulty, "nodeState", n.getCachedState()) + if liveNodes < 2 { + lggr.Criticalf("RPC endpoint has fallen behind; %s %s", msgCannotDisable, msgDegradedState) + continue + } + n.declareOutOfSync(syncStatusNotInSyncWithPool) + return + } + case bh, open := <-headsSub.Heads: + if !open { + lggr.Errorw("Subscription channel unexpectedly closed", "nodeState", n.getCachedState()) + n.declareUnreachable() + return + } + receivedNewHead := n.onNewHead(lggr, &localHighestChainInfo, bh) + if receivedNewHead && noNewHeadsTimeoutThreshold > 0 { + headsSub.ResetTimer(noNewHeadsTimeoutThreshold) + } + case err = <-headsSub.Errors: + lggr.Errorw("Subscription was terminated", "err", err, "nodeState", n.getCachedState()) + n.declareUnreachable() + return + case <-headsSub.NoNewHeads: + // We haven't received a head on the channel for at least the + // threshold amount of time, mark it broken + lggr.Errorw(fmt.Sprintf("RPC endpoint detected out of sync; no new heads received for %s (last head received was %v)", noNewHeadsTimeoutThreshold, localHighestChainInfo.BlockNumber), "nodeState", n.getCachedState(), "latestReceivedBlockNumber", localHighestChainInfo.BlockNumber, "noNewHeadsTimeoutThreshold", noNewHeadsTimeoutThreshold) + if n.poolInfoProvider != nil { + if l, _ := n.poolInfoProvider.LatestChainInfo(); l < 2 { + lggr.Criticalf("RPC endpoint detected out of sync; %s %s", msgCannotDisable, msgDegradedState) + // We don't necessarily want to wait the full timeout to check again, we should + // check regularly and log noisily in this state + headsSub.ResetTimer(zombieNodeCheckInterval(noNewHeadsTimeoutThreshold)) + continue + } + } + n.declareOutOfSync(syncStatusNoNewHead) + return + case latestFinalized, open := <-finalizedHeadsSub.Heads: + if !open { + lggr.Errorw("Finalized heads subscription channel unexpectedly closed") + n.declareUnreachable() + return + } + if !latestFinalized.IsValid() { + lggr.Warn("Latest finalized block is not valid") + continue + } + + latestFinalizedBN := latestFinalized.BlockNumber() + if latestFinalizedBN > localHighestChainInfo.FinalizedBlockNumber { + promPoolRPCNodeHighestFinalizedBlock.WithLabelValues(n.chainID.String(), n.name).Set(float64(latestFinalizedBN)) + localHighestChainInfo.FinalizedBlockNumber = latestFinalizedBN + } + + case <-finalizedHeadsSub.NoNewHeads: + // We haven't received a finalized head on the channel for at least the + // threshold amount of time, mark it broken + lggr.Errorw(fmt.Sprintf("RPC's finalized state is out of sync; no new finalized heads received for %s (last finalized head received was %v)", noNewFinalizedBlocksTimeoutThreshold, localHighestChainInfo.FinalizedBlockNumber), "latestReceivedBlockNumber", localHighestChainInfo.BlockNumber) + if n.poolInfoProvider != nil { + if l, _ := n.poolInfoProvider.LatestChainInfo(); l < 2 { + lggr.Criticalf("RPC's finalized state is out of sync; %s %s", msgCannotDisable, msgDegradedState) + // We don't necessarily want to wait the full timeout to check again, we should + // check regularly and log noisily in this state + finalizedHeadsSub.ResetTimer(zombieNodeCheckInterval(noNewFinalizedBlocksTimeoutThreshold)) + continue + } + } + n.declareOutOfSync(syncStatusNoNewFinalizedHead) + return + case <-finalizedHeadsSub.Errors: + lggr.Errorw("Finalized heads subscription was terminated", "err", err) + n.declareUnreachable() + return + } + } +} + +type headSubscription[HEAD any] struct { + Heads <-chan HEAD + Errors <-chan error + NoNewHeads <-chan time.Time + + noNewHeadsTicker *time.Ticker + sub types.Subscription + cleanUpTasks []func() +} + +func (sub *headSubscription[HEAD]) ResetTimer(duration time.Duration) { + sub.noNewHeadsTicker.Reset(duration) +} + +func (sub *headSubscription[HEAD]) Unsubscribe() { + for _, doCleanUp := range sub.cleanUpTasks { + doCleanUp() + } +} + +func (n *node[CHAIN_ID, HEAD, PRC]) registerNewSubscription(ctx context.Context, lggr logger.SugaredLogger, + noNewDataThreshold time.Duration, newSub func(ctx context.Context) (<-chan HEAD, types.Subscription, error)) (headSubscription[HEAD], error) { + result := headSubscription[HEAD]{} + var err error + var sub types.Subscription + result.Heads, sub, err = newSub(ctx) + if err != nil { + return result, err + } + + result.Errors = sub.Err() + lggr.Debug("Successfully subscribed") + + // TODO: will be removed as part of merging effort with BCI-2875 + result.sub = sub + //n.stateMu.Lock() + //n.healthCheckSubs = append(n.healthCheckSubs, sub) + //n.stateMu.Unlock() + + result.cleanUpTasks = append(result.cleanUpTasks, sub.Unsubscribe) + + if noNewDataThreshold > 0 { + lggr.Debugw("Subscription liveness checking enabled") + result.noNewHeadsTicker = time.NewTicker(noNewDataThreshold) + result.NoNewHeads = result.noNewHeadsTicker.C + result.cleanUpTasks = append(result.cleanUpTasks, result.noNewHeadsTicker.Stop) + } else { + lggr.Debug("Subscription liveness checking disabled") + } + + return result, nil +} + +func (n *node[CHAIN_ID, HEAD, RPC]) onNewFinalizedHead(lggr logger.SugaredLogger, chainInfo *ChainInfo, latestFinalized HEAD) bool { + if !latestFinalized.IsValid() { + lggr.Warn("Latest finalized block is not valid") + return false + } + + latestFinalizedBN := latestFinalized.BlockNumber() + lggr.Tracew("Got latest finalized head", "latestFinalized", latestFinalized) + if latestFinalizedBN <= chainInfo.FinalizedBlockNumber { + lggr.Tracew("Ignoring previously seen finalized block number") + return false + } + + promPoolRPCNodeHighestFinalizedBlock.WithLabelValues(n.chainID.String(), n.name).Set(float64(latestFinalizedBN)) + chainInfo.FinalizedBlockNumber = latestFinalizedBN + return true +} + +func (n *node[CHAIN_ID, HEAD, RPC]) onNewHead(lggr logger.SugaredLogger, chainInfo *ChainInfo, head HEAD) bool { + if !head.IsValid() { + lggr.Warn("Latest head is not valid") + return false + } + + promPoolRPCNodeNumSeenBlocks.WithLabelValues(n.chainID.String(), n.name).Inc() + lggr.Tracew("Got head", "head", head) + lggr = lggr.With("latestReceivedBlockNumber", chainInfo.BlockNumber, "blockNumber", head.BlockNumber(), "nodeState", n.getCachedState()) + if head.BlockNumber() <= chainInfo.BlockNumber { + lggr.Tracew("Ignoring previously seen block number") + return false + } + + promPoolRPCNodeHighestSeenBlock.WithLabelValues(n.chainID.String(), n.name).Set(float64(head.BlockNumber())) + chainInfo.BlockNumber = head.BlockNumber() + + if !n.chainCfg.FinalityTagEnabled() { + latestFinalizedBN := max(head.BlockNumber()-int64(n.chainCfg.FinalityDepth()), 0) + if latestFinalizedBN > chainInfo.FinalizedBlockNumber { + promPoolRPCNodeHighestFinalizedBlock.WithLabelValues(n.chainID.String(), n.name).Set(float64(latestFinalizedBN)) + chainInfo.FinalizedBlockNumber = latestFinalizedBN + } + } + + return true +} + +// syncStatus returns outOfSync true if num or td is more than SyncThresold behind the best node. +// Always returns outOfSync false for SyncThreshold 0. +// liveNodes is only included when outOfSync is true. +func (n *node[CHAIN_ID, HEAD, RPC]) syncStatus(num int64, td *big.Int) (outOfSync bool, liveNodes int) { + if n.poolInfoProvider == nil { + return // skip for tests + } + threshold := n.nodePoolCfg.SyncThreshold() + if threshold == 0 { + return // disabled + } + // Check against best node + ln, ci := n.poolInfoProvider.LatestChainInfo() + mode := n.nodePoolCfg.SelectionMode() + switch mode { + case NodeSelectionModeHighestHead, NodeSelectionModeRoundRobin, NodeSelectionModePriorityLevel: + return num < ci.BlockNumber-int64(threshold), ln + case NodeSelectionModeTotalDifficulty: + bigThreshold := big.NewInt(int64(threshold)) + return td.Cmp(bigmath.Sub(ci.TotalDifficulty, bigThreshold)) < 0, ln + default: + panic("unrecognized NodeSelectionMode: " + mode) + } +} + +const ( + msgReceivedBlock = "Received block for RPC node, waiting until back in-sync to mark as live again" + msgReceivedFinalizedBlock = "Received new finalized block for RPC node, waiting until back in-sync to mark as live again" + msgInSync = "RPC node back in sync" +) + +// isOutOfSyncWithPool returns outOfSync true if num or td is more than SyncThresold behind the best node. +// Always returns outOfSync false for SyncThreshold 0. +// liveNodes is only included when outOfSync is true. +func (n *node[CHAIN_ID, HEAD, RPC]) isOutOfSyncWithPool(localState ChainInfo) (outOfSync bool, liveNodes int) { + if n.poolInfoProvider == nil { + n.lfcLog.Warn("skipping sync state against the pool - should only occur in tests") + return // skip for tests + } + threshold := n.nodePoolCfg.SyncThreshold() + if threshold == 0 { + return // disabled + } + // Check against best node + ln, ci := n.poolInfoProvider.LatestChainInfo() + mode := n.nodePoolCfg.SelectionMode() + switch mode { + case NodeSelectionModeHighestHead, NodeSelectionModeRoundRobin, NodeSelectionModePriorityLevel: + return localState.BlockNumber < ci.BlockNumber-int64(threshold), ln + case NodeSelectionModeTotalDifficulty: + bigThreshold := big.NewInt(int64(threshold)) + return localState.TotalDifficulty.Cmp(bigmath.Sub(ci.TotalDifficulty, bigThreshold)) < 0, ln + default: + panic("unrecognized NodeSelectionMode: " + mode) + } +} + +// outOfSyncLoop takes an OutOfSync node and waits until isOutOfSync returns false to go back to live status +func (n *node[CHAIN_ID, HEAD, RPC]) outOfSyncLoop(syncIssues syncStatus) { + defer n.wg.Done() + ctx, cancel := n.newCtx() + defer cancel() + + { + // sanity check + state := n.getCachedState() + switch state { + case NodeStateOutOfSync: + case NodeStateClosed: + return + default: + panic(fmt.Sprintf("outOfSyncLoop can only run for node in OutOfSync state, got: %s", state)) + } + } + + outOfSyncAt := time.Now() + + // set logger name to OutOfSync or FinalizedBlockOutOfSync + lggr := logger.Sugared(logger.Named(n.lfcLog, n.getCachedState().String())).With("nodeState", n.getCachedState()) + lggr.Debugw("Trying to revive out-of-sync RPC node") + + // Need to redial since out-of-sync nodes are automatically disconnected + state := n.createVerifiedConn(ctx, lggr) + if state != NodeStateAlive { + n.declareState(state) + return + } + + noNewHeadsTimeoutThreshold := n.chainCfg.NodeNoNewHeadsThreshold() + headsSub, err := n.registerNewSubscription(ctx, lggr.With("subscriptionType", "heads"), + noNewHeadsTimeoutThreshold, n.rpc.SubscribeToHeads) + if err != nil { + lggr.Errorw("Failed to subscribe heads on out-of-sync RPC node", "err", err) + n.declareUnreachable() + return + } + + lggr.Tracew("Successfully subscribed to heads feed on out-of-sync RPC node") + defer headsSub.Unsubscribe() + + noNewFinalizedBlocksTimeoutThreshold := n.chainCfg.NoNewFinalizedHeadsThreshold() + var finalizedHeadsSub headSubscription[HEAD] + if n.chainCfg.FinalityTagEnabled() { + finalizedHeadsSub, err = n.registerNewSubscription(ctx, lggr.With("subscriptionType", "finalizedHeads"), + noNewFinalizedBlocksTimeoutThreshold, n.rpc.SubscribeToFinalizedHeads) + if err != nil { + lggr.Errorw("Subscribe to finalized heads failed on out-of-sync RPC node", "err", err) + n.declareUnreachable() + return + } + + lggr.Tracew("Successfully subscribed to finalized heads feed on out-of-sync RPC node") + defer finalizedHeadsSub.Unsubscribe() + } + + _, localHighestChainInfo := n.rpc.GetInterceptedChainInfo() + for { + if syncIssues == syncStatusSynced { + // back in-sync! flip back into alive loop + lggr.Infow(fmt.Sprintf("%s: %s. Node was out-of-sync for %s", msgInSync, n.String(), time.Since(outOfSyncAt))) + n.declareInSync() + return + } + + select { + case <-ctx.Done(): + return + case head, open := <-headsSub.Heads: + if !open { + lggr.Errorw("Subscription channel unexpectedly closed", "nodeState", n.getCachedState()) + n.declareUnreachable() + return + } + + if !n.onNewHead(lggr, &localHighestChainInfo, head) { + continue + } + + // received a new head - clear NoNewHead flag + syncIssues &= ^syncStatusNoNewHead + if outOfSync, _ := n.isOutOfSyncWithPool(localHighestChainInfo); !outOfSync { + // we caught up with the pool - clear NotInSyncWithPool flag + syncIssues &= ^syncStatusNotInSyncWithPool + } else { + // we've received new head, but lagging behind the pool, add NotInSyncWithPool flag to prevent false transition to alive + syncIssues |= syncStatusNotInSyncWithPool + } + + if noNewHeadsTimeoutThreshold > 0 { + headsSub.ResetTimer(noNewHeadsTimeoutThreshold) + } + + lggr.Debugw(msgReceivedBlock, "blockNumber", head.BlockNumber(), "blockDifficulty", head.BlockDifficulty(), "syncIssues", syncIssues) + case <-time.After(zombieNodeCheckInterval(noNewHeadsTimeoutThreshold)): + if n.poolInfoProvider != nil { + if l, _ := n.poolInfoProvider.LatestChainInfo(); l < 1 { + lggr.Criticalw("RPC endpoint is still out of sync, but there are no other available nodes. This RPC node will be forcibly moved back into the live pool in a degraded state", "syncIssues", syncIssues) + n.declareInSync() + return + } + } + case err := <-headsSub.Errors: + lggr.Errorw("Subscription was terminated", "err", err) + n.declareUnreachable() + return + case <-headsSub.NoNewHeads: + // we are not resetting the timer, as there is no need to add syncStatusNoNewHead until it's removed on new head. + syncIssues |= syncStatusNoNewHead + lggr.Debugw(fmt.Sprintf("No new heads received for %s. Node stays out-of-sync due to sync issues: %s", noNewHeadsTimeoutThreshold, syncIssues)) + case latestFinalized, open := <-finalizedHeadsSub.Heads: + if !open { + lggr.Errorw("Finalized heads subscription channel unexpectedly closed") + n.declareUnreachable() + return + } + if !latestFinalized.IsValid() { + lggr.Warn("Latest finalized block is not valid") + continue + } + + receivedNewHead := n.onNewFinalizedHead(lggr, &localHighestChainInfo, latestFinalized) + if !receivedNewHead { + continue + } + + // on new finalized head remove NoNewFinalizedHead flag from the mask + syncIssues &= ^syncStatusNoNewFinalizedHead + if noNewFinalizedBlocksTimeoutThreshold > 0 { + finalizedHeadsSub.ResetTimer(noNewFinalizedBlocksTimeoutThreshold) + } + + lggr.Debugw(msgReceivedFinalizedBlock, "blockNumber", latestFinalized.BlockNumber(), "syncIssues", syncIssues) + case err := <-finalizedHeadsSub.Errors: + lggr.Errorw("Finalized head subscription was terminated", "err", err) + n.declareUnreachable() + return + case <-finalizedHeadsSub.NoNewHeads: + // we are not resetting the timer, as there is no need to add syncStatusNoNewFinalizedHead until it's removed on new finalized head. + syncIssues |= syncStatusNoNewFinalizedHead + lggr.Debugw(fmt.Sprintf("No new finalized heads received for %s. Node stays out-of-sync due to sync issues: %s", noNewFinalizedBlocksTimeoutThreshold, syncIssues)) + } + } +} + +func (n *node[CHAIN_ID, HEAD, RPC]) unreachableLoop() { + defer n.wg.Done() + ctx, cancel := n.newCtx() + defer cancel() + + { + // sanity check + state := n.getCachedState() + switch state { + case NodeStateUnreachable: + case NodeStateClosed: + return + default: + panic(fmt.Sprintf("unreachableLoop can only run for node in Unreachable state, got: %s", state)) + } + } + + unreachableAt := time.Now() + + lggr := logger.Sugared(logger.Named(n.lfcLog, "Unreachable")) + lggr.Debugw("Trying to revive unreachable RPC node", "nodeState", n.getCachedState()) + + dialRetryBackoff := iutils.NewRedialBackoff() + + for { + select { + case <-ctx.Done(): + return + case <-time.After(dialRetryBackoff.Duration()): + lggr.Tracew("Trying to re-dial RPC node", "nodeState", n.getCachedState()) + + err := n.rpc.Dial(ctx) + if err != nil { + lggr.Errorw(fmt.Sprintf("Failed to redial RPC node; still unreachable: %v", err), "err", err, "nodeState", n.getCachedState()) + continue + } + + n.setState(NodeStateDialed) + + state := n.verifyConn(ctx, lggr) + switch state { + case NodeStateUnreachable: + n.setState(NodeStateUnreachable) + continue + case NodeStateAlive: + lggr.Infow(fmt.Sprintf("Successfully redialled and verified RPC node %s. Node was offline for %s", n.String(), time.Since(unreachableAt)), "nodeState", n.getCachedState()) + fallthrough + default: + n.declareState(state) + return + } + } + } +} + +func (n *node[CHAIN_ID, HEAD, RPC]) invalidChainIDLoop() { + defer n.wg.Done() + ctx, cancel := n.newCtx() + defer cancel() + + { + // sanity check + state := n.getCachedState() + switch state { + case NodeStateInvalidChainID: + case NodeStateClosed: + return + default: + panic(fmt.Sprintf("invalidChainIDLoop can only run for node in InvalidChainID state, got: %s", state)) + } + } + + invalidAt := time.Now() + + lggr := logger.Named(n.lfcLog, "InvalidChainID") + + // Need to redial since invalid chain ID nodes are automatically disconnected + state := n.createVerifiedConn(ctx, lggr) + if state != NodeStateInvalidChainID { + n.declareState(state) + return + } + + lggr.Debugw(fmt.Sprintf("Periodically re-checking RPC node %s with invalid chain ID", n.String()), "nodeState", n.getCachedState()) + + chainIDRecheckBackoff := iutils.NewRedialBackoff() + + for { + select { + case <-ctx.Done(): + return + case <-time.After(chainIDRecheckBackoff.Duration()): + state := n.verifyConn(ctx, lggr) + switch state { + case NodeStateInvalidChainID: + continue + case NodeStateAlive: + lggr.Infow(fmt.Sprintf("Successfully verified RPC node. Node was offline for %s", time.Since(invalidAt)), "nodeState", n.getCachedState()) + fallthrough + default: + n.declareState(state) + return + } + } + } +} + +func (n *node[CHAIN_ID, HEAD, RPC]) syncingLoop() { + defer n.wg.Done() + ctx, cancel := n.newCtx() + defer cancel() + + { + // sanity check + state := n.getCachedState() + switch state { + case NodeStateSyncing: + case NodeStateClosed: + return + default: + panic(fmt.Sprintf("syncingLoop can only run for node in NodeStateSyncing state, got: %s", state)) + } + } + + syncingAt := time.Now() + + lggr := logger.Sugared(logger.Named(n.lfcLog, "Syncing")) + lggr.Debugw(fmt.Sprintf("Periodically re-checking RPC node %s with syncing status", n.String()), "nodeState", n.getCachedState()) + // Need to redial since syncing nodes are automatically disconnected + state := n.createVerifiedConn(ctx, lggr) + if state != NodeStateSyncing { + n.declareState(state) + return + } + + recheckBackoff := iutils.NewRedialBackoff() + + for { + select { + case <-ctx.Done(): + return + case <-time.After(recheckBackoff.Duration()): + lggr.Tracew("Trying to recheck if the node is still syncing", "nodeState", n.getCachedState()) + isSyncing, err := n.rpc.IsSyncing(ctx) + if err != nil { + lggr.Errorw("Unexpected error while verifying RPC node synchronization status", "err", err, "nodeState", n.getCachedState()) + n.declareUnreachable() + return + } + + if isSyncing { + lggr.Errorw("Verification failed: Node is syncing", "nodeState", n.getCachedState()) + continue + } + + lggr.Infow(fmt.Sprintf("Successfully verified RPC node. Node was syncing for %s", time.Since(syncingAt)), "nodeState", n.getCachedState()) + n.declareAlive() + return + } + } +} diff --git a/pkg/solana/client/multinode/node_selector.go b/pkg/solana/client/multinode/node_selector.go new file mode 100644 index 000000000..372b521bb --- /dev/null +++ b/pkg/solana/client/multinode/node_selector.go @@ -0,0 +1,43 @@ +package client + +import ( + "fmt" + + "github.com/smartcontractkit/chainlink/v2/common/types" +) + +const ( + NodeSelectionModeHighestHead = "HighestHead" + NodeSelectionModeRoundRobin = "RoundRobin" + NodeSelectionModeTotalDifficulty = "TotalDifficulty" + NodeSelectionModePriorityLevel = "PriorityLevel" +) + +type NodeSelector[ + CHAIN_ID types.ID, + RPC any, +] interface { + // Select returns a Node, or nil if none can be selected. + // Implementation must be thread-safe. + Select() Node[CHAIN_ID, RPC] + // Name returns the strategy name, e.g. "HighestHead" or "RoundRobin" + Name() string +} + +func newNodeSelector[ + CHAIN_ID types.ID, + RPC any, +](selectionMode string, nodes []Node[CHAIN_ID, RPC]) NodeSelector[CHAIN_ID, RPC] { + switch selectionMode { + case NodeSelectionModeHighestHead: + return NewHighestHeadNodeSelector[CHAIN_ID, RPC](nodes) + case NodeSelectionModeRoundRobin: + return NewRoundRobinSelector[CHAIN_ID, RPC](nodes) + case NodeSelectionModeTotalDifficulty: + return NewTotalDifficultyNodeSelector[CHAIN_ID, RPC](nodes) + case NodeSelectionModePriorityLevel: + return NewPriorityLevelNodeSelector[CHAIN_ID, RPC](nodes) + default: + panic(fmt.Sprintf("unsupported NodeSelectionMode: %s", selectionMode)) + } +} diff --git a/pkg/solana/client/multinode/poller.go b/pkg/solana/client/multinode/poller.go new file mode 100644 index 000000000..d6080722c --- /dev/null +++ b/pkg/solana/client/multinode/poller.go @@ -0,0 +1,99 @@ +package client + +import ( + "context" + "sync" + "time" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/services" + + "github.com/smartcontractkit/chainlink/v2/common/types" +) + +// Poller is a component that polls a function at a given interval +// and delivers the result to a channel. It is used by multinode to poll +// for new heads and implements the Subscription interface. +type Poller[T any] struct { + services.StateMachine + pollingInterval time.Duration + pollingFunc func(ctx context.Context) (T, error) + pollingTimeout time.Duration + logger logger.Logger + channel chan<- T + errCh chan error + + stopCh services.StopChan + wg sync.WaitGroup +} + +// NewPoller creates a new Poller instance and returns a channel to receive the polled data +func NewPoller[ + T any, +](pollingInterval time.Duration, pollingFunc func(ctx context.Context) (T, error), pollingTimeout time.Duration, logger logger.Logger) (Poller[T], <-chan T) { + channel := make(chan T) + return Poller[T]{ + pollingInterval: pollingInterval, + pollingFunc: pollingFunc, + pollingTimeout: pollingTimeout, + channel: channel, + logger: logger, + errCh: make(chan error), + stopCh: make(chan struct{}), + }, channel +} + +var _ types.Subscription = &Poller[any]{} + +func (p *Poller[T]) Start() error { + return p.StartOnce("Poller", func() error { + p.wg.Add(1) + go p.pollingLoop() + return nil + }) +} + +// Unsubscribe cancels the sending of events to the data channel +func (p *Poller[T]) Unsubscribe() { + _ = p.StopOnce("Poller", func() error { + close(p.stopCh) + p.wg.Wait() + close(p.errCh) + close(p.channel) + return nil + }) +} + +func (p *Poller[T]) Err() <-chan error { + return p.errCh +} + +func (p *Poller[T]) pollingLoop() { + defer p.wg.Done() + + ticker := time.NewTicker(p.pollingInterval) + defer ticker.Stop() + + for { + select { + case <-p.stopCh: + return + case <-ticker.C: + // Set polling timeout + pollingCtx, cancelPolling := p.stopCh.CtxCancel(context.WithTimeout(context.Background(), p.pollingTimeout)) + // Execute polling function + result, err := p.pollingFunc(pollingCtx) + cancelPolling() + if err != nil { + p.logger.Warnf("polling error: %v", err) + continue + } + // Send result to channel or block if channel is full + select { + case p.channel <- result: + case <-p.stopCh: + return + } + } + } +} diff --git a/pkg/solana/client/multinode/poller_test.go b/pkg/solana/client/multinode/poller_test.go new file mode 100644 index 000000000..91af57930 --- /dev/null +++ b/pkg/solana/client/multinode/poller_test.go @@ -0,0 +1,187 @@ +package client + +import ( + "context" + "fmt" + "math/big" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" +) + +func Test_Poller(t *testing.T) { + lggr := logger.Test(t) + + t.Run("Test multiple start", func(t *testing.T) { + pollFunc := func(ctx context.Context) (Head, error) { + return nil, nil + } + + poller, _ := NewPoller[Head](time.Millisecond, pollFunc, time.Second, lggr) + err := poller.Start() + require.NoError(t, err) + + err = poller.Start() + require.Error(t, err) + poller.Unsubscribe() + }) + + t.Run("Test polling for heads", func(t *testing.T) { + // Mock polling function that returns a new value every time it's called + var pollNumber int + pollLock := sync.Mutex{} + pollFunc := func(ctx context.Context) (Head, error) { + pollLock.Lock() + defer pollLock.Unlock() + pollNumber++ + h := head{ + BlockNumber: int64(pollNumber), + BlockDifficulty: big.NewInt(int64(pollNumber)), + } + return h.ToMockHead(t), nil + } + + // Create poller and start to receive data + poller, channel := NewPoller[Head](time.Millisecond, pollFunc, time.Second, lggr) + require.NoError(t, poller.Start()) + defer poller.Unsubscribe() + + // Receive updates from the poller + pollCount := 0 + pollMax := 50 + for ; pollCount < pollMax; pollCount++ { + h := <-channel + assert.Equal(t, int64(pollCount+1), h.BlockNumber()) + } + }) + + t.Run("Test polling errors", func(t *testing.T) { + // Mock polling function that returns an error + var pollNumber int + pollLock := sync.Mutex{} + pollFunc := func(ctx context.Context) (Head, error) { + pollLock.Lock() + defer pollLock.Unlock() + pollNumber++ + return nil, fmt.Errorf("polling error %d", pollNumber) + } + + olggr, observedLogs := logger.TestObserved(t, zap.WarnLevel) + + // Create poller and subscribe to receive data + poller, _ := NewPoller[Head](time.Millisecond, pollFunc, time.Second, olggr) + require.NoError(t, poller.Start()) + defer poller.Unsubscribe() + + // Ensure that all errors were logged as expected + logsSeen := func() bool { + for pollCount := 0; pollCount < 50; pollCount++ { + numLogs := observedLogs.FilterMessage(fmt.Sprintf("polling error: polling error %d", pollCount+1)).Len() + if numLogs != 1 { + return false + } + } + return true + } + require.Eventually(t, logsSeen, tests.WaitTimeout(t), 100*time.Millisecond) + }) + + t.Run("Test polling timeout", func(t *testing.T) { + pollFunc := func(ctx context.Context) (Head, error) { + if <-ctx.Done(); true { + return nil, ctx.Err() + } + return nil, nil + } + + // Set instant timeout + pollingTimeout := time.Duration(0) + + olggr, observedLogs := logger.TestObserved(t, zap.WarnLevel) + + // Create poller and subscribe to receive data + poller, _ := NewPoller[Head](time.Millisecond, pollFunc, pollingTimeout, olggr) + require.NoError(t, poller.Start()) + defer poller.Unsubscribe() + + // Ensure that timeout errors were logged as expected + logsSeen := func() bool { + return observedLogs.FilterMessage("polling error: context deadline exceeded").Len() >= 1 + } + require.Eventually(t, logsSeen, tests.WaitTimeout(t), 100*time.Millisecond) + }) + + t.Run("Test unsubscribe during polling", func(t *testing.T) { + wait := make(chan struct{}) + closeOnce := sync.OnceFunc(func() { close(wait) }) + pollFunc := func(ctx context.Context) (Head, error) { + closeOnce() + // Block in polling function until context is cancelled + if <-ctx.Done(); true { + return nil, ctx.Err() + } + return nil, nil + } + + // Set long timeout + pollingTimeout := time.Minute + + olggr, observedLogs := logger.TestObserved(t, zap.WarnLevel) + + // Create poller and subscribe to receive data + poller, _ := NewPoller[Head](time.Millisecond, pollFunc, pollingTimeout, olggr) + require.NoError(t, poller.Start()) + + // Unsubscribe while blocked in polling function + <-wait + poller.Unsubscribe() + + // Ensure error was logged + logsSeen := func() bool { + return observedLogs.FilterMessage("polling error: context canceled").Len() >= 1 + } + require.Eventually(t, logsSeen, tests.WaitTimeout(t), 100*time.Millisecond) + }) +} + +func Test_Poller_Unsubscribe(t *testing.T) { + lggr := logger.Test(t) + pollFunc := func(ctx context.Context) (Head, error) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + h := head{ + BlockNumber: 0, + BlockDifficulty: big.NewInt(0), + } + return h.ToMockHead(t), nil + } + } + + t.Run("Test multiple unsubscribe", func(t *testing.T) { + poller, channel := NewPoller[Head](time.Millisecond, pollFunc, time.Second, lggr) + err := poller.Start() + require.NoError(t, err) + + <-channel + poller.Unsubscribe() + poller.Unsubscribe() + }) + + t.Run("Read channel after unsubscribe", func(t *testing.T) { + poller, channel := NewPoller[Head](time.Millisecond, pollFunc, time.Second, lggr) + err := poller.Start() + require.NoError(t, err) + + poller.Unsubscribe() + require.Equal(t, <-channel, nil) + }) +} diff --git a/pkg/solana/client/multinode/send_only_node.go b/pkg/solana/client/multinode/send_only_node.go new file mode 100644 index 000000000..069911c78 --- /dev/null +++ b/pkg/solana/client/multinode/send_only_node.go @@ -0,0 +1,183 @@ +package client + +import ( + "context" + "fmt" + "net/url" + "sync" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/services" + + "github.com/smartcontractkit/chainlink/v2/common/types" +) + +type sendOnlyClient[ + CHAIN_ID types.ID, +] interface { + Close() + ChainID(context.Context) (CHAIN_ID, error) + Dial(ctx context.Context) error +} + +// SendOnlyNode represents one node used as a sendonly +type SendOnlyNode[ + CHAIN_ID types.ID, + RPC any, +] interface { + // Start may attempt to connect to the node, but should only return error for misconfiguration - never for temporary errors. + Start(context.Context) error + Close() error + + ConfiguredChainID() CHAIN_ID + RPC() RPC + + String() string + // State returns NodeState + State() NodeState + // Name is a unique identifier for this node. + Name() string +} + +// It only supports sending transactions +// It must use an http(s) url +type sendOnlyNode[ + CHAIN_ID types.ID, + RPC sendOnlyClient[CHAIN_ID], +] struct { + services.StateMachine + + stateMu sync.RWMutex // protects state* fields + state NodeState + + rpc RPC + uri url.URL + log logger.Logger + name string + chainID CHAIN_ID + chStop services.StopChan + wg sync.WaitGroup +} + +// NewSendOnlyNode returns a new sendonly node +func NewSendOnlyNode[ + CHAIN_ID types.ID, + RPC sendOnlyClient[CHAIN_ID], +]( + lggr logger.Logger, + httpuri url.URL, + name string, + chainID CHAIN_ID, + rpc RPC, +) SendOnlyNode[CHAIN_ID, RPC] { + s := new(sendOnlyNode[CHAIN_ID, RPC]) + s.name = name + s.log = logger.Named(logger.Named(lggr, "SendOnlyNode"), name) + s.log = logger.With(s.log, + "nodeTier", "sendonly", + ) + s.rpc = rpc + s.uri = httpuri + s.chainID = chainID + s.chStop = make(chan struct{}) + return s +} + +func (s *sendOnlyNode[CHAIN_ID, RPC]) Start(ctx context.Context) error { + return s.StartOnce(s.name, func() error { + s.start(ctx) + return nil + }) +} + +// Start setups up and verifies the sendonly node +// Should only be called once in a node's lifecycle +func (s *sendOnlyNode[CHAIN_ID, RPC]) start(startCtx context.Context) { + if s.State() != NodeStateUndialed { + panic(fmt.Sprintf("cannot dial node with state %v", s.state)) + } + + err := s.rpc.Dial(startCtx) + if err != nil { + promPoolRPCNodeTransitionsToUnusable.WithLabelValues(s.chainID.String(), s.name).Inc() + s.log.Errorw("Dial failed: SendOnly Node is unusable", "err", err) + s.setState(NodeStateUnusable) + return + } + s.setState(NodeStateDialed) + + if s.chainID.String() == "0" { + // Skip verification if chainID is zero + s.log.Warn("sendonly rpc ChainID verification skipped") + } else { + chainID, err := s.rpc.ChainID(startCtx) + if err != nil || chainID.String() != s.chainID.String() { + promPoolRPCNodeTransitionsToUnreachable.WithLabelValues(s.chainID.String(), s.name).Inc() + if err != nil { + promPoolRPCNodeTransitionsToUnreachable.WithLabelValues(s.chainID.String(), s.name).Inc() + s.log.Errorw(fmt.Sprintf("Verify failed: %v", err), "err", err) + s.setState(NodeStateUnreachable) + } else { + promPoolRPCNodeTransitionsToInvalidChainID.WithLabelValues(s.chainID.String(), s.name).Inc() + s.log.Errorf( + "sendonly rpc ChainID doesn't match local chain ID: RPC ID=%s, local ID=%s, node name=%s", + chainID.String(), + s.chainID.String(), + s.name, + ) + s.setState(NodeStateInvalidChainID) + } + // Since it has failed, spin up the verifyLoop that will keep + // retrying until success + s.wg.Add(1) + go s.verifyLoop() + return + } + } + + promPoolRPCNodeTransitionsToAlive.WithLabelValues(s.chainID.String(), s.name).Inc() + s.setState(NodeStateAlive) + s.log.Infow("Sendonly RPC Node is online", "NodeState", s.state) +} + +func (s *sendOnlyNode[CHAIN_ID, RPC]) Close() error { + return s.StopOnce(s.name, func() error { + s.rpc.Close() + close(s.chStop) + s.wg.Wait() + s.setState(NodeStateClosed) + return nil + }) +} + +func (s *sendOnlyNode[CHAIN_ID, RPC]) ConfiguredChainID() CHAIN_ID { + return s.chainID +} + +func (s *sendOnlyNode[CHAIN_ID, RPC]) RPC() RPC { + return s.rpc +} + +func (s *sendOnlyNode[CHAIN_ID, RPC]) String() string { + return fmt.Sprintf("(%s)%s:%s", Secondary.String(), s.name, s.uri.Redacted()) +} + +func (s *sendOnlyNode[CHAIN_ID, RPC]) setState(state NodeState) (changed bool) { + s.stateMu.Lock() + defer s.stateMu.Unlock() + if s.state == state { + return false + } + s.state = state + return true +} + +func (s *sendOnlyNode[CHAIN_ID, RPC]) State() NodeState { + s.stateMu.RLock() + defer s.stateMu.RUnlock() + return s.state +} + +func (s *sendOnlyNode[CHAIN_ID, RPC]) Name() string { + return s.name +} diff --git a/pkg/solana/client/multinode/send_only_node_lifecycle.go b/pkg/solana/client/multinode/send_only_node_lifecycle.go new file mode 100644 index 000000000..a6ac11248 --- /dev/null +++ b/pkg/solana/client/multinode/send_only_node_lifecycle.go @@ -0,0 +1,67 @@ +package client + +import ( + "fmt" + "time" + + "github.com/smartcontractkit/chainlink/v2/common/internal/utils" +) + +// verifyLoop may only be triggered once, on Start, if initial chain ID check +// fails. +// +// It will continue checking until success and then exit permanently. +func (s *sendOnlyNode[CHAIN_ID, RPC]) verifyLoop() { + defer s.wg.Done() + ctx, cancel := s.chStop.NewCtx() + defer cancel() + + backoff := utils.NewRedialBackoff() + for { + select { + case <-ctx.Done(): + return + case <-time.After(backoff.Duration()): + } + chainID, err := s.rpc.ChainID(ctx) + if err != nil { + ok := s.IfStarted(func() { + if changed := s.setState(NodeStateUnreachable); changed { + promPoolRPCNodeTransitionsToUnreachable.WithLabelValues(s.chainID.String(), s.name).Inc() + } + }) + if !ok { + return + } + s.log.Errorw(fmt.Sprintf("Verify failed: %v", err), "err", err) + continue + } else if chainID.String() != s.chainID.String() { + ok := s.IfStarted(func() { + if changed := s.setState(NodeStateInvalidChainID); changed { + promPoolRPCNodeTransitionsToInvalidChainID.WithLabelValues(s.chainID.String(), s.name).Inc() + } + }) + if !ok { + return + } + s.log.Errorf( + "sendonly rpc ChainID doesn't match local chain ID: RPC ID=%s, local ID=%s, node name=%s", + chainID.String(), + s.chainID.String(), + s.name, + ) + + continue + } + ok := s.IfStarted(func() { + if changed := s.setState(NodeStateAlive); changed { + promPoolRPCNodeTransitionsToAlive.WithLabelValues(s.chainID.String(), s.name).Inc() + } + }) + if !ok { + return + } + s.log.Infow("Sendonly RPC Node is online", "NodeState", s.state) + return + } +} diff --git a/pkg/solana/client/multinode/send_only_node_test.go b/pkg/solana/client/multinode/send_only_node_test.go new file mode 100644 index 000000000..352fb5b92 --- /dev/null +++ b/pkg/solana/client/multinode/send_only_node_test.go @@ -0,0 +1,139 @@ +package client + +import ( + "errors" + "fmt" + "net/url" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" + + "github.com/smartcontractkit/chainlink/v2/common/types" +) + +func TestNewSendOnlyNode(t *testing.T) { + t.Parallel() + + urlFormat := "http://user:%s@testurl.com" + password := "pass" + u, err := url.Parse(fmt.Sprintf(urlFormat, password)) + require.NoError(t, err) + redacted := fmt.Sprintf(urlFormat, "xxxxx") + lggr := logger.Test(t) + name := "TestNewSendOnlyNode" + chainID := types.RandomID() + client := newMockSendOnlyClient[types.ID](t) + + node := NewSendOnlyNode(lggr, *u, name, chainID, client) + assert.NotNil(t, node) + + // Must contain name & url with redacted password + assert.Contains(t, node.String(), fmt.Sprintf("%s:%s", name, redacted)) + assert.Equal(t, node.ConfiguredChainID(), chainID) +} + +func TestStartSendOnlyNode(t *testing.T) { + t.Parallel() + t.Run("becomes unusable if initial dial fails", func(t *testing.T) { + t.Parallel() + lggr, observedLogs := logger.TestObserved(t, zap.WarnLevel) + client := newMockSendOnlyClient[types.ID](t) + client.On("Close").Once() + expectedError := errors.New("some http error") + client.On("Dial", mock.Anything).Return(expectedError).Once() + s := NewSendOnlyNode(lggr, url.URL{}, t.Name(), types.RandomID(), client) + + defer func() { assert.NoError(t, s.Close()) }() + err := s.Start(tests.Context(t)) + require.NoError(t, err) + + assert.Equal(t, NodeStateUnusable, s.State()) + tests.RequireLogMessage(t, observedLogs, "Dial failed: SendOnly Node is unusable") + }) + t.Run("Default ChainID(0) produces warn and skips checks", func(t *testing.T) { + t.Parallel() + lggr, observedLogs := logger.TestObserved(t, zap.WarnLevel) + client := newMockSendOnlyClient[types.ID](t) + client.On("Close").Once() + client.On("Dial", mock.Anything).Return(nil).Once() + s := NewSendOnlyNode(lggr, url.URL{}, t.Name(), types.NewIDFromInt(0), client) + + defer func() { assert.NoError(t, s.Close()) }() + err := s.Start(tests.Context(t)) + require.NoError(t, err) + + assert.Equal(t, NodeStateAlive, s.State()) + tests.RequireLogMessage(t, observedLogs, "sendonly rpc ChainID verification skipped") + }) + t.Run("Can recover from chainID verification failure", func(t *testing.T) { + t.Parallel() + lggr, observedLogs := logger.TestObserved(t, zap.WarnLevel) + client := newMockSendOnlyClient[types.ID](t) + client.On("Close").Once() + client.On("Dial", mock.Anything).Return(nil) + expectedError := errors.New("failed to get chain ID") + chainID := types.RandomID() + const failuresCount = 2 + client.On("ChainID", mock.Anything).Return(types.RandomID(), expectedError).Times(failuresCount) + client.On("ChainID", mock.Anything).Return(chainID, nil) + + s := NewSendOnlyNode(lggr, url.URL{}, t.Name(), chainID, client) + + defer func() { assert.NoError(t, s.Close()) }() + err := s.Start(tests.Context(t)) + require.NoError(t, err) + + assert.Equal(t, NodeStateUnreachable, s.State()) + tests.AssertLogCountEventually(t, observedLogs, fmt.Sprintf("Verify failed: %v", expectedError), failuresCount) + tests.AssertEventually(t, func() bool { + return s.State() == NodeStateAlive + }) + }) + t.Run("Can recover from chainID mismatch", func(t *testing.T) { + t.Parallel() + lggr, observedLogs := logger.TestObserved(t, zap.WarnLevel) + client := newMockSendOnlyClient[types.ID](t) + client.On("Close").Once() + client.On("Dial", mock.Anything).Return(nil).Once() + configuredChainID := types.NewIDFromInt(11) + rpcChainID := types.NewIDFromInt(20) + const failuresCount = 2 + client.On("ChainID", mock.Anything).Return(rpcChainID, nil).Times(failuresCount) + client.On("ChainID", mock.Anything).Return(configuredChainID, nil) + s := NewSendOnlyNode(lggr, url.URL{}, t.Name(), configuredChainID, client) + + defer func() { assert.NoError(t, s.Close()) }() + err := s.Start(tests.Context(t)) + require.NoError(t, err) + + assert.Equal(t, NodeStateInvalidChainID, s.State()) + tests.AssertLogCountEventually(t, observedLogs, "sendonly rpc ChainID doesn't match local chain ID", failuresCount) + tests.AssertEventually(t, func() bool { + return s.State() == NodeStateAlive + }) + }) + t.Run("Start with Random ChainID", func(t *testing.T) { + t.Parallel() + lggr, observedLogs := logger.TestObserved(t, zap.WarnLevel) + client := newMockSendOnlyClient[types.ID](t) + client.On("Close").Once() + client.On("Dial", mock.Anything).Return(nil).Once() + configuredChainID := types.RandomID() + client.On("ChainID", mock.Anything).Return(configuredChainID, nil) + s := NewSendOnlyNode(lggr, url.URL{}, t.Name(), configuredChainID, client) + + defer func() { assert.NoError(t, s.Close()) }() + err := s.Start(tests.Context(t)) + assert.NoError(t, err) + tests.AssertEventually(t, func() bool { + return s.State() == NodeStateAlive + }) + assert.Equal(t, 0, observedLogs.Len()) // No warnings expected + }) +} diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go new file mode 100644 index 000000000..a4c5e2b3d --- /dev/null +++ b/pkg/solana/client/multinode/transaction_sender.go @@ -0,0 +1,277 @@ +package client + +import ( + "context" + "fmt" + "math" + "slices" + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/services" +) + +var ( + // PromMultiNodeInvariantViolations reports violation of our assumptions + PromMultiNodeInvariantViolations = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "multi_node_invariant_violations", + Help: "The number of invariant violations", + }, []string{"network", "chainId", "invariant"}) +) + +// TxErrorClassifier - defines interface of a function that transforms raw RPC error into the SendTxReturnCode enum +// (e.g. Successful, Fatal, Retryable, etc.) +type TxErrorClassifier[TX any] func(tx TX, err error) SendTxReturnCode + +type sendTxResult struct { + Err error + ResultCode SendTxReturnCode +} + +const sendTxQuorum = 0.7 + +// SendTxRPCClient - defines interface of an RPC used by TransactionSender to broadcast transaction +type SendTxRPCClient[TX any] interface { + // SendTransaction errors returned should include name or other unique identifier of the RPC + SendTransaction(ctx context.Context, tx TX) error +} + +func NewTransactionSender[TX any, CHAIN_ID ID, RPC SendTxRPCClient[TX]]( + lggr logger.Logger, + chainID CHAIN_ID, + chainFamily string, + multiNode *MultiNode[CHAIN_ID, RPC], + txErrorClassifier TxErrorClassifier[TX], + sendTxSoftTimeout time.Duration, +) *TransactionSender[TX, CHAIN_ID, RPC] { + if sendTxSoftTimeout == 0 { + sendTxSoftTimeout = QueryTimeout / 2 + } + return &TransactionSender[TX, CHAIN_ID, RPC]{ + chainID: chainID, + chainFamily: chainFamily, + lggr: logger.Sugared(lggr).Named("TransactionSender").With("chainID", chainID.String()), + multiNode: multiNode, + txErrorClassifier: txErrorClassifier, + sendTxSoftTimeout: sendTxSoftTimeout, + chStop: make(services.StopChan), + } +} + +type TransactionSender[TX any, CHAIN_ID ID, RPC SendTxRPCClient[TX]] struct { + services.StateMachine + chainID CHAIN_ID + chainFamily string + lggr logger.SugaredLogger + multiNode *MultiNode[CHAIN_ID, RPC] + txErrorClassifier TxErrorClassifier[TX] + sendTxSoftTimeout time.Duration // defines max waiting time from first response til responses evaluation + + wg sync.WaitGroup // waits for all reporting goroutines to finish + chStop services.StopChan +} + +// SendTransaction - broadcasts transaction to all the send-only and primary nodes in MultiNode. +// A returned nil or error does not guarantee that the transaction will or won't be included. Additional checks must be +// performed to determine the final state. +// +// Send-only nodes' results are ignored as they tend to return false-positive responses. Broadcast to them is necessary +// to speed up the propagation of TX in the network. +// +// Handling of primary nodes' results consists of collection and aggregation. +// In the collection step, we gather as many results as possible while minimizing waiting time. This operation succeeds +// on one of the following conditions: +// * Received at least one success +// * Received at least one result and `sendTxSoftTimeout` expired +// * Received results from the sufficient number of nodes defined by sendTxQuorum. +// The aggregation is based on the following conditions: +// * If there is at least one success - returns success +// * If there is at least one terminal error - returns terminal error +// * If there is both success and terminal error - returns success and reports invariant violation +// * Otherwise, returns any (effectively random) of the errors. +func (txSender *TransactionSender[TX, CHAIN_ID, RPC]) SendTransaction(ctx context.Context, tx TX) (SendTxReturnCode, error) { + txResults := make(chan sendTxResult) + txResultsToReport := make(chan sendTxResult) + primaryNodeWg := sync.WaitGroup{} + + ctx, cancel := txSender.chStop.Ctx(ctx) + defer cancel() + + healthyNodesNum := 0 + err := txSender.multiNode.DoAll(ctx, func(ctx context.Context, rpc RPC, isSendOnly bool) { + if isSendOnly { + txSender.wg.Add(1) + go func() { + defer txSender.wg.Done() + // Send-only nodes' results are ignored as they tend to return false-positive responses. + // Broadcast to them is necessary to speed up the propagation of TX in the network. + _ = txSender.broadcastTxAsync(ctx, rpc, tx) + }() + return + } + + // Primary Nodes + healthyNodesNum++ + primaryNodeWg.Add(1) + go func() { + defer primaryNodeWg.Done() + result := txSender.broadcastTxAsync(ctx, rpc, tx) + select { + case <-ctx.Done(): + return + case txResults <- result: + } + + select { + case <-ctx.Done(): + return + case txResultsToReport <- result: + } + }() + }) + if err != nil { + primaryNodeWg.Wait() + close(txResultsToReport) + close(txResults) + return 0, err + } + + // This needs to be done in parallel so the reporting knows when it's done (when the channel is closed) + txSender.wg.Add(1) + go func() { + defer txSender.wg.Done() + primaryNodeWg.Wait() + close(txResultsToReport) + close(txResults) + }() + + txSender.wg.Add(1) + go txSender.reportSendTxAnomalies(tx, txResultsToReport) + + return txSender.collectTxResults(ctx, tx, healthyNodesNum, txResults) +} + +func (txSender *TransactionSender[TX, CHAIN_ID, RPC]) broadcastTxAsync(ctx context.Context, rpc RPC, tx TX) sendTxResult { + txErr := rpc.SendTransaction(ctx, tx) + txSender.lggr.Debugw("Node sent transaction", "tx", tx, "err", txErr) + resultCode := txSender.txErrorClassifier(tx, txErr) + if !slices.Contains(sendTxSuccessfulCodes, resultCode) { + txSender.lggr.Warnw("RPC returned error", "tx", tx, "err", txErr) + } + return sendTxResult{Err: txErr, ResultCode: resultCode} +} + +func (txSender *TransactionSender[TX, CHAIN_ID, RPC]) reportSendTxAnomalies(tx TX, txResults <-chan sendTxResult) { + defer txSender.wg.Done() + resultsByCode := sendTxErrors{} + // txResults eventually will be closed + for txResult := range txResults { + resultsByCode[txResult.ResultCode] = append(resultsByCode[txResult.ResultCode], txResult.Err) + } + + _, _, criticalErr := aggregateTxResults(resultsByCode) + if criticalErr != nil { + txSender.lggr.Criticalw("observed invariant violation on SendTransaction", "tx", tx, "resultsByCode", resultsByCode, "err", criticalErr) + PromMultiNodeInvariantViolations.WithLabelValues(txSender.chainFamily, txSender.chainID.String(), criticalErr.Error()).Inc() + } +} + +type sendTxErrors map[SendTxReturnCode][]error + +func aggregateTxResults(resultsByCode sendTxErrors) (returnCode SendTxReturnCode, txResult error, err error) { + severeCode, severeErrors, hasSevereErrors := findFirstIn(resultsByCode, sendTxSevereErrors) + successCode, successResults, hasSuccess := findFirstIn(resultsByCode, sendTxSuccessfulCodes) + if hasSuccess { + // We assume that primary node would never report false positive txResult for a transaction. + // Thus, if such case occurs it's probably due to misconfiguration or a bug and requires manual intervention. + if hasSevereErrors { + const errMsg = "found contradictions in nodes replies on SendTransaction: got success and severe error" + // return success, since at least 1 node has accepted our broadcasted Tx, and thus it can now be included onchain + return successCode, successResults[0], fmt.Errorf(errMsg) + } + + // other errors are temporary - we are safe to return success + return successCode, successResults[0], nil + } + + if hasSevereErrors { + return severeCode, severeErrors[0], nil + } + + // return temporary error + for code, result := range resultsByCode { + return code, result[0], nil + } + + err = fmt.Errorf("expected at least one response on SendTransaction") + return 0, err, err +} + +func (txSender *TransactionSender[TX, CHAIN_ID, RPC]) collectTxResults(ctx context.Context, tx TX, healthyNodesNum int, txResults <-chan sendTxResult) (SendTxReturnCode, error) { + if healthyNodesNum == 0 { + return 0, ErroringNodeError + } + requiredResults := int(math.Ceil(float64(healthyNodesNum) * sendTxQuorum)) + errorsByCode := sendTxErrors{} + var softTimeoutChan <-chan time.Time + var resultsCount int +loop: + for { + select { + case <-ctx.Done(): + txSender.lggr.Debugw("Failed to collect of the results before context was done", "tx", tx, "errorsByCode", errorsByCode) + return 0, ctx.Err() + case result := <-txResults: + errorsByCode[result.ResultCode] = append(errorsByCode[result.ResultCode], result.Err) + resultsCount++ + if slices.Contains(sendTxSuccessfulCodes, result.ResultCode) || resultsCount >= requiredResults { + break loop + } + case <-softTimeoutChan: + txSender.lggr.Debugw("Send Tx soft timeout expired - returning responses we've collected so far", "tx", tx, "resultsCount", resultsCount, "requiredResults", requiredResults) + break loop + } + + if softTimeoutChan == nil { + tm := time.NewTimer(txSender.sendTxSoftTimeout) + softTimeoutChan = tm.C + // we are fine with stopping timer at the end of function + //nolint + defer tm.Stop() + } + } + + // ignore critical error as it's reported in reportSendTxAnomalies + returnCode, result, _ := aggregateTxResults(errorsByCode) + return returnCode, result +} + +func (txSender *TransactionSender[TX, CHAIN_ID, RPC]) Start(ctx context.Context) error { + return txSender.StartOnce("TransactionSender", func() error { + return nil + }) +} + +func (txSender *TransactionSender[TX, CHAIN_ID, RPC]) Close() error { + return txSender.StopOnce("TransactionSender", func() error { + close(txSender.chStop) + txSender.wg.Wait() + return nil + }) +} + +// findFirstIn - returns the first existing key and value for the slice of keys +func findFirstIn[K comparable, V any](set map[K]V, keys []K) (K, V, bool) { + for _, k := range keys { + if v, ok := set[k]; ok { + return k, v, true + } + } + var zeroK K + var zeroV V + return zeroK, zeroV, false +} diff --git a/pkg/solana/client/multinode/transaction_sender_test.go b/pkg/solana/client/multinode/transaction_sender_test.go new file mode 100644 index 000000000..e4387abee --- /dev/null +++ b/pkg/solana/client/multinode/transaction_sender_test.go @@ -0,0 +1,360 @@ +package client + +import ( + "context" + "fmt" + "testing" + + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" + "github.com/smartcontractkit/chainlink/v2/common/types" +) + +type sendTxMultiNode struct { + *MultiNode[types.ID, SendTxRPCClient[any]] +} + +type sendTxRPC struct { + sendTxRun func(args mock.Arguments) + sendTxErr error +} + +var _ SendTxRPCClient[any] = (*sendTxRPC)(nil) + +func newSendTxRPC(sendTxErr error, sendTxRun func(args mock.Arguments)) *sendTxRPC { + return &sendTxRPC{sendTxErr: sendTxErr, sendTxRun: sendTxRun} +} + +func (rpc *sendTxRPC) SendTransaction(ctx context.Context, _ any) error { + if rpc.sendTxRun != nil { + rpc.sendTxRun(mock.Arguments{ctx}) + } + return rpc.sendTxErr +} + +func newTestTransactionSender(t *testing.T, chainID types.ID, lggr logger.Logger, + nodes []Node[types.ID, SendTxRPCClient[any]], + sendOnlyNodes []SendOnlyNode[types.ID, SendTxRPCClient[any]], +) (*sendTxMultiNode, *TransactionSender[any, types.ID, SendTxRPCClient[any]]) { + mn := sendTxMultiNode{NewMultiNode[types.ID, SendTxRPCClient[any]]( + lggr, NodeSelectionModeRoundRobin, 0, nodes, sendOnlyNodes, chainID, "chainFamily", 0)} + err := mn.StartOnce("startedTestMultiNode", func() error { return nil }) + require.NoError(t, err) + + txSender := NewTransactionSender[any, types.ID, SendTxRPCClient[any]](lggr, chainID, mn.chainFamily, mn.MultiNode, classifySendTxError, tests.TestInterval) + err = txSender.Start(tests.Context(t)) + require.NoError(t, err) + + t.Cleanup(func() { + err := mn.Close() + if err != nil { + // Allow MultiNode to be closed early for testing + require.EqualError(t, err, "MultiNode has already been stopped: already stopped") + } + err = txSender.Close() + if err != nil { + // Allow TransactionSender to be closed early for testing + require.EqualError(t, err, "TransactionSender has already been stopped: already stopped") + } + }) + return &mn, txSender +} + +func classifySendTxError(_ any, err error) SendTxReturnCode { + if err != nil { + return Fatal + } + return Successful +} + +func TestTransactionSender_SendTransaction(t *testing.T) { + t.Parallel() + + newNodeWithState := func(t *testing.T, state NodeState, txErr error, sendTxRun func(args mock.Arguments)) *mockNode[types.ID, SendTxRPCClient[any]] { + rpc := newSendTxRPC(txErr, sendTxRun) + node := newMockNode[types.ID, SendTxRPCClient[any]](t) + node.On("String").Return("node name").Maybe() + node.On("RPC").Return(rpc).Maybe() + node.On("State").Return(state).Maybe() + node.On("Close").Return(nil).Once() + return node + } + + newNode := func(t *testing.T, txErr error, sendTxRun func(args mock.Arguments)) *mockNode[types.ID, SendTxRPCClient[any]] { + return newNodeWithState(t, NodeStateAlive, txErr, sendTxRun) + } + + t.Run("Fails if there is no nodes available", func(t *testing.T) { + lggr, _ := logger.TestObserved(t, zap.DebugLevel) + _, txSender := newTestTransactionSender(t, types.RandomID(), lggr, nil, nil) + _, err := txSender.SendTransaction(tests.Context(t), nil) + assert.EqualError(t, err, ErroringNodeError.Error()) + }) + + t.Run("Transaction failure happy path", func(t *testing.T) { + expectedError := errors.New("transaction failed") + mainNode := newNode(t, expectedError, nil) + lggr, observedLogs := logger.TestObserved(t, zap.DebugLevel) + + _, txSender := newTestTransactionSender(t, types.RandomID(), lggr, + []Node[types.ID, SendTxRPCClient[any]]{mainNode}, + []SendOnlyNode[types.ID, SendTxRPCClient[any]]{newNode(t, errors.New("unexpected error"), nil)}) + + result, sendErr := txSender.SendTransaction(tests.Context(t), nil) + require.ErrorIs(t, sendErr, expectedError) + require.Equal(t, Fatal, result) + tests.AssertLogCountEventually(t, observedLogs, "Node sent transaction", 2) + tests.AssertLogCountEventually(t, observedLogs, "RPC returned error", 2) + }) + + t.Run("Transaction success happy path", func(t *testing.T) { + mainNode := newNode(t, nil, nil) + + lggr, observedLogs := logger.TestObserved(t, zap.DebugLevel) + _, txSender := newTestTransactionSender(t, types.RandomID(), lggr, + []Node[types.ID, SendTxRPCClient[any]]{mainNode}, + []SendOnlyNode[types.ID, SendTxRPCClient[any]]{newNode(t, errors.New("unexpected error"), nil)}) + + result, sendErr := txSender.SendTransaction(tests.Context(t), nil) + require.NoError(t, sendErr) + require.Equal(t, Successful, result) + tests.AssertLogCountEventually(t, observedLogs, "Node sent transaction", 2) + tests.AssertLogCountEventually(t, observedLogs, "RPC returned error", 1) + }) + + t.Run("Context expired before collecting sufficient results", func(t *testing.T) { + testContext, testCancel := context.WithCancel(tests.Context(t)) + defer testCancel() + + mainNode := newNode(t, nil, func(_ mock.Arguments) { + // block caller til end of the test + <-testContext.Done() + }) + + lggr, _ := logger.TestObserved(t, zap.DebugLevel) + + _, txSender := newTestTransactionSender(t, types.RandomID(), lggr, + []Node[types.ID, SendTxRPCClient[any]]{mainNode}, nil) + + requestContext, cancel := context.WithCancel(tests.Context(t)) + cancel() + _, sendErr := txSender.SendTransaction(requestContext, nil) + require.EqualError(t, sendErr, "context canceled") + }) + + t.Run("Soft timeout stops results collection", func(t *testing.T) { + chainID := types.RandomID() + expectedError := errors.New("transaction failed") + fastNode := newNode(t, expectedError, nil) + + // hold reply from the node till end of the test + testContext, testCancel := context.WithCancel(tests.Context(t)) + defer testCancel() + slowNode := newNode(t, errors.New("transaction failed"), func(_ mock.Arguments) { + // block caller til end of the test + <-testContext.Done() + }) + + lggr, _ := logger.TestObserved(t, zap.DebugLevel) + + _, txSender := newTestTransactionSender(t, chainID, lggr, []Node[types.ID, SendTxRPCClient[any]]{fastNode, slowNode}, nil) + _, sendErr := txSender.SendTransaction(tests.Context(t), nil) + require.EqualError(t, sendErr, expectedError.Error()) + }) + t.Run("Fails when multinode is closed", func(t *testing.T) { + chainID := types.RandomID() + fastNode := newNode(t, nil, nil) + // hold reply from the node till end of the test + testContext, testCancel := context.WithCancel(tests.Context(t)) + defer testCancel() + slowNode := newNode(t, errors.New("transaction failed"), func(_ mock.Arguments) { + // block caller til end of the test + <-testContext.Done() + }) + slowSendOnly := newNode(t, errors.New("send only failed"), func(_ mock.Arguments) { + // block caller til end of the test + <-testContext.Done() + }) + + lggr, _ := logger.TestObserved(t, zap.DebugLevel) + + mn, txSender := newTestTransactionSender(t, chainID, lggr, + []Node[types.ID, SendTxRPCClient[any]]{fastNode, slowNode}, + []SendOnlyNode[types.ID, SendTxRPCClient[any]]{slowSendOnly}) + + require.NoError(t, mn.Close()) + _, err := txSender.SendTransaction(tests.Context(t), nil) + require.EqualError(t, err, "MultiNode is stopped") + }) + t.Run("Fails when closed", func(t *testing.T) { + chainID := types.RandomID() + fastNode := newNode(t, nil, nil) + // hold reply from the node till end of the test + testContext, testCancel := context.WithCancel(tests.Context(t)) + defer testCancel() + slowNode := newNode(t, errors.New("transaction failed"), func(_ mock.Arguments) { + // block caller til end of the test + <-testContext.Done() + }) + slowSendOnly := newNode(t, errors.New("send only failed"), func(_ mock.Arguments) { + // block caller til end of the test + <-testContext.Done() + }) + + lggr, _ := logger.TestObserved(t, zap.DebugLevel) + + _, txSender := newTestTransactionSender(t, chainID, lggr, + []Node[types.ID, SendTxRPCClient[any]]{fastNode, slowNode}, + []SendOnlyNode[types.ID, SendTxRPCClient[any]]{slowSendOnly}) + + require.NoError(t, txSender.Close()) + _, err := txSender.SendTransaction(tests.Context(t), nil) + require.EqualError(t, err, "context canceled") + }) + t.Run("Returns error if there is no healthy primary nodes", func(t *testing.T) { + chainID := types.RandomID() + primary := newNodeWithState(t, NodeStateUnreachable, nil, nil) + sendOnly := newNodeWithState(t, NodeStateUnreachable, nil, nil) + + lggr, _ := logger.TestObserved(t, zap.DebugLevel) + + _, txSender := newTestTransactionSender(t, chainID, lggr, + []Node[types.ID, SendTxRPCClient[any]]{primary}, + []SendOnlyNode[types.ID, SendTxRPCClient[any]]{sendOnly}) + + _, sendErr := txSender.SendTransaction(tests.Context(t), nil) + assert.EqualError(t, sendErr, ErroringNodeError.Error()) + }) + + t.Run("Transaction success even if one of the nodes is unhealthy", func(t *testing.T) { + chainID := types.RandomID() + mainNode := newNode(t, nil, nil) + unexpectedCall := func(args mock.Arguments) { + panic("SendTx must not be called for unhealthy node") + } + unhealthyNode := newNodeWithState(t, NodeStateUnreachable, nil, unexpectedCall) + unhealthySendOnlyNode := newNodeWithState(t, NodeStateUnreachable, nil, unexpectedCall) + + lggr, _ := logger.TestObserved(t, zap.DebugLevel) + + _, txSender := newTestTransactionSender(t, chainID, lggr, + []Node[types.ID, SendTxRPCClient[any]]{mainNode, unhealthyNode}, + []SendOnlyNode[types.ID, SendTxRPCClient[any]]{unhealthySendOnlyNode}) + + returnCode, sendErr := txSender.SendTransaction(tests.Context(t), nil) + require.NoError(t, sendErr) + require.Equal(t, Successful, returnCode) + }) +} + +func TestTransactionSender_SendTransaction_aggregateTxResults(t *testing.T) { + t.Parallel() + // ensure failure on new SendTxReturnCode + codesToCover := map[SendTxReturnCode]struct{}{} + for code := Successful; code < sendTxReturnCodeLen; code++ { + codesToCover[code] = struct{}{} + } + + testCases := []struct { + Name string + ExpectedTxResult string + ExpectedCriticalErr string + ResultsByCode sendTxErrors + }{ + { + Name: "Returns success and logs critical error on success and Fatal", + ExpectedTxResult: "success", + ExpectedCriticalErr: "found contradictions in nodes replies on SendTransaction: got success and severe error", + ResultsByCode: sendTxErrors{ + Successful: {errors.New("success")}, + Fatal: {errors.New("fatal")}, + }, + }, + { + Name: "Returns TransactionAlreadyKnown and logs critical error on TransactionAlreadyKnown and Fatal", + ExpectedTxResult: "tx_already_known", + ExpectedCriticalErr: "found contradictions in nodes replies on SendTransaction: got success and severe error", + ResultsByCode: sendTxErrors{ + TransactionAlreadyKnown: {errors.New("tx_already_known")}, + Unsupported: {errors.New("unsupported")}, + }, + }, + { + Name: "Prefers sever error to temporary", + ExpectedTxResult: "underpriced", + ExpectedCriticalErr: "", + ResultsByCode: sendTxErrors{ + Retryable: {errors.New("retryable")}, + Underpriced: {errors.New("underpriced")}, + }, + }, + { + Name: "Returns temporary error", + ExpectedTxResult: "retryable", + ExpectedCriticalErr: "", + ResultsByCode: sendTxErrors{ + Retryable: {errors.New("retryable")}, + }, + }, + { + Name: "Insufficient funds is treated as error", + ExpectedTxResult: "", + ExpectedCriticalErr: "", + ResultsByCode: sendTxErrors{ + Successful: {nil}, + InsufficientFunds: {errors.New("insufficientFunds")}, + }, + }, + { + Name: "Logs critical error on empty ResultsByCode", + ExpectedTxResult: "expected at least one response on SendTransaction", + ExpectedCriticalErr: "expected at least one response on SendTransaction", + ResultsByCode: sendTxErrors{}, + }, + { + Name: "Zk terminally stuck", + ExpectedTxResult: "not enough keccak counters to continue the execution", + ExpectedCriticalErr: "", + ResultsByCode: sendTxErrors{ + TerminallyStuck: {errors.New("not enough keccak counters to continue the execution")}, + }, + }, + } + + for _, testCase := range testCases { + for code := range testCase.ResultsByCode { + delete(codesToCover, code) + } + + t.Run(testCase.Name, func(t *testing.T) { + _, txResult, err := aggregateTxResults(testCase.ResultsByCode) + if testCase.ExpectedTxResult == "" { + assert.NoError(t, err) + } else { + assert.EqualError(t, txResult, testCase.ExpectedTxResult) + } + + logger.Sugared(logger.Test(t)).Info("Map: " + fmt.Sprint(testCase.ResultsByCode)) + logger.Sugared(logger.Test(t)).Criticalw("observed invariant violation on SendTransaction", "resultsByCode", testCase.ResultsByCode, "err", err) + + if testCase.ExpectedCriticalErr == "" { + assert.NoError(t, err) + } else { + assert.EqualError(t, err, testCase.ExpectedCriticalErr) + } + }) + } + + // explicitly signal that following codes are properly handled in aggregateTxResults, + // but dedicated test cases won't be beneficial + for _, codeToIgnore := range []SendTxReturnCode{Unknown, ExceedsMaxFee, FeeOutOfValidRange} { + delete(codesToCover, codeToIgnore) + } + assert.Empty(t, codesToCover, "all of the SendTxReturnCode must be covered by this test") +} diff --git a/pkg/solana/client/multinode/types.go b/pkg/solana/client/multinode/types.go new file mode 100644 index 000000000..5cd831fc1 --- /dev/null +++ b/pkg/solana/client/multinode/types.go @@ -0,0 +1,124 @@ +package client + +import ( + "context" + "fmt" + "math/big" +) + +// A chain-agnostic generic interface to represent the following native types on various chains: +// PublicKey, Address, Account, BlockHash, TxHash +type Hashable interface { + fmt.Stringer + comparable + + Bytes() []byte +} + +// Subscription represents an event subscription where events are +// delivered on a data channel. +// This is a generic interface for Subscription to represent used by clients. +type Subscription interface { + // Unsubscribe cancels the sending of events to the data channel + // and closes the error channel. Unsubscribe should be callable multiple + // times without causing an error. + Unsubscribe() + // Err returns the subscription error channel. The error channel receives + // a value if there is an issue with the subscription (e.g. the network connection + // delivering the events has been closed). Only one value will ever be sent. + // The error channel is closed by Unsubscribe. + Err() <-chan error +} + +// RPCClient includes all the necessary generalized RPC methods along with any additional chain-specific methods. +type RPCClient[ + CHAIN_ID ID, + HEAD Head, +] interface { + // ChainID - fetches ChainID from the RPC to verify that it matches config + ChainID(ctx context.Context) (CHAIN_ID, error) + // Dial - prepares the RPC for usage. Can be called on fresh or closed RPC + Dial(ctx context.Context) error + // SubscribeToHeads - returns channel and subscription for new heads. + SubscribeToHeads(ctx context.Context) (<-chan HEAD, Subscription, error) + // SubscribeToFinalizedHeads - returns channel and subscription for finalized heads. + SubscribeToFinalizedHeads(ctx context.Context) (<-chan HEAD, Subscription, error) + // Ping - returns error if RPC is not reachable + Ping(context.Context) error + // IsSyncing - returns true if the RPC is in Syncing state and can not process calls + IsSyncing(ctx context.Context) (bool, error) + // UnsubscribeAllExcept - close all subscriptions except `subs` + UnsubscribeAllExcept(subs ...Subscription) + // Close - closes all subscriptions and aborts all RPC calls + Close() + // GetInterceptedChainInfo - returns latest and highest observed by application layer ChainInfo. + // latest ChainInfo is the most recent value received within a NodeClient's current lifecycle between Dial and DisconnectAll. + // highestUserObservations ChainInfo is the highest ChainInfo observed excluding health checks calls. + // Its values must not be reset. + // The results of corresponding calls, to get the most recent head and the latest finalized head, must be + // intercepted and reflected in ChainInfo before being returned to a caller. Otherwise, MultiNode is not able to + // provide repeatable read guarantee. + // DisconnectAll must reset latest ChainInfo to default value. + // Ensure implementation does not have a race condition when values are reset before request completion and as + // a result latest ChainInfo contains information from the previous cycle. + GetInterceptedChainInfo() (latest, highestUserObservations ChainInfo) +} + +// Head is the interface required by the NodeClient +type Head interface { + BlockNumber() int64 + BlockDifficulty() *big.Int + IsValid() bool +} + +// PoolChainInfoProvider - provides aggregation of nodes pool ChainInfo +type PoolChainInfoProvider interface { + // LatestChainInfo - returns number of live nodes available in the pool, so we can prevent the last alive node in a pool from being + // moved to out-of-sync state. It is better to have one out-of-sync node than no nodes at all. + // Returns highest latest ChainInfo within the alive nodes. E.g. most recent block number and highest block number + // observed by Node A are 10 and 15; Node B - 12 and 14. This method will return 12. + LatestChainInfo() (int, ChainInfo) + // HighestUserObservations - returns highest ChainInfo ever observed by any user of MultiNode. + HighestUserObservations() ChainInfo +} + +// ChainInfo - defines RPC's or MultiNode's view on the chain +type ChainInfo struct { + BlockNumber int64 + FinalizedBlockNumber int64 + TotalDifficulty *big.Int +} + +func MaxTotalDifficulty(a, b *big.Int) *big.Int { + if a == nil { + if b == nil { + return nil + } + + return big.NewInt(0).Set(b) + } + + if b == nil || a.Cmp(b) >= 0 { + return big.NewInt(0).Set(a) + } + + return big.NewInt(0).Set(b) +} + +// ID represents the base type, for any chain's ID. +// It should be convertible to a string, that can uniquely identify this chain +type ID fmt.Stringer + +type multiNodeContextKey int + +const ( + contextKeyHeathCheckRequest multiNodeContextKey = iota + 1 +) + +func CtxAddHealthCheckFlag(ctx context.Context) context.Context { + return context.WithValue(ctx, contextKeyHeathCheckRequest, struct{}{}) +} + +func CtxIsHeathCheckRequest(ctx context.Context) bool { + return ctx.Value(contextKeyHeathCheckRequest) != nil +} diff --git a/pkg/solana/client/rpc_client.go b/pkg/solana/client/rpc_client.go new file mode 100644 index 000000000..0db8d9062 --- /dev/null +++ b/pkg/solana/client/rpc_client.go @@ -0,0 +1,318 @@ +package client + +import ( + "context" + "errors" + "fmt" + "math/big" + "time" + + "github.com/gagliardetto/solana-go" + "github.com/gagliardetto/solana-go/rpc" + "golang.org/x/sync/singleflight" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + + mn "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" + "github.com/smartcontractkit/chainlink-solana/pkg/solana/config" + "github.com/smartcontractkit/chainlink-solana/pkg/solana/monitor" +) + +type StringID string + +func (s StringID) String() string { + return string(s) +} + +// TODO: ChainReaderWriter needs ChainID() (string, error) +// TODO: We probably don't need this though? +var _ ReaderWriter = (*RpcClient)(nil) + +type Head struct { + rpc.GetBlockResult +} + +func (h *Head) BlockNumber() int64 { + if h.BlockHeight == nil { + return 0 + } + return int64(*h.BlockHeight) +} + +func (h *Head) BlockDifficulty() *big.Int { + return nil +} + +func (h *Head) IsValid() bool { + return true +} + +type RpcClient struct { + url string + rpc *rpc.Client + skipPreflight bool // to enable or disable preflight checks + commitment rpc.CommitmentType + maxRetries *uint + txTimeout time.Duration + contextDuration time.Duration + log logger.Logger + + // provides a duplicate function call suppression mechanism + requestGroup *singleflight.Group +} + +func (c *RpcClient) Dial(ctx context.Context) error { + //TODO implement me + panic("implement me") +} + +func (c *RpcClient) SubscribeToHeads(ctx context.Context) (<-chan *Head, mn.Subscription, error) { + //TODO implement me + panic("implement me") +} + +func (c *RpcClient) SubscribeToFinalizedHeads(ctx context.Context) (<-chan *Head, mn.Subscription, error) { + //TODO implement me + panic("implement me") +} + +func (c *RpcClient) Ping(ctx context.Context) error { + //TODO implement me + panic("implement me") +} + +func (c *RpcClient) IsSyncing(ctx context.Context) (bool, error) { + //TODO implement me + panic("implement me") +} + +func (c *RpcClient) UnsubscribeAllExcept(subs ...mn.Subscription) { + //TODO implement me + panic("implement me") +} + +func (c *RpcClient) Close() { + //TODO implement me + panic("implement me") +} + +func (c *RpcClient) GetInterceptedChainInfo() (latest, highestUserObservations mn.ChainInfo) { + //TODO implement me + panic("implement me") +} + +func NewRpcClient(endpoint string, cfg config.Config, requestTimeout time.Duration, log logger.Logger) (*RpcClient, error) { + return &RpcClient{ + url: endpoint, + rpc: rpc.New(endpoint), + skipPreflight: cfg.SkipPreflight(), + commitment: cfg.Commitment(), + maxRetries: cfg.MaxRetries(), + txTimeout: cfg.TxTimeout(), + contextDuration: requestTimeout, + log: log, + requestGroup: &singleflight.Group{}, + }, nil +} + +func (c *RpcClient) latency(name string) func() { + start := time.Now() + return func() { + monitor.SetClientLatency(time.Since(start), name, c.url) + } +} + +func (c *RpcClient) Balance(addr solana.PublicKey) (uint64, error) { + done := c.latency("balance") + defer done() + + ctx, cancel := context.WithTimeout(context.Background(), c.contextDuration) + defer cancel() + + v, err, _ := c.requestGroup.Do(fmt.Sprintf("GetBalance(%s)", addr.String()), func() (interface{}, error) { + return c.rpc.GetBalance(ctx, addr, c.commitment) + }) + if err != nil { + return 0, err + } + res := v.(*rpc.GetBalanceResult) + return res.Value, err +} + +func (c *RpcClient) SlotHeight() (uint64, error) { + return c.SlotHeightWithCommitment(rpc.CommitmentProcessed) // get the latest slot height +} + +func (c *RpcClient) SlotHeightWithCommitment(commitment rpc.CommitmentType) (uint64, error) { + done := c.latency("slot_height") + defer done() + + ctx, cancel := context.WithTimeout(context.Background(), c.contextDuration) + defer cancel() + v, err, _ := c.requestGroup.Do("GetSlotHeight", func() (interface{}, error) { + return c.rpc.GetSlot(ctx, commitment) + }) + return v.(uint64), err +} + +func (c *RpcClient) GetAccountInfoWithOpts(ctx context.Context, addr solana.PublicKey, opts *rpc.GetAccountInfoOpts) (*rpc.GetAccountInfoResult, error) { + done := c.latency("account_info") + defer done() + + ctx, cancel := context.WithTimeout(ctx, c.contextDuration) + defer cancel() + opts.Commitment = c.commitment // overrides passed in value - use defined client commitment type + return c.rpc.GetAccountInfoWithOpts(ctx, addr, opts) +} + +func (c *RpcClient) LatestBlockhash() (*rpc.GetLatestBlockhashResult, error) { + done := c.latency("latest_blockhash") + defer done() + + ctx, cancel := context.WithTimeout(context.Background(), c.contextDuration) + defer cancel() + + v, err, _ := c.requestGroup.Do("GetLatestBlockhash", func() (interface{}, error) { + return c.rpc.GetLatestBlockhash(ctx, c.commitment) + }) + return v.(*rpc.GetLatestBlockhashResult), err +} + +func (c *RpcClient) ChainID(ctx context.Context) (StringID, error) { + done := c.latency("chain_id") + defer done() + + ctx, cancel := context.WithTimeout(ctx, c.contextDuration) + defer cancel() + v, err, _ := c.requestGroup.Do("GetGenesisHash", func() (interface{}, error) { + return c.rpc.GetGenesisHash(ctx) + }) + if err != nil { + return "", err + } + hash := v.(solana.Hash) + + var network string + switch hash.String() { + case DevnetGenesisHash: + network = "devnet" + case TestnetGenesisHash: + network = "testnet" + case MainnetGenesisHash: + network = "mainnet" + default: + c.log.Warnf("unknown genesis hash - assuming solana chain is 'localnet'") + network = "localnet" + } + return StringID(network), nil +} + +func (c *RpcClient) GetFeeForMessage(msg string) (uint64, error) { + done := c.latency("fee_for_message") + defer done() + + // msg is base58 encoded data + + ctx, cancel := context.WithTimeout(context.Background(), c.contextDuration) + defer cancel() + res, err := c.rpc.GetFeeForMessage(ctx, msg, c.commitment) + if err != nil { + return 0, fmt.Errorf("error in GetFeeForMessage: %w", err) + } + + if res == nil || res.Value == nil { + return 0, errors.New("nil pointer in GetFeeForMessage") + } + return *res.Value, nil +} + +// https://docs.solana.com/developing/clients/jsonrpc-api#getsignaturestatuses +func (c *RpcClient) SignatureStatuses(ctx context.Context, sigs []solana.Signature) ([]*rpc.SignatureStatusesResult, error) { + done := c.latency("signature_statuses") + defer done() + + ctx, cancel := context.WithTimeout(ctx, c.contextDuration) + defer cancel() + + // searchTransactionHistory = false + res, err := c.rpc.GetSignatureStatuses(ctx, false, sigs...) + if err != nil { + return nil, fmt.Errorf("error in GetSignatureStatuses: %w", err) + } + + if res == nil || res.Value == nil { + return nil, errors.New("nil pointer in GetSignatureStatuses") + } + return res.Value, nil +} + +// https://docs.solana.com/developing/clients/jsonrpc-api#simulatetransaction +// opts - (optional) use `nil` to use defaults +func (c *RpcClient) SimulateTx(ctx context.Context, tx *solana.Transaction, opts *rpc.SimulateTransactionOpts) (*rpc.SimulateTransactionResult, error) { + done := c.latency("simulate_tx") + defer done() + + ctx, cancel := context.WithTimeout(ctx, c.contextDuration) + defer cancel() + + if opts == nil { + opts = &rpc.SimulateTransactionOpts{ + SigVerify: true, // verify signature + Commitment: c.commitment, + } + } + + res, err := c.rpc.SimulateTransactionWithOpts(ctx, tx, opts) + if err != nil { + return nil, fmt.Errorf("error in SimulateTransactionWithOpts: %w", err) + } + + if res == nil || res.Value == nil { + return nil, errors.New("nil pointer in SimulateTransactionWithOpts") + } + + return res.Value, nil +} + +func (c *RpcClient) SendTransaction(ctx context.Context, tx *solana.Transaction) error { + // TODO: Implement + return nil +} + +func (c *RpcClient) SendTx(ctx context.Context, tx *solana.Transaction) (solana.Signature, error) { + done := c.latency("send_tx") + defer done() + + ctx, cancel := context.WithTimeout(ctx, c.txTimeout) + defer cancel() + + opts := rpc.TransactionOpts{ + SkipPreflight: c.skipPreflight, + PreflightCommitment: c.commitment, + MaxRetries: c.maxRetries, + } + + return c.rpc.SendTransactionWithOpts(ctx, tx, opts) +} + +func (c *RpcClient) GetLatestBlock() (*rpc.GetBlockResult, error) { + // get latest confirmed slot + slot, err := c.SlotHeightWithCommitment(c.commitment) + if err != nil { + return nil, fmt.Errorf("GetLatestBlock.SlotHeight: %w", err) + } + + // get block based on slot + done := c.latency("latest_block") + defer done() + ctx, cancel := context.WithTimeout(context.Background(), c.txTimeout) + defer cancel() + v, err, _ := c.requestGroup.Do("GetBlockWithOpts", func() (interface{}, error) { + version := uint64(0) // pull all tx types (legacy + v0) + return c.rpc.GetBlockWithOpts(ctx, slot, &rpc.GetBlockOpts{ + Commitment: c.commitment, + MaxSupportedTransactionVersion: &version, + }) + }) + return v.(*rpc.GetBlockResult), err +} diff --git a/pkg/solana/config/multinode.go b/pkg/solana/config/multinode.go new file mode 100644 index 000000000..72e345728 --- /dev/null +++ b/pkg/solana/config/multinode.go @@ -0,0 +1,86 @@ +package config + +import "time" + +type MultiNode struct { + // TODO: Determine current config overlap https://smartcontract-it.atlassian.net/browse/BCI-4065 + // Feature flag + multiNodeEnabled bool + + // Node Configs + pollFailureThreshold uint32 + pollInterval time.Duration + selectionMode string + syncThreshold uint32 + nodeIsSyncingEnabled bool + finalizedBlockPollInterval time.Duration + enforceRepeatableRead bool + deathDeclarationDelay time.Duration + + // Chain Configs + nodeNoNewHeadsThreshold time.Duration + noNewFinalizedHeadsThreshold time.Duration + finalityDepth uint32 + finalityTagEnabled bool + finalizedBlockOffset uint32 +} + +func (c *MultiNode) MultiNodeEnabled() bool { + return c.multiNodeEnabled +} + +func (c *MultiNode) PollFailureThreshold() uint32 { + return c.pollFailureThreshold +} + +func (c *MultiNode) PollInterval() time.Duration { + return c.pollInterval +} + +func (c *MultiNode) SelectionMode() string { + return c.selectionMode +} + +func (c *MultiNode) SyncThreshold() uint32 { + return c.syncThreshold +} + +func (c *MultiNode) NodeIsSyncingEnabled() bool { + return c.nodeIsSyncingEnabled +} + +func (c *MultiNode) FinalizedBlockPollInterval() time.Duration { + return c.finalizedBlockPollInterval +} + +func (c *MultiNode) EnforceRepeatableRead() bool { + return c.enforceRepeatableRead +} + +func (c *MultiNode) DeathDeclarationDelay() time.Duration { + return c.deathDeclarationDelay +} + +func (c *MultiNode) NodeNoNewHeadsThreshold() time.Duration { + return c.nodeNoNewHeadsThreshold +} + +func (c *MultiNode) NoNewFinalizedHeadsThreshold() time.Duration { + return c.noNewFinalizedHeadsThreshold +} + +func (c *MultiNode) FinalityDepth() uint32 { + return c.finalityDepth +} + +func (c *MultiNode) FinalityTagEnabled() bool { + return c.finalityTagEnabled +} + +func (c *MultiNode) FinalizedBlockOffset() uint32 { + return c.finalizedBlockOffset +} + +func (c *MultiNode) SetDefaults() { + // TODO: Set defaults for MultiNode config https://smartcontract-it.atlassian.net/browse/BCI-4065 +} diff --git a/pkg/solana/config/toml.go b/pkg/solana/config/toml.go index e5eb705e6..b1cdfc7f5 100644 --- a/pkg/solana/config/toml.go +++ b/pkg/solana/config/toml.go @@ -112,6 +112,7 @@ type TOMLConfig struct { // Do not access directly, use [IsEnabled] Enabled *bool Chain + MultiNode Nodes Nodes } @@ -279,8 +280,13 @@ func (c *TOMLConfig) ListNodes() Nodes { return c.Nodes } +func (c *TOMLConfig) MultiNodeConfig() *MultiNode { + return &c.MultiNode +} + func NewDefault() *TOMLConfig { cfg := &TOMLConfig{} - cfg.SetDefaults() + cfg.Chain.SetDefaults() + cfg.MultiNode.SetDefaults() return cfg } From 1c684859c1d49f1d4248028585932edb82b1fba5 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 3 Sep 2024 11:48:41 -0400 Subject: [PATCH 002/174] Update MultiNode files --- pkg/solana/client/multinode/ctx.go | 17 +++ pkg/solana/client/multinode/node.go | 7 +- pkg/solana/client/multinode/node_fsm.go | 8 +- pkg/solana/client/multinode/node_lifecycle.go | 95 ++++---------- pkg/solana/client/multinode/node_selector.go | 6 +- .../multinode/node_selector_highest_head.go | 38 ++++++ .../multinode/node_selector_priority_level.go | 121 ++++++++++++++++++ .../multinode/node_selector_round_robin.go | 46 +++++++ .../node_selector_total_difficulty.go | 51 ++++++++ pkg/solana/client/multinode/poller.go | 58 ++++----- pkg/solana/client/multinode/redialbackoff.go | 17 +++ pkg/solana/client/multinode/send_only_node.go | 2 +- .../client/multinode/transaction_sender.go | 28 ++-- pkg/solana/client/multinode/types.go | 31 +---- pkg/solana/client/rpc_client.go | 4 +- 15 files changed, 371 insertions(+), 158 deletions(-) create mode 100644 pkg/solana/client/multinode/ctx.go create mode 100644 pkg/solana/client/multinode/node_selector_highest_head.go create mode 100644 pkg/solana/client/multinode/node_selector_priority_level.go create mode 100644 pkg/solana/client/multinode/node_selector_round_robin.go create mode 100644 pkg/solana/client/multinode/node_selector_total_difficulty.go create mode 100644 pkg/solana/client/multinode/redialbackoff.go diff --git a/pkg/solana/client/multinode/ctx.go b/pkg/solana/client/multinode/ctx.go new file mode 100644 index 000000000..57b2fc8a8 --- /dev/null +++ b/pkg/solana/client/multinode/ctx.go @@ -0,0 +1,17 @@ +package client + +import "context" + +type multiNodeContextKey int + +const ( + contextKeyHeathCheckRequest multiNodeContextKey = iota + 1 +) + +func CtxAddHealthCheckFlag(ctx context.Context) context.Context { + return context.WithValue(ctx, contextKeyHeathCheckRequest, struct{}{}) +} + +func CtxIsHeathCheckRequest(ctx context.Context) bool { + return ctx.Value(contextKeyHeathCheckRequest) != nil +} diff --git a/pkg/solana/client/multinode/node.go b/pkg/solana/client/multinode/node.go index c3532b1a1..8ab30f856 100644 --- a/pkg/solana/client/multinode/node.go +++ b/pkg/solana/client/multinode/node.go @@ -110,8 +110,7 @@ type node[ // wg waits for subsidiary goroutines wg sync.WaitGroup - aliveLoopSub Subscription - finalizedBlockSub Subscription + healthCheckSubs []Subscription } func NewNode[ @@ -180,9 +179,7 @@ func (n *node[CHAIN_ID, HEAD, RPC]) RPC() RPC { // unsubscribeAllExceptAliveLoop is not thread-safe; it should only be called // while holding the stateMu lock. func (n *node[CHAIN_ID, HEAD, RPC]) unsubscribeAllExceptAliveLoop() { - aliveLoopSub := n.aliveLoopSub - finalizedBlockSub := n.finalizedBlockSub - n.rpc.UnsubscribeAllExcept(aliveLoopSub, finalizedBlockSub) + n.rpc.UnsubscribeAllExcept(n.healthCheckSubs...) } func (n *node[CHAIN_ID, HEAD, RPC]) UnsubscribeAllExceptAliveLoop() { diff --git a/pkg/solana/client/multinode/node_fsm.go b/pkg/solana/client/multinode/node_fsm.go index 1111210c4..981e325da 100644 --- a/pkg/solana/client/multinode/node_fsm.go +++ b/pkg/solana/client/multinode/node_fsm.go @@ -256,7 +256,7 @@ func (n *node[CHAIN_ID, HEAD, RPC]) transitionToOutOfSync(fn func()) { } switch n.state { case NodeStateAlive: - n.unsubscribeAllExceptAliveLoop() + n.rpc.Close() n.state = NodeStateOutOfSync default: panic(transitionFail(n.state, NodeStateOutOfSync)) @@ -281,7 +281,7 @@ func (n *node[CHAIN_ID, HEAD, RPC]) transitionToUnreachable(fn func()) { } switch n.state { case NodeStateUndialed, NodeStateDialed, NodeStateAlive, NodeStateOutOfSync, NodeStateInvalidChainID, NodeStateSyncing: - n.unsubscribeAllExceptAliveLoop() + n.rpc.Close() n.state = NodeStateUnreachable default: panic(transitionFail(n.state, NodeStateUnreachable)) @@ -324,7 +324,7 @@ func (n *node[CHAIN_ID, HEAD, RPC]) transitionToInvalidChainID(fn func()) { } switch n.state { case NodeStateDialed, NodeStateOutOfSync, NodeStateSyncing: - n.unsubscribeAllExceptAliveLoop() + n.rpc.Close() n.state = NodeStateInvalidChainID default: panic(transitionFail(n.state, NodeStateInvalidChainID)) @@ -349,7 +349,7 @@ func (n *node[CHAIN_ID, HEAD, RPC]) transitionToSyncing(fn func()) { } switch n.state { case NodeStateDialed, NodeStateOutOfSync, NodeStateInvalidChainID: - n.unsubscribeAllExceptAliveLoop() + n.rpc.Close() n.state = NodeStateSyncing default: panic(transitionFail(n.state, NodeStateSyncing)) diff --git a/pkg/solana/client/multinode/node_lifecycle.go b/pkg/solana/client/multinode/node_lifecycle.go index 823a1abc3..44203bf97 100644 --- a/pkg/solana/client/multinode/node_lifecycle.go +++ b/pkg/solana/client/multinode/node_lifecycle.go @@ -7,16 +7,12 @@ import ( "math/big" "time" - "github.com/smartcontractkit/chainlink/v2/common/types" - "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/utils" bigmath "github.com/smartcontractkit/chainlink-common/pkg/utils/big_math" - - iutils "github.com/smartcontractkit/chainlink/v2/common/internal/utils" ) var ( @@ -103,15 +99,7 @@ func (n *node[CHAIN_ID, HEAD, RPC]) aliveLoop() { return } - n.stateMu.Lock() - n.aliveLoopSub = headsSub.sub - n.stateMu.Unlock() - defer func() { - defer headsSub.sub.Unsubscribe() - n.stateMu.Lock() - n.aliveLoopSub = nil - n.stateMu.Unlock() - }() + defer n.unsubscribeHealthChecks() var pollCh <-chan time.Time if pollInterval > 0 { @@ -138,16 +126,6 @@ func (n *node[CHAIN_ID, HEAD, RPC]) aliveLoop() { n.declareUnreachable() return } - - n.stateMu.Lock() - n.finalizedBlockSub = finalizedHeadsSub.sub - n.stateMu.Unlock() - defer func() { - finalizedHeadsSub.Unsubscribe() - n.stateMu.Lock() - n.finalizedBlockSub = nil - n.stateMu.Unlock() - }() } localHighestChainInfo, _ := n.rpc.GetInterceptedChainInfo() @@ -187,7 +165,7 @@ func (n *node[CHAIN_ID, HEAD, RPC]) aliveLoop() { return } _, latestChainInfo := n.StateAndLatest() - if outOfSync, liveNodes := n.syncStatus(latestChainInfo.BlockNumber, latestChainInfo.TotalDifficulty); outOfSync { + if outOfSync, liveNodes := n.isOutOfSyncWithPool(latestChainInfo); outOfSync { // note: there must be another live node for us to be out of sync lggr.Errorw("RPC endpoint has fallen behind", "blockNumber", latestChainInfo.BlockNumber, "totalDifficulty", latestChainInfo.TotalDifficulty, "nodeState", n.getCachedState()) if liveNodes < 2 { @@ -232,17 +210,11 @@ func (n *node[CHAIN_ID, HEAD, RPC]) aliveLoop() { n.declareUnreachable() return } - if !latestFinalized.IsValid() { - lggr.Warn("Latest finalized block is not valid") - continue - } - latestFinalizedBN := latestFinalized.BlockNumber() - if latestFinalizedBN > localHighestChainInfo.FinalizedBlockNumber { - promPoolRPCNodeHighestFinalizedBlock.WithLabelValues(n.chainID.String(), n.name).Set(float64(latestFinalizedBN)) - localHighestChainInfo.FinalizedBlockNumber = latestFinalizedBN + receivedNewHead := n.onNewFinalizedHead(lggr, &localHighestChainInfo, latestFinalized) + if receivedNewHead && noNewFinalizedBlocksTimeoutThreshold > 0 { + finalizedHeadsSub.ResetTimer(noNewFinalizedBlocksTimeoutThreshold) } - case <-finalizedHeadsSub.NoNewHeads: // We haven't received a finalized head on the channel for at least the // threshold amount of time, mark it broken @@ -266,13 +238,22 @@ func (n *node[CHAIN_ID, HEAD, RPC]) aliveLoop() { } } +func (n *node[CHAIN_ID, HEAD, RPC]) unsubscribeHealthChecks() { + n.stateMu.Lock() + for _, sub := range n.healthCheckSubs { + sub.Unsubscribe() + } + n.healthCheckSubs = []Subscription{} + n.stateMu.Unlock() +} + type headSubscription[HEAD any] struct { Heads <-chan HEAD Errors <-chan error NoNewHeads <-chan time.Time noNewHeadsTicker *time.Ticker - sub types.Subscription + sub Subscription cleanUpTasks []func() } @@ -287,10 +268,10 @@ func (sub *headSubscription[HEAD]) Unsubscribe() { } func (n *node[CHAIN_ID, HEAD, PRC]) registerNewSubscription(ctx context.Context, lggr logger.SugaredLogger, - noNewDataThreshold time.Duration, newSub func(ctx context.Context) (<-chan HEAD, types.Subscription, error)) (headSubscription[HEAD], error) { + noNewDataThreshold time.Duration, newSub func(ctx context.Context) (<-chan HEAD, Subscription, error)) (headSubscription[HEAD], error) { result := headSubscription[HEAD]{} var err error - var sub types.Subscription + var sub Subscription result.Heads, sub, err = newSub(ctx) if err != nil { return result, err @@ -299,11 +280,10 @@ func (n *node[CHAIN_ID, HEAD, PRC]) registerNewSubscription(ctx context.Context, result.Errors = sub.Err() lggr.Debug("Successfully subscribed") - // TODO: will be removed as part of merging effort with BCI-2875 result.sub = sub - //n.stateMu.Lock() - //n.healthCheckSubs = append(n.healthCheckSubs, sub) - //n.stateMu.Unlock() + n.stateMu.Lock() + n.healthCheckSubs = append(n.healthCheckSubs, sub) + n.stateMu.Unlock() result.cleanUpTasks = append(result.cleanUpTasks, sub.Unsubscribe) @@ -365,31 +345,6 @@ func (n *node[CHAIN_ID, HEAD, RPC]) onNewHead(lggr logger.SugaredLogger, chainIn return true } -// syncStatus returns outOfSync true if num or td is more than SyncThresold behind the best node. -// Always returns outOfSync false for SyncThreshold 0. -// liveNodes is only included when outOfSync is true. -func (n *node[CHAIN_ID, HEAD, RPC]) syncStatus(num int64, td *big.Int) (outOfSync bool, liveNodes int) { - if n.poolInfoProvider == nil { - return // skip for tests - } - threshold := n.nodePoolCfg.SyncThreshold() - if threshold == 0 { - return // disabled - } - // Check against best node - ln, ci := n.poolInfoProvider.LatestChainInfo() - mode := n.nodePoolCfg.SelectionMode() - switch mode { - case NodeSelectionModeHighestHead, NodeSelectionModeRoundRobin, NodeSelectionModePriorityLevel: - return num < ci.BlockNumber-int64(threshold), ln - case NodeSelectionModeTotalDifficulty: - bigThreshold := big.NewInt(int64(threshold)) - return td.Cmp(bigmath.Sub(ci.TotalDifficulty, bigThreshold)) < 0, ln - default: - panic("unrecognized NodeSelectionMode: " + mode) - } -} - const ( msgReceivedBlock = "Received block for RPC node, waiting until back in-sync to mark as live again" msgReceivedFinalizedBlock = "Received new finalized block for RPC node, waiting until back in-sync to mark as live again" @@ -462,8 +417,9 @@ func (n *node[CHAIN_ID, HEAD, RPC]) outOfSyncLoop(syncIssues syncStatus) { return } + defer n.unsubscribeHealthChecks() + lggr.Tracew("Successfully subscribed to heads feed on out-of-sync RPC node") - defer headsSub.Unsubscribe() noNewFinalizedBlocksTimeoutThreshold := n.chainCfg.NoNewFinalizedHeadsThreshold() var finalizedHeadsSub headSubscription[HEAD] @@ -477,7 +433,6 @@ func (n *node[CHAIN_ID, HEAD, RPC]) outOfSyncLoop(syncIssues syncStatus) { } lggr.Tracew("Successfully subscribed to finalized heads feed on out-of-sync RPC node") - defer finalizedHeadsSub.Unsubscribe() } _, localHighestChainInfo := n.rpc.GetInterceptedChainInfo() @@ -591,7 +546,7 @@ func (n *node[CHAIN_ID, HEAD, RPC]) unreachableLoop() { lggr := logger.Sugared(logger.Named(n.lfcLog, "Unreachable")) lggr.Debugw("Trying to revive unreachable RPC node", "nodeState", n.getCachedState()) - dialRetryBackoff := iutils.NewRedialBackoff() + dialRetryBackoff := NewRedialBackoff() for { select { @@ -654,7 +609,7 @@ func (n *node[CHAIN_ID, HEAD, RPC]) invalidChainIDLoop() { lggr.Debugw(fmt.Sprintf("Periodically re-checking RPC node %s with invalid chain ID", n.String()), "nodeState", n.getCachedState()) - chainIDRecheckBackoff := iutils.NewRedialBackoff() + chainIDRecheckBackoff := NewRedialBackoff() for { select { @@ -704,7 +659,7 @@ func (n *node[CHAIN_ID, HEAD, RPC]) syncingLoop() { return } - recheckBackoff := iutils.NewRedialBackoff() + recheckBackoff := NewRedialBackoff() for { select { diff --git a/pkg/solana/client/multinode/node_selector.go b/pkg/solana/client/multinode/node_selector.go index 372b521bb..872026fe2 100644 --- a/pkg/solana/client/multinode/node_selector.go +++ b/pkg/solana/client/multinode/node_selector.go @@ -2,8 +2,6 @@ package client import ( "fmt" - - "github.com/smartcontractkit/chainlink/v2/common/types" ) const ( @@ -14,7 +12,7 @@ const ( ) type NodeSelector[ - CHAIN_ID types.ID, + CHAIN_ID ID, RPC any, ] interface { // Select returns a Node, or nil if none can be selected. @@ -25,7 +23,7 @@ type NodeSelector[ } func newNodeSelector[ - CHAIN_ID types.ID, + CHAIN_ID ID, RPC any, ](selectionMode string, nodes []Node[CHAIN_ID, RPC]) NodeSelector[CHAIN_ID, RPC] { switch selectionMode { diff --git a/pkg/solana/client/multinode/node_selector_highest_head.go b/pkg/solana/client/multinode/node_selector_highest_head.go new file mode 100644 index 000000000..52188bbdf --- /dev/null +++ b/pkg/solana/client/multinode/node_selector_highest_head.go @@ -0,0 +1,38 @@ +package client + +import ( + "math" +) + +type highestHeadNodeSelector[ + CHAIN_ID ID, + RPC any, +] []Node[CHAIN_ID, RPC] + +func NewHighestHeadNodeSelector[ + CHAIN_ID ID, + RPC any, +](nodes []Node[CHAIN_ID, RPC]) NodeSelector[CHAIN_ID, RPC] { + return highestHeadNodeSelector[CHAIN_ID, RPC](nodes) +} + +func (s highestHeadNodeSelector[CHAIN_ID, RPC]) Select() Node[CHAIN_ID, RPC] { + var highestHeadNumber int64 = math.MinInt64 + var highestHeadNodes []Node[CHAIN_ID, RPC] + for _, n := range s { + state, currentChainInfo := n.StateAndLatest() + currentHeadNumber := currentChainInfo.BlockNumber + if state == NodeStateAlive && currentHeadNumber >= highestHeadNumber { + if highestHeadNumber < currentHeadNumber { + highestHeadNumber = currentHeadNumber + highestHeadNodes = nil + } + highestHeadNodes = append(highestHeadNodes, n) + } + } + return firstOrHighestPriority(highestHeadNodes) +} + +func (s highestHeadNodeSelector[CHAIN_ID, RPC]) Name() string { + return NodeSelectionModeHighestHead +} diff --git a/pkg/solana/client/multinode/node_selector_priority_level.go b/pkg/solana/client/multinode/node_selector_priority_level.go new file mode 100644 index 000000000..3e171b98b --- /dev/null +++ b/pkg/solana/client/multinode/node_selector_priority_level.go @@ -0,0 +1,121 @@ +package client + +import ( + "math" + "sort" + "sync/atomic" +) + +type priorityLevelNodeSelector[ + CHAIN_ID ID, + RPC any, +] struct { + nodes []Node[CHAIN_ID, RPC] + roundRobinCount []atomic.Uint32 +} + +type nodeWithPriority[ + CHAIN_ID ID, + RPC any, +] struct { + node Node[CHAIN_ID, RPC] + priority int32 +} + +func NewPriorityLevelNodeSelector[ + CHAIN_ID ID, + RPC any, +](nodes []Node[CHAIN_ID, RPC]) NodeSelector[CHAIN_ID, RPC] { + return &priorityLevelNodeSelector[CHAIN_ID, RPC]{ + nodes: nodes, + roundRobinCount: make([]atomic.Uint32, nrOfPriorityTiers(nodes)), + } +} + +func (s priorityLevelNodeSelector[CHAIN_ID, RPC]) Select() Node[CHAIN_ID, RPC] { + nodes := s.getHighestPriorityAliveTier() + + if len(nodes) == 0 { + return nil + } + priorityLevel := nodes[len(nodes)-1].priority + + // NOTE: Inc returns the number after addition, so we must -1 to get the "current" counter + count := s.roundRobinCount[priorityLevel].Add(1) - 1 + idx := int(count % uint32(len(nodes))) + + return nodes[idx].node +} + +func (s priorityLevelNodeSelector[CHAIN_ID, RPC]) Name() string { + return NodeSelectionModePriorityLevel +} + +// getHighestPriorityAliveTier filters nodes that are not in state NodeStateAlive and +// returns only the highest tier of alive nodes +func (s priorityLevelNodeSelector[CHAIN_ID, RPC]) getHighestPriorityAliveTier() []nodeWithPriority[CHAIN_ID, RPC] { + var nodes []nodeWithPriority[CHAIN_ID, RPC] + for _, n := range s.nodes { + if n.State() == NodeStateAlive { + nodes = append(nodes, nodeWithPriority[CHAIN_ID, RPC]{n, n.Order()}) + } + } + + if len(nodes) == 0 { + return nil + } + + return removeLowerTiers(nodes) +} + +// removeLowerTiers take a slice of nodeWithPriority[CHAIN_ID, BLOCK_HASH, HEAD, RPC] and keeps only the highest tier +func removeLowerTiers[ + CHAIN_ID ID, + RPC any, +](nodes []nodeWithPriority[CHAIN_ID, RPC]) []nodeWithPriority[CHAIN_ID, RPC] { + sort.SliceStable(nodes, func(i, j int) bool { + return nodes[i].priority > nodes[j].priority + }) + + var nodes2 []nodeWithPriority[CHAIN_ID, RPC] + currentPriority := nodes[len(nodes)-1].priority + + for _, n := range nodes { + if n.priority == currentPriority { + nodes2 = append(nodes2, n) + } + } + + return nodes2 +} + +// nrOfPriorityTiers calculates the total number of priority tiers +func nrOfPriorityTiers[ + CHAIN_ID ID, + RPC any, +](nodes []Node[CHAIN_ID, RPC]) int32 { + highestPriority := int32(0) + for _, n := range nodes { + priority := n.Order() + if highestPriority < priority { + highestPriority = priority + } + } + return highestPriority + 1 +} + +// firstOrHighestPriority takes a list of nodes and returns the first one with the highest priority +func firstOrHighestPriority[ + CHAIN_ID ID, + RPC any, +](nodes []Node[CHAIN_ID, RPC]) Node[CHAIN_ID, RPC] { + hp := int32(math.MaxInt32) + var node Node[CHAIN_ID, RPC] + for _, n := range nodes { + if n.Order() < hp { + hp = n.Order() + node = n + } + } + return node +} diff --git a/pkg/solana/client/multinode/node_selector_round_robin.go b/pkg/solana/client/multinode/node_selector_round_robin.go new file mode 100644 index 000000000..52fa9d6c8 --- /dev/null +++ b/pkg/solana/client/multinode/node_selector_round_robin.go @@ -0,0 +1,46 @@ +package client + +import ( + "sync/atomic" +) + +type roundRobinSelector[ + CHAIN_ID ID, + RPC any, +] struct { + nodes []Node[CHAIN_ID, RPC] + roundRobinCount atomic.Uint32 +} + +func NewRoundRobinSelector[ + CHAIN_ID ID, + RPC any, +](nodes []Node[CHAIN_ID, RPC]) NodeSelector[CHAIN_ID, RPC] { + return &roundRobinSelector[CHAIN_ID, RPC]{ + nodes: nodes, + } +} + +func (s *roundRobinSelector[CHAIN_ID, RPC]) Select() Node[CHAIN_ID, RPC] { + var liveNodes []Node[CHAIN_ID, RPC] + for _, n := range s.nodes { + if n.State() == NodeStateAlive { + liveNodes = append(liveNodes, n) + } + } + + nNodes := len(liveNodes) + if nNodes == 0 { + return nil + } + + // NOTE: Inc returns the number after addition, so we must -1 to get the "current" counter + count := s.roundRobinCount.Add(1) - 1 + idx := int(count % uint32(nNodes)) + + return liveNodes[idx] +} + +func (s *roundRobinSelector[CHAIN_ID, RPC]) Name() string { + return NodeSelectionModeRoundRobin +} diff --git a/pkg/solana/client/multinode/node_selector_total_difficulty.go b/pkg/solana/client/multinode/node_selector_total_difficulty.go new file mode 100644 index 000000000..3f3c79de9 --- /dev/null +++ b/pkg/solana/client/multinode/node_selector_total_difficulty.go @@ -0,0 +1,51 @@ +package client + +import ( + "math/big" +) + +type totalDifficultyNodeSelector[ + CHAIN_ID ID, + RPC any, +] []Node[CHAIN_ID, RPC] + +func NewTotalDifficultyNodeSelector[ + CHAIN_ID ID, + RPC any, +](nodes []Node[CHAIN_ID, RPC]) NodeSelector[CHAIN_ID, RPC] { + return totalDifficultyNodeSelector[CHAIN_ID, RPC](nodes) +} + +func (s totalDifficultyNodeSelector[CHAIN_ID, RPC]) Select() Node[CHAIN_ID, RPC] { + // NodeNoNewHeadsThreshold may not be enabled, in this case all nodes have td == nil + var highestTD *big.Int + var nodes []Node[CHAIN_ID, RPC] + var aliveNodes []Node[CHAIN_ID, RPC] + + for _, n := range s { + state, currentChainInfo := n.StateAndLatest() + if state != NodeStateAlive { + continue + } + + currentTD := currentChainInfo.TotalDifficulty + aliveNodes = append(aliveNodes, n) + if currentTD != nil && (highestTD == nil || currentTD.Cmp(highestTD) >= 0) { + if highestTD == nil || currentTD.Cmp(highestTD) > 0 { + highestTD = currentTD + nodes = nil + } + nodes = append(nodes, n) + } + } + + //If all nodes have td == nil pick one from the nodes that are alive + if len(nodes) == 0 { + return firstOrHighestPriority(aliveNodes) + } + return firstOrHighestPriority(nodes) +} + +func (s totalDifficultyNodeSelector[CHAIN_ID, RPC]) Name() string { + return NodeSelectionModeTotalDifficulty +} diff --git a/pkg/solana/client/multinode/poller.go b/pkg/solana/client/multinode/poller.go index d6080722c..eeb6c3af5 100644 --- a/pkg/solana/client/multinode/poller.go +++ b/pkg/solana/client/multinode/poller.go @@ -2,7 +2,6 @@ package client import ( "context" - "sync" "time" "github.com/smartcontractkit/chainlink-common/pkg/logger" @@ -15,83 +14,80 @@ import ( // and delivers the result to a channel. It is used by multinode to poll // for new heads and implements the Subscription interface. type Poller[T any] struct { - services.StateMachine + services.Service + eng *services.Engine + pollingInterval time.Duration pollingFunc func(ctx context.Context) (T, error) pollingTimeout time.Duration - logger logger.Logger channel chan<- T errCh chan error - - stopCh services.StopChan - wg sync.WaitGroup } // NewPoller creates a new Poller instance and returns a channel to receive the polled data func NewPoller[ T any, -](pollingInterval time.Duration, pollingFunc func(ctx context.Context) (T, error), pollingTimeout time.Duration, logger logger.Logger) (Poller[T], <-chan T) { +](pollingInterval time.Duration, pollingFunc func(ctx context.Context) (T, error), pollingTimeout time.Duration, lggr logger.Logger) (Poller[T], <-chan T) { channel := make(chan T) - return Poller[T]{ + p := Poller[T]{ pollingInterval: pollingInterval, pollingFunc: pollingFunc, pollingTimeout: pollingTimeout, channel: channel, - logger: logger, errCh: make(chan error), - stopCh: make(chan struct{}), - }, channel + } + p.Service, p.eng = services.Config{ + Name: "Poller", + Start: p.start, + Close: p.close, + }.NewServiceEngine(lggr) + return p, channel } var _ types.Subscription = &Poller[any]{} -func (p *Poller[T]) Start() error { - return p.StartOnce("Poller", func() error { - p.wg.Add(1) - go p.pollingLoop() - return nil - }) +func (p *Poller[T]) start(ctx context.Context) error { + p.eng.Go(p.pollingLoop) + return nil } // Unsubscribe cancels the sending of events to the data channel func (p *Poller[T]) Unsubscribe() { - _ = p.StopOnce("Poller", func() error { - close(p.stopCh) - p.wg.Wait() - close(p.errCh) - close(p.channel) - return nil - }) + _ = p.Close() +} + +func (p *Poller[T]) close() error { + close(p.errCh) + close(p.channel) + return nil } func (p *Poller[T]) Err() <-chan error { return p.errCh } -func (p *Poller[T]) pollingLoop() { - defer p.wg.Done() - +func (p *Poller[T]) pollingLoop(ctx context.Context) { ticker := time.NewTicker(p.pollingInterval) defer ticker.Stop() for { select { - case <-p.stopCh: + case <-ctx.Done(): return case <-ticker.C: // Set polling timeout - pollingCtx, cancelPolling := p.stopCh.CtxCancel(context.WithTimeout(context.Background(), p.pollingTimeout)) + pollingCtx, cancelPolling := context.WithTimeout(ctx, p.pollingTimeout) // Execute polling function result, err := p.pollingFunc(pollingCtx) cancelPolling() if err != nil { - p.logger.Warnf("polling error: %v", err) + p.eng.Warnf("polling error: %v", err) continue } // Send result to channel or block if channel is full select { case p.channel <- result: - case <-p.stopCh: + case <-ctx.Done(): return } } diff --git a/pkg/solana/client/multinode/redialbackoff.go b/pkg/solana/client/multinode/redialbackoff.go new file mode 100644 index 000000000..41be2232d --- /dev/null +++ b/pkg/solana/client/multinode/redialbackoff.go @@ -0,0 +1,17 @@ +package client + +import ( + "time" + + "github.com/jpillora/backoff" +) + +// NewRedialBackoff is a standard backoff to use for redialling or reconnecting to +// unreachable network endpoints +func NewRedialBackoff() backoff.Backoff { + return backoff.Backoff{ + Min: 1 * time.Second, + Max: 15 * time.Second, + Jitter: true, + } +} diff --git a/pkg/solana/client/multinode/send_only_node.go b/pkg/solana/client/multinode/send_only_node.go index 069911c78..1ff8efa79 100644 --- a/pkg/solana/client/multinode/send_only_node.go +++ b/pkg/solana/client/multinode/send_only_node.go @@ -137,7 +137,7 @@ func (s *sendOnlyNode[CHAIN_ID, RPC]) start(startCtx context.Context) { promPoolRPCNodeTransitionsToAlive.WithLabelValues(s.chainID.String(), s.name).Inc() s.setState(NodeStateAlive) - s.log.Infow("Sendonly RPC Node is online", "NodeState", s.state) + s.log.Infow("Sendonly RPC Node is online", "nodeState", s.state) } func (s *sendOnlyNode[CHAIN_ID, RPC]) Close() error { diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go index a4c5e2b3d..1408e7417 100644 --- a/pkg/solana/client/multinode/transaction_sender.go +++ b/pkg/solana/client/multinode/transaction_sender.go @@ -2,6 +2,7 @@ package client import ( "context" + "errors" "fmt" "math" "slices" @@ -13,6 +14,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" + "github.com/smartcontractkit/chainlink/v2/common/types" ) var ( @@ -40,7 +42,7 @@ type SendTxRPCClient[TX any] interface { SendTransaction(ctx context.Context, tx TX) error } -func NewTransactionSender[TX any, CHAIN_ID ID, RPC SendTxRPCClient[TX]]( +func NewTransactionSender[TX any, CHAIN_ID types.ID, RPC SendTxRPCClient[TX]]( lggr logger.Logger, chainID CHAIN_ID, chainFamily string, @@ -62,7 +64,7 @@ func NewTransactionSender[TX any, CHAIN_ID ID, RPC SendTxRPCClient[TX]]( } } -type TransactionSender[TX any, CHAIN_ID ID, RPC SendTxRPCClient[TX]] struct { +type TransactionSender[TX any, CHAIN_ID types.ID, RPC SendTxRPCClient[TX]] struct { services.StateMachine chainID CHAIN_ID chainFamily string @@ -133,12 +135,6 @@ func (txSender *TransactionSender[TX, CHAIN_ID, RPC]) SendTransaction(ctx contex } }() }) - if err != nil { - primaryNodeWg.Wait() - close(txResultsToReport) - close(txResults) - return 0, err - } // This needs to be done in parallel so the reporting knows when it's done (when the channel is closed) txSender.wg.Add(1) @@ -149,6 +145,10 @@ func (txSender *TransactionSender[TX, CHAIN_ID, RPC]) SendTransaction(ctx contex close(txResults) }() + if err != nil { + return 0, err + } + txSender.wg.Add(1) go txSender.reportSendTxAnomalies(tx, txResultsToReport) @@ -167,7 +167,7 @@ func (txSender *TransactionSender[TX, CHAIN_ID, RPC]) broadcastTxAsync(ctx conte func (txSender *TransactionSender[TX, CHAIN_ID, RPC]) reportSendTxAnomalies(tx TX, txResults <-chan sendTxResult) { defer txSender.wg.Done() - resultsByCode := sendTxErrors{} + resultsByCode := sendTxResults{} // txResults eventually will be closed for txResult := range txResults { resultsByCode[txResult.ResultCode] = append(resultsByCode[txResult.ResultCode], txResult.Err) @@ -180,9 +180,9 @@ func (txSender *TransactionSender[TX, CHAIN_ID, RPC]) reportSendTxAnomalies(tx T } } -type sendTxErrors map[SendTxReturnCode][]error +type sendTxResults map[SendTxReturnCode][]error -func aggregateTxResults(resultsByCode sendTxErrors) (returnCode SendTxReturnCode, txResult error, err error) { +func aggregateTxResults(resultsByCode sendTxResults) (returnCode SendTxReturnCode, txResult error, err error) { severeCode, severeErrors, hasSevereErrors := findFirstIn(resultsByCode, sendTxSevereErrors) successCode, successResults, hasSuccess := findFirstIn(resultsByCode, sendTxSuccessfulCodes) if hasSuccess { @@ -191,7 +191,7 @@ func aggregateTxResults(resultsByCode sendTxErrors) (returnCode SendTxReturnCode if hasSevereErrors { const errMsg = "found contradictions in nodes replies on SendTransaction: got success and severe error" // return success, since at least 1 node has accepted our broadcasted Tx, and thus it can now be included onchain - return successCode, successResults[0], fmt.Errorf(errMsg) + return successCode, successResults[0], errors.New(errMsg) } // other errors are temporary - we are safe to return success @@ -208,7 +208,7 @@ func aggregateTxResults(resultsByCode sendTxErrors) (returnCode SendTxReturnCode } err = fmt.Errorf("expected at least one response on SendTransaction") - return 0, err, err + return Retryable, err, err } func (txSender *TransactionSender[TX, CHAIN_ID, RPC]) collectTxResults(ctx context.Context, tx TX, healthyNodesNum int, txResults <-chan sendTxResult) (SendTxReturnCode, error) { @@ -216,7 +216,7 @@ func (txSender *TransactionSender[TX, CHAIN_ID, RPC]) collectTxResults(ctx conte return 0, ErroringNodeError } requiredResults := int(math.Ceil(float64(healthyNodesNum) * sendTxQuorum)) - errorsByCode := sendTxErrors{} + errorsByCode := sendTxResults{} var softTimeoutChan <-chan time.Time var resultsCount int loop: diff --git a/pkg/solana/client/multinode/types.go b/pkg/solana/client/multinode/types.go index 5cd831fc1..6c863c867 100644 --- a/pkg/solana/client/multinode/types.go +++ b/pkg/solana/client/multinode/types.go @@ -6,14 +6,9 @@ import ( "math/big" ) -// A chain-agnostic generic interface to represent the following native types on various chains: -// PublicKey, Address, Account, BlockHash, TxHash -type Hashable interface { - fmt.Stringer - comparable - - Bytes() []byte -} +// ID represents the base type, for any chain's ID. +// It should be convertible to a string, that can uniquely identify this chain +type ID fmt.Stringer // Subscription represents an event subscription where events are // delivered on a data channel. @@ -30,7 +25,7 @@ type Subscription interface { Err() <-chan error } -// RPCClient includes all the necessary generalized RPC methods along with any additional chain-specific methods. +// RPCClient includes all the necessary generalized RPC methods used by Node to perform health checks type RPCClient[ CHAIN_ID ID, HEAD Head, @@ -104,21 +99,3 @@ func MaxTotalDifficulty(a, b *big.Int) *big.Int { return big.NewInt(0).Set(b) } - -// ID represents the base type, for any chain's ID. -// It should be convertible to a string, that can uniquely identify this chain -type ID fmt.Stringer - -type multiNodeContextKey int - -const ( - contextKeyHeathCheckRequest multiNodeContextKey = iota + 1 -) - -func CtxAddHealthCheckFlag(ctx context.Context) context.Context { - return context.WithValue(ctx, contextKeyHeathCheckRequest, struct{}{}) -} - -func CtxIsHeathCheckRequest(ctx context.Context) bool { - return ctx.Value(contextKeyHeathCheckRequest) != nil -} diff --git a/pkg/solana/client/rpc_client.go b/pkg/solana/client/rpc_client.go index 0db8d9062..ec89d2b66 100644 --- a/pkg/solana/client/rpc_client.go +++ b/pkg/solana/client/rpc_client.go @@ -24,8 +24,6 @@ func (s StringID) String() string { return string(s) } -// TODO: ChainReaderWriter needs ChainID() (string, error) -// TODO: We probably don't need this though? var _ ReaderWriter = (*RpcClient)(nil) type Head struct { @@ -61,6 +59,8 @@ type RpcClient struct { requestGroup *singleflight.Group } +// TODO: BCI-4061: Implement RPC Client for MultiNode + func (c *RpcClient) Dial(ctx context.Context) error { //TODO implement me panic("implement me") From c44a6ce77c5e71cccb4312af2b8a4a5e2113b176 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 3 Sep 2024 11:58:39 -0400 Subject: [PATCH 003/174] Add MultiNode flag --- pkg/solana/cmd/chainlink-solana/main.go | 12 +++++++++++- pkg/solana/config/multinode.go | 1 + 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/pkg/solana/cmd/chainlink-solana/main.go b/pkg/solana/cmd/chainlink-solana/main.go index 08893e7de..6a966a693 100644 --- a/pkg/solana/cmd/chainlink-solana/main.go +++ b/pkg/solana/cmd/chainlink-solana/main.go @@ -66,10 +66,20 @@ func (c *pluginRelayer) NewRelayer(ctx context.Context, config string, keystore Logger: c.Logger, KeyStore: keystore, } - chain, err := solana.NewChain(&cfg.Solana, opts) + + var chain solana.Chain + var err error + + if cfg.Solana.MultiNodeConfig().MultiNodeEnabled() { + chain, err = solana.NewMultiNodeChain(&cfg.Solana, opts) + } else { + chain, err = solana.NewChain(&cfg.Solana, opts) + } + if err != nil { return nil, fmt.Errorf("failed to create chain: %w", err) } + ra := &loop.RelayerAdapter{Relayer: solana.NewRelayer(c.Logger, chain, capRegistry), RelayerExt: chain} c.SubService(ra) diff --git a/pkg/solana/config/multinode.go b/pkg/solana/config/multinode.go index 72e345728..1755e6ee6 100644 --- a/pkg/solana/config/multinode.go +++ b/pkg/solana/config/multinode.go @@ -83,4 +83,5 @@ func (c *MultiNode) FinalizedBlockOffset() uint32 { func (c *MultiNode) SetDefaults() { // TODO: Set defaults for MultiNode config https://smartcontract-it.atlassian.net/browse/BCI-4065 + c.multiNodeEnabled = false } From 64db86a4a00c8366864d18798df593cb3daab70c Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 3 Sep 2024 12:05:13 -0400 Subject: [PATCH 004/174] Remove internal dependency --- pkg/solana/client/multinode/send_only_node.go | 10 ++++------ .../client/multinode/send_only_node_lifecycle.go | 4 +--- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/pkg/solana/client/multinode/send_only_node.go b/pkg/solana/client/multinode/send_only_node.go index 1ff8efa79..4f60f566d 100644 --- a/pkg/solana/client/multinode/send_only_node.go +++ b/pkg/solana/client/multinode/send_only_node.go @@ -8,12 +8,10 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" - - "github.com/smartcontractkit/chainlink/v2/common/types" ) type sendOnlyClient[ - CHAIN_ID types.ID, + CHAIN_ID ID, ] interface { Close() ChainID(context.Context) (CHAIN_ID, error) @@ -22,7 +20,7 @@ type sendOnlyClient[ // SendOnlyNode represents one node used as a sendonly type SendOnlyNode[ - CHAIN_ID types.ID, + CHAIN_ID ID, RPC any, ] interface { // Start may attempt to connect to the node, but should only return error for misconfiguration - never for temporary errors. @@ -42,7 +40,7 @@ type SendOnlyNode[ // It only supports sending transactions // It must use an http(s) url type sendOnlyNode[ - CHAIN_ID types.ID, + CHAIN_ID ID, RPC sendOnlyClient[CHAIN_ID], ] struct { services.StateMachine @@ -61,7 +59,7 @@ type sendOnlyNode[ // NewSendOnlyNode returns a new sendonly node func NewSendOnlyNode[ - CHAIN_ID types.ID, + CHAIN_ID ID, RPC sendOnlyClient[CHAIN_ID], ]( lggr logger.Logger, diff --git a/pkg/solana/client/multinode/send_only_node_lifecycle.go b/pkg/solana/client/multinode/send_only_node_lifecycle.go index a6ac11248..83642feba 100644 --- a/pkg/solana/client/multinode/send_only_node_lifecycle.go +++ b/pkg/solana/client/multinode/send_only_node_lifecycle.go @@ -3,8 +3,6 @@ package client import ( "fmt" "time" - - "github.com/smartcontractkit/chainlink/v2/common/internal/utils" ) // verifyLoop may only be triggered once, on Start, if initial chain ID check @@ -16,7 +14,7 @@ func (s *sendOnlyNode[CHAIN_ID, RPC]) verifyLoop() { ctx, cancel := s.chStop.NewCtx() defer cancel() - backoff := utils.NewRedialBackoff() + backoff := NewRedialBackoff() for { select { case <-ctx.Done(): From f7c1bc95b193cfb3e1e4ecbda2e46fba85d88002 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 3 Sep 2024 12:10:52 -0400 Subject: [PATCH 005/174] Fix build --- pkg/solana/client/multinode/poller.go | 4 +--- pkg/solana/client/multinode/transaction_sender.go | 5 ++--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/pkg/solana/client/multinode/poller.go b/pkg/solana/client/multinode/poller.go index eeb6c3af5..9ebe1dcfc 100644 --- a/pkg/solana/client/multinode/poller.go +++ b/pkg/solana/client/multinode/poller.go @@ -6,8 +6,6 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" - - "github.com/smartcontractkit/chainlink/v2/common/types" ) // Poller is a component that polls a function at a given interval @@ -44,7 +42,7 @@ func NewPoller[ return p, channel } -var _ types.Subscription = &Poller[any]{} +var _ Subscription = &Poller[any]{} func (p *Poller[T]) start(ctx context.Context) error { p.eng.Go(p.pollingLoop) diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go index 1408e7417..d567e164f 100644 --- a/pkg/solana/client/multinode/transaction_sender.go +++ b/pkg/solana/client/multinode/transaction_sender.go @@ -14,7 +14,6 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" - "github.com/smartcontractkit/chainlink/v2/common/types" ) var ( @@ -42,7 +41,7 @@ type SendTxRPCClient[TX any] interface { SendTransaction(ctx context.Context, tx TX) error } -func NewTransactionSender[TX any, CHAIN_ID types.ID, RPC SendTxRPCClient[TX]]( +func NewTransactionSender[TX any, CHAIN_ID ID, RPC SendTxRPCClient[TX]]( lggr logger.Logger, chainID CHAIN_ID, chainFamily string, @@ -64,7 +63,7 @@ func NewTransactionSender[TX any, CHAIN_ID types.ID, RPC SendTxRPCClient[TX]]( } } -type TransactionSender[TX any, CHAIN_ID types.ID, RPC SendTxRPCClient[TX]] struct { +type TransactionSender[TX any, CHAIN_ID ID, RPC SendTxRPCClient[TX]] struct { services.StateMachine chainID CHAIN_ID chainFamily string From 9e91b476f7939c79e99a9f2ded8450a9c5260f76 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 4 Sep 2024 12:40:03 -0400 Subject: [PATCH 006/174] Fix import cycle --- pkg/solana/chain.go | 6 +- pkg/solana/chain_multinode.go | 21 +- pkg/solana/client/client.go | 8 +- pkg/solana/client/client_test.go | 3 +- pkg/solana/client/mocks/ReaderWriter.go | 23 +- pkg/solana/client/multinode/poller_test.go | 187 --------- .../client/multinode/send_only_node_test.go | 139 ------- .../multinode/transaction_sender_test.go | 360 ------------------ pkg/solana/client/multinode/types.go | 7 + pkg/solana/client/rpc_client.go | 10 +- 10 files changed, 43 insertions(+), 721 deletions(-) delete mode 100644 pkg/solana/client/multinode/poller_test.go delete mode 100644 pkg/solana/client/multinode/send_only_node_test.go delete mode 100644 pkg/solana/client/multinode/transaction_sender_test.go diff --git a/pkg/solana/chain.go b/pkg/solana/chain.go index 20f6322d5..bc2dd845a 100644 --- a/pkg/solana/chain.go +++ b/pkg/solana/chain.go @@ -22,6 +22,8 @@ import ( relaytypes "github.com/smartcontractkit/chainlink-common/pkg/types" "github.com/smartcontractkit/chainlink-common/pkg/types/core" + mn "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" + "github.com/smartcontractkit/chainlink-solana/pkg/solana/client" "github.com/smartcontractkit/chainlink-solana/pkg/solana/config" "github.com/smartcontractkit/chainlink-solana/pkg/solana/monitor" @@ -187,13 +189,13 @@ func (v *verifiedCachedClient) LatestBlockhash() (*rpc.GetLatestBlockhashResult, return v.ReaderWriter.LatestBlockhash() } -func (v *verifiedCachedClient) ChainID(ctx context.Context) (client.StringID, error) { +func (v *verifiedCachedClient) ChainID(ctx context.Context) (mn.StringID, error) { verified, err := v.verifyChainID() if !verified { return "", err } - return client.StringID(v.chainID), nil + return mn.StringID(v.chainID), nil } func (v *verifiedCachedClient) GetFeeForMessage(msg string) (uint64, error) { diff --git a/pkg/solana/chain_multinode.go b/pkg/solana/chain_multinode.go index 82fb5b23f..cb7161a55 100644 --- a/pkg/solana/chain_multinode.go +++ b/pkg/solana/chain_multinode.go @@ -15,6 +15,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/loop" "github.com/smartcontractkit/chainlink-common/pkg/services" relaytypes "github.com/smartcontractkit/chainlink-common/pkg/types" + "github.com/smartcontractkit/chainlink-solana/pkg/solana/client" mn "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" "github.com/smartcontractkit/chainlink-solana/pkg/solana/config" @@ -39,8 +40,8 @@ type multiNodeChain struct { services.StateMachine id string cfg *config.TOMLConfig - multiNode *mn.MultiNode[client.StringID, *client.RpcClient] - txSender *mn.TransactionSender[*solanago.Transaction, client.StringID, *client.RpcClient] + multiNode *mn.MultiNode[mn.StringID, *client.RpcClient] + txSender *mn.TransactionSender[*solanago.Transaction, mn.StringID, *client.RpcClient] txm *txm.Txm balanceMonitor services.Service lggr logger.Logger @@ -57,7 +58,7 @@ func newMultiNodeChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr mnCfg := cfg.MultiNodeConfig() - var nodes []mn.Node[client.StringID, *client.RpcClient] + var nodes []mn.Node[mn.StringID, *client.RpcClient] for i, nodeInfo := range cfg.ListNodes() { // create client and check @@ -67,20 +68,20 @@ func newMultiNodeChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr continue } - newNode := mn.NewNode[client.StringID, *client.Head, *client.RpcClient]( + newNode := mn.NewNode[mn.StringID, *client.Head, *client.RpcClient]( mnCfg, mnCfg, lggr, *nodeInfo.URL.URL(), nil, *nodeInfo.Name, - int32(i), client.StringID(id), 0, rpcClient, chainFamily) + int32(i), mn.StringID(id), 0, rpcClient, chainFamily) nodes = append(nodes, newNode) } - multiNode := mn.NewMultiNode[client.StringID, *client.RpcClient]( + multiNode := mn.NewMultiNode[mn.StringID, *client.RpcClient]( lggr, mn.NodeSelectionModeRoundRobin, time.Duration(0), // TODO: set lease duration nodes, - []mn.SendOnlyNode[client.StringID, *client.RpcClient]{}, // TODO: no send only nodes? - client.StringID(id), + []mn.SendOnlyNode[mn.StringID, *client.RpcClient]{}, // TODO: no send only nodes? + mn.StringID(id), chainFamily, time.Duration(0), // TODO: set deathDeclarationDelay ) @@ -89,9 +90,9 @@ func newMultiNodeChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr return 0 // TODO ClassifySendError(err, clientErrors, logger.Sugared(logger.Nop()), tx, common.Address{}, false) } - txSender := mn.NewTransactionSender[*solanago.Transaction, client.StringID, *client.RpcClient]( + txSender := mn.NewTransactionSender[*solanago.Transaction, mn.StringID, *client.RpcClient]( lggr, - client.StringID(id), + mn.StringID(id), chainFamily, multiNode, classifySendError, diff --git a/pkg/solana/client/client.go b/pkg/solana/client/client.go index d007e3c4c..d2294824d 100644 --- a/pkg/solana/client/client.go +++ b/pkg/solana/client/client.go @@ -6,6 +6,8 @@ import ( "fmt" "time" + mn "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" + "github.com/gagliardetto/solana-go" "github.com/gagliardetto/solana-go/rpc" "golang.org/x/sync/singleflight" @@ -33,7 +35,7 @@ type Reader interface { Balance(addr solana.PublicKey) (uint64, error) SlotHeight() (uint64, error) LatestBlockhash() (*rpc.GetLatestBlockhashResult, error) - ChainID(ctx context.Context) (StringID, error) + ChainID(ctx context.Context) (mn.StringID, error) GetFeeForMessage(msg string) (uint64, error) GetLatestBlock() (*rpc.GetBlockResult, error) } @@ -142,7 +144,7 @@ func (c *Client) LatestBlockhash() (*rpc.GetLatestBlockhashResult, error) { return v.(*rpc.GetLatestBlockhashResult), err } -func (c *Client) ChainID(ctx context.Context) (StringID, error) { +func (c *Client) ChainID(ctx context.Context) (mn.StringID, error) { done := c.latency("chain_id") defer done() @@ -168,7 +170,7 @@ func (c *Client) ChainID(ctx context.Context) (StringID, error) { c.log.Warnf("unknown genesis hash - assuming solana chain is 'localnet'") network = "localnet" } - return StringID(network), nil + return mn.StringID(network), nil } func (c *Client) GetFeeForMessage(msg string) (uint64, error) { diff --git a/pkg/solana/client/client_test.go b/pkg/solana/client/client_test.go index 6f2276bd3..ed4e1dba4 100644 --- a/pkg/solana/client/client_test.go +++ b/pkg/solana/client/client_test.go @@ -19,6 +19,7 @@ import ( "github.com/stretchr/testify/require" "github.com/smartcontractkit/chainlink-common/pkg/logger" + mn "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" "github.com/smartcontractkit/chainlink-solana/pkg/solana/config" "github.com/smartcontractkit/chainlink-solana/pkg/solana/monitor" @@ -122,7 +123,7 @@ func TestClient_Reader_ChainID(t *testing.T) { for _, n := range networks { network, err := c.ChainID(context.Background()) assert.NoError(t, err) - assert.Equal(t, n, network) + assert.Equal(t, mn.StringID(n), network) } } diff --git a/pkg/solana/client/mocks/ReaderWriter.go b/pkg/solana/client/mocks/ReaderWriter.go index 2bbb82fef..b6cd6808a 100644 --- a/pkg/solana/client/mocks/ReaderWriter.go +++ b/pkg/solana/client/mocks/ReaderWriter.go @@ -6,6 +6,7 @@ import ( context "context" rpc "github.com/gagliardetto/solana-go/rpc" + multinode "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" mock "github.com/stretchr/testify/mock" solana "github.com/gagliardetto/solana-go" @@ -44,27 +45,27 @@ func (_m *ReaderWriter) Balance(addr solana.PublicKey) (uint64, error) { return r0, r1 } -// ChainID provides a mock function with given fields: -func (_m *ReaderWriter) ChainID() (string, error) { - ret := _m.Called() +// ChainID provides a mock function with given fields: ctx +func (_m *ReaderWriter) ChainID(ctx context.Context) (multinode.StringID, error) { + ret := _m.Called(ctx) if len(ret) == 0 { panic("no return value specified for ChainID") } - var r0 string + var r0 multinode.StringID var r1 error - if rf, ok := ret.Get(0).(func() (string, error)); ok { - return rf() + if rf, ok := ret.Get(0).(func(context.Context) (multinode.StringID, error)); ok { + return rf(ctx) } - if rf, ok := ret.Get(0).(func() string); ok { - r0 = rf() + if rf, ok := ret.Get(0).(func(context.Context) multinode.StringID); ok { + r0 = rf(ctx) } else { - r0 = ret.Get(0).(string) + r0 = ret.Get(0).(multinode.StringID) } - if rf, ok := ret.Get(1).(func() error); ok { - r1 = rf() + if rf, ok := ret.Get(1).(func(context.Context) error); ok { + r1 = rf(ctx) } else { r1 = ret.Error(1) } diff --git a/pkg/solana/client/multinode/poller_test.go b/pkg/solana/client/multinode/poller_test.go deleted file mode 100644 index 91af57930..000000000 --- a/pkg/solana/client/multinode/poller_test.go +++ /dev/null @@ -1,187 +0,0 @@ -package client - -import ( - "context" - "fmt" - "math/big" - "sync" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.uber.org/zap" - - "github.com/smartcontractkit/chainlink-common/pkg/logger" - "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" -) - -func Test_Poller(t *testing.T) { - lggr := logger.Test(t) - - t.Run("Test multiple start", func(t *testing.T) { - pollFunc := func(ctx context.Context) (Head, error) { - return nil, nil - } - - poller, _ := NewPoller[Head](time.Millisecond, pollFunc, time.Second, lggr) - err := poller.Start() - require.NoError(t, err) - - err = poller.Start() - require.Error(t, err) - poller.Unsubscribe() - }) - - t.Run("Test polling for heads", func(t *testing.T) { - // Mock polling function that returns a new value every time it's called - var pollNumber int - pollLock := sync.Mutex{} - pollFunc := func(ctx context.Context) (Head, error) { - pollLock.Lock() - defer pollLock.Unlock() - pollNumber++ - h := head{ - BlockNumber: int64(pollNumber), - BlockDifficulty: big.NewInt(int64(pollNumber)), - } - return h.ToMockHead(t), nil - } - - // Create poller and start to receive data - poller, channel := NewPoller[Head](time.Millisecond, pollFunc, time.Second, lggr) - require.NoError(t, poller.Start()) - defer poller.Unsubscribe() - - // Receive updates from the poller - pollCount := 0 - pollMax := 50 - for ; pollCount < pollMax; pollCount++ { - h := <-channel - assert.Equal(t, int64(pollCount+1), h.BlockNumber()) - } - }) - - t.Run("Test polling errors", func(t *testing.T) { - // Mock polling function that returns an error - var pollNumber int - pollLock := sync.Mutex{} - pollFunc := func(ctx context.Context) (Head, error) { - pollLock.Lock() - defer pollLock.Unlock() - pollNumber++ - return nil, fmt.Errorf("polling error %d", pollNumber) - } - - olggr, observedLogs := logger.TestObserved(t, zap.WarnLevel) - - // Create poller and subscribe to receive data - poller, _ := NewPoller[Head](time.Millisecond, pollFunc, time.Second, olggr) - require.NoError(t, poller.Start()) - defer poller.Unsubscribe() - - // Ensure that all errors were logged as expected - logsSeen := func() bool { - for pollCount := 0; pollCount < 50; pollCount++ { - numLogs := observedLogs.FilterMessage(fmt.Sprintf("polling error: polling error %d", pollCount+1)).Len() - if numLogs != 1 { - return false - } - } - return true - } - require.Eventually(t, logsSeen, tests.WaitTimeout(t), 100*time.Millisecond) - }) - - t.Run("Test polling timeout", func(t *testing.T) { - pollFunc := func(ctx context.Context) (Head, error) { - if <-ctx.Done(); true { - return nil, ctx.Err() - } - return nil, nil - } - - // Set instant timeout - pollingTimeout := time.Duration(0) - - olggr, observedLogs := logger.TestObserved(t, zap.WarnLevel) - - // Create poller and subscribe to receive data - poller, _ := NewPoller[Head](time.Millisecond, pollFunc, pollingTimeout, olggr) - require.NoError(t, poller.Start()) - defer poller.Unsubscribe() - - // Ensure that timeout errors were logged as expected - logsSeen := func() bool { - return observedLogs.FilterMessage("polling error: context deadline exceeded").Len() >= 1 - } - require.Eventually(t, logsSeen, tests.WaitTimeout(t), 100*time.Millisecond) - }) - - t.Run("Test unsubscribe during polling", func(t *testing.T) { - wait := make(chan struct{}) - closeOnce := sync.OnceFunc(func() { close(wait) }) - pollFunc := func(ctx context.Context) (Head, error) { - closeOnce() - // Block in polling function until context is cancelled - if <-ctx.Done(); true { - return nil, ctx.Err() - } - return nil, nil - } - - // Set long timeout - pollingTimeout := time.Minute - - olggr, observedLogs := logger.TestObserved(t, zap.WarnLevel) - - // Create poller and subscribe to receive data - poller, _ := NewPoller[Head](time.Millisecond, pollFunc, pollingTimeout, olggr) - require.NoError(t, poller.Start()) - - // Unsubscribe while blocked in polling function - <-wait - poller.Unsubscribe() - - // Ensure error was logged - logsSeen := func() bool { - return observedLogs.FilterMessage("polling error: context canceled").Len() >= 1 - } - require.Eventually(t, logsSeen, tests.WaitTimeout(t), 100*time.Millisecond) - }) -} - -func Test_Poller_Unsubscribe(t *testing.T) { - lggr := logger.Test(t) - pollFunc := func(ctx context.Context) (Head, error) { - select { - case <-ctx.Done(): - return nil, ctx.Err() - default: - h := head{ - BlockNumber: 0, - BlockDifficulty: big.NewInt(0), - } - return h.ToMockHead(t), nil - } - } - - t.Run("Test multiple unsubscribe", func(t *testing.T) { - poller, channel := NewPoller[Head](time.Millisecond, pollFunc, time.Second, lggr) - err := poller.Start() - require.NoError(t, err) - - <-channel - poller.Unsubscribe() - poller.Unsubscribe() - }) - - t.Run("Read channel after unsubscribe", func(t *testing.T) { - poller, channel := NewPoller[Head](time.Millisecond, pollFunc, time.Second, lggr) - err := poller.Start() - require.NoError(t, err) - - poller.Unsubscribe() - require.Equal(t, <-channel, nil) - }) -} diff --git a/pkg/solana/client/multinode/send_only_node_test.go b/pkg/solana/client/multinode/send_only_node_test.go deleted file mode 100644 index 352fb5b92..000000000 --- a/pkg/solana/client/multinode/send_only_node_test.go +++ /dev/null @@ -1,139 +0,0 @@ -package client - -import ( - "errors" - "fmt" - "net/url" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" - "go.uber.org/zap" - - "github.com/smartcontractkit/chainlink-common/pkg/logger" - "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" - - "github.com/smartcontractkit/chainlink/v2/common/types" -) - -func TestNewSendOnlyNode(t *testing.T) { - t.Parallel() - - urlFormat := "http://user:%s@testurl.com" - password := "pass" - u, err := url.Parse(fmt.Sprintf(urlFormat, password)) - require.NoError(t, err) - redacted := fmt.Sprintf(urlFormat, "xxxxx") - lggr := logger.Test(t) - name := "TestNewSendOnlyNode" - chainID := types.RandomID() - client := newMockSendOnlyClient[types.ID](t) - - node := NewSendOnlyNode(lggr, *u, name, chainID, client) - assert.NotNil(t, node) - - // Must contain name & url with redacted password - assert.Contains(t, node.String(), fmt.Sprintf("%s:%s", name, redacted)) - assert.Equal(t, node.ConfiguredChainID(), chainID) -} - -func TestStartSendOnlyNode(t *testing.T) { - t.Parallel() - t.Run("becomes unusable if initial dial fails", func(t *testing.T) { - t.Parallel() - lggr, observedLogs := logger.TestObserved(t, zap.WarnLevel) - client := newMockSendOnlyClient[types.ID](t) - client.On("Close").Once() - expectedError := errors.New("some http error") - client.On("Dial", mock.Anything).Return(expectedError).Once() - s := NewSendOnlyNode(lggr, url.URL{}, t.Name(), types.RandomID(), client) - - defer func() { assert.NoError(t, s.Close()) }() - err := s.Start(tests.Context(t)) - require.NoError(t, err) - - assert.Equal(t, NodeStateUnusable, s.State()) - tests.RequireLogMessage(t, observedLogs, "Dial failed: SendOnly Node is unusable") - }) - t.Run("Default ChainID(0) produces warn and skips checks", func(t *testing.T) { - t.Parallel() - lggr, observedLogs := logger.TestObserved(t, zap.WarnLevel) - client := newMockSendOnlyClient[types.ID](t) - client.On("Close").Once() - client.On("Dial", mock.Anything).Return(nil).Once() - s := NewSendOnlyNode(lggr, url.URL{}, t.Name(), types.NewIDFromInt(0), client) - - defer func() { assert.NoError(t, s.Close()) }() - err := s.Start(tests.Context(t)) - require.NoError(t, err) - - assert.Equal(t, NodeStateAlive, s.State()) - tests.RequireLogMessage(t, observedLogs, "sendonly rpc ChainID verification skipped") - }) - t.Run("Can recover from chainID verification failure", func(t *testing.T) { - t.Parallel() - lggr, observedLogs := logger.TestObserved(t, zap.WarnLevel) - client := newMockSendOnlyClient[types.ID](t) - client.On("Close").Once() - client.On("Dial", mock.Anything).Return(nil) - expectedError := errors.New("failed to get chain ID") - chainID := types.RandomID() - const failuresCount = 2 - client.On("ChainID", mock.Anything).Return(types.RandomID(), expectedError).Times(failuresCount) - client.On("ChainID", mock.Anything).Return(chainID, nil) - - s := NewSendOnlyNode(lggr, url.URL{}, t.Name(), chainID, client) - - defer func() { assert.NoError(t, s.Close()) }() - err := s.Start(tests.Context(t)) - require.NoError(t, err) - - assert.Equal(t, NodeStateUnreachable, s.State()) - tests.AssertLogCountEventually(t, observedLogs, fmt.Sprintf("Verify failed: %v", expectedError), failuresCount) - tests.AssertEventually(t, func() bool { - return s.State() == NodeStateAlive - }) - }) - t.Run("Can recover from chainID mismatch", func(t *testing.T) { - t.Parallel() - lggr, observedLogs := logger.TestObserved(t, zap.WarnLevel) - client := newMockSendOnlyClient[types.ID](t) - client.On("Close").Once() - client.On("Dial", mock.Anything).Return(nil).Once() - configuredChainID := types.NewIDFromInt(11) - rpcChainID := types.NewIDFromInt(20) - const failuresCount = 2 - client.On("ChainID", mock.Anything).Return(rpcChainID, nil).Times(failuresCount) - client.On("ChainID", mock.Anything).Return(configuredChainID, nil) - s := NewSendOnlyNode(lggr, url.URL{}, t.Name(), configuredChainID, client) - - defer func() { assert.NoError(t, s.Close()) }() - err := s.Start(tests.Context(t)) - require.NoError(t, err) - - assert.Equal(t, NodeStateInvalidChainID, s.State()) - tests.AssertLogCountEventually(t, observedLogs, "sendonly rpc ChainID doesn't match local chain ID", failuresCount) - tests.AssertEventually(t, func() bool { - return s.State() == NodeStateAlive - }) - }) - t.Run("Start with Random ChainID", func(t *testing.T) { - t.Parallel() - lggr, observedLogs := logger.TestObserved(t, zap.WarnLevel) - client := newMockSendOnlyClient[types.ID](t) - client.On("Close").Once() - client.On("Dial", mock.Anything).Return(nil).Once() - configuredChainID := types.RandomID() - client.On("ChainID", mock.Anything).Return(configuredChainID, nil) - s := NewSendOnlyNode(lggr, url.URL{}, t.Name(), configuredChainID, client) - - defer func() { assert.NoError(t, s.Close()) }() - err := s.Start(tests.Context(t)) - assert.NoError(t, err) - tests.AssertEventually(t, func() bool { - return s.State() == NodeStateAlive - }) - assert.Equal(t, 0, observedLogs.Len()) // No warnings expected - }) -} diff --git a/pkg/solana/client/multinode/transaction_sender_test.go b/pkg/solana/client/multinode/transaction_sender_test.go deleted file mode 100644 index e4387abee..000000000 --- a/pkg/solana/client/multinode/transaction_sender_test.go +++ /dev/null @@ -1,360 +0,0 @@ -package client - -import ( - "context" - "fmt" - "testing" - - "github.com/pkg/errors" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" - "go.uber.org/zap" - - "github.com/smartcontractkit/chainlink-common/pkg/logger" - "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" - "github.com/smartcontractkit/chainlink/v2/common/types" -) - -type sendTxMultiNode struct { - *MultiNode[types.ID, SendTxRPCClient[any]] -} - -type sendTxRPC struct { - sendTxRun func(args mock.Arguments) - sendTxErr error -} - -var _ SendTxRPCClient[any] = (*sendTxRPC)(nil) - -func newSendTxRPC(sendTxErr error, sendTxRun func(args mock.Arguments)) *sendTxRPC { - return &sendTxRPC{sendTxErr: sendTxErr, sendTxRun: sendTxRun} -} - -func (rpc *sendTxRPC) SendTransaction(ctx context.Context, _ any) error { - if rpc.sendTxRun != nil { - rpc.sendTxRun(mock.Arguments{ctx}) - } - return rpc.sendTxErr -} - -func newTestTransactionSender(t *testing.T, chainID types.ID, lggr logger.Logger, - nodes []Node[types.ID, SendTxRPCClient[any]], - sendOnlyNodes []SendOnlyNode[types.ID, SendTxRPCClient[any]], -) (*sendTxMultiNode, *TransactionSender[any, types.ID, SendTxRPCClient[any]]) { - mn := sendTxMultiNode{NewMultiNode[types.ID, SendTxRPCClient[any]]( - lggr, NodeSelectionModeRoundRobin, 0, nodes, sendOnlyNodes, chainID, "chainFamily", 0)} - err := mn.StartOnce("startedTestMultiNode", func() error { return nil }) - require.NoError(t, err) - - txSender := NewTransactionSender[any, types.ID, SendTxRPCClient[any]](lggr, chainID, mn.chainFamily, mn.MultiNode, classifySendTxError, tests.TestInterval) - err = txSender.Start(tests.Context(t)) - require.NoError(t, err) - - t.Cleanup(func() { - err := mn.Close() - if err != nil { - // Allow MultiNode to be closed early for testing - require.EqualError(t, err, "MultiNode has already been stopped: already stopped") - } - err = txSender.Close() - if err != nil { - // Allow TransactionSender to be closed early for testing - require.EqualError(t, err, "TransactionSender has already been stopped: already stopped") - } - }) - return &mn, txSender -} - -func classifySendTxError(_ any, err error) SendTxReturnCode { - if err != nil { - return Fatal - } - return Successful -} - -func TestTransactionSender_SendTransaction(t *testing.T) { - t.Parallel() - - newNodeWithState := func(t *testing.T, state NodeState, txErr error, sendTxRun func(args mock.Arguments)) *mockNode[types.ID, SendTxRPCClient[any]] { - rpc := newSendTxRPC(txErr, sendTxRun) - node := newMockNode[types.ID, SendTxRPCClient[any]](t) - node.On("String").Return("node name").Maybe() - node.On("RPC").Return(rpc).Maybe() - node.On("State").Return(state).Maybe() - node.On("Close").Return(nil).Once() - return node - } - - newNode := func(t *testing.T, txErr error, sendTxRun func(args mock.Arguments)) *mockNode[types.ID, SendTxRPCClient[any]] { - return newNodeWithState(t, NodeStateAlive, txErr, sendTxRun) - } - - t.Run("Fails if there is no nodes available", func(t *testing.T) { - lggr, _ := logger.TestObserved(t, zap.DebugLevel) - _, txSender := newTestTransactionSender(t, types.RandomID(), lggr, nil, nil) - _, err := txSender.SendTransaction(tests.Context(t), nil) - assert.EqualError(t, err, ErroringNodeError.Error()) - }) - - t.Run("Transaction failure happy path", func(t *testing.T) { - expectedError := errors.New("transaction failed") - mainNode := newNode(t, expectedError, nil) - lggr, observedLogs := logger.TestObserved(t, zap.DebugLevel) - - _, txSender := newTestTransactionSender(t, types.RandomID(), lggr, - []Node[types.ID, SendTxRPCClient[any]]{mainNode}, - []SendOnlyNode[types.ID, SendTxRPCClient[any]]{newNode(t, errors.New("unexpected error"), nil)}) - - result, sendErr := txSender.SendTransaction(tests.Context(t), nil) - require.ErrorIs(t, sendErr, expectedError) - require.Equal(t, Fatal, result) - tests.AssertLogCountEventually(t, observedLogs, "Node sent transaction", 2) - tests.AssertLogCountEventually(t, observedLogs, "RPC returned error", 2) - }) - - t.Run("Transaction success happy path", func(t *testing.T) { - mainNode := newNode(t, nil, nil) - - lggr, observedLogs := logger.TestObserved(t, zap.DebugLevel) - _, txSender := newTestTransactionSender(t, types.RandomID(), lggr, - []Node[types.ID, SendTxRPCClient[any]]{mainNode}, - []SendOnlyNode[types.ID, SendTxRPCClient[any]]{newNode(t, errors.New("unexpected error"), nil)}) - - result, sendErr := txSender.SendTransaction(tests.Context(t), nil) - require.NoError(t, sendErr) - require.Equal(t, Successful, result) - tests.AssertLogCountEventually(t, observedLogs, "Node sent transaction", 2) - tests.AssertLogCountEventually(t, observedLogs, "RPC returned error", 1) - }) - - t.Run("Context expired before collecting sufficient results", func(t *testing.T) { - testContext, testCancel := context.WithCancel(tests.Context(t)) - defer testCancel() - - mainNode := newNode(t, nil, func(_ mock.Arguments) { - // block caller til end of the test - <-testContext.Done() - }) - - lggr, _ := logger.TestObserved(t, zap.DebugLevel) - - _, txSender := newTestTransactionSender(t, types.RandomID(), lggr, - []Node[types.ID, SendTxRPCClient[any]]{mainNode}, nil) - - requestContext, cancel := context.WithCancel(tests.Context(t)) - cancel() - _, sendErr := txSender.SendTransaction(requestContext, nil) - require.EqualError(t, sendErr, "context canceled") - }) - - t.Run("Soft timeout stops results collection", func(t *testing.T) { - chainID := types.RandomID() - expectedError := errors.New("transaction failed") - fastNode := newNode(t, expectedError, nil) - - // hold reply from the node till end of the test - testContext, testCancel := context.WithCancel(tests.Context(t)) - defer testCancel() - slowNode := newNode(t, errors.New("transaction failed"), func(_ mock.Arguments) { - // block caller til end of the test - <-testContext.Done() - }) - - lggr, _ := logger.TestObserved(t, zap.DebugLevel) - - _, txSender := newTestTransactionSender(t, chainID, lggr, []Node[types.ID, SendTxRPCClient[any]]{fastNode, slowNode}, nil) - _, sendErr := txSender.SendTransaction(tests.Context(t), nil) - require.EqualError(t, sendErr, expectedError.Error()) - }) - t.Run("Fails when multinode is closed", func(t *testing.T) { - chainID := types.RandomID() - fastNode := newNode(t, nil, nil) - // hold reply from the node till end of the test - testContext, testCancel := context.WithCancel(tests.Context(t)) - defer testCancel() - slowNode := newNode(t, errors.New("transaction failed"), func(_ mock.Arguments) { - // block caller til end of the test - <-testContext.Done() - }) - slowSendOnly := newNode(t, errors.New("send only failed"), func(_ mock.Arguments) { - // block caller til end of the test - <-testContext.Done() - }) - - lggr, _ := logger.TestObserved(t, zap.DebugLevel) - - mn, txSender := newTestTransactionSender(t, chainID, lggr, - []Node[types.ID, SendTxRPCClient[any]]{fastNode, slowNode}, - []SendOnlyNode[types.ID, SendTxRPCClient[any]]{slowSendOnly}) - - require.NoError(t, mn.Close()) - _, err := txSender.SendTransaction(tests.Context(t), nil) - require.EqualError(t, err, "MultiNode is stopped") - }) - t.Run("Fails when closed", func(t *testing.T) { - chainID := types.RandomID() - fastNode := newNode(t, nil, nil) - // hold reply from the node till end of the test - testContext, testCancel := context.WithCancel(tests.Context(t)) - defer testCancel() - slowNode := newNode(t, errors.New("transaction failed"), func(_ mock.Arguments) { - // block caller til end of the test - <-testContext.Done() - }) - slowSendOnly := newNode(t, errors.New("send only failed"), func(_ mock.Arguments) { - // block caller til end of the test - <-testContext.Done() - }) - - lggr, _ := logger.TestObserved(t, zap.DebugLevel) - - _, txSender := newTestTransactionSender(t, chainID, lggr, - []Node[types.ID, SendTxRPCClient[any]]{fastNode, slowNode}, - []SendOnlyNode[types.ID, SendTxRPCClient[any]]{slowSendOnly}) - - require.NoError(t, txSender.Close()) - _, err := txSender.SendTransaction(tests.Context(t), nil) - require.EqualError(t, err, "context canceled") - }) - t.Run("Returns error if there is no healthy primary nodes", func(t *testing.T) { - chainID := types.RandomID() - primary := newNodeWithState(t, NodeStateUnreachable, nil, nil) - sendOnly := newNodeWithState(t, NodeStateUnreachable, nil, nil) - - lggr, _ := logger.TestObserved(t, zap.DebugLevel) - - _, txSender := newTestTransactionSender(t, chainID, lggr, - []Node[types.ID, SendTxRPCClient[any]]{primary}, - []SendOnlyNode[types.ID, SendTxRPCClient[any]]{sendOnly}) - - _, sendErr := txSender.SendTransaction(tests.Context(t), nil) - assert.EqualError(t, sendErr, ErroringNodeError.Error()) - }) - - t.Run("Transaction success even if one of the nodes is unhealthy", func(t *testing.T) { - chainID := types.RandomID() - mainNode := newNode(t, nil, nil) - unexpectedCall := func(args mock.Arguments) { - panic("SendTx must not be called for unhealthy node") - } - unhealthyNode := newNodeWithState(t, NodeStateUnreachable, nil, unexpectedCall) - unhealthySendOnlyNode := newNodeWithState(t, NodeStateUnreachable, nil, unexpectedCall) - - lggr, _ := logger.TestObserved(t, zap.DebugLevel) - - _, txSender := newTestTransactionSender(t, chainID, lggr, - []Node[types.ID, SendTxRPCClient[any]]{mainNode, unhealthyNode}, - []SendOnlyNode[types.ID, SendTxRPCClient[any]]{unhealthySendOnlyNode}) - - returnCode, sendErr := txSender.SendTransaction(tests.Context(t), nil) - require.NoError(t, sendErr) - require.Equal(t, Successful, returnCode) - }) -} - -func TestTransactionSender_SendTransaction_aggregateTxResults(t *testing.T) { - t.Parallel() - // ensure failure on new SendTxReturnCode - codesToCover := map[SendTxReturnCode]struct{}{} - for code := Successful; code < sendTxReturnCodeLen; code++ { - codesToCover[code] = struct{}{} - } - - testCases := []struct { - Name string - ExpectedTxResult string - ExpectedCriticalErr string - ResultsByCode sendTxErrors - }{ - { - Name: "Returns success and logs critical error on success and Fatal", - ExpectedTxResult: "success", - ExpectedCriticalErr: "found contradictions in nodes replies on SendTransaction: got success and severe error", - ResultsByCode: sendTxErrors{ - Successful: {errors.New("success")}, - Fatal: {errors.New("fatal")}, - }, - }, - { - Name: "Returns TransactionAlreadyKnown and logs critical error on TransactionAlreadyKnown and Fatal", - ExpectedTxResult: "tx_already_known", - ExpectedCriticalErr: "found contradictions in nodes replies on SendTransaction: got success and severe error", - ResultsByCode: sendTxErrors{ - TransactionAlreadyKnown: {errors.New("tx_already_known")}, - Unsupported: {errors.New("unsupported")}, - }, - }, - { - Name: "Prefers sever error to temporary", - ExpectedTxResult: "underpriced", - ExpectedCriticalErr: "", - ResultsByCode: sendTxErrors{ - Retryable: {errors.New("retryable")}, - Underpriced: {errors.New("underpriced")}, - }, - }, - { - Name: "Returns temporary error", - ExpectedTxResult: "retryable", - ExpectedCriticalErr: "", - ResultsByCode: sendTxErrors{ - Retryable: {errors.New("retryable")}, - }, - }, - { - Name: "Insufficient funds is treated as error", - ExpectedTxResult: "", - ExpectedCriticalErr: "", - ResultsByCode: sendTxErrors{ - Successful: {nil}, - InsufficientFunds: {errors.New("insufficientFunds")}, - }, - }, - { - Name: "Logs critical error on empty ResultsByCode", - ExpectedTxResult: "expected at least one response on SendTransaction", - ExpectedCriticalErr: "expected at least one response on SendTransaction", - ResultsByCode: sendTxErrors{}, - }, - { - Name: "Zk terminally stuck", - ExpectedTxResult: "not enough keccak counters to continue the execution", - ExpectedCriticalErr: "", - ResultsByCode: sendTxErrors{ - TerminallyStuck: {errors.New("not enough keccak counters to continue the execution")}, - }, - }, - } - - for _, testCase := range testCases { - for code := range testCase.ResultsByCode { - delete(codesToCover, code) - } - - t.Run(testCase.Name, func(t *testing.T) { - _, txResult, err := aggregateTxResults(testCase.ResultsByCode) - if testCase.ExpectedTxResult == "" { - assert.NoError(t, err) - } else { - assert.EqualError(t, txResult, testCase.ExpectedTxResult) - } - - logger.Sugared(logger.Test(t)).Info("Map: " + fmt.Sprint(testCase.ResultsByCode)) - logger.Sugared(logger.Test(t)).Criticalw("observed invariant violation on SendTransaction", "resultsByCode", testCase.ResultsByCode, "err", err) - - if testCase.ExpectedCriticalErr == "" { - assert.NoError(t, err) - } else { - assert.EqualError(t, err, testCase.ExpectedCriticalErr) - } - }) - } - - // explicitly signal that following codes are properly handled in aggregateTxResults, - // but dedicated test cases won't be beneficial - for _, codeToIgnore := range []SendTxReturnCode{Unknown, ExceedsMaxFee, FeeOutOfValidRange} { - delete(codesToCover, codeToIgnore) - } - assert.Empty(t, codesToCover, "all of the SendTxReturnCode must be covered by this test") -} diff --git a/pkg/solana/client/multinode/types.go b/pkg/solana/client/multinode/types.go index 6c863c867..51b70e573 100644 --- a/pkg/solana/client/multinode/types.go +++ b/pkg/solana/client/multinode/types.go @@ -10,6 +10,13 @@ import ( // It should be convertible to a string, that can uniquely identify this chain type ID fmt.Stringer +// StringID enables using string directly as a ChainID +type StringID string + +func (s StringID) String() string { + return string(s) +} + // Subscription represents an event subscription where events are // delivered on a data channel. // This is a generic interface for Subscription to represent used by clients. diff --git a/pkg/solana/client/rpc_client.go b/pkg/solana/client/rpc_client.go index ec89d2b66..424c48b67 100644 --- a/pkg/solana/client/rpc_client.go +++ b/pkg/solana/client/rpc_client.go @@ -18,12 +18,6 @@ import ( "github.com/smartcontractkit/chainlink-solana/pkg/solana/monitor" ) -type StringID string - -func (s StringID) String() string { - return string(s) -} - var _ ReaderWriter = (*RpcClient)(nil) type Head struct { @@ -178,7 +172,7 @@ func (c *RpcClient) LatestBlockhash() (*rpc.GetLatestBlockhashResult, error) { return v.(*rpc.GetLatestBlockhashResult), err } -func (c *RpcClient) ChainID(ctx context.Context) (StringID, error) { +func (c *RpcClient) ChainID(ctx context.Context) (mn.StringID, error) { done := c.latency("chain_id") defer done() @@ -204,7 +198,7 @@ func (c *RpcClient) ChainID(ctx context.Context) (StringID, error) { c.log.Warnf("unknown genesis hash - assuming solana chain is 'localnet'") network = "localnet" } - return StringID(network), nil + return mn.StringID(network), nil } func (c *RpcClient) GetFeeForMessage(msg string) (uint64, error) { From 354dc50a80d185a9ea53d166105fc07565a07dfb Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 4 Sep 2024 12:52:52 -0400 Subject: [PATCH 007/174] tidy --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 12c2ea189..c88476699 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/go-viper/mapstructure/v2 v2.1.0 github.com/google/uuid v1.6.0 github.com/hashicorp/go-plugin v1.6.2-0.20240829161738-06afb6d7ae99 + github.com/jpillora/backoff v1.0.0 github.com/pelletier/go-toml/v2 v2.2.0 github.com/prometheus/client_golang v1.17.0 github.com/smartcontractkit/chainlink-common v0.2.2-0.20240829145110-4a45c426fbe8 @@ -55,7 +56,6 @@ require ( github.com/hashicorp/go-hclog v1.5.0 // indirect github.com/hashicorp/yamux v0.1.1 // indirect github.com/invopop/jsonschema v0.12.0 // indirect - github.com/jpillora/backoff v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.15.15 // indirect github.com/kr/pretty v0.3.1 // indirect From 60c33524323aa898121cd5d9dc3613392c030c89 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 4 Sep 2024 13:00:05 -0400 Subject: [PATCH 008/174] Update client_test.go --- pkg/solana/client/client_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/solana/client/client_test.go b/pkg/solana/client/client_test.go index ed4e1dba4..f41f4773a 100644 --- a/pkg/solana/client/client_test.go +++ b/pkg/solana/client/client_test.go @@ -79,7 +79,7 @@ func TestClient_Reader_Integration(t *testing.T) { // get chain ID based on gensis hash network, err := c.ChainID(context.Background()) assert.NoError(t, err) - assert.Equal(t, "localnet", network) + assert.Equal(t, mn.StringID("localnet"), network) // get account info (also tested inside contract_test) res, err := c.GetAccountInfoWithOpts(context.TODO(), solana.PublicKey{}, &rpc.GetAccountInfoOpts{Commitment: rpc.CommitmentFinalized}) From 8e2306bfc12dbbf6bdce8604577b04a777478dd3 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 4 Sep 2024 13:06:44 -0400 Subject: [PATCH 009/174] lint --- pkg/solana/chain_multinode.go | 19 +++++------- pkg/solana/client/client_test.go | 2 +- pkg/solana/client/rpc_client.go | 50 ++++++++++++++++---------------- 3 files changed, 34 insertions(+), 37 deletions(-) diff --git a/pkg/solana/chain_multinode.go b/pkg/solana/chain_multinode.go index cb7161a55..d8e4f133c 100644 --- a/pkg/solana/chain_multinode.go +++ b/pkg/solana/chain_multinode.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "math/big" - "sync" "time" solanago "github.com/gagliardetto/solana-go" @@ -40,13 +39,11 @@ type multiNodeChain struct { services.StateMachine id string cfg *config.TOMLConfig - multiNode *mn.MultiNode[mn.StringID, *client.RpcClient] - txSender *mn.TransactionSender[*solanago.Transaction, mn.StringID, *client.RpcClient] + multiNode *mn.MultiNode[mn.StringID, *client.RPCClient] + txSender *mn.TransactionSender[*solanago.Transaction, mn.StringID, *client.RPCClient] txm *txm.Txm balanceMonitor services.Service lggr logger.Logger - - clientLock sync.RWMutex } func newMultiNodeChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr logger.Logger) (*multiNodeChain, error) { @@ -58,29 +55,29 @@ func newMultiNodeChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr mnCfg := cfg.MultiNodeConfig() - var nodes []mn.Node[mn.StringID, *client.RpcClient] + var nodes []mn.Node[mn.StringID, *client.RPCClient] for i, nodeInfo := range cfg.ListNodes() { // create client and check - rpcClient, err := client.NewRpcClient(nodeInfo.URL.String(), cfg, DefaultRequestTimeout, logger.Named(lggr, "Client."+*nodeInfo.Name)) + rpcClient, err := client.NewRPCClient(nodeInfo.URL.String(), cfg, DefaultRequestTimeout, logger.Named(lggr, "Client."+*nodeInfo.Name)) if err != nil { lggr.Warnw("failed to create client", "name", *nodeInfo.Name, "solana-url", nodeInfo.URL.String(), "err", err.Error()) continue } - newNode := mn.NewNode[mn.StringID, *client.Head, *client.RpcClient]( + newNode := mn.NewNode[mn.StringID, *client.Head, *client.RPCClient]( mnCfg, mnCfg, lggr, *nodeInfo.URL.URL(), nil, *nodeInfo.Name, int32(i), mn.StringID(id), 0, rpcClient, chainFamily) nodes = append(nodes, newNode) } - multiNode := mn.NewMultiNode[mn.StringID, *client.RpcClient]( + multiNode := mn.NewMultiNode[mn.StringID, *client.RPCClient]( lggr, mn.NodeSelectionModeRoundRobin, time.Duration(0), // TODO: set lease duration nodes, - []mn.SendOnlyNode[mn.StringID, *client.RpcClient]{}, // TODO: no send only nodes? + []mn.SendOnlyNode[mn.StringID, *client.RPCClient]{}, // TODO: no send only nodes? mn.StringID(id), chainFamily, time.Duration(0), // TODO: set deathDeclarationDelay @@ -90,7 +87,7 @@ func newMultiNodeChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr return 0 // TODO ClassifySendError(err, clientErrors, logger.Sugared(logger.Nop()), tx, common.Address{}, false) } - txSender := mn.NewTransactionSender[*solanago.Transaction, mn.StringID, *client.RpcClient]( + txSender := mn.NewTransactionSender[*solanago.Transaction, mn.StringID, *client.RPCClient]( lggr, mn.StringID(id), chainFamily, diff --git a/pkg/solana/client/client_test.go b/pkg/solana/client/client_test.go index f41f4773a..6a4feb61f 100644 --- a/pkg/solana/client/client_test.go +++ b/pkg/solana/client/client_test.go @@ -19,8 +19,8 @@ import ( "github.com/stretchr/testify/require" "github.com/smartcontractkit/chainlink-common/pkg/logger" - mn "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" + mn "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" "github.com/smartcontractkit/chainlink-solana/pkg/solana/config" "github.com/smartcontractkit/chainlink-solana/pkg/solana/monitor" ) diff --git a/pkg/solana/client/rpc_client.go b/pkg/solana/client/rpc_client.go index 424c48b67..c9ceeab6a 100644 --- a/pkg/solana/client/rpc_client.go +++ b/pkg/solana/client/rpc_client.go @@ -18,7 +18,7 @@ import ( "github.com/smartcontractkit/chainlink-solana/pkg/solana/monitor" ) -var _ ReaderWriter = (*RpcClient)(nil) +var _ ReaderWriter = (*RPCClient)(nil) type Head struct { rpc.GetBlockResult @@ -39,7 +39,7 @@ func (h *Head) IsValid() bool { return true } -type RpcClient struct { +type RPCClient struct { url string rpc *rpc.Client skipPreflight bool // to enable or disable preflight checks @@ -55,48 +55,48 @@ type RpcClient struct { // TODO: BCI-4061: Implement RPC Client for MultiNode -func (c *RpcClient) Dial(ctx context.Context) error { +func (c *RPCClient) Dial(ctx context.Context) error { //TODO implement me panic("implement me") } -func (c *RpcClient) SubscribeToHeads(ctx context.Context) (<-chan *Head, mn.Subscription, error) { +func (c *RPCClient) SubscribeToHeads(ctx context.Context) (<-chan *Head, mn.Subscription, error) { //TODO implement me panic("implement me") } -func (c *RpcClient) SubscribeToFinalizedHeads(ctx context.Context) (<-chan *Head, mn.Subscription, error) { +func (c *RPCClient) SubscribeToFinalizedHeads(ctx context.Context) (<-chan *Head, mn.Subscription, error) { //TODO implement me panic("implement me") } -func (c *RpcClient) Ping(ctx context.Context) error { +func (c *RPCClient) Ping(ctx context.Context) error { //TODO implement me panic("implement me") } -func (c *RpcClient) IsSyncing(ctx context.Context) (bool, error) { +func (c *RPCClient) IsSyncing(ctx context.Context) (bool, error) { //TODO implement me panic("implement me") } -func (c *RpcClient) UnsubscribeAllExcept(subs ...mn.Subscription) { +func (c *RPCClient) UnsubscribeAllExcept(subs ...mn.Subscription) { //TODO implement me panic("implement me") } -func (c *RpcClient) Close() { +func (c *RPCClient) Close() { //TODO implement me panic("implement me") } -func (c *RpcClient) GetInterceptedChainInfo() (latest, highestUserObservations mn.ChainInfo) { +func (c *RPCClient) GetInterceptedChainInfo() (latest, highestUserObservations mn.ChainInfo) { //TODO implement me panic("implement me") } -func NewRpcClient(endpoint string, cfg config.Config, requestTimeout time.Duration, log logger.Logger) (*RpcClient, error) { - return &RpcClient{ +func NewRPCClient(endpoint string, cfg config.Config, requestTimeout time.Duration, log logger.Logger) (*RPCClient, error) { + return &RPCClient{ url: endpoint, rpc: rpc.New(endpoint), skipPreflight: cfg.SkipPreflight(), @@ -109,14 +109,14 @@ func NewRpcClient(endpoint string, cfg config.Config, requestTimeout time.Durati }, nil } -func (c *RpcClient) latency(name string) func() { +func (c *RPCClient) latency(name string) func() { start := time.Now() return func() { monitor.SetClientLatency(time.Since(start), name, c.url) } } -func (c *RpcClient) Balance(addr solana.PublicKey) (uint64, error) { +func (c *RPCClient) Balance(addr solana.PublicKey) (uint64, error) { done := c.latency("balance") defer done() @@ -133,11 +133,11 @@ func (c *RpcClient) Balance(addr solana.PublicKey) (uint64, error) { return res.Value, err } -func (c *RpcClient) SlotHeight() (uint64, error) { +func (c *RPCClient) SlotHeight() (uint64, error) { return c.SlotHeightWithCommitment(rpc.CommitmentProcessed) // get the latest slot height } -func (c *RpcClient) SlotHeightWithCommitment(commitment rpc.CommitmentType) (uint64, error) { +func (c *RPCClient) SlotHeightWithCommitment(commitment rpc.CommitmentType) (uint64, error) { done := c.latency("slot_height") defer done() @@ -149,7 +149,7 @@ func (c *RpcClient) SlotHeightWithCommitment(commitment rpc.CommitmentType) (uin return v.(uint64), err } -func (c *RpcClient) GetAccountInfoWithOpts(ctx context.Context, addr solana.PublicKey, opts *rpc.GetAccountInfoOpts) (*rpc.GetAccountInfoResult, error) { +func (c *RPCClient) GetAccountInfoWithOpts(ctx context.Context, addr solana.PublicKey, opts *rpc.GetAccountInfoOpts) (*rpc.GetAccountInfoResult, error) { done := c.latency("account_info") defer done() @@ -159,7 +159,7 @@ func (c *RpcClient) GetAccountInfoWithOpts(ctx context.Context, addr solana.Publ return c.rpc.GetAccountInfoWithOpts(ctx, addr, opts) } -func (c *RpcClient) LatestBlockhash() (*rpc.GetLatestBlockhashResult, error) { +func (c *RPCClient) LatestBlockhash() (*rpc.GetLatestBlockhashResult, error) { done := c.latency("latest_blockhash") defer done() @@ -172,7 +172,7 @@ func (c *RpcClient) LatestBlockhash() (*rpc.GetLatestBlockhashResult, error) { return v.(*rpc.GetLatestBlockhashResult), err } -func (c *RpcClient) ChainID(ctx context.Context) (mn.StringID, error) { +func (c *RPCClient) ChainID(ctx context.Context) (mn.StringID, error) { done := c.latency("chain_id") defer done() @@ -201,7 +201,7 @@ func (c *RpcClient) ChainID(ctx context.Context) (mn.StringID, error) { return mn.StringID(network), nil } -func (c *RpcClient) GetFeeForMessage(msg string) (uint64, error) { +func (c *RPCClient) GetFeeForMessage(msg string) (uint64, error) { done := c.latency("fee_for_message") defer done() @@ -221,7 +221,7 @@ func (c *RpcClient) GetFeeForMessage(msg string) (uint64, error) { } // https://docs.solana.com/developing/clients/jsonrpc-api#getsignaturestatuses -func (c *RpcClient) SignatureStatuses(ctx context.Context, sigs []solana.Signature) ([]*rpc.SignatureStatusesResult, error) { +func (c *RPCClient) SignatureStatuses(ctx context.Context, sigs []solana.Signature) ([]*rpc.SignatureStatusesResult, error) { done := c.latency("signature_statuses") defer done() @@ -242,7 +242,7 @@ func (c *RpcClient) SignatureStatuses(ctx context.Context, sigs []solana.Signatu // https://docs.solana.com/developing/clients/jsonrpc-api#simulatetransaction // opts - (optional) use `nil` to use defaults -func (c *RpcClient) SimulateTx(ctx context.Context, tx *solana.Transaction, opts *rpc.SimulateTransactionOpts) (*rpc.SimulateTransactionResult, error) { +func (c *RPCClient) SimulateTx(ctx context.Context, tx *solana.Transaction, opts *rpc.SimulateTransactionOpts) (*rpc.SimulateTransactionResult, error) { done := c.latency("simulate_tx") defer done() @@ -268,12 +268,12 @@ func (c *RpcClient) SimulateTx(ctx context.Context, tx *solana.Transaction, opts return res.Value, nil } -func (c *RpcClient) SendTransaction(ctx context.Context, tx *solana.Transaction) error { +func (c *RPCClient) SendTransaction(ctx context.Context, tx *solana.Transaction) error { // TODO: Implement return nil } -func (c *RpcClient) SendTx(ctx context.Context, tx *solana.Transaction) (solana.Signature, error) { +func (c *RPCClient) SendTx(ctx context.Context, tx *solana.Transaction) (solana.Signature, error) { done := c.latency("send_tx") defer done() @@ -289,7 +289,7 @@ func (c *RpcClient) SendTx(ctx context.Context, tx *solana.Transaction) (solana. return c.rpc.SendTransactionWithOpts(ctx, tx, opts) } -func (c *RpcClient) GetLatestBlock() (*rpc.GetBlockResult, error) { +func (c *RPCClient) GetLatestBlock() (*rpc.GetBlockResult, error) { // get latest confirmed slot slot, err := c.SlotHeightWithCommitment(c.commitment) if err != nil { From b8d67550b482edb32e31e3149738ef107351b540 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 4 Sep 2024 13:27:58 -0400 Subject: [PATCH 010/174] Fix duplicate metrics --- pkg/solana/client/multinode/multi_node.go | 2 +- pkg/solana/client/multinode/node.go | 6 +++--- pkg/solana/client/multinode/node_fsm.go | 14 +++++++------- pkg/solana/client/multinode/node_lifecycle.go | 12 ++++++------ pkg/solana/client/multinode/transaction_sender.go | 2 +- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/pkg/solana/client/multinode/multi_node.go b/pkg/solana/client/multinode/multi_node.go index 386e09554..1a4846edf 100644 --- a/pkg/solana/client/multinode/multi_node.go +++ b/pkg/solana/client/multinode/multi_node.go @@ -18,7 +18,7 @@ import ( var ( // PromMultiNodeRPCNodeStates reports current RPC node state PromMultiNodeRPCNodeStates = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Name: "multi_node_states", + Name: "solana_multi_node_states", Help: "The number of RPC nodes currently in the given state for the given chain", }, []string{"network", "chainId", "state"}) ErroringNodeError = fmt.Errorf("no live nodes available") diff --git a/pkg/solana/client/multinode/node.go b/pkg/solana/client/multinode/node.go index 8ab30f856..7d2b02ce2 100644 --- a/pkg/solana/client/multinode/node.go +++ b/pkg/solana/client/multinode/node.go @@ -21,15 +21,15 @@ var errInvalidChainID = errors.New("invalid chain id") var ( promPoolRPCNodeVerifies = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_verifies", + Name: "solana_pool_rpc_node_verifies", Help: "The total number of chain ID verifications for the given RPC node", }, []string{"network", "chainID", "nodeName"}) promPoolRPCNodeVerifiesFailed = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_verifies_failed", + Name: "solana_pool_rpc_node_verifies_failed", Help: "The total number of failed chain ID verifications for the given RPC node", }, []string{"network", "chainID", "nodeName"}) promPoolRPCNodeVerifiesSuccess = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_verifies_success", + Name: "solana_pool_rpc_node_verifies_success", Help: "The total number of successful chain ID verifications for the given RPC node", }, []string{"network", "chainID", "nodeName"}) ) diff --git a/pkg/solana/client/multinode/node_fsm.go b/pkg/solana/client/multinode/node_fsm.go index 981e325da..136910868 100644 --- a/pkg/solana/client/multinode/node_fsm.go +++ b/pkg/solana/client/multinode/node_fsm.go @@ -9,31 +9,31 @@ import ( var ( promPoolRPCNodeTransitionsToAlive = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_num_transitions_to_alive", + Name: "solana_pool_rpc_node_num_transitions_to_alive", Help: transitionString(NodeStateAlive), }, []string{"chainID", "nodeName"}) promPoolRPCNodeTransitionsToInSync = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_num_transitions_to_in_sync", + Name: "solana_pool_rpc_node_num_transitions_to_in_sync", Help: fmt.Sprintf("%s to %s", transitionString(NodeStateOutOfSync), NodeStateAlive), }, []string{"chainID", "nodeName"}) promPoolRPCNodeTransitionsToOutOfSync = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_num_transitions_to_out_of_sync", + Name: "solana_pool_rpc_node_num_transitions_to_out_of_sync", Help: transitionString(NodeStateOutOfSync), }, []string{"chainID", "nodeName"}) promPoolRPCNodeTransitionsToUnreachable = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_num_transitions_to_unreachable", + Name: "solana_pool_rpc_node_num_transitions_to_unreachable", Help: transitionString(NodeStateUnreachable), }, []string{"chainID", "nodeName"}) promPoolRPCNodeTransitionsToInvalidChainID = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_num_transitions_to_invalid_chain_id", + Name: "solana_pool_rpc_node_num_transitions_to_invalid_chain_id", Help: transitionString(NodeStateInvalidChainID), }, []string{"chainID", "nodeName"}) promPoolRPCNodeTransitionsToUnusable = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_num_transitions_to_unusable", + Name: "solana_pool_rpc_node_num_transitions_to_unusable", Help: transitionString(NodeStateUnusable), }, []string{"chainID", "nodeName"}) promPoolRPCNodeTransitionsToSyncing = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_num_transitions_to_syncing", + Name: "solana_pool_rpc_node_num_transitions_to_syncing", Help: transitionString(NodeStateSyncing), }, []string{"chainID", "nodeName"}) ) diff --git a/pkg/solana/client/multinode/node_lifecycle.go b/pkg/solana/client/multinode/node_lifecycle.go index 44203bf97..d6b150690 100644 --- a/pkg/solana/client/multinode/node_lifecycle.go +++ b/pkg/solana/client/multinode/node_lifecycle.go @@ -17,27 +17,27 @@ import ( var ( promPoolRPCNodeHighestSeenBlock = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Name: "pool_rpc_node_highest_seen_block", + Name: "solana_pool_rpc_node_highest_seen_block", Help: "The highest seen block for the given RPC node", }, []string{"chainID", "nodeName"}) promPoolRPCNodeHighestFinalizedBlock = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Name: "pool_rpc_node_highest_finalized_block", + Name: "solana_pool_rpc_node_highest_finalized_block", Help: "The highest seen finalized block for the given RPC node", }, []string{"chainID", "nodeName"}) promPoolRPCNodeNumSeenBlocks = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_num_seen_blocks", + Name: "solana_pool_rpc_node_num_seen_blocks", Help: "The total number of new blocks seen by the given RPC node", }, []string{"chainID", "nodeName"}) promPoolRPCNodePolls = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_polls_total", + Name: "solana_pool_rpc_node_polls_total", Help: "The total number of poll checks for the given RPC node", }, []string{"chainID", "nodeName"}) promPoolRPCNodePollsFailed = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_polls_failed", + Name: "solana_pool_rpc_node_polls_failed", Help: "The total number of failed poll checks for the given RPC node", }, []string{"chainID", "nodeName"}) promPoolRPCNodePollsSuccess = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_polls_success", + Name: "solana_pool_rpc_node_polls_success", Help: "The total number of successful poll checks for the given RPC node", }, []string{"chainID", "nodeName"}) ) diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go index d567e164f..71de153ae 100644 --- a/pkg/solana/client/multinode/transaction_sender.go +++ b/pkg/solana/client/multinode/transaction_sender.go @@ -19,7 +19,7 @@ import ( var ( // PromMultiNodeInvariantViolations reports violation of our assumptions PromMultiNodeInvariantViolations = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "multi_node_invariant_violations", + Name: "solana_multi_node_invariant_violations", Help: "The number of invariant violations", }, []string{"network", "chainId", "invariant"}) ) From 2cb4d77bf71db88ec70b1605ed6b5adacef3c763 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 5 Sep 2024 12:19:03 -0400 Subject: [PATCH 011/174] Add chain multinode flag --- pkg/solana/chain.go | 91 +++++++- pkg/solana/chain_multinode.go | 266 ------------------------ pkg/solana/cmd/chainlink-solana/main.go | 10 +- 3 files changed, 86 insertions(+), 281 deletions(-) delete mode 100644 pkg/solana/chain_multinode.go diff --git a/pkg/solana/chain.go b/pkg/solana/chain.go index bc2dd845a..7360f7c0c 100644 --- a/pkg/solana/chain.go +++ b/pkg/solana/chain.go @@ -70,7 +70,7 @@ func NewChain(cfg *config.TOMLConfig, opts ChainOpts) (Chain, error) { if !cfg.IsEnabled() { return nil, fmt.Errorf("cannot create new chain with ID %s: chain is disabled", *cfg.ChainID) } - c, err := newChain(*cfg.ChainID, cfg, opts.KeyStore, opts.Logger) + c, err := newChain(*cfg.ChainID, cfg, cfg.MultiNodeEnabled(), opts.KeyStore, opts.Logger) if err != nil { return nil, err } @@ -87,6 +87,11 @@ type chain struct { balanceMonitor services.Service lggr logger.Logger + // if multiNode is enabled, the clientCache will not be used + multiNodeEnabled bool + multiNode *mn.MultiNode[mn.StringID, *client.RPCClient] + txSender *mn.TransactionSender[*solanago.Transaction, mn.StringID, *client.RPCClient] + // tracking node chain id for verification clientCache map[string]*verifiedCachedClient // map URL -> {client, chainId} [mainnet/testnet/devnet/localnet] clientLock sync.RWMutex @@ -216,14 +221,70 @@ func (v *verifiedCachedClient) GetAccountInfoWithOpts(ctx context.Context, addr return v.ReaderWriter.GetAccountInfoWithOpts(ctx, addr, opts) } -func newChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr logger.Logger) (*chain, error) { +func newChain(id string, cfg *config.TOMLConfig, multiNodeEnabled bool, ks loop.Keystore, lggr logger.Logger) (*chain, error) { lggr = logger.With(lggr, "chainID", id, "chain", "solana") var ch = chain{ - id: id, - cfg: cfg, - lggr: logger.Named(lggr, "Chain"), - clientCache: map[string]*verifiedCachedClient{}, + id: id, + cfg: cfg, + lggr: logger.Named(lggr, "Chain"), + multiNodeEnabled: multiNodeEnabled, + clientCache: map[string]*verifiedCachedClient{}, } + + if multiNodeEnabled { + chainFamily := "solana" + + mnCfg := cfg.MultiNodeConfig() + + var nodes []mn.Node[mn.StringID, *client.RPCClient] + + for i, nodeInfo := range cfg.ListNodes() { + // create client and check + rpcClient, err := client.NewRPCClient(nodeInfo.URL.String(), cfg, DefaultRequestTimeout, logger.Named(lggr, "Client."+*nodeInfo.Name)) + if err != nil { + lggr.Warnw("failed to create client", "name", *nodeInfo.Name, "solana-url", nodeInfo.URL.String(), "err", err.Error()) + continue + } + + newNode := mn.NewNode[mn.StringID, *client.Head, *client.RPCClient]( + mnCfg, mnCfg, lggr, *nodeInfo.URL.URL(), nil, *nodeInfo.Name, + int32(i), mn.StringID(id), 0, rpcClient, chainFamily) + + nodes = append(nodes, newNode) + } + + multiNode := mn.NewMultiNode[mn.StringID, *client.RPCClient]( + lggr, + mn.NodeSelectionModeRoundRobin, + time.Minute, // TODO: set lease duration + nodes, + []mn.SendOnlyNode[mn.StringID, *client.RPCClient]{}, + mn.StringID(id), + chainFamily, + mnCfg.DeathDeclarationDelay(), + ) + + // TODO: implement error classification + classifySendError := func(tx *solanago.Transaction, err error) mn.SendTxReturnCode { + return 0 // TODO ClassifySendError(err, clientErrors, logger.Sugared(logger.Nop()), tx, common.Address{}, false) + } + + txSender := mn.NewTransactionSender[*solanago.Transaction, mn.StringID, *client.RPCClient]( + lggr, + mn.StringID(id), + chainFamily, + multiNode, + classifySendError, + 0, // use the default value provided by the implementation + ) + + ch.multiNode = multiNode + ch.txSender = txSender + + // clientCache will not be used if multinode is enabled + ch.clientCache = nil + } + tc := func() (client.ReaderWriter, error) { return ch.getClient() } @@ -302,6 +363,10 @@ func (c *chain) ChainID() string { // getClient returns a client, randomly selecting one from available and valid nodes func (c *chain) getClient() (client.ReaderWriter, error) { + if c.multiNodeEnabled { + return c.multiNode.SelectRPC() + } + var node *config.Node var client client.ReaderWriter nodes := c.cfg.ListNodes() @@ -381,6 +446,13 @@ func (c *chain) Start(ctx context.Context) error { c.lggr.Debug("Starting txm") c.lggr.Debug("Starting balance monitor") var ms services.MultiStart + if c.multiNodeEnabled { + c.lggr.Debug("Starting multinode") + err := ms.Start(ctx, c.multiNode, c.txSender) + if err != nil { + return err + } + } return ms.Start(ctx, c.txm, c.balanceMonitor) }) } @@ -390,6 +462,13 @@ func (c *chain) Close() error { c.lggr.Debug("Stopping") c.lggr.Debug("Stopping txm") c.lggr.Debug("Stopping balance monitor") + if c.multiNodeEnabled { + c.lggr.Debug("Stopping multinode") + err := services.CloseAll(c.multiNode, c.txSender) + if err != nil { + return err + } + } return services.CloseAll(c.txm, c.balanceMonitor) }) } diff --git a/pkg/solana/chain_multinode.go b/pkg/solana/chain_multinode.go deleted file mode 100644 index d8e4f133c..000000000 --- a/pkg/solana/chain_multinode.go +++ /dev/null @@ -1,266 +0,0 @@ -package solana - -import ( - "context" - "errors" - "fmt" - "math/big" - "time" - - solanago "github.com/gagliardetto/solana-go" - "github.com/gagliardetto/solana-go/programs/system" - "github.com/smartcontractkit/chainlink-common/pkg/chains" - "github.com/smartcontractkit/chainlink-common/pkg/logger" - "github.com/smartcontractkit/chainlink-common/pkg/loop" - "github.com/smartcontractkit/chainlink-common/pkg/services" - relaytypes "github.com/smartcontractkit/chainlink-common/pkg/types" - - "github.com/smartcontractkit/chainlink-solana/pkg/solana/client" - mn "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" - "github.com/smartcontractkit/chainlink-solana/pkg/solana/config" - "github.com/smartcontractkit/chainlink-solana/pkg/solana/monitor" - "github.com/smartcontractkit/chainlink-solana/pkg/solana/txm" -) - -func NewMultiNodeChain(cfg *config.TOMLConfig, opts ChainOpts) (Chain, error) { - if !cfg.IsEnabled() { - return nil, fmt.Errorf("cannot create new chain with ID %s: chain is disabled", *cfg.ChainID) - } - c, err := newMultiNodeChain(*cfg.ChainID, cfg, opts.KeyStore, opts.Logger) - if err != nil { - return nil, err - } - return c, nil -} - -var _ Chain = (*multiNodeChain)(nil) - -type multiNodeChain struct { - services.StateMachine - id string - cfg *config.TOMLConfig - multiNode *mn.MultiNode[mn.StringID, *client.RPCClient] - txSender *mn.TransactionSender[*solanago.Transaction, mn.StringID, *client.RPCClient] - txm *txm.Txm - balanceMonitor services.Service - lggr logger.Logger -} - -func newMultiNodeChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr logger.Logger) (*multiNodeChain, error) { - lggr = logger.With(lggr, "chainID", id, "chain", "solana") - - chainFamily := "solana" - - cfg.BlockHistoryPollPeriod() - - mnCfg := cfg.MultiNodeConfig() - - var nodes []mn.Node[mn.StringID, *client.RPCClient] - - for i, nodeInfo := range cfg.ListNodes() { - // create client and check - rpcClient, err := client.NewRPCClient(nodeInfo.URL.String(), cfg, DefaultRequestTimeout, logger.Named(lggr, "Client."+*nodeInfo.Name)) - if err != nil { - lggr.Warnw("failed to create client", "name", *nodeInfo.Name, "solana-url", nodeInfo.URL.String(), "err", err.Error()) - continue - } - - newNode := mn.NewNode[mn.StringID, *client.Head, *client.RPCClient]( - mnCfg, mnCfg, lggr, *nodeInfo.URL.URL(), nil, *nodeInfo.Name, - int32(i), mn.StringID(id), 0, rpcClient, chainFamily) - - nodes = append(nodes, newNode) - } - - multiNode := mn.NewMultiNode[mn.StringID, *client.RPCClient]( - lggr, - mn.NodeSelectionModeRoundRobin, - time.Duration(0), // TODO: set lease duration - nodes, - []mn.SendOnlyNode[mn.StringID, *client.RPCClient]{}, // TODO: no send only nodes? - mn.StringID(id), - chainFamily, - time.Duration(0), // TODO: set deathDeclarationDelay - ) - - classifySendError := func(tx *solanago.Transaction, err error) mn.SendTxReturnCode { - return 0 // TODO ClassifySendError(err, clientErrors, logger.Sugared(logger.Nop()), tx, common.Address{}, false) - } - - txSender := mn.NewTransactionSender[*solanago.Transaction, mn.StringID, *client.RPCClient]( - lggr, - mn.StringID(id), - chainFamily, - multiNode, - classifySendError, - 0, // use the default value provided by the implementation - ) - - var ch = multiNodeChain{ - id: id, - cfg: cfg, - multiNode: multiNode, - txSender: txSender, - lggr: logger.Named(lggr, "Chain"), - } - - tc := func() (client.ReaderWriter, error) { - return ch.multiNode.SelectRPC() - } - - ch.txm = txm.NewTxm(ch.id, tc, cfg, ks, lggr) - bc := func() (monitor.BalanceClient, error) { - return ch.multiNode.SelectRPC() - } - ch.balanceMonitor = monitor.NewBalanceMonitor(ch.id, cfg, lggr, ks, bc) - return &ch, nil -} - -// ChainService interface -func (c *multiNodeChain) GetChainStatus(ctx context.Context) (relaytypes.ChainStatus, error) { - toml, err := c.cfg.TOMLString() - if err != nil { - return relaytypes.ChainStatus{}, err - } - return relaytypes.ChainStatus{ - ID: c.id, - Enabled: c.cfg.IsEnabled(), - Config: toml, - }, nil -} - -func (c *multiNodeChain) ListNodeStatuses(ctx context.Context, pageSize int32, pageToken string) (stats []relaytypes.NodeStatus, nextPageToken string, total int, err error) { - return chains.ListNodeStatuses(int(pageSize), pageToken, c.listNodeStatuses) -} - -func (c *multiNodeChain) Transact(ctx context.Context, from, to string, amount *big.Int, balanceCheck bool) error { - return c.sendTx(ctx, from, to, amount, balanceCheck) -} - -func (c *multiNodeChain) listNodeStatuses(start, end int) ([]relaytypes.NodeStatus, int, error) { - stats := make([]relaytypes.NodeStatus, 0) - total := len(c.cfg.Nodes) - if start >= total { - return stats, total, chains.ErrOutOfRange - } - if end > total { - end = total - } - nodes := c.cfg.Nodes[start:end] - for _, node := range nodes { - stat, err := config.NodeStatus(node, c.ChainID()) - if err != nil { - return stats, total, err - } - stats = append(stats, stat) - } - return stats, total, nil -} - -func (c *multiNodeChain) Name() string { - return c.lggr.Name() -} - -func (c *multiNodeChain) ID() string { - return c.id -} - -func (c *multiNodeChain) Config() config.Config { - return c.cfg -} - -func (c *multiNodeChain) TxManager() TxManager { - return c.txm -} - -func (c *multiNodeChain) Reader() (client.Reader, error) { - return c.multiNode.SelectRPC() -} - -func (c *multiNodeChain) ChainID() string { - return c.id -} - -func (c *multiNodeChain) Start(ctx context.Context) error { - return c.StartOnce("Chain", func() error { - c.lggr.Debug("Starting") - c.lggr.Debug("Starting txm") - c.lggr.Debug("Starting balance monitor") - var ms services.MultiStart - return ms.Start(ctx, c.txm, c.balanceMonitor) - }) -} - -func (c *multiNodeChain) Close() error { - return c.StopOnce("Chain", func() error { - c.lggr.Debug("Stopping") - c.lggr.Debug("Stopping txm") - c.lggr.Debug("Stopping balance monitor") - return services.CloseAll(c.txm, c.balanceMonitor) - }) -} - -func (c *multiNodeChain) Ready() error { - return errors.Join( - c.StateMachine.Ready(), - c.txm.Ready(), - ) -} - -func (c *multiNodeChain) HealthReport() map[string]error { - report := map[string]error{c.Name(): c.Healthy()} - services.CopyHealth(report, c.txm.HealthReport()) - return report -} - -func (c *multiNodeChain) sendTx(ctx context.Context, from, to string, amount *big.Int, balanceCheck bool) error { - reader, err := c.Reader() - if err != nil { - return fmt.Errorf("chain unreachable: %w", err) - } - - fromKey, err := solanago.PublicKeyFromBase58(from) - if err != nil { - return fmt.Errorf("failed to parse from key: %w", err) - } - toKey, err := solanago.PublicKeyFromBase58(to) - if err != nil { - return fmt.Errorf("failed to parse to key: %w", err) - } - if !amount.IsUint64() { - return fmt.Errorf("amount %s overflows uint64", amount) - } - amountI := amount.Uint64() - - blockhash, err := reader.LatestBlockhash() - if err != nil { - return fmt.Errorf("failed to get latest block hash: %w", err) - } - tx, err := solanago.NewTransaction( - []solanago.Instruction{ - system.NewTransferInstruction( - amountI, - fromKey, - toKey, - ).Build(), - }, - blockhash.Value.Blockhash, - solanago.TransactionPayer(fromKey), - ) - if err != nil { - return fmt.Errorf("failed to create tx: %w", err) - } - - if balanceCheck { - if err = solanaValidateBalance(reader, fromKey, amountI, tx.Message.ToBase64()); err != nil { - return fmt.Errorf("failed to validate balance: %w", err) - } - } - - txm := c.TxManager() - err = txm.Enqueue("", tx) - if err != nil { - return fmt.Errorf("transaction failed: %w", err) - } - return nil -} diff --git a/pkg/solana/cmd/chainlink-solana/main.go b/pkg/solana/cmd/chainlink-solana/main.go index 6a966a693..d65f6cbc9 100644 --- a/pkg/solana/cmd/chainlink-solana/main.go +++ b/pkg/solana/cmd/chainlink-solana/main.go @@ -67,15 +67,7 @@ func (c *pluginRelayer) NewRelayer(ctx context.Context, config string, keystore KeyStore: keystore, } - var chain solana.Chain - var err error - - if cfg.Solana.MultiNodeConfig().MultiNodeEnabled() { - chain, err = solana.NewMultiNodeChain(&cfg.Solana, opts) - } else { - chain, err = solana.NewChain(&cfg.Solana, opts) - } - + chain, err := solana.NewChain(&cfg.Solana, opts) if err != nil { return nil, fmt.Errorf("failed to create chain: %w", err) } From 0b33b1f9f8384de519a021af80e90c5b0e193c09 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 6 Sep 2024 12:07:39 -0400 Subject: [PATCH 012/174] Extend client --- pkg/solana/chain.go | 55 +++--- pkg/solana/client/client.go | 70 +++++++ pkg/solana/client/rpc_client.go | 312 -------------------------------- 3 files changed, 95 insertions(+), 342 deletions(-) delete mode 100644 pkg/solana/client/rpc_client.go diff --git a/pkg/solana/chain.go b/pkg/solana/chain.go index 7360f7c0c..9a6068f03 100644 --- a/pkg/solana/chain.go +++ b/pkg/solana/chain.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "io" "math/big" "math/rand" "strings" @@ -70,7 +71,7 @@ func NewChain(cfg *config.TOMLConfig, opts ChainOpts) (Chain, error) { if !cfg.IsEnabled() { return nil, fmt.Errorf("cannot create new chain with ID %s: chain is disabled", *cfg.ChainID) } - c, err := newChain(*cfg.ChainID, cfg, cfg.MultiNodeEnabled(), opts.KeyStore, opts.Logger) + c, err := newChain(*cfg.ChainID, cfg, opts.KeyStore, opts.Logger) if err != nil { return nil, err } @@ -88,9 +89,8 @@ type chain struct { lggr logger.Logger // if multiNode is enabled, the clientCache will not be used - multiNodeEnabled bool - multiNode *mn.MultiNode[mn.StringID, *client.RPCClient] - txSender *mn.TransactionSender[*solanago.Transaction, mn.StringID, *client.RPCClient] + multiNode *mn.MultiNode[mn.StringID, *client.Client] + txSender *mn.TransactionSender[*solanago.Transaction, mn.StringID, *client.Client] // tracking node chain id for verification clientCache map[string]*verifiedCachedClient // map URL -> {client, chainId} [mainnet/testnet/devnet/localnet] @@ -221,44 +221,43 @@ func (v *verifiedCachedClient) GetAccountInfoWithOpts(ctx context.Context, addr return v.ReaderWriter.GetAccountInfoWithOpts(ctx, addr, opts) } -func newChain(id string, cfg *config.TOMLConfig, multiNodeEnabled bool, ks loop.Keystore, lggr logger.Logger) (*chain, error) { +func newChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr logger.Logger) (*chain, error) { lggr = logger.With(lggr, "chainID", id, "chain", "solana") var ch = chain{ - id: id, - cfg: cfg, - lggr: logger.Named(lggr, "Chain"), - multiNodeEnabled: multiNodeEnabled, - clientCache: map[string]*verifiedCachedClient{}, + id: id, + cfg: cfg, + lggr: logger.Named(lggr, "Chain"), + clientCache: map[string]*verifiedCachedClient{}, } - if multiNodeEnabled { + if cfg.MultiNodeEnabled() { chainFamily := "solana" mnCfg := cfg.MultiNodeConfig() - var nodes []mn.Node[mn.StringID, *client.RPCClient] + var nodes []mn.Node[mn.StringID, *client.Client] for i, nodeInfo := range cfg.ListNodes() { // create client and check - rpcClient, err := client.NewRPCClient(nodeInfo.URL.String(), cfg, DefaultRequestTimeout, logger.Named(lggr, "Client."+*nodeInfo.Name)) + rpcClient, err := client.NewClient(nodeInfo.URL.String(), cfg, DefaultRequestTimeout, logger.Named(lggr, "Client."+*nodeInfo.Name)) if err != nil { lggr.Warnw("failed to create client", "name", *nodeInfo.Name, "solana-url", nodeInfo.URL.String(), "err", err.Error()) continue } - newNode := mn.NewNode[mn.StringID, *client.Head, *client.RPCClient]( + newNode := mn.NewNode[mn.StringID, *client.Head, *client.Client]( mnCfg, mnCfg, lggr, *nodeInfo.URL.URL(), nil, *nodeInfo.Name, int32(i), mn.StringID(id), 0, rpcClient, chainFamily) nodes = append(nodes, newNode) } - multiNode := mn.NewMultiNode[mn.StringID, *client.RPCClient]( + multiNode := mn.NewMultiNode[mn.StringID, *client.Client]( lggr, mn.NodeSelectionModeRoundRobin, time.Minute, // TODO: set lease duration nodes, - []mn.SendOnlyNode[mn.StringID, *client.RPCClient]{}, + []mn.SendOnlyNode[mn.StringID, *client.Client]{}, mn.StringID(id), chainFamily, mnCfg.DeathDeclarationDelay(), @@ -269,7 +268,7 @@ func newChain(id string, cfg *config.TOMLConfig, multiNodeEnabled bool, ks loop. return 0 // TODO ClassifySendError(err, clientErrors, logger.Sugared(logger.Nop()), tx, common.Address{}, false) } - txSender := mn.NewTransactionSender[*solanago.Transaction, mn.StringID, *client.RPCClient]( + txSender := mn.NewTransactionSender[*solanago.Transaction, mn.StringID, *client.Client]( lggr, mn.StringID(id), chainFamily, @@ -363,7 +362,7 @@ func (c *chain) ChainID() string { // getClient returns a client, randomly selecting one from available and valid nodes func (c *chain) getClient() (client.ReaderWriter, error) { - if c.multiNodeEnabled { + if c.cfg.MultiNodeEnabled() { return c.multiNode.SelectRPC() } @@ -446,14 +445,12 @@ func (c *chain) Start(ctx context.Context) error { c.lggr.Debug("Starting txm") c.lggr.Debug("Starting balance monitor") var ms services.MultiStart - if c.multiNodeEnabled { + startAll := []services.StartClose{c.txm, c.balanceMonitor} + if c.cfg.MultiNodeEnabled() { c.lggr.Debug("Starting multinode") - err := ms.Start(ctx, c.multiNode, c.txSender) - if err != nil { - return err - } + startAll = append(startAll, c.multiNode, c.txSender) } - return ms.Start(ctx, c.txm, c.balanceMonitor) + return ms.Start(ctx, startAll...) }) } @@ -462,14 +459,12 @@ func (c *chain) Close() error { c.lggr.Debug("Stopping") c.lggr.Debug("Stopping txm") c.lggr.Debug("Stopping balance monitor") - if c.multiNodeEnabled { + closeAll := []io.Closer{c.txm, c.balanceMonitor} + if c.cfg.MultiNodeEnabled() { c.lggr.Debug("Stopping multinode") - err := services.CloseAll(c.multiNode, c.txSender) - if err != nil { - return err - } + closeAll = append(closeAll, c.multiNode, c.txSender) } - return services.CloseAll(c.txm, c.balanceMonitor) + return services.CloseAll(closeAll...) }) } diff --git a/pkg/solana/client/client.go b/pkg/solana/client/client.go index d2294824d..21111fab4 100644 --- a/pkg/solana/client/client.go +++ b/pkg/solana/client/client.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "math/big" "time" mn "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" @@ -67,6 +68,25 @@ type Client struct { requestGroup *singleflight.Group } +type Head struct { + rpc.GetBlockResult +} + +func (h *Head) BlockNumber() int64 { + if h.BlockHeight == nil { + return 0 + } + return int64(*h.BlockHeight) +} + +func (h *Head) BlockDifficulty() *big.Int { + return nil +} + +func (h *Head) IsValid() bool { + return true +} + func NewClient(endpoint string, cfg config.Config, requestTimeout time.Duration, log logger.Logger) (*Client, error) { return &Client{ url: endpoint, @@ -81,6 +101,56 @@ func NewClient(endpoint string, cfg config.Config, requestTimeout time.Duration, }, nil } +var _ mn.RPCClient[mn.StringID, *Head] = (*Client)(nil) +var _ mn.SendTxRPCClient[*solana.Transaction] = (*Client)(nil) + +// TODO: BCI-4061: Implement Client for MultiNode + +func (c *Client) Dial(ctx context.Context) error { + //TODO implement me + panic("implement me") +} + +func (c *Client) SubscribeToHeads(ctx context.Context) (<-chan *Head, mn.Subscription, error) { + //TODO implement me + panic("implement me") +} + +func (c *Client) SubscribeToFinalizedHeads(ctx context.Context) (<-chan *Head, mn.Subscription, error) { + //TODO implement me + panic("implement me") +} + +func (c *Client) Ping(ctx context.Context) error { + //TODO implement me + panic("implement me") +} + +func (c *Client) IsSyncing(ctx context.Context) (bool, error) { + //TODO implement me + panic("implement me") +} + +func (c *Client) UnsubscribeAllExcept(subs ...mn.Subscription) { + //TODO implement me + panic("implement me") +} + +func (c *Client) Close() { + //TODO implement me + panic("implement me") +} + +func (c *Client) GetInterceptedChainInfo() (latest, highestUserObservations mn.ChainInfo) { + //TODO implement me + panic("implement me") +} + +func (c *Client) SendTransaction(ctx context.Context, tx *solana.Transaction) error { + // TODO: Implement + return nil +} + func (c *Client) latency(name string) func() { start := time.Now() return func() { diff --git a/pkg/solana/client/rpc_client.go b/pkg/solana/client/rpc_client.go deleted file mode 100644 index c9ceeab6a..000000000 --- a/pkg/solana/client/rpc_client.go +++ /dev/null @@ -1,312 +0,0 @@ -package client - -import ( - "context" - "errors" - "fmt" - "math/big" - "time" - - "github.com/gagliardetto/solana-go" - "github.com/gagliardetto/solana-go/rpc" - "golang.org/x/sync/singleflight" - - "github.com/smartcontractkit/chainlink-common/pkg/logger" - - mn "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" - "github.com/smartcontractkit/chainlink-solana/pkg/solana/config" - "github.com/smartcontractkit/chainlink-solana/pkg/solana/monitor" -) - -var _ ReaderWriter = (*RPCClient)(nil) - -type Head struct { - rpc.GetBlockResult -} - -func (h *Head) BlockNumber() int64 { - if h.BlockHeight == nil { - return 0 - } - return int64(*h.BlockHeight) -} - -func (h *Head) BlockDifficulty() *big.Int { - return nil -} - -func (h *Head) IsValid() bool { - return true -} - -type RPCClient struct { - url string - rpc *rpc.Client - skipPreflight bool // to enable or disable preflight checks - commitment rpc.CommitmentType - maxRetries *uint - txTimeout time.Duration - contextDuration time.Duration - log logger.Logger - - // provides a duplicate function call suppression mechanism - requestGroup *singleflight.Group -} - -// TODO: BCI-4061: Implement RPC Client for MultiNode - -func (c *RPCClient) Dial(ctx context.Context) error { - //TODO implement me - panic("implement me") -} - -func (c *RPCClient) SubscribeToHeads(ctx context.Context) (<-chan *Head, mn.Subscription, error) { - //TODO implement me - panic("implement me") -} - -func (c *RPCClient) SubscribeToFinalizedHeads(ctx context.Context) (<-chan *Head, mn.Subscription, error) { - //TODO implement me - panic("implement me") -} - -func (c *RPCClient) Ping(ctx context.Context) error { - //TODO implement me - panic("implement me") -} - -func (c *RPCClient) IsSyncing(ctx context.Context) (bool, error) { - //TODO implement me - panic("implement me") -} - -func (c *RPCClient) UnsubscribeAllExcept(subs ...mn.Subscription) { - //TODO implement me - panic("implement me") -} - -func (c *RPCClient) Close() { - //TODO implement me - panic("implement me") -} - -func (c *RPCClient) GetInterceptedChainInfo() (latest, highestUserObservations mn.ChainInfo) { - //TODO implement me - panic("implement me") -} - -func NewRPCClient(endpoint string, cfg config.Config, requestTimeout time.Duration, log logger.Logger) (*RPCClient, error) { - return &RPCClient{ - url: endpoint, - rpc: rpc.New(endpoint), - skipPreflight: cfg.SkipPreflight(), - commitment: cfg.Commitment(), - maxRetries: cfg.MaxRetries(), - txTimeout: cfg.TxTimeout(), - contextDuration: requestTimeout, - log: log, - requestGroup: &singleflight.Group{}, - }, nil -} - -func (c *RPCClient) latency(name string) func() { - start := time.Now() - return func() { - monitor.SetClientLatency(time.Since(start), name, c.url) - } -} - -func (c *RPCClient) Balance(addr solana.PublicKey) (uint64, error) { - done := c.latency("balance") - defer done() - - ctx, cancel := context.WithTimeout(context.Background(), c.contextDuration) - defer cancel() - - v, err, _ := c.requestGroup.Do(fmt.Sprintf("GetBalance(%s)", addr.String()), func() (interface{}, error) { - return c.rpc.GetBalance(ctx, addr, c.commitment) - }) - if err != nil { - return 0, err - } - res := v.(*rpc.GetBalanceResult) - return res.Value, err -} - -func (c *RPCClient) SlotHeight() (uint64, error) { - return c.SlotHeightWithCommitment(rpc.CommitmentProcessed) // get the latest slot height -} - -func (c *RPCClient) SlotHeightWithCommitment(commitment rpc.CommitmentType) (uint64, error) { - done := c.latency("slot_height") - defer done() - - ctx, cancel := context.WithTimeout(context.Background(), c.contextDuration) - defer cancel() - v, err, _ := c.requestGroup.Do("GetSlotHeight", func() (interface{}, error) { - return c.rpc.GetSlot(ctx, commitment) - }) - return v.(uint64), err -} - -func (c *RPCClient) GetAccountInfoWithOpts(ctx context.Context, addr solana.PublicKey, opts *rpc.GetAccountInfoOpts) (*rpc.GetAccountInfoResult, error) { - done := c.latency("account_info") - defer done() - - ctx, cancel := context.WithTimeout(ctx, c.contextDuration) - defer cancel() - opts.Commitment = c.commitment // overrides passed in value - use defined client commitment type - return c.rpc.GetAccountInfoWithOpts(ctx, addr, opts) -} - -func (c *RPCClient) LatestBlockhash() (*rpc.GetLatestBlockhashResult, error) { - done := c.latency("latest_blockhash") - defer done() - - ctx, cancel := context.WithTimeout(context.Background(), c.contextDuration) - defer cancel() - - v, err, _ := c.requestGroup.Do("GetLatestBlockhash", func() (interface{}, error) { - return c.rpc.GetLatestBlockhash(ctx, c.commitment) - }) - return v.(*rpc.GetLatestBlockhashResult), err -} - -func (c *RPCClient) ChainID(ctx context.Context) (mn.StringID, error) { - done := c.latency("chain_id") - defer done() - - ctx, cancel := context.WithTimeout(ctx, c.contextDuration) - defer cancel() - v, err, _ := c.requestGroup.Do("GetGenesisHash", func() (interface{}, error) { - return c.rpc.GetGenesisHash(ctx) - }) - if err != nil { - return "", err - } - hash := v.(solana.Hash) - - var network string - switch hash.String() { - case DevnetGenesisHash: - network = "devnet" - case TestnetGenesisHash: - network = "testnet" - case MainnetGenesisHash: - network = "mainnet" - default: - c.log.Warnf("unknown genesis hash - assuming solana chain is 'localnet'") - network = "localnet" - } - return mn.StringID(network), nil -} - -func (c *RPCClient) GetFeeForMessage(msg string) (uint64, error) { - done := c.latency("fee_for_message") - defer done() - - // msg is base58 encoded data - - ctx, cancel := context.WithTimeout(context.Background(), c.contextDuration) - defer cancel() - res, err := c.rpc.GetFeeForMessage(ctx, msg, c.commitment) - if err != nil { - return 0, fmt.Errorf("error in GetFeeForMessage: %w", err) - } - - if res == nil || res.Value == nil { - return 0, errors.New("nil pointer in GetFeeForMessage") - } - return *res.Value, nil -} - -// https://docs.solana.com/developing/clients/jsonrpc-api#getsignaturestatuses -func (c *RPCClient) SignatureStatuses(ctx context.Context, sigs []solana.Signature) ([]*rpc.SignatureStatusesResult, error) { - done := c.latency("signature_statuses") - defer done() - - ctx, cancel := context.WithTimeout(ctx, c.contextDuration) - defer cancel() - - // searchTransactionHistory = false - res, err := c.rpc.GetSignatureStatuses(ctx, false, sigs...) - if err != nil { - return nil, fmt.Errorf("error in GetSignatureStatuses: %w", err) - } - - if res == nil || res.Value == nil { - return nil, errors.New("nil pointer in GetSignatureStatuses") - } - return res.Value, nil -} - -// https://docs.solana.com/developing/clients/jsonrpc-api#simulatetransaction -// opts - (optional) use `nil` to use defaults -func (c *RPCClient) SimulateTx(ctx context.Context, tx *solana.Transaction, opts *rpc.SimulateTransactionOpts) (*rpc.SimulateTransactionResult, error) { - done := c.latency("simulate_tx") - defer done() - - ctx, cancel := context.WithTimeout(ctx, c.contextDuration) - defer cancel() - - if opts == nil { - opts = &rpc.SimulateTransactionOpts{ - SigVerify: true, // verify signature - Commitment: c.commitment, - } - } - - res, err := c.rpc.SimulateTransactionWithOpts(ctx, tx, opts) - if err != nil { - return nil, fmt.Errorf("error in SimulateTransactionWithOpts: %w", err) - } - - if res == nil || res.Value == nil { - return nil, errors.New("nil pointer in SimulateTransactionWithOpts") - } - - return res.Value, nil -} - -func (c *RPCClient) SendTransaction(ctx context.Context, tx *solana.Transaction) error { - // TODO: Implement - return nil -} - -func (c *RPCClient) SendTx(ctx context.Context, tx *solana.Transaction) (solana.Signature, error) { - done := c.latency("send_tx") - defer done() - - ctx, cancel := context.WithTimeout(ctx, c.txTimeout) - defer cancel() - - opts := rpc.TransactionOpts{ - SkipPreflight: c.skipPreflight, - PreflightCommitment: c.commitment, - MaxRetries: c.maxRetries, - } - - return c.rpc.SendTransactionWithOpts(ctx, tx, opts) -} - -func (c *RPCClient) GetLatestBlock() (*rpc.GetBlockResult, error) { - // get latest confirmed slot - slot, err := c.SlotHeightWithCommitment(c.commitment) - if err != nil { - return nil, fmt.Errorf("GetLatestBlock.SlotHeight: %w", err) - } - - // get block based on slot - done := c.latency("latest_block") - defer done() - ctx, cancel := context.WithTimeout(context.Background(), c.txTimeout) - defer cancel() - v, err, _ := c.requestGroup.Do("GetBlockWithOpts", func() (interface{}, error) { - version := uint64(0) // pull all tx types (legacy + v0) - return c.rpc.GetBlockWithOpts(ctx, slot, &rpc.GetBlockOpts{ - Commitment: c.commitment, - MaxSupportedTransactionVersion: &version, - }) - }) - return v.(*rpc.GetBlockResult), err -} From 389010386951df0fe1d2ec6b86e544aa356371c4 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 6 Sep 2024 14:56:53 -0400 Subject: [PATCH 013/174] Implement rpc client methods --- pkg/solana/client/client.go | 180 +++++++++++++++++++++++++++------ pkg/solana/config/multinode.go | 21 +++- 2 files changed, 167 insertions(+), 34 deletions(-) diff --git a/pkg/solana/client/client.go b/pkg/solana/client/client.go index 21111fab4..e917c5fc8 100644 --- a/pkg/solana/client/client.go +++ b/pkg/solana/client/client.go @@ -4,7 +4,9 @@ import ( "context" "errors" "fmt" + "github.com/ethereum/go-ethereum" "math/big" + "sync" "time" mn "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" @@ -66,6 +68,37 @@ type Client struct { // provides a duplicate function call suppression mechanism requestGroup *singleflight.Group + + // MultiNode + finalizedBlockPollInterval time.Duration + stateMu sync.RWMutex // protects state* fields + subs map[ethereum.Subscription]struct{} + + // chStopInFlight can be closed to immediately cancel all in-flight requests on + // this RpcClient. Closing and replacing should be serialized through + // stateMu since it can happen on state transitions as well as RpcClient Close. + chStopInFlight chan struct{} + + chainInfoLock sync.RWMutex + // intercepted values seen by callers of the rpcClient excluding health check calls. Need to ensure MultiNode provides repeatable read guarantee + highestUserObservations mn.ChainInfo + // most recent chain info observed during current lifecycle (reseted on DisconnectAll) + latestChainInfo mn.ChainInfo +} + +func NewClient(endpoint string, cfg *config.TOMLConfig, requestTimeout time.Duration, log logger.Logger) (*Client, error) { + return &Client{ + url: endpoint, + rpc: rpc.New(endpoint), + skipPreflight: cfg.SkipPreflight(), + commitment: cfg.Commitment(), + maxRetries: cfg.MaxRetries(), + txTimeout: cfg.TxTimeout(), + contextDuration: requestTimeout, + log: log, + requestGroup: &singleflight.Group{}, + finalizedBlockPollInterval: cfg.FinalizedBlockPollInterval(), + }, nil } type Head struct { @@ -87,68 +120,151 @@ func (h *Head) IsValid() bool { return true } -func NewClient(endpoint string, cfg config.Config, requestTimeout time.Duration, log logger.Logger) (*Client, error) { - return &Client{ - url: endpoint, - rpc: rpc.New(endpoint), - skipPreflight: cfg.SkipPreflight(), - commitment: cfg.Commitment(), - maxRetries: cfg.MaxRetries(), - txTimeout: cfg.TxTimeout(), - contextDuration: requestTimeout, - log: log, - requestGroup: &singleflight.Group{}, - }, nil -} - var _ mn.RPCClient[mn.StringID, *Head] = (*Client)(nil) var _ mn.SendTxRPCClient[*solana.Transaction] = (*Client)(nil) -// TODO: BCI-4061: Implement Client for MultiNode - func (c *Client) Dial(ctx context.Context) error { - //TODO implement me + // TODO: is there any work to do here? Doesn't seem like we have to dial anything + // TODO: Could maybe do a version check here? panic("implement me") } func (c *Client) SubscribeToHeads(ctx context.Context) (<-chan *Head, mn.Subscription, error) { - //TODO implement me + // TODO: BlockPollInternal + if c.finalizedBlockPollInterval == 0 { + return nil, nil, errors.New("FinalizedBlockPollInterval is 0") + } + timeout := c.finalizedBlockPollInterval + poller, channel := mn.NewPoller[*Head](c.finalizedBlockPollInterval, c.LatestFinalizedBlock, timeout, c.log) + if err := poller.Start(ctx); err != nil { + return nil, nil, err + } + return channel, &poller, nil panic("implement me") } +func (c *Client) LatestFinalizedBlock(ctx context.Context) (*Head, error) { + // TODO: Do we need this? + // capture chStopInFlight to ensure we are not updating chainInfo with observations related to previous life cycle + //ctx, cancel, chStopInFlight, _, _ := c.acquireQueryCtx(ctx, c.rpcTimeout) + + // TODO: Is this really the way to implement this? + latestBH, err := c.rpc.GetLatestBlockhash(ctx, c.commitment) + if err != nil { + return nil, err + } + + var finalityDepth uint64 = 1 // TODO: Value? + + latestFinalizedBH := latestBH.Value.LastValidBlockHeight - finalityDepth // TODO: subtract finality depth? + + resp, err := c.rpc.GetBlock(ctx, latestFinalizedBH) + if err != nil { + return nil, err + } + return &Head{GetBlockResult: *resp}, nil +} + func (c *Client) SubscribeToFinalizedHeads(ctx context.Context) (<-chan *Head, mn.Subscription, error) { - //TODO implement me - panic("implement me") + if c.finalizedBlockPollInterval == 0 { + return nil, nil, errors.New("FinalizedBlockPollInterval is 0") + } + timeout := c.finalizedBlockPollInterval + poller, channel := mn.NewPoller[*Head](c.finalizedBlockPollInterval, c.LatestFinalizedBlock, timeout, c.log) + if err := poller.Start(ctx); err != nil { + return nil, nil, err + } + return channel, &poller, nil +} + +func (c *Client) onNewFinalizedHead(ctx context.Context, requestCh <-chan struct{}, head *Head) { + if head == nil { + return + } + c.chainInfoLock.Lock() + defer c.chainInfoLock.Unlock() + if !mn.CtxIsHeathCheckRequest(ctx) { + c.highestUserObservations.FinalizedBlockNumber = max(c.highestUserObservations.FinalizedBlockNumber, head.BlockNumber()) + } + select { + case <-requestCh: // no need to update latestChainInfo, as rpcClient already started new life cycle + return + default: + c.latestChainInfo.FinalizedBlockNumber = head.BlockNumber() + } } func (c *Client) Ping(ctx context.Context) error { - //TODO implement me - panic("implement me") + /* TODO: Should we use this health check for ping or somewhere? + health, err := c.rpc.GetHealth(ctx) + if err != nil { + return false, err + } + return health == rpc.HealthOk, nil + */ + + version, err := c.rpc.GetVersion(ctx) + if err != nil { + return fmt.Errorf("ping failed: %v", err) + } + c.log.Debugf("ping client version: %s", version.SolanaCore) + return err } func (c *Client) IsSyncing(ctx context.Context) (bool, error) { - //TODO implement me - panic("implement me") + // TODO: is this relevant? + return false, nil } func (c *Client) UnsubscribeAllExcept(subs ...mn.Subscription) { - //TODO implement me - panic("implement me") + c.stateMu.Lock() + defer c.stateMu.Unlock() + + keepSubs := map[mn.Subscription]struct{}{} + for _, sub := range subs { + keepSubs[sub] = struct{}{} + } + + for sub := range c.subs { + if _, keep := keepSubs[sub]; !keep { + sub.Unsubscribe() + delete(c.subs, sub) + } + } +} + +// cancelInflightRequests closes and replaces the chStopInFlight +func (c *Client) cancelInflightRequests() { + c.stateMu.Lock() + defer c.stateMu.Unlock() + close(c.chStopInFlight) + c.chStopInFlight = make(chan struct{}) } func (c *Client) Close() { - //TODO implement me - panic("implement me") + defer func() { + err := c.rpc.Close() + if err != nil { + c.log.Errorf("error closing rpc: %v", err) + } + }() + c.cancelInflightRequests() + c.UnsubscribeAllExcept() + c.chainInfoLock.Lock() + c.latestChainInfo = mn.ChainInfo{} + c.chainInfoLock.Unlock() } func (c *Client) GetInterceptedChainInfo() (latest, highestUserObservations mn.ChainInfo) { - //TODO implement me - panic("implement me") + c.chainInfoLock.Lock() + defer c.chainInfoLock.Unlock() + return c.latestChainInfo, c.highestUserObservations } func (c *Client) SendTransaction(ctx context.Context, tx *solana.Transaction) error { - // TODO: Implement - return nil + // TODO: Is this all we need to do here? + _, err := c.SendTx(ctx, tx) + return err } func (c *Client) latency(name string) func() { diff --git a/pkg/solana/config/multinode.go b/pkg/solana/config/multinode.go index 1755e6ee6..42828cd9a 100644 --- a/pkg/solana/config/multinode.go +++ b/pkg/solana/config/multinode.go @@ -3,7 +3,6 @@ package config import "time" type MultiNode struct { - // TODO: Determine current config overlap https://smartcontract-it.atlassian.net/browse/BCI-4065 // Feature flag multiNodeEnabled bool @@ -13,6 +12,7 @@ type MultiNode struct { selectionMode string syncThreshold uint32 nodeIsSyncingEnabled bool + leaseDuration time.Duration finalizedBlockPollInterval time.Duration enforceRepeatableRead bool deathDeclarationDelay time.Duration @@ -82,6 +82,23 @@ func (c *MultiNode) FinalizedBlockOffset() uint32 { } func (c *MultiNode) SetDefaults() { - // TODO: Set defaults for MultiNode config https://smartcontract-it.atlassian.net/browse/BCI-4065 c.multiNodeEnabled = false + + // Node Configs + c.pollFailureThreshold = 5 + c.pollInterval = 10 * time.Second + c.selectionMode = "HighestHead" + c.syncThreshold = 5 + c.leaseDuration = 0 + c.nodeIsSyncingEnabled = false + c.finalizedBlockPollInterval = 5 * time.Second + c.enforceRepeatableRead = false + c.deathDeclarationDelay = 10 * time.Second + + // Chain Configs + c.nodeNoNewHeadsThreshold = 10 * time.Second // TODO: Value? + c.noNewFinalizedHeadsThreshold = 10 * time.Second // TODO: Value? + c.finalityDepth = 0 // TODO: Value? + c.finalityTagEnabled = false // TODO: Value? + c.finalizedBlockOffset = 0 // TODO: Value? } From 95e1c9a381b37baf6bdf3c43fc2a64cefcdf2e27 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 6 Sep 2024 14:58:32 -0400 Subject: [PATCH 014/174] Add defaults --- pkg/solana/config/multinode.go | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/pkg/solana/config/multinode.go b/pkg/solana/config/multinode.go index 1755e6ee6..42828cd9a 100644 --- a/pkg/solana/config/multinode.go +++ b/pkg/solana/config/multinode.go @@ -3,7 +3,6 @@ package config import "time" type MultiNode struct { - // TODO: Determine current config overlap https://smartcontract-it.atlassian.net/browse/BCI-4065 // Feature flag multiNodeEnabled bool @@ -13,6 +12,7 @@ type MultiNode struct { selectionMode string syncThreshold uint32 nodeIsSyncingEnabled bool + leaseDuration time.Duration finalizedBlockPollInterval time.Duration enforceRepeatableRead bool deathDeclarationDelay time.Duration @@ -82,6 +82,23 @@ func (c *MultiNode) FinalizedBlockOffset() uint32 { } func (c *MultiNode) SetDefaults() { - // TODO: Set defaults for MultiNode config https://smartcontract-it.atlassian.net/browse/BCI-4065 c.multiNodeEnabled = false + + // Node Configs + c.pollFailureThreshold = 5 + c.pollInterval = 10 * time.Second + c.selectionMode = "HighestHead" + c.syncThreshold = 5 + c.leaseDuration = 0 + c.nodeIsSyncingEnabled = false + c.finalizedBlockPollInterval = 5 * time.Second + c.enforceRepeatableRead = false + c.deathDeclarationDelay = 10 * time.Second + + // Chain Configs + c.nodeNoNewHeadsThreshold = 10 * time.Second // TODO: Value? + c.noNewFinalizedHeadsThreshold = 10 * time.Second // TODO: Value? + c.finalityDepth = 0 // TODO: Value? + c.finalityTagEnabled = false // TODO: Value? + c.finalizedBlockOffset = 0 // TODO: Value? } From 45a7596b6f6067faccad1b718ee1feef85103503 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 6 Sep 2024 15:15:20 -0400 Subject: [PATCH 015/174] Add latest block methods --- pkg/solana/client/client.go | 63 ++++++++++++++++++++++++++++++------- 1 file changed, 52 insertions(+), 11 deletions(-) diff --git a/pkg/solana/client/client.go b/pkg/solana/client/client.go index e917c5fc8..7b769911e 100644 --- a/pkg/solana/client/client.go +++ b/pkg/solana/client/client.go @@ -70,6 +70,7 @@ type Client struct { requestGroup *singleflight.Group // MultiNode + pollInterval time.Duration finalizedBlockPollInterval time.Duration stateMu sync.RWMutex // protects state* fields subs map[ethereum.Subscription]struct{} @@ -97,6 +98,7 @@ func NewClient(endpoint string, cfg *config.TOMLConfig, requestTimeout time.Dura contextDuration: requestTimeout, log: log, requestGroup: &singleflight.Group{}, + pollInterval: cfg.PollInterval(), finalizedBlockPollInterval: cfg.FinalizedBlockPollInterval(), }, nil } @@ -113,6 +115,8 @@ func (h *Head) BlockNumber() int64 { } func (h *Head) BlockDifficulty() *big.Int { + // TODO: Is difficulty relevant for Solana? + // TODO: If not, then remove changes to it in latestBlockInfo return nil } @@ -130,7 +134,18 @@ func (c *Client) Dial(ctx context.Context) error { } func (c *Client) SubscribeToHeads(ctx context.Context) (<-chan *Head, mn.Subscription, error) { - // TODO: BlockPollInternal + if c.pollInterval == 0 { + return nil, nil, errors.New("PollInterval is 0") + } + timeout := c.pollInterval + poller, channel := mn.NewPoller[*Head](c.pollInterval, c.LatestBlock, timeout, c.log) + if err := poller.Start(ctx); err != nil { + return nil, nil, err + } + return channel, &poller, nil +} + +func (c *Client) SubscribeToFinalizedHeads(ctx context.Context) (<-chan *Head, mn.Subscription, error) { if c.finalizedBlockPollInterval == 0 { return nil, nil, errors.New("FinalizedBlockPollInterval is 0") } @@ -140,7 +155,22 @@ func (c *Client) SubscribeToHeads(ctx context.Context) (<-chan *Head, mn.Subscri return nil, nil, err } return channel, &poller, nil - panic("implement me") +} + +func (c *Client) LatestBlock(ctx context.Context) (*Head, error) { + latestBlockHash, err := c.rpc.GetLatestBlockhash(ctx, c.commitment) + if err != nil { + return nil, err + } + + latestBlock, err := c.rpc.GetBlock(ctx, latestBlockHash.Value.LastValidBlockHeight) + if err != nil { + return nil, err + } + + head := &Head{GetBlockResult: *latestBlock} + c.onNewHead(ctx, c.chStopInFlight, head) + return head, nil } func (c *Client) LatestFinalizedBlock(ctx context.Context) (*Head, error) { @@ -162,19 +192,30 @@ func (c *Client) LatestFinalizedBlock(ctx context.Context) (*Head, error) { if err != nil { return nil, err } - return &Head{GetBlockResult: *resp}, nil + + head := &Head{GetBlockResult: *resp} + c.onNewFinalizedHead(ctx, c.chStopInFlight, head) + return head, nil } -func (c *Client) SubscribeToFinalizedHeads(ctx context.Context) (<-chan *Head, mn.Subscription, error) { - if c.finalizedBlockPollInterval == 0 { - return nil, nil, errors.New("FinalizedBlockPollInterval is 0") +func (c *Client) onNewHead(ctx context.Context, requestCh <-chan struct{}, head *Head) { + if head == nil { + return } - timeout := c.finalizedBlockPollInterval - poller, channel := mn.NewPoller[*Head](c.finalizedBlockPollInterval, c.LatestFinalizedBlock, timeout, c.log) - if err := poller.Start(ctx); err != nil { - return nil, nil, err + + c.chainInfoLock.Lock() + defer c.chainInfoLock.Unlock() + if !mn.CtxIsHeathCheckRequest(ctx) { + c.highestUserObservations.BlockNumber = max(c.highestUserObservations.BlockNumber, head.BlockNumber()) + c.highestUserObservations.TotalDifficulty = mn.MaxTotalDifficulty(c.highestUserObservations.TotalDifficulty, head.BlockDifficulty()) + } + select { + case <-requestCh: // no need to update latestChainInfo, as rpcClient already started new life cycle + return + default: + c.latestChainInfo.BlockNumber = head.BlockNumber() + c.latestChainInfo.TotalDifficulty = head.BlockDifficulty() } - return channel, &poller, nil } func (c *Client) onNewFinalizedHead(ctx context.Context, requestCh <-chan struct{}, head *Head) { From d8d312ccfad986ae3324bc4a29f1c5060ff30459 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 10 Sep 2024 11:38:24 -0400 Subject: [PATCH 016/174] Address comments --- pkg/solana/chain.go | 10 +++++++--- pkg/solana/client/client.go | 3 +++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/pkg/solana/chain.go b/pkg/solana/chain.go index 9a6068f03..c7cc09e7f 100644 --- a/pkg/solana/chain.go +++ b/pkg/solana/chain.go @@ -238,11 +238,14 @@ func newChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr logger.L var nodes []mn.Node[mn.StringID, *client.Client] for i, nodeInfo := range cfg.ListNodes() { + if nodeInfo == nil || nodeInfo.Name == nil || nodeInfo.URL == nil { + return nil, fmt.Errorf("node config contains nil: %+v", nodeInfo) + } // create client and check rpcClient, err := client.NewClient(nodeInfo.URL.String(), cfg, DefaultRequestTimeout, logger.Named(lggr, "Client."+*nodeInfo.Name)) if err != nil { lggr.Warnw("failed to create client", "name", *nodeInfo.Name, "solana-url", nodeInfo.URL.String(), "err", err.Error()) - continue + return nil, fmt.Errorf("failed to create client: %w", err) } newNode := mn.NewNode[mn.StringID, *client.Head, *client.Client]( @@ -255,7 +258,7 @@ func newChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr logger.L multiNode := mn.NewMultiNode[mn.StringID, *client.Client]( lggr, mn.NodeSelectionModeRoundRobin, - time.Minute, // TODO: set lease duration + 0, nodes, []mn.SendOnlyNode[mn.StringID, *client.Client]{}, mn.StringID(id), @@ -263,7 +266,8 @@ func newChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr logger.L mnCfg.DeathDeclarationDelay(), ) - // TODO: implement error classification + // TODO: implement error classification; move logic to separate file if large + // TODO: might be useful to reference anza-xyz/agave@master/sdk/src/transaction/error.rs classifySendError := func(tx *solanago.Transaction, err error) mn.SendTxReturnCode { return 0 // TODO ClassifySendError(err, clientErrors, logger.Sugared(logger.Nop()), tx, common.Address{}, false) } diff --git a/pkg/solana/client/client.go b/pkg/solana/client/client.go index 21111fab4..d2c4a6b08 100644 --- a/pkg/solana/client/client.go +++ b/pkg/solana/client/client.go @@ -84,6 +84,9 @@ func (h *Head) BlockDifficulty() *big.Int { } func (h *Head) IsValid() bool { + if h.BlockHeight == nil { + return false + } return true } From 3c3756e0b3301c8a524e84310ea7955fc91eb365 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 12 Sep 2024 13:07:08 -0400 Subject: [PATCH 017/174] lint --- .golangci.yml | 1 + pkg/solana/client/client.go | 5 +---- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 0e66a8650..02b479dbf 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -30,6 +30,7 @@ linters-settings: excludes: - G101 - G104 + - G115 # - G204 # - G304 # - G404 diff --git a/pkg/solana/client/client.go b/pkg/solana/client/client.go index d2c4a6b08..183ff7a92 100644 --- a/pkg/solana/client/client.go +++ b/pkg/solana/client/client.go @@ -84,10 +84,7 @@ func (h *Head) BlockDifficulty() *big.Int { } func (h *Head) IsValid() bool { - if h.BlockHeight == nil { - return false - } - return true + return h.BlockHeight != nil } func NewClient(endpoint string, cfg config.Config, requestTimeout time.Duration, log logger.Logger) (*Client, error) { From 2521670c879b95c28a786f71e9d9cc70d7f68a30 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 12 Sep 2024 13:24:45 -0400 Subject: [PATCH 018/174] Fix lint overflow issues --- .golangci.yml | 1 - pkg/solana/chain.go | 2 +- pkg/solana/client/client.go | 4 ++-- pkg/solana/client/multinode/node.go | 4 ++-- pkg/solana/client/multinode/node_fsm.go | 4 ++-- pkg/solana/client/multinode/node_lifecycle.go | 4 ++-- pkg/solana/client/multinode/node_selector_highest_head.go | 6 +----- pkg/solana/client/multinode/node_selector_priority_level.go | 4 ++-- pkg/solana/client/multinode/node_selector_round_robin.go | 4 ++-- pkg/solana/client/multinode/types.go | 6 +++--- 10 files changed, 17 insertions(+), 22 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 02b479dbf..0e66a8650 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -30,7 +30,6 @@ linters-settings: excludes: - G101 - G104 - - G115 # - G204 # - G304 # - G404 diff --git a/pkg/solana/chain.go b/pkg/solana/chain.go index 152e9e304..e2747014c 100644 --- a/pkg/solana/chain.go +++ b/pkg/solana/chain.go @@ -250,7 +250,7 @@ func newChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr logger.L newNode := mn.NewNode[mn.StringID, *client.Head, *client.Client]( mnCfg, mnCfg, lggr, *nodeInfo.URL.URL(), nil, *nodeInfo.Name, - int32(i), mn.StringID(id), 0, rpcClient, chainFamily) + i, mn.StringID(id), 0, rpcClient, chainFamily) nodes = append(nodes, newNode) } diff --git a/pkg/solana/client/client.go b/pkg/solana/client/client.go index 183ff7a92..3d6e3fb98 100644 --- a/pkg/solana/client/client.go +++ b/pkg/solana/client/client.go @@ -72,11 +72,11 @@ type Head struct { rpc.GetBlockResult } -func (h *Head) BlockNumber() int64 { +func (h *Head) BlockNumber() uint64 { if h.BlockHeight == nil { return 0 } - return int64(*h.BlockHeight) + return *h.BlockHeight } func (h *Head) BlockDifficulty() *big.Int { diff --git a/pkg/solana/client/multinode/node.go b/pkg/solana/client/multinode/node.go index 7d2b02ce2..afdece741 100644 --- a/pkg/solana/client/multinode/node.go +++ b/pkg/solana/client/multinode/node.go @@ -89,7 +89,7 @@ type node[ services.StateMachine lfcLog logger.Logger name string - id int32 + id int chainID CHAIN_ID nodePoolCfg NodeConfig chainCfg ChainConfig @@ -124,7 +124,7 @@ func NewNode[ wsuri url.URL, httpuri *url.URL, name string, - id int32, + id int, chainID CHAIN_ID, nodeOrder int32, rpc RPC, diff --git a/pkg/solana/client/multinode/node_fsm.go b/pkg/solana/client/multinode/node_fsm.go index 136910868..5d0176c02 100644 --- a/pkg/solana/client/multinode/node_fsm.go +++ b/pkg/solana/client/multinode/node_fsm.go @@ -150,10 +150,10 @@ func (n *node[CHAIN_ID, HEAD, RPC]) isFinalizedBlockOutOfSync() bool { highestObservedByCaller := n.poolInfoProvider.HighestUserObservations() latest, _ := n.rpc.GetInterceptedChainInfo() if n.chainCfg.FinalityTagEnabled() { - return latest.FinalizedBlockNumber < highestObservedByCaller.FinalizedBlockNumber-int64(n.chainCfg.FinalizedBlockOffset()) + return latest.FinalizedBlockNumber < highestObservedByCaller.FinalizedBlockNumber-uint64(n.chainCfg.FinalizedBlockOffset()) } - return latest.BlockNumber < highestObservedByCaller.BlockNumber-int64(n.chainCfg.FinalizedBlockOffset()) + return latest.BlockNumber < highestObservedByCaller.BlockNumber-uint64(n.chainCfg.FinalizedBlockOffset()) } // StateAndLatest returns nodeState with the latest ChainInfo observed by Node during current lifecycle. diff --git a/pkg/solana/client/multinode/node_lifecycle.go b/pkg/solana/client/multinode/node_lifecycle.go index d6b150690..427ccb216 100644 --- a/pkg/solana/client/multinode/node_lifecycle.go +++ b/pkg/solana/client/multinode/node_lifecycle.go @@ -335,7 +335,7 @@ func (n *node[CHAIN_ID, HEAD, RPC]) onNewHead(lggr logger.SugaredLogger, chainIn chainInfo.BlockNumber = head.BlockNumber() if !n.chainCfg.FinalityTagEnabled() { - latestFinalizedBN := max(head.BlockNumber()-int64(n.chainCfg.FinalityDepth()), 0) + latestFinalizedBN := max(head.BlockNumber()-uint64(n.chainCfg.FinalityDepth()), 0) if latestFinalizedBN > chainInfo.FinalizedBlockNumber { promPoolRPCNodeHighestFinalizedBlock.WithLabelValues(n.chainID.String(), n.name).Set(float64(latestFinalizedBN)) chainInfo.FinalizedBlockNumber = latestFinalizedBN @@ -368,7 +368,7 @@ func (n *node[CHAIN_ID, HEAD, RPC]) isOutOfSyncWithPool(localState ChainInfo) (o mode := n.nodePoolCfg.SelectionMode() switch mode { case NodeSelectionModeHighestHead, NodeSelectionModeRoundRobin, NodeSelectionModePriorityLevel: - return localState.BlockNumber < ci.BlockNumber-int64(threshold), ln + return localState.BlockNumber < ci.BlockNumber-uint64(threshold), ln case NodeSelectionModeTotalDifficulty: bigThreshold := big.NewInt(int64(threshold)) return localState.TotalDifficulty.Cmp(bigmath.Sub(ci.TotalDifficulty, bigThreshold)) < 0, ln diff --git a/pkg/solana/client/multinode/node_selector_highest_head.go b/pkg/solana/client/multinode/node_selector_highest_head.go index 52188bbdf..c7d0d1e3d 100644 --- a/pkg/solana/client/multinode/node_selector_highest_head.go +++ b/pkg/solana/client/multinode/node_selector_highest_head.go @@ -1,9 +1,5 @@ package client -import ( - "math" -) - type highestHeadNodeSelector[ CHAIN_ID ID, RPC any, @@ -17,7 +13,7 @@ func NewHighestHeadNodeSelector[ } func (s highestHeadNodeSelector[CHAIN_ID, RPC]) Select() Node[CHAIN_ID, RPC] { - var highestHeadNumber int64 = math.MinInt64 + var highestHeadNumber uint64 var highestHeadNodes []Node[CHAIN_ID, RPC] for _, n := range s { state, currentChainInfo := n.StateAndLatest() diff --git a/pkg/solana/client/multinode/node_selector_priority_level.go b/pkg/solana/client/multinode/node_selector_priority_level.go index 3e171b98b..ead720976 100644 --- a/pkg/solana/client/multinode/node_selector_priority_level.go +++ b/pkg/solana/client/multinode/node_selector_priority_level.go @@ -41,8 +41,8 @@ func (s priorityLevelNodeSelector[CHAIN_ID, RPC]) Select() Node[CHAIN_ID, RPC] { priorityLevel := nodes[len(nodes)-1].priority // NOTE: Inc returns the number after addition, so we must -1 to get the "current" counter - count := s.roundRobinCount[priorityLevel].Add(1) - 1 - idx := int(count % uint32(len(nodes))) + count := int(s.roundRobinCount[priorityLevel].Add(1) - 1) + idx := count % len(nodes) return nodes[idx].node } diff --git a/pkg/solana/client/multinode/node_selector_round_robin.go b/pkg/solana/client/multinode/node_selector_round_robin.go index 52fa9d6c8..c5ed8d853 100644 --- a/pkg/solana/client/multinode/node_selector_round_robin.go +++ b/pkg/solana/client/multinode/node_selector_round_robin.go @@ -35,8 +35,8 @@ func (s *roundRobinSelector[CHAIN_ID, RPC]) Select() Node[CHAIN_ID, RPC] { } // NOTE: Inc returns the number after addition, so we must -1 to get the "current" counter - count := s.roundRobinCount.Add(1) - 1 - idx := int(count % uint32(nNodes)) + count := int(s.roundRobinCount.Add(1) - 1) + idx := count % nNodes return liveNodes[idx] } diff --git a/pkg/solana/client/multinode/types.go b/pkg/solana/client/multinode/types.go index 51b70e573..2c177a9dc 100644 --- a/pkg/solana/client/multinode/types.go +++ b/pkg/solana/client/multinode/types.go @@ -68,7 +68,7 @@ type RPCClient[ // Head is the interface required by the NodeClient type Head interface { - BlockNumber() int64 + BlockNumber() uint64 BlockDifficulty() *big.Int IsValid() bool } @@ -86,8 +86,8 @@ type PoolChainInfoProvider interface { // ChainInfo - defines RPC's or MultiNode's view on the chain type ChainInfo struct { - BlockNumber int64 - FinalizedBlockNumber int64 + BlockNumber uint64 + FinalizedBlockNumber uint64 TotalDifficulty *big.Int } From 5b5cfd671379471d512233c835fdb38c072f232d Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 12 Sep 2024 13:33:00 -0400 Subject: [PATCH 019/174] Update transaction_sender.go --- pkg/solana/client/multinode/transaction_sender.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go index 71de153ae..fbd5acca5 100644 --- a/pkg/solana/client/multinode/transaction_sender.go +++ b/pkg/solana/client/multinode/transaction_sender.go @@ -145,7 +145,7 @@ func (txSender *TransactionSender[TX, CHAIN_ID, RPC]) SendTransaction(ctx contex }() if err != nil { - return 0, err + return Retryable, err } txSender.wg.Add(1) @@ -212,7 +212,7 @@ func aggregateTxResults(resultsByCode sendTxResults) (returnCode SendTxReturnCod func (txSender *TransactionSender[TX, CHAIN_ID, RPC]) collectTxResults(ctx context.Context, tx TX, healthyNodesNum int, txResults <-chan sendTxResult) (SendTxReturnCode, error) { if healthyNodesNum == 0 { - return 0, ErroringNodeError + return Retryable, ErroringNodeError } requiredResults := int(math.Ceil(float64(healthyNodesNum) * sendTxQuorum)) errorsByCode := sendTxResults{} @@ -223,7 +223,7 @@ loop: select { case <-ctx.Done(): txSender.lggr.Debugw("Failed to collect of the results before context was done", "tx", tx, "errorsByCode", errorsByCode) - return 0, ctx.Err() + return Retryable, ctx.Err() case result := <-txResults: errorsByCode[result.ResultCode] = append(errorsByCode[result.ResultCode], result.Err) resultsCount++ From 690f8124b9c17d25b01f0696f3c648bb3df8ab31 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 12 Sep 2024 13:53:28 -0400 Subject: [PATCH 020/174] Fix lint --- pkg/solana/client/client.go | 5 +++-- pkg/solana/client/multinode/node_fsm.go | 4 ++-- pkg/solana/client/multinode/node_lifecycle.go | 4 ++-- pkg/solana/client/multinode/node_selector_highest_head.go | 4 +++- pkg/solana/client/multinode/types.go | 6 +++--- 5 files changed, 13 insertions(+), 10 deletions(-) diff --git a/pkg/solana/client/client.go b/pkg/solana/client/client.go index 3d6e3fb98..35ab96e62 100644 --- a/pkg/solana/client/client.go +++ b/pkg/solana/client/client.go @@ -72,11 +72,12 @@ type Head struct { rpc.GetBlockResult } -func (h *Head) BlockNumber() uint64 { +func (h *Head) BlockNumber() int64 { if h.BlockHeight == nil { return 0 } - return *h.BlockHeight + //nolint:gosec + return int64(*h.BlockHeight) } func (h *Head) BlockDifficulty() *big.Int { diff --git a/pkg/solana/client/multinode/node_fsm.go b/pkg/solana/client/multinode/node_fsm.go index 5d0176c02..136910868 100644 --- a/pkg/solana/client/multinode/node_fsm.go +++ b/pkg/solana/client/multinode/node_fsm.go @@ -150,10 +150,10 @@ func (n *node[CHAIN_ID, HEAD, RPC]) isFinalizedBlockOutOfSync() bool { highestObservedByCaller := n.poolInfoProvider.HighestUserObservations() latest, _ := n.rpc.GetInterceptedChainInfo() if n.chainCfg.FinalityTagEnabled() { - return latest.FinalizedBlockNumber < highestObservedByCaller.FinalizedBlockNumber-uint64(n.chainCfg.FinalizedBlockOffset()) + return latest.FinalizedBlockNumber < highestObservedByCaller.FinalizedBlockNumber-int64(n.chainCfg.FinalizedBlockOffset()) } - return latest.BlockNumber < highestObservedByCaller.BlockNumber-uint64(n.chainCfg.FinalizedBlockOffset()) + return latest.BlockNumber < highestObservedByCaller.BlockNumber-int64(n.chainCfg.FinalizedBlockOffset()) } // StateAndLatest returns nodeState with the latest ChainInfo observed by Node during current lifecycle. diff --git a/pkg/solana/client/multinode/node_lifecycle.go b/pkg/solana/client/multinode/node_lifecycle.go index 427ccb216..d6b150690 100644 --- a/pkg/solana/client/multinode/node_lifecycle.go +++ b/pkg/solana/client/multinode/node_lifecycle.go @@ -335,7 +335,7 @@ func (n *node[CHAIN_ID, HEAD, RPC]) onNewHead(lggr logger.SugaredLogger, chainIn chainInfo.BlockNumber = head.BlockNumber() if !n.chainCfg.FinalityTagEnabled() { - latestFinalizedBN := max(head.BlockNumber()-uint64(n.chainCfg.FinalityDepth()), 0) + latestFinalizedBN := max(head.BlockNumber()-int64(n.chainCfg.FinalityDepth()), 0) if latestFinalizedBN > chainInfo.FinalizedBlockNumber { promPoolRPCNodeHighestFinalizedBlock.WithLabelValues(n.chainID.String(), n.name).Set(float64(latestFinalizedBN)) chainInfo.FinalizedBlockNumber = latestFinalizedBN @@ -368,7 +368,7 @@ func (n *node[CHAIN_ID, HEAD, RPC]) isOutOfSyncWithPool(localState ChainInfo) (o mode := n.nodePoolCfg.SelectionMode() switch mode { case NodeSelectionModeHighestHead, NodeSelectionModeRoundRobin, NodeSelectionModePriorityLevel: - return localState.BlockNumber < ci.BlockNumber-uint64(threshold), ln + return localState.BlockNumber < ci.BlockNumber-int64(threshold), ln case NodeSelectionModeTotalDifficulty: bigThreshold := big.NewInt(int64(threshold)) return localState.TotalDifficulty.Cmp(bigmath.Sub(ci.TotalDifficulty, bigThreshold)) < 0, ln diff --git a/pkg/solana/client/multinode/node_selector_highest_head.go b/pkg/solana/client/multinode/node_selector_highest_head.go index c7d0d1e3d..68901cba3 100644 --- a/pkg/solana/client/multinode/node_selector_highest_head.go +++ b/pkg/solana/client/multinode/node_selector_highest_head.go @@ -1,5 +1,7 @@ package client +import "math" + type highestHeadNodeSelector[ CHAIN_ID ID, RPC any, @@ -13,7 +15,7 @@ func NewHighestHeadNodeSelector[ } func (s highestHeadNodeSelector[CHAIN_ID, RPC]) Select() Node[CHAIN_ID, RPC] { - var highestHeadNumber uint64 + var highestHeadNumber int64 = math.MinInt64 var highestHeadNodes []Node[CHAIN_ID, RPC] for _, n := range s { state, currentChainInfo := n.StateAndLatest() diff --git a/pkg/solana/client/multinode/types.go b/pkg/solana/client/multinode/types.go index 2c177a9dc..51b70e573 100644 --- a/pkg/solana/client/multinode/types.go +++ b/pkg/solana/client/multinode/types.go @@ -68,7 +68,7 @@ type RPCClient[ // Head is the interface required by the NodeClient type Head interface { - BlockNumber() uint64 + BlockNumber() int64 BlockDifficulty() *big.Int IsValid() bool } @@ -86,8 +86,8 @@ type PoolChainInfoProvider interface { // ChainInfo - defines RPC's or MultiNode's view on the chain type ChainInfo struct { - BlockNumber uint64 - FinalizedBlockNumber uint64 + BlockNumber int64 + FinalizedBlockNumber int64 TotalDifficulty *big.Int } From fd3823bfb9c35c6b7e49e675419fcff04da05a75 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 12 Sep 2024 14:17:14 -0400 Subject: [PATCH 021/174] Validate node config --- pkg/solana/chain.go | 4 ---- pkg/solana/config/toml.go | 17 +++++++++++++++++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/pkg/solana/chain.go b/pkg/solana/chain.go index e2747014c..d63b6ab3b 100644 --- a/pkg/solana/chain.go +++ b/pkg/solana/chain.go @@ -238,10 +238,6 @@ func newChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr logger.L var nodes []mn.Node[mn.StringID, *client.Client] for i, nodeInfo := range cfg.ListNodes() { - if nodeInfo == nil || nodeInfo.Name == nil || nodeInfo.URL == nil { - return nil, fmt.Errorf("node config contains nil: %+v", nodeInfo) - } - // create client and check rpcClient, err := client.NewClient(nodeInfo.URL.String(), cfg, DefaultRequestTimeout, logger.Named(lggr, "Client."+*nodeInfo.Name)) if err != nil { lggr.Warnw("failed to create client", "name", *nodeInfo.Name, "solana-url", nodeInfo.URL.String(), "err", err.Error()) diff --git a/pkg/solana/config/toml.go b/pkg/solana/config/toml.go index b1cdfc7f5..a39b9f297 100644 --- a/pkg/solana/config/toml.go +++ b/pkg/solana/config/toml.go @@ -192,6 +192,23 @@ func (c *TOMLConfig) ValidateConfig() (err error) { if len(c.Nodes) == 0 { err = errors.Join(err, config.ErrMissing{Name: "Nodes", Msg: "must have at least one node"}) } + + for _, node := range c.Nodes { + if node == nil { + err = errors.Join(err, config.ErrMissing{Name: "Node", Msg: "required for all nodes"}) + } + if node.Name == nil { + err = errors.Join(err, config.ErrMissing{Name: "Name", Msg: "required for all nodes"}) + } else if *node.Name == "" { + err = errors.Join(err, config.ErrEmpty{Name: "Name", Msg: "required for all nodes"}) + } + if node.URL == nil { + err = errors.Join(err, config.ErrMissing{Name: "URL", Msg: "required for all nodes"}) + } else if (*url.URL)(node.URL) == nil { + err = errors.Join(err, config.ErrEmpty{Name: "URL", Msg: "required for all nodes"}) + } + } + return } From 4bf96b7f8763e471cb1ab5076040b2e94c79fb24 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 12 Sep 2024 14:25:10 -0400 Subject: [PATCH 022/174] Update toml.go --- pkg/solana/config/toml.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/solana/config/toml.go b/pkg/solana/config/toml.go index a39b9f297..599729ec5 100644 --- a/pkg/solana/config/toml.go +++ b/pkg/solana/config/toml.go @@ -194,9 +194,6 @@ func (c *TOMLConfig) ValidateConfig() (err error) { } for _, node := range c.Nodes { - if node == nil { - err = errors.Join(err, config.ErrMissing{Name: "Node", Msg: "required for all nodes"}) - } if node.Name == nil { err = errors.Join(err, config.ErrMissing{Name: "Name", Msg: "required for all nodes"}) } else if *node.Name == "" { From c1b83a5a3ae779790f64b6a9f5711760bf268da1 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 18 Sep 2024 10:14:04 -0400 Subject: [PATCH 023/174] Add SendOnly nodes --- pkg/solana/chain.go | 9 +++++++-- pkg/solana/client/client.go | 5 +++-- pkg/solana/config/config.go | 5 +++-- pkg/solana/config/toml.go | 15 +-------------- 4 files changed, 14 insertions(+), 20 deletions(-) diff --git a/pkg/solana/chain.go b/pkg/solana/chain.go index d63b6ab3b..8b4ad5787 100644 --- a/pkg/solana/chain.go +++ b/pkg/solana/chain.go @@ -236,6 +236,7 @@ func newChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr logger.L mnCfg := cfg.MultiNodeConfig() var nodes []mn.Node[mn.StringID, *client.Client] + var sendOnlyNodes []mn.SendOnlyNode[mn.StringID, *client.Client] for i, nodeInfo := range cfg.ListNodes() { rpcClient, err := client.NewClient(nodeInfo.URL.String(), cfg, DefaultRequestTimeout, logger.Named(lggr, "Client."+*nodeInfo.Name)) @@ -248,7 +249,11 @@ func newChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr logger.L mnCfg, mnCfg, lggr, *nodeInfo.URL.URL(), nil, *nodeInfo.Name, i, mn.StringID(id), 0, rpcClient, chainFamily) - nodes = append(nodes, newNode) + if nodeInfo.SendOnly { + sendOnlyNodes = append(sendOnlyNodes, newNode) + } else { + nodes = append(nodes, newNode) + } } multiNode := mn.NewMultiNode[mn.StringID, *client.Client]( @@ -256,7 +261,7 @@ func newChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr logger.L mn.NodeSelectionModeRoundRobin, 0, nodes, - []mn.SendOnlyNode[mn.StringID, *client.Client]{}, + sendOnlyNodes, mn.StringID(id), chainFamily, mnCfg.DeathDeclarationDelay(), diff --git a/pkg/solana/client/client.go b/pkg/solana/client/client.go index 35ab96e62..785e7e508 100644 --- a/pkg/solana/client/client.go +++ b/pkg/solana/client/client.go @@ -73,10 +73,11 @@ type Head struct { } func (h *Head) BlockNumber() int64 { - if h.BlockHeight == nil { + if !h.IsValid() { return 0 } - //nolint:gosec + // nolint:gosec + // G115: integer overflow conversion uint64 -> int64 return int64(*h.BlockHeight) } diff --git a/pkg/solana/config/config.go b/pkg/solana/config/config.go index 9d5cdc5a9..28698c7c3 100644 --- a/pkg/solana/config/config.go +++ b/pkg/solana/config/config.go @@ -146,8 +146,9 @@ func (c *Chain) SetDefaults() { } type Node struct { - Name *string - URL *config.URL + Name *string + URL *config.URL + SendOnly bool } func (n *Node) ValidateConfig() (err error) { diff --git a/pkg/solana/config/toml.go b/pkg/solana/config/toml.go index 599729ec5..90657fd2c 100644 --- a/pkg/solana/config/toml.go +++ b/pkg/solana/config/toml.go @@ -105,6 +105,7 @@ func setFromNode(n, f *Node) { if f.URL != nil { n.URL = f.URL } + n.SendOnly = f.SendOnly } type TOMLConfig struct { @@ -192,20 +193,6 @@ func (c *TOMLConfig) ValidateConfig() (err error) { if len(c.Nodes) == 0 { err = errors.Join(err, config.ErrMissing{Name: "Nodes", Msg: "must have at least one node"}) } - - for _, node := range c.Nodes { - if node.Name == nil { - err = errors.Join(err, config.ErrMissing{Name: "Name", Msg: "required for all nodes"}) - } else if *node.Name == "" { - err = errors.Join(err, config.ErrEmpty{Name: "Name", Msg: "required for all nodes"}) - } - if node.URL == nil { - err = errors.Join(err, config.ErrMissing{Name: "URL", Msg: "required for all nodes"}) - } else if (*url.URL)(node.URL) == nil { - err = errors.Join(err, config.ErrEmpty{Name: "URL", Msg: "required for all nodes"}) - } - } - return } From e0231b0614921313bb8ea96743ed3d60ec052ad1 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 18 Sep 2024 10:56:03 -0400 Subject: [PATCH 024/174] Use pointers on config --- pkg/solana/chain.go | 2 +- pkg/solana/config/multinode.go | 164 ++++++++++++++++++++++----------- pkg/solana/config/toml.go | 1 + 3 files changed, 111 insertions(+), 56 deletions(-) diff --git a/pkg/solana/chain.go b/pkg/solana/chain.go index 9a6068f03..0ba7e597b 100644 --- a/pkg/solana/chain.go +++ b/pkg/solana/chain.go @@ -255,7 +255,7 @@ func newChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr logger.L multiNode := mn.NewMultiNode[mn.StringID, *client.Client]( lggr, mn.NodeSelectionModeRoundRobin, - time.Minute, // TODO: set lease duration + mnCfg.LeaseDuration(), nodes, []mn.SendOnlyNode[mn.StringID, *client.Client]{}, mn.StringID(id), diff --git a/pkg/solana/config/multinode.go b/pkg/solana/config/multinode.go index 42828cd9a..a946e6303 100644 --- a/pkg/solana/config/multinode.go +++ b/pkg/solana/config/multinode.go @@ -1,104 +1,158 @@ package config -import "time" +import ( + "github.com/smartcontractkit/chainlink-common/pkg/config" + "time" +) type MultiNode struct { + // TODO: Make these pointers; update SetFrom and SetDefaults to handle nil values // Feature flag - multiNodeEnabled bool + multiNodeEnabled *bool // Node Configs - pollFailureThreshold uint32 - pollInterval time.Duration - selectionMode string - syncThreshold uint32 - nodeIsSyncingEnabled bool - leaseDuration time.Duration - finalizedBlockPollInterval time.Duration - enforceRepeatableRead bool - deathDeclarationDelay time.Duration + pollFailureThreshold *uint32 + pollInterval *config.Duration + selectionMode *string + syncThreshold *uint32 + nodeIsSyncingEnabled *bool + leaseDuration *config.Duration + finalizedBlockPollInterval *config.Duration + enforceRepeatableRead *bool + deathDeclarationDelay *config.Duration // Chain Configs - nodeNoNewHeadsThreshold time.Duration - noNewFinalizedHeadsThreshold time.Duration - finalityDepth uint32 - finalityTagEnabled bool - finalizedBlockOffset uint32 + nodeNoNewHeadsThreshold *config.Duration + noNewFinalizedHeadsThreshold *config.Duration + finalityDepth *uint32 + finalityTagEnabled *bool + finalizedBlockOffset *uint32 } func (c *MultiNode) MultiNodeEnabled() bool { - return c.multiNodeEnabled + return *c.multiNodeEnabled } func (c *MultiNode) PollFailureThreshold() uint32 { - return c.pollFailureThreshold + return *c.pollFailureThreshold } func (c *MultiNode) PollInterval() time.Duration { - return c.pollInterval + return c.pollInterval.Duration() } func (c *MultiNode) SelectionMode() string { - return c.selectionMode + return *c.selectionMode } func (c *MultiNode) SyncThreshold() uint32 { - return c.syncThreshold + return *c.syncThreshold } func (c *MultiNode) NodeIsSyncingEnabled() bool { - return c.nodeIsSyncingEnabled + return *c.nodeIsSyncingEnabled } +func (c *MultiNode) LeaseDuration() time.Duration { return c.leaseDuration.Duration() } + func (c *MultiNode) FinalizedBlockPollInterval() time.Duration { - return c.finalizedBlockPollInterval + return c.finalizedBlockPollInterval.Duration() } -func (c *MultiNode) EnforceRepeatableRead() bool { - return c.enforceRepeatableRead -} +func (c *MultiNode) EnforceRepeatableRead() bool { return *c.enforceRepeatableRead } -func (c *MultiNode) DeathDeclarationDelay() time.Duration { - return c.deathDeclarationDelay -} +func (c *MultiNode) DeathDeclarationDelay() time.Duration { return c.deathDeclarationDelay.Duration() } func (c *MultiNode) NodeNoNewHeadsThreshold() time.Duration { - return c.nodeNoNewHeadsThreshold + return c.nodeNoNewHeadsThreshold.Duration() } func (c *MultiNode) NoNewFinalizedHeadsThreshold() time.Duration { - return c.noNewFinalizedHeadsThreshold + return c.noNewFinalizedHeadsThreshold.Duration() } -func (c *MultiNode) FinalityDepth() uint32 { - return c.finalityDepth -} +func (c *MultiNode) FinalityDepth() uint32 { return *c.finalityDepth } -func (c *MultiNode) FinalityTagEnabled() bool { - return c.finalityTagEnabled -} +func (c *MultiNode) FinalityTagEnabled() bool { return *c.finalityTagEnabled } -func (c *MultiNode) FinalizedBlockOffset() uint32 { - return c.finalizedBlockOffset -} +func (c *MultiNode) FinalizedBlockOffset() uint32 { return *c.finalizedBlockOffset } func (c *MultiNode) SetDefaults() { - c.multiNodeEnabled = false + c.multiNodeEnabled = new(bool) + + // Node Configs + defaultPollFailureThreshold := uint32(5) + c.pollFailureThreshold = &defaultPollFailureThreshold + c.pollInterval = config.MustNewDuration(10 * time.Second) + + highestHead := "HighestHead" + c.selectionMode = &highestHead + + syncThreshold := uint32(5) + c.syncThreshold = &syncThreshold + + c.leaseDuration = config.MustNewDuration(time.Minute) // TODO: default value? + c.nodeIsSyncingEnabled = new(bool) // // TODO: default false? + c.finalizedBlockPollInterval = config.MustNewDuration(5 * time.Second) + c.enforceRepeatableRead = new(bool) + c.deathDeclarationDelay = config.MustNewDuration(10 * time.Second) + + // Chain Configs + c.nodeNoNewHeadsThreshold = config.MustNewDuration(10 * time.Second) // TODO: Value? + c.noNewFinalizedHeadsThreshold = config.MustNewDuration(10 * time.Second) // TODO: Value? + c.finalityDepth = new(uint32) // TODO: default value? + c.finalityTagEnabled = new(bool) // TODO: default false? + c.finalizedBlockOffset = new(uint32) // TODO: default value? +} + +func (mn *MultiNode) SetFrom(fs *MultiNode) { + if fs.multiNodeEnabled != nil { + mn.multiNodeEnabled = fs.multiNodeEnabled + } // Node Configs - c.pollFailureThreshold = 5 - c.pollInterval = 10 * time.Second - c.selectionMode = "HighestHead" - c.syncThreshold = 5 - c.leaseDuration = 0 - c.nodeIsSyncingEnabled = false - c.finalizedBlockPollInterval = 5 * time.Second - c.enforceRepeatableRead = false - c.deathDeclarationDelay = 10 * time.Second + if fs.pollFailureThreshold != nil { + mn.pollFailureThreshold = fs.pollFailureThreshold + } + if fs.pollInterval != nil { + mn.pollInterval = fs.pollInterval + } + if fs.selectionMode != nil { + mn.selectionMode = fs.selectionMode + } + if fs.syncThreshold != nil { + mn.syncThreshold = fs.syncThreshold + } + if fs.nodeIsSyncingEnabled != nil { + mn.nodeIsSyncingEnabled = fs.nodeIsSyncingEnabled + } + if fs.leaseDuration != nil { + mn.leaseDuration = fs.leaseDuration + } + if fs.finalizedBlockPollInterval != nil { + mn.finalizedBlockPollInterval = fs.finalizedBlockPollInterval + } + if fs.enforceRepeatableRead != nil { + mn.enforceRepeatableRead = fs.enforceRepeatableRead + } + if fs.deathDeclarationDelay != nil { + mn.deathDeclarationDelay = fs.deathDeclarationDelay + } // Chain Configs - c.nodeNoNewHeadsThreshold = 10 * time.Second // TODO: Value? - c.noNewFinalizedHeadsThreshold = 10 * time.Second // TODO: Value? - c.finalityDepth = 0 // TODO: Value? - c.finalityTagEnabled = false // TODO: Value? - c.finalizedBlockOffset = 0 // TODO: Value? + if fs.nodeNoNewHeadsThreshold != nil { + mn.nodeNoNewHeadsThreshold = fs.nodeNoNewHeadsThreshold + } + if fs.noNewFinalizedHeadsThreshold != nil { + mn.noNewFinalizedHeadsThreshold = fs.noNewFinalizedHeadsThreshold + } + if fs.finalityDepth != nil { + mn.finalityDepth = fs.finalityDepth + } + if fs.finalityTagEnabled != nil { + mn.finalityTagEnabled = fs.finalityTagEnabled + } + if fs.finalizedBlockOffset != nil { + mn.finalizedBlockOffset = fs.finalizedBlockOffset + } } diff --git a/pkg/solana/config/toml.go b/pkg/solana/config/toml.go index b1cdfc7f5..4e626ed08 100644 --- a/pkg/solana/config/toml.go +++ b/pkg/solana/config/toml.go @@ -129,6 +129,7 @@ func (c *TOMLConfig) SetFrom(f *TOMLConfig) { } setFromChain(&c.Chain, &f.Chain) c.Nodes.SetFrom(&f.Nodes) + c.MultiNode.SetFrom(&f.MultiNode) } func setFromChain(c, f *Chain) { From 8b135488627cb14d25018988728239f54400fb0a Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 19 Sep 2024 11:58:29 -0400 Subject: [PATCH 025/174] Add test outlines --- go.mod | 12 +++++++++ go.sum | 28 ++++++++++++++++++++ pkg/solana/chain_test.go | 38 +++++++++++++++++++++++++++ pkg/solana/client/client.go | 17 ++++++------ pkg/solana/client/client_test.go | 44 ++++++++++++++++++++++++++++++++ 5 files changed, 130 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 484603e25..dfc24a151 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ toolchain go1.22.3 require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc + github.com/ethereum/go-ethereum v1.13.8 github.com/gagliardetto/binary v0.7.7 github.com/gagliardetto/gofuzz v1.2.2 github.com/gagliardetto/solana-go v1.8.4 @@ -33,13 +34,20 @@ require ( github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/bits-and-blooms/bitset v1.10.0 // indirect github.com/blendle/zapdriver v1.3.1 // indirect + github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect github.com/buger/goterm v0.0.0-20200322175922-2f3e71b85129 // indirect github.com/buger/jsonparser v1.1.1 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/confluentinc/confluent-kafka-go/v2 v2.3.0 // indirect + github.com/consensys/bavard v0.1.13 // indirect + github.com/consensys/gnark-crypto v0.12.1 // indirect + github.com/crate-crypto/go-kzg-4844 v0.7.0 // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect github.com/dfuse-io/logging v0.0.0-20210109005628-b97a57253f70 // indirect + github.com/ethereum/c-kzg-4844 v0.4.0 // indirect github.com/fatih/color v1.16.0 // indirect github.com/fxamacker/cbor/v2 v2.5.0 // indirect github.com/go-json-experiment/json v0.0.0-20231102232822-2e55bd4e08b0 // indirect @@ -58,6 +66,7 @@ require ( github.com/hako/durafmt v0.0.0-20200710122514-c0fb7b4da026 // indirect github.com/hashicorp/go-hclog v1.5.0 // indirect github.com/hashicorp/yamux v0.1.1 // indirect + github.com/holiman/uint256 v1.2.4 // indirect github.com/invopop/jsonschema v0.12.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.15.15 // indirect @@ -72,6 +81,7 @@ require ( github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/miekg/dns v1.1.35 // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect + github.com/mmcloughlin/addchain v0.4.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mostynb/zstdpool-freelist v0.0.0-20201229113212-927304c0c3b1 // indirect @@ -91,6 +101,7 @@ require ( github.com/smartcontractkit/grpc-proxy v0.0.0-20240830132753-a7e17fec5ab7 // indirect github.com/streamingfast/logging v0.0.0-20220405224725-2755dab2ce75 // indirect github.com/stretchr/objx v0.5.2 // indirect + github.com/supranational/blst v0.3.11 // indirect github.com/teris-io/shortid v0.0.0-20201117134242-e59966efd125 // indirect github.com/tidwall/gjson v1.14.4 // indirect github.com/tidwall/match v1.1.1 // indirect @@ -128,6 +139,7 @@ require ( google.golang.org/protobuf v1.34.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + rsc.io/tmplfunc v0.0.3 // indirect ) // until merged upstream: https://github.com/mwitkow/grpc-proxy/pull/69 diff --git a/go.sum b/go.sum index a4e3f137a..ddc7d6d7a 100644 --- a/go.sum +++ b/go.sum @@ -61,9 +61,14 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bits-and-blooms/bitset v1.10.0 h1:ePXTeiPEazB5+opbv5fr8umg2R/1NlzgDsyepwsSr88= +github.com/bits-and-blooms/bitset v1.10.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/blendle/zapdriver v1.3.1 h1:C3dydBOWYRiOk+B8X9IVZ5IOe+7cl+tGOexN4QqHfpE= github.com/blendle/zapdriver v1.3.1/go.mod h1:mdXfREi6u5MArG4j9fewC+FGnXaBR+T4Ox4J2u4eHCc= +github.com/btcsuite/btcd v0.22.0-beta h1:LTDpDKUM5EeOFBPM8IXpinEcmZ6FWfNZbE3lfrfdnWo= +github.com/btcsuite/btcd/btcec/v2 v2.2.0 h1:fzn1qaOt32TuLjFlkzYSsBC35Q3KUjT1SwPxiMSCF5k= +github.com/btcsuite/btcd/btcec/v2 v2.2.0/go.mod h1:U7MHm051Al6XmscBQ0BoNydpOTsFAn707034b5nY8zU= github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZlaQsNA= github.com/bufbuild/protocompile v0.4.0/go.mod h1:3v93+mbWn/v3xzN+31nwkJfrEpAUwp+BagBSZWx+TP8= github.com/buger/goterm v0.0.0-20200322175922-2f3e71b85129 h1:gfAMKE626QEuKG3si0pdTRcr/YEbBoxY+3GOH3gWvl4= @@ -84,6 +89,10 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/confluentinc/confluent-kafka-go/v2 v2.3.0 h1:icCHutJouWlQREayFwCc7lxDAhws08td+W3/gdqgZts= github.com/confluentinc/confluent-kafka-go/v2 v2.3.0/go.mod h1:/VTy8iEpe6mD9pkCH5BhijlUl8ulUXymKv1Qig5Rgb8= +github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= +github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= +github.com/consensys/gnark-crypto v0.12.1 h1:lHH39WuuFgVHONRl3J0LRBtuYdQTumFSDtJF7HpyG8M= +github.com/consensys/gnark-crypto v0.12.1/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY= github.com/containerd/cgroups v1.0.4 h1:jN/mbWBEaz+T1pi5OFtnkQ+8qnmEbAr1Oo1FRm5B0dA= github.com/containerd/cgroups v1.0.4/go.mod h1:nLNQtsF7Sl2HxNebu77i1R0oDlhiTG+kO4JTrUzo6IA= github.com/containerd/containerd v1.6.8 h1:h4dOFDwzHmqFEP754PgfgTeVXFnLiRc6kiqC7tplDJs= @@ -94,12 +103,17 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/crate-crypto/go-kzg-4844 v0.7.0 h1:C0vgZRk4q4EZ/JgPfzuSoxdCq3C3mOZMBShovmncxvA= +github.com/crate-crypto/go-kzg-4844 v0.7.0/go.mod h1:1kMhvPgI0Ky3yIa+9lFySEBUBXkYxeOi8ZF1sYioxhc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/daaku/go.zipexe v1.0.0/go.mod h1:z8IiR6TsVLEYKwXAoE/I+8ys/sDkgTzSL0CLnGVd57E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1m5sE92cU+pd5Mcc= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= github.com/dfuse-io/logging v0.0.0-20201110202154-26697de88c79/go.mod h1:V+ED4kT/t/lKtH99JQmKIb0v9WL3VaYkJ36CfHlVECI= github.com/dfuse-io/logging v0.0.0-20210109005628-b97a57253f70 h1:CuJS05R9jmNlUK8GOxrEELPbfXm0EuGh/30LjkjN5vo= github.com/dfuse-io/logging v0.0.0-20210109005628-b97a57253f70/go.mod h1:EoK/8RFbMEteaCaz89uessDTnCWjbbcr+DXcBh4el5o= @@ -118,6 +132,10 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/ethereum/c-kzg-4844 v0.4.0 h1:3MS1s4JtA868KpJxroZoepdV0ZKBp3u/O5HcZ7R3nlY= +github.com/ethereum/c-kzg-4844 v0.4.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= +github.com/ethereum/go-ethereum v1.13.8 h1:1od+thJel3tM52ZUNQwvpYOeRHlbkVFZ5S8fhi0Lgsg= +github.com/ethereum/go-ethereum v1.13.8/go.mod h1:sc48XYQxCzH3fG9BcrXCOOgQk2JfZzNAmIKnceogzsA= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= @@ -219,6 +237,7 @@ github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -267,6 +286,8 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hashicorp/yamux v0.1.1 h1:yrQxtgseBDrq9Y652vSRDvsKCJKOUD+GzTS4Y0Y8pvE= github.com/hashicorp/yamux v0.1.1/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ= +github.com/holiman/uint256 v1.2.4 h1:jUc4Nk8fm9jZabQuqr2JzednajVmBpC+oiTiXZJEApU= +github.com/holiman/uint256 v1.2.4/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= @@ -345,6 +366,9 @@ github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS4 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY= +github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU= +github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFVejUS1/tS/qU= github.com/moby/sys/mount v0.3.3 h1:fX1SVkXFJ47XWDoeFW4Sq7PdQJnV2QIDZAqjNqgEjUs= github.com/moby/sys/mount v0.3.3/go.mod h1:PBaEorSNTLG5t/+4EgukEQVlAvVEc6ZjTySwKdqp5K0= github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78= @@ -475,6 +499,8 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/supranational/blst v0.3.11 h1:LyU6FolezeWAhvQk0k6O/d49jqgO52MSDDfYgbeoEm4= +github.com/supranational/blst v0.3.11/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/teris-io/shortid v0.0.0-20171029131806-771a37caa5cf/go.mod h1:M8agBzgqHIhgj7wEn9/0hJUZcrvt9VY+Ln+S1I5Mha0= github.com/teris-io/shortid v0.0.0-20201117134242-e59966efd125 h1:3SNcvBmEPE1YlB1JpVZouslJpI3GBNoiqW7+wb0Rz7w= github.com/teris-io/shortid v0.0.0-20201117134242-e59966efd125/go.mod h1:M8agBzgqHIhgj7wEn9/0hJUZcrvt9VY+Ln+S1I5Mha0= @@ -887,3 +913,5 @@ honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU= +rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA= diff --git a/pkg/solana/chain_test.go b/pkg/solana/chain_test.go index 3f1fdaf23..8869ae3c1 100644 --- a/pkg/solana/chain_test.go +++ b/pkg/solana/chain_test.go @@ -117,6 +117,44 @@ func TestSolanaChain_GetClient(t *testing.T) { assert.NoError(t, err) } +func TestSolanaChain_MultiNode(t *testing.T) { + // TODO: Set up chain with MultiNode enabled + + ch := solcfg.Chain{} + ch.SetDefaults() + cfg := &solcfg.TOMLConfig{ + ChainID: ptr("devnet"), + Chain: ch, + } + testChain := chain{ + id: "devnet", + cfg: cfg, + lggr: logger.Test(t), + clientCache: map[string]*verifiedCachedClient{}, + } + + + + cfg.Nodes = []*solcfg.Node{ + { + Name: ptr("devnet"), + URL: config.MustParseURL(mockServer.URL + "/1"), + }, + { + Name: ptr("devnet"), + URL: config.MustParseURL(mockServer.URL + "/2"), + }, + } + _, err := testChain.getClient() + assert.NoError(t, err) + + // TODO: Start MultiNode and ensure we can call getClient() + + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + + } +} + func TestSolanaChain_VerifiedClient(t *testing.T) { called := false mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/pkg/solana/client/client.go b/pkg/solana/client/client.go index 998fb7d1d..1ff818d29 100644 --- a/pkg/solana/client/client.go +++ b/pkg/solana/client/client.go @@ -132,6 +132,13 @@ var _ mn.SendTxRPCClient[*solana.Transaction] = (*Client)(nil) func (c *Client) Dial(ctx context.Context) error { // TODO: is there any work to do here? Doesn't seem like we have to dial anything // TODO: Could maybe do a version check here? + /* TODO: Should we use this health check? + health, err := c.rpc.GetHealth(ctx) + if err != nil { + return false, err + } + return health == rpc.HealthOk, nil + */ panic("implement me") } @@ -186,7 +193,7 @@ func (c *Client) LatestFinalizedBlock(ctx context.Context) (*Head, error) { return nil, err } - var finalityDepth uint64 = 1 // TODO: Value? + var finalityDepth uint64 = 1 // TODO: Get value from config latestFinalizedBH := latestBH.Value.LastValidBlockHeight - finalityDepth // TODO: subtract finality depth? @@ -238,14 +245,6 @@ func (c *Client) onNewFinalizedHead(ctx context.Context, requestCh <-chan struct } func (c *Client) Ping(ctx context.Context) error { - /* TODO: Should we use this health check for ping or somewhere? - health, err := c.rpc.GetHealth(ctx) - if err != nil { - return false, err - } - return health == rpc.HealthOk, nil - */ - version, err := c.rpc.GetVersion(ctx) if err != nil { return fmt.Errorf("ping failed: %v", err) diff --git a/pkg/solana/client/client_test.go b/pkg/solana/client/client_test.go index 6a4feb61f..0705615b7 100644 --- a/pkg/solana/client/client_test.go +++ b/pkg/solana/client/client_test.go @@ -297,6 +297,50 @@ func TestClient_SendTxDuplicates_Integration(t *testing.T) { assert.Equal(t, uint64(5_000), initBal-endBal) } +func TestClient_Subscriptions_Integration(t *testing.T) { + // TODO: Test subscribing to heads and finalized heads + // TODO: Ensure chain info is updated on new heads + // TODO: Test Dial, Close, IsSyncing, GetInterceptedChainInfo + + // TODO: Create server for testing?? + url := SetupLocalSolNode(t) + privKey, err := solana.NewRandomPrivateKey() + require.NoError(t, err) + pubKey := privKey.PublicKey() + FundTestAccounts(t, []solana.PublicKey{pubKey}, url) + + requestTimeout := 5 * time.Second + lggr := logger.Test(t) + cfg := config.NewDefault() + + ctx := context.Background() + c, err := NewClient(url, cfg, requestTimeout, lggr) + require.NoError(t, err) + + ch, sub, err := c.SubscribeToHeads(ctx) + defer sub.Unsubscribe() + require.NoError(t, err) + + // TODO: How do we test this? + // check for new heads + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + for { + select { + case head := <-ch: + t.Logf("New head: %v", head) + wg.Done() + case <-ctx.Done(): + return + } + } + }() + + wg.Wait() + +} + func TestClientLatency(t *testing.T) { c := Client{} v := 100 From 8aa39f61386a169b22b86796de40ba79e7834389 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 19 Sep 2024 11:59:27 -0400 Subject: [PATCH 026/174] Use test context --- pkg/solana/chain_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/solana/chain_test.go b/pkg/solana/chain_test.go index 3f1fdaf23..c5ccc7307 100644 --- a/pkg/solana/chain_test.go +++ b/pkg/solana/chain_test.go @@ -1,9 +1,9 @@ package solana import ( - "context" "errors" "fmt" + "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" "io" "net/http" "net/http/httptest" @@ -175,7 +175,7 @@ func TestSolanaChain_VerifiedClient(t *testing.T) { testChain.id = "incorrect" c, err = testChain.verifiedClient(node) assert.NoError(t, err) - _, err = c.ChainID(context.Background()) + _, err = c.ChainID(tests.Context(t)) // expect error from id mismatch (even if using a cached client) when performing RPC calls assert.Error(t, err) assert.Equal(t, fmt.Sprintf("client returned mismatched chain id (expected: %s, got: %s): %s", "incorrect", "devnet", node.URL), err.Error()) From b5ff16d1a84bd8d6328921d50063845982ce1e90 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 19 Sep 2024 12:17:53 -0400 Subject: [PATCH 027/174] Use configured selection mode --- pkg/solana/chain.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/solana/chain.go b/pkg/solana/chain.go index 2f2697b95..44b72764e 100644 --- a/pkg/solana/chain.go +++ b/pkg/solana/chain.go @@ -258,7 +258,7 @@ func newChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr logger.L multiNode := mn.NewMultiNode[mn.StringID, *client.Client]( lggr, - mn.NodeSelectionModeRoundRobin, + mnCfg.SelectionMode(), mnCfg.LeaseDuration(), nodes, sendOnlyNodes, From 1b3a101d02604ad8fa9d9decb5c7d677af098f11 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 24 Sep 2024 11:45:02 -0400 Subject: [PATCH 028/174] Set defaults --- pkg/solana/config/multinode.go | 39 +++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/pkg/solana/config/multinode.go b/pkg/solana/config/multinode.go index a946e6303..4c343cc3f 100644 --- a/pkg/solana/config/multinode.go +++ b/pkg/solana/config/multinode.go @@ -1,12 +1,14 @@ package config import ( - "github.com/smartcontractkit/chainlink-common/pkg/config" "time" + + "github.com/smartcontractkit/chainlink-common/pkg/config" + + client "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" ) type MultiNode struct { - // TODO: Make these pointers; update SetFrom and SetDefaults to handle nil values // Feature flag multiNodeEnabled *bool @@ -78,31 +80,30 @@ func (c *MultiNode) FinalityTagEnabled() bool { return *c.finalityTagEnabled } func (c *MultiNode) FinalizedBlockOffset() uint32 { return *c.finalizedBlockOffset } func (c *MultiNode) SetDefaults() { - c.multiNodeEnabled = new(bool) + c.multiNodeEnabled = ptr(false) // Node Configs - defaultPollFailureThreshold := uint32(5) - c.pollFailureThreshold = &defaultPollFailureThreshold + c.pollFailureThreshold = ptr(uint32(5)) c.pollInterval = config.MustNewDuration(10 * time.Second) - highestHead := "HighestHead" - c.selectionMode = &highestHead + c.selectionMode = ptr(client.NodeSelectionModePriorityLevel) - syncThreshold := uint32(5) - c.syncThreshold = &syncThreshold + c.syncThreshold = ptr(uint32(5)) - c.leaseDuration = config.MustNewDuration(time.Minute) // TODO: default value? - c.nodeIsSyncingEnabled = new(bool) // // TODO: default false? + // Period at which we verify if active node is still highest block number + c.leaseDuration = config.MustNewDuration(time.Minute) + + c.nodeIsSyncingEnabled = ptr(false) c.finalizedBlockPollInterval = config.MustNewDuration(5 * time.Second) - c.enforceRepeatableRead = new(bool) + c.enforceRepeatableRead = ptr(true) c.deathDeclarationDelay = config.MustNewDuration(10 * time.Second) // Chain Configs - c.nodeNoNewHeadsThreshold = config.MustNewDuration(10 * time.Second) // TODO: Value? - c.noNewFinalizedHeadsThreshold = config.MustNewDuration(10 * time.Second) // TODO: Value? - c.finalityDepth = new(uint32) // TODO: default value? - c.finalityTagEnabled = new(bool) // TODO: default false? - c.finalizedBlockOffset = new(uint32) // TODO: default value? + c.nodeNoNewHeadsThreshold = config.MustNewDuration(10 * time.Second) + c.noNewFinalizedHeadsThreshold = config.MustNewDuration(10 * time.Second) + c.finalityDepth = ptr(uint32(0)) + c.finalityTagEnabled = ptr(true) + c.finalizedBlockOffset = ptr(uint32(0)) } func (mn *MultiNode) SetFrom(fs *MultiNode) { @@ -156,3 +157,7 @@ func (mn *MultiNode) SetFrom(fs *MultiNode) { mn.finalizedBlockOffset = fs.finalizedBlockOffset } } + +func ptr[T any](v T) *T { + return &v +} From 0afd8da74834e1adec96e7425f9332d64e67953f Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 24 Sep 2024 11:57:05 -0400 Subject: [PATCH 029/174] lint --- pkg/solana/chain_test.go | 1 - pkg/solana/config/multinode.go | 32 ++++++++++++++++---------------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/pkg/solana/chain_test.go b/pkg/solana/chain_test.go index 7127c0827..6fb966740 100644 --- a/pkg/solana/chain_test.go +++ b/pkg/solana/chain_test.go @@ -3,7 +3,6 @@ package solana import ( "errors" "fmt" - "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" "io" "net/http" "net/http/httptest" diff --git a/pkg/solana/config/multinode.go b/pkg/solana/config/multinode.go index 4c343cc3f..8c7e72b02 100644 --- a/pkg/solana/config/multinode.go +++ b/pkg/solana/config/multinode.go @@ -106,55 +106,55 @@ func (c *MultiNode) SetDefaults() { c.finalizedBlockOffset = ptr(uint32(0)) } -func (mn *MultiNode) SetFrom(fs *MultiNode) { +func (c *MultiNode) SetFrom(fs *MultiNode) { if fs.multiNodeEnabled != nil { - mn.multiNodeEnabled = fs.multiNodeEnabled + c.multiNodeEnabled = fs.multiNodeEnabled } // Node Configs if fs.pollFailureThreshold != nil { - mn.pollFailureThreshold = fs.pollFailureThreshold + c.pollFailureThreshold = fs.pollFailureThreshold } if fs.pollInterval != nil { - mn.pollInterval = fs.pollInterval + c.pollInterval = fs.pollInterval } if fs.selectionMode != nil { - mn.selectionMode = fs.selectionMode + c.selectionMode = fs.selectionMode } if fs.syncThreshold != nil { - mn.syncThreshold = fs.syncThreshold + c.syncThreshold = fs.syncThreshold } if fs.nodeIsSyncingEnabled != nil { - mn.nodeIsSyncingEnabled = fs.nodeIsSyncingEnabled + c.nodeIsSyncingEnabled = fs.nodeIsSyncingEnabled } if fs.leaseDuration != nil { - mn.leaseDuration = fs.leaseDuration + c.leaseDuration = fs.leaseDuration } if fs.finalizedBlockPollInterval != nil { - mn.finalizedBlockPollInterval = fs.finalizedBlockPollInterval + c.finalizedBlockPollInterval = fs.finalizedBlockPollInterval } if fs.enforceRepeatableRead != nil { - mn.enforceRepeatableRead = fs.enforceRepeatableRead + c.enforceRepeatableRead = fs.enforceRepeatableRead } if fs.deathDeclarationDelay != nil { - mn.deathDeclarationDelay = fs.deathDeclarationDelay + c.deathDeclarationDelay = fs.deathDeclarationDelay } // Chain Configs if fs.nodeNoNewHeadsThreshold != nil { - mn.nodeNoNewHeadsThreshold = fs.nodeNoNewHeadsThreshold + c.nodeNoNewHeadsThreshold = fs.nodeNoNewHeadsThreshold } if fs.noNewFinalizedHeadsThreshold != nil { - mn.noNewFinalizedHeadsThreshold = fs.noNewFinalizedHeadsThreshold + c.noNewFinalizedHeadsThreshold = fs.noNewFinalizedHeadsThreshold } if fs.finalityDepth != nil { - mn.finalityDepth = fs.finalityDepth + c.finalityDepth = fs.finalityDepth } if fs.finalityTagEnabled != nil { - mn.finalityTagEnabled = fs.finalityTagEnabled + c.finalityTagEnabled = fs.finalityTagEnabled } if fs.finalizedBlockOffset != nil { - mn.finalizedBlockOffset = fs.finalizedBlockOffset + c.finalizedBlockOffset = fs.finalizedBlockOffset } } From b99b90cdd2dd83d2fa02825d4f2307979f4518b2 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 24 Sep 2024 12:18:26 -0400 Subject: [PATCH 030/174] Add nil check --- pkg/solana/chain.go | 8 ++++---- pkg/solana/config/multinode.go | 2 +- pkg/solana/config/toml.go | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/solana/chain.go b/pkg/solana/chain.go index 44b72764e..560d6c94d 100644 --- a/pkg/solana/chain.go +++ b/pkg/solana/chain.go @@ -230,7 +230,7 @@ func newChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr logger.L clientCache: map[string]*verifiedCachedClient{}, } - if cfg.MultiNodeEnabled() { + if cfg.MultiNode.MultiNodeEnabled() { chainFamily := "solana" mnCfg := cfg.MultiNodeConfig() @@ -398,7 +398,7 @@ func (c *chain) ChainID() string { // getClient returns a client, randomly selecting one from available and valid nodes func (c *chain) getClient() (client.ReaderWriter, error) { - if c.cfg.MultiNodeEnabled() { + if c.cfg.MultiNode.MultiNodeEnabled() { return c.multiNode.SelectRPC() } @@ -482,7 +482,7 @@ func (c *chain) Start(ctx context.Context) error { c.lggr.Debug("Starting balance monitor") var ms services.MultiStart startAll := []services.StartClose{c.txm, c.balanceMonitor} - if c.cfg.MultiNodeEnabled() { + if c.cfg.MultiNode.MultiNodeEnabled() { c.lggr.Debug("Starting multinode") startAll = append(startAll, c.multiNode, c.txSender) } @@ -496,7 +496,7 @@ func (c *chain) Close() error { c.lggr.Debug("Stopping txm") c.lggr.Debug("Stopping balance monitor") closeAll := []io.Closer{c.txm, c.balanceMonitor} - if c.cfg.MultiNodeEnabled() { + if c.cfg.MultiNode.MultiNodeEnabled() { c.lggr.Debug("Stopping multinode") closeAll = append(closeAll, c.multiNode, c.txSender) } diff --git a/pkg/solana/config/multinode.go b/pkg/solana/config/multinode.go index 8c7e72b02..912e87f77 100644 --- a/pkg/solana/config/multinode.go +++ b/pkg/solana/config/multinode.go @@ -32,7 +32,7 @@ type MultiNode struct { } func (c *MultiNode) MultiNodeEnabled() bool { - return *c.multiNodeEnabled + return c.multiNodeEnabled != nil && *c.multiNodeEnabled } func (c *MultiNode) PollFailureThreshold() uint32 { diff --git a/pkg/solana/config/toml.go b/pkg/solana/config/toml.go index df8506ea6..e3c0bb01a 100644 --- a/pkg/solana/config/toml.go +++ b/pkg/solana/config/toml.go @@ -113,8 +113,8 @@ type TOMLConfig struct { // Do not access directly, use [IsEnabled] Enabled *bool Chain - MultiNode - Nodes Nodes + MultiNode MultiNode + Nodes Nodes } func (c *TOMLConfig) IsEnabled() bool { From 05524a1339be93afbe4fd7b73e29e1b177cbdcb0 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 24 Sep 2024 13:05:18 -0400 Subject: [PATCH 031/174] Add client test --- pkg/solana/chain_test.go | 52 ++++++++++++++++++++------------ pkg/solana/client/client.go | 31 +++++-------------- pkg/solana/client/client_test.go | 2 ++ pkg/solana/config/multinode.go | 4 +-- pkg/solana/config/toml.go | 2 +- 5 files changed, 45 insertions(+), 46 deletions(-) diff --git a/pkg/solana/chain_test.go b/pkg/solana/chain_test.go index b9f926011..39c3e5e7a 100644 --- a/pkg/solana/chain_test.go +++ b/pkg/solana/chain_test.go @@ -47,9 +47,12 @@ func TestSolanaChain_GetClient(t *testing.T) { ch := solcfg.Chain{} ch.SetDefaults() + mn := solcfg.MultiNode{} + mn.SetDefaults(false) cfg := &solcfg.TOMLConfig{ - ChainID: ptr("devnet"), - Chain: ch, + ChainID: ptr("devnet"), + Chain: ch, + MultiNode: mn, } testChain := chain{ id: "devnet", @@ -117,24 +120,28 @@ func TestSolanaChain_GetClient(t *testing.T) { assert.NoError(t, err) } -func TestSolanaChain_MultiNode(t *testing.T) { - // TODO: Set up chain with MultiNode enabled +func TestSolanaChain_MultiNode_GetClient(t *testing.T) { + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + out := fmt.Sprintf(TestSolanaGenesisHashTemplate, client.MainnetGenesisHash) // mainnet genesis hash + if !strings.Contains(r.URL.Path, "/mismatch") { + // devnet gensis hash + out = fmt.Sprintf(TestSolanaGenesisHashTemplate, client.DevnetGenesisHash) + } + _, err := w.Write([]byte(out)) + require.NoError(t, err) + })) + defer mockServer.Close() ch := solcfg.Chain{} ch.SetDefaults() + mn := solcfg.MultiNode{} + mn.SetDefaults(true) + cfg := &solcfg.TOMLConfig{ - ChainID: ptr("devnet"), - Chain: ch, + ChainID: ptr("devnet"), + Chain: ch, + MultiNode: mn, } - testChain := chain{ - id: "devnet", - cfg: cfg, - lggr: logger.Test(t), - clientCache: map[string]*verifiedCachedClient{}, - } - - - cfg.Nodes = []*solcfg.Node{ { Name: ptr("devnet"), @@ -145,14 +152,19 @@ func TestSolanaChain_MultiNode(t *testing.T) { URL: config.MustParseURL(mockServer.URL + "/2"), }, } - _, err := testChain.getClient() - assert.NoError(t, err) - // TODO: Start MultiNode and ensure we can call getClient() + testChain, err := newChain("devnet", cfg, nil, logger.Test(t)) + require.NoError(t, err) - mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + err = testChain.multiNode.Start(tests.Context(t)) + assert.NoError(t, err) - } + selectedClient, err := testChain.getClient() + assert.NoError(t, err) + + id, err := selectedClient.ChainID(tests.Context(t)) + assert.NoError(t, err) + assert.Equal(t, "devnet", id.String()) } func TestSolanaChain_VerifiedClient(t *testing.T) { diff --git a/pkg/solana/client/client.go b/pkg/solana/client/client.go index 1ff818d29..c7e81cb84 100644 --- a/pkg/solana/client/client.go +++ b/pkg/solana/client/client.go @@ -98,8 +98,8 @@ func NewClient(endpoint string, cfg *config.TOMLConfig, requestTimeout time.Dura contextDuration: requestTimeout, log: log, requestGroup: &singleflight.Group{}, - pollInterval: cfg.PollInterval(), - finalizedBlockPollInterval: cfg.FinalizedBlockPollInterval(), + pollInterval: cfg.MultiNode.PollInterval(), + finalizedBlockPollInterval: cfg.MultiNode.FinalizedBlockPollInterval(), }, nil } @@ -117,8 +117,7 @@ func (h *Head) BlockNumber() int64 { } func (h *Head) BlockDifficulty() *big.Int { - // TODO: Is difficulty relevant for Solana? - // TODO: If not, then remove changes to it in latestBlockInfo + // TODO: Not relevant for Solana? return nil } @@ -130,16 +129,7 @@ var _ mn.RPCClient[mn.StringID, *Head] = (*Client)(nil) var _ mn.SendTxRPCClient[*solana.Transaction] = (*Client)(nil) func (c *Client) Dial(ctx context.Context) error { - // TODO: is there any work to do here? Doesn't seem like we have to dial anything - // TODO: Could maybe do a version check here? - /* TODO: Should we use this health check? - health, err := c.rpc.GetHealth(ctx) - if err != nil { - return false, err - } - return health == rpc.HealthOk, nil - */ - panic("implement me") + return nil } func (c *Client) SubscribeToHeads(ctx context.Context) (<-chan *Head, mn.Subscription, error) { @@ -187,22 +177,17 @@ func (c *Client) LatestFinalizedBlock(ctx context.Context) (*Head, error) { // capture chStopInFlight to ensure we are not updating chainInfo with observations related to previous life cycle //ctx, cancel, chStopInFlight, _, _ := c.acquireQueryCtx(ctx, c.rpcTimeout) - // TODO: Is this really the way to implement this? - latestBH, err := c.rpc.GetLatestBlockhash(ctx, c.commitment) + finalizedBlockHeight, err := c.rpc.GetBlockHeight(ctx, rpc.CommitmentFinalized) if err != nil { return nil, err } - var finalityDepth uint64 = 1 // TODO: Get value from config - - latestFinalizedBH := latestBH.Value.LastValidBlockHeight - finalityDepth // TODO: subtract finality depth? - - resp, err := c.rpc.GetBlock(ctx, latestFinalizedBH) + block, err := c.rpc.GetBlock(ctx, finalizedBlockHeight) if err != nil { return nil, err } - head := &Head{GetBlockResult: *resp} + head := &Head{GetBlockResult: *block} c.onNewFinalizedHead(ctx, c.chStopInFlight, head) return head, nil } @@ -254,7 +239,7 @@ func (c *Client) Ping(ctx context.Context) error { } func (c *Client) IsSyncing(ctx context.Context) (bool, error) { - // TODO: is this relevant? + // Not relevant for Solana return false, nil } diff --git a/pkg/solana/client/client_test.go b/pkg/solana/client/client_test.go index 0705615b7..00effeb45 100644 --- a/pkg/solana/client/client_test.go +++ b/pkg/solana/client/client_test.go @@ -297,6 +297,7 @@ func TestClient_SendTxDuplicates_Integration(t *testing.T) { assert.Equal(t, uint64(5_000), initBal-endBal) } +/* func TestClient_Subscriptions_Integration(t *testing.T) { // TODO: Test subscribing to heads and finalized heads // TODO: Ensure chain info is updated on new heads @@ -340,6 +341,7 @@ func TestClient_Subscriptions_Integration(t *testing.T) { wg.Wait() } +*/ func TestClientLatency(t *testing.T) { c := Client{} diff --git a/pkg/solana/config/multinode.go b/pkg/solana/config/multinode.go index 912e87f77..b625dddf4 100644 --- a/pkg/solana/config/multinode.go +++ b/pkg/solana/config/multinode.go @@ -79,8 +79,8 @@ func (c *MultiNode) FinalityTagEnabled() bool { return *c.finalityTagEnabled } func (c *MultiNode) FinalizedBlockOffset() uint32 { return *c.finalizedBlockOffset } -func (c *MultiNode) SetDefaults() { - c.multiNodeEnabled = ptr(false) +func (c *MultiNode) SetDefaults(enabled bool) { + c.multiNodeEnabled = ptr(enabled) // Node Configs c.pollFailureThreshold = ptr(uint32(5)) diff --git a/pkg/solana/config/toml.go b/pkg/solana/config/toml.go index e3c0bb01a..4b060f290 100644 --- a/pkg/solana/config/toml.go +++ b/pkg/solana/config/toml.go @@ -289,6 +289,6 @@ func (c *TOMLConfig) MultiNodeConfig() *MultiNode { func NewDefault() *TOMLConfig { cfg := &TOMLConfig{} cfg.Chain.SetDefaults() - cfg.MultiNode.SetDefaults() + cfg.MultiNode.SetDefaults(false) return cfg } From 6b92e70ca8eebd379f22ea888995750fdf87e41c Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 24 Sep 2024 13:57:11 -0400 Subject: [PATCH 032/174] Add subscription test --- pkg/solana/client/client.go | 8 ++--- pkg/solana/client/client_test.go | 57 +++++++++++++++++--------------- 2 files changed, 35 insertions(+), 30 deletions(-) diff --git a/pkg/solana/client/client.go b/pkg/solana/client/client.go index c7e81cb84..2e31f5944 100644 --- a/pkg/solana/client/client.go +++ b/pkg/solana/client/client.go @@ -117,7 +117,7 @@ func (h *Head) BlockNumber() int64 { } func (h *Head) BlockDifficulty() *big.Int { - // TODO: Not relevant for Solana? + // Not relevant for Solana return nil } @@ -157,17 +157,17 @@ func (c *Client) SubscribeToFinalizedHeads(ctx context.Context) (<-chan *Head, m } func (c *Client) LatestBlock(ctx context.Context) (*Head, error) { - latestBlockHash, err := c.rpc.GetLatestBlockhash(ctx, c.commitment) + latestBlockHeight, err := c.rpc.GetBlockHeight(ctx, rpc.CommitmentConfirmed) if err != nil { return nil, err } - latestBlock, err := c.rpc.GetBlock(ctx, latestBlockHash.Value.LastValidBlockHeight) + block, err := c.rpc.GetBlock(ctx, latestBlockHeight) if err != nil { return nil, err } - head := &Head{GetBlockResult: *latestBlock} + head := &Head{GetBlockResult: *block} c.onNewHead(ctx, c.chStopInFlight, head) return head, nil } diff --git a/pkg/solana/client/client_test.go b/pkg/solana/client/client_test.go index 00effeb45..cb14f8ddb 100644 --- a/pkg/solana/client/client_test.go +++ b/pkg/solana/client/client_test.go @@ -3,6 +3,7 @@ package client import ( "context" "fmt" + "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" "math/rand" "net/http" "net/http/httptest" @@ -297,13 +298,7 @@ func TestClient_SendTxDuplicates_Integration(t *testing.T) { assert.Equal(t, uint64(5_000), initBal-endBal) } -/* func TestClient_Subscriptions_Integration(t *testing.T) { - // TODO: Test subscribing to heads and finalized heads - // TODO: Ensure chain info is updated on new heads - // TODO: Test Dial, Close, IsSyncing, GetInterceptedChainInfo - - // TODO: Create server for testing?? url := SetupLocalSolNode(t) privKey, err := solana.NewRandomPrivateKey() require.NoError(t, err) @@ -313,35 +308,45 @@ func TestClient_Subscriptions_Integration(t *testing.T) { requestTimeout := 5 * time.Second lggr := logger.Test(t) cfg := config.NewDefault() + // Enable MultiNode + cfg.MultiNode.SetDefaults(true) - ctx := context.Background() c, err := NewClient(url, cfg, requestTimeout, lggr) require.NoError(t, err) - ch, sub, err := c.SubscribeToHeads(ctx) - defer sub.Unsubscribe() + err = c.Ping(tests.Context(t)) require.NoError(t, err) - // TODO: How do we test this? - // check for new heads - wg := sync.WaitGroup{} - wg.Add(1) - go func() { - for { - select { - case head := <-ch: - t.Logf("New head: %v", head) - wg.Done() - case <-ctx.Done(): - return - } - } - }() + ch, sub, err := c.SubscribeToHeads(tests.Context(t)) + require.NoError(t, err) + defer sub.Unsubscribe() - wg.Wait() + finalizedCh, finalizedSub, err := c.SubscribeToFinalizedHeads(tests.Context(t)) + require.NoError(t, err) + defer finalizedSub.Unsubscribe() + + require.NoError(t, err) + ctx, cancel := context.WithTimeout(tests.Context(t), time.Minute) + defer cancel() + + select { + case head := <-ch: + require.NotEqual(t, solana.Hash{}, head.Blockhash) + latest, _ := c.GetInterceptedChainInfo() + require.Equal(t, head.BlockNumber(), latest.BlockNumber) + case <-ctx.Done(): + t.Fatal("failed to receive head: ", ctx.Err()) + } + select { + case finalizedHead := <-finalizedCh: + require.NotEqual(t, solana.Hash{}, finalizedHead.Blockhash) + latest, _ := c.GetInterceptedChainInfo() + require.Equal(t, finalizedHead.BlockNumber(), latest.FinalizedBlockNumber) + case <-ctx.Done(): + t.Fatal("failed to receive finalized head: ", ctx.Err()) + } } -*/ func TestClientLatency(t *testing.T) { c := Client{} From 500233a3c7780b4e4d6679af3decd9ab00165e23 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 24 Sep 2024 14:02:25 -0400 Subject: [PATCH 033/174] tidy --- go.mod | 1 + go.sum | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/go.mod b/go.mod index 5ac569076..5c11a226b 100644 --- a/go.mod +++ b/go.mod @@ -36,6 +36,7 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.10.0 // indirect github.com/blendle/zapdriver v1.3.1 // indirect + github.com/btcsuite/btcd v0.22.0-beta // indirect github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect github.com/buger/goterm v0.0.0-20200322175922-2f3e71b85129 // indirect github.com/buger/jsonparser v1.1.1 // indirect diff --git a/go.sum b/go.sum index f976a95c0..e08d28643 100644 --- a/go.sum +++ b/go.sum @@ -34,6 +34,8 @@ github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOEl github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8= +github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0= github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= @@ -41,6 +43,11 @@ github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5 github.com/Microsoft/hcsshim v0.9.4 h1:mnUj0ivWy6UzbB1uLFqKR6F+ZyiDc7j4iGgHTpO+5+I= github.com/Microsoft/hcsshim v0.9.4/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA= +github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= +github.com/VictoriaMetrics/fastcache v1.12.1 h1:i0mICQuojGDL3KblA7wUNlY5lOK6a4bwt3uRKnkZU40= +github.com/VictoriaMetrics/fastcache v1.12.1/go.mod h1:tX04vaqcNoQeGLD+ra5pU5sWkuxnzWhEzLwhP9w653o= +github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -66,9 +73,21 @@ github.com/bits-and-blooms/bitset v1.10.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6 github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/blendle/zapdriver v1.3.1 h1:C3dydBOWYRiOk+B8X9IVZ5IOe+7cl+tGOexN4QqHfpE= github.com/blendle/zapdriver v1.3.1/go.mod h1:mdXfREi6u5MArG4j9fewC+FGnXaBR+T4Ox4J2u4eHCc= +github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= github.com/btcsuite/btcd v0.22.0-beta h1:LTDpDKUM5EeOFBPM8IXpinEcmZ6FWfNZbE3lfrfdnWo= +github.com/btcsuite/btcd v0.22.0-beta/go.mod h1:9n5ntfhhHQBIhUvlhDvD3Qg6fRUj4jkN0VB8L8svzOA= github.com/btcsuite/btcd/btcec/v2 v2.2.0 h1:fzn1qaOt32TuLjFlkzYSsBC35Q3KUjT1SwPxiMSCF5k= github.com/btcsuite/btcd/btcec/v2 v2.2.0/go.mod h1:U7MHm051Al6XmscBQ0BoNydpOTsFAn707034b5nY8zU= +github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= +github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= +github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce/go.mod h1:0DVlHczLPewLcPGEIeUEzfOJhqGPQ0mJJRDBtD307+o= +github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= +github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= +github.com/btcsuite/goleveldb v1.0.0/go.mod h1:QiK9vBlgftBg6rWQIj6wFzbPfRjiykIEhBH4obrXJ/I= +github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= +github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= +github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= +github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZlaQsNA= github.com/bufbuild/protocompile v0.4.0/go.mod h1:3v93+mbWn/v3xzN+31nwkJfrEpAUwp+BagBSZWx+TP8= github.com/buger/goterm v0.0.0-20200322175922-2f3e71b85129 h1:gfAMKE626QEuKG3si0pdTRcr/YEbBoxY+3GOH3gWvl4= @@ -87,6 +106,16 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cockroachdb/errors v1.9.1 h1:yFVvsI0VxmRShfawbt/laCIDy/mtTqqnvoNgiy5bEV8= +github.com/cockroachdb/errors v1.9.1/go.mod h1:2sxOtL2WIc096WSZqZ5h8fa17rdDq9HZOZLBCor4mBk= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= +github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593 h1:aPEJyR4rPBvDmeyi+l/FS/VtA00IWvjeFvjen1m1l1A= +github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593/go.mod h1:6hk1eMY/u5t+Cf18q5lFMUA1Rc+Sm5I6Ra1QuPyxXCo= +github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ= +github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= +github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= github.com/confluentinc/confluent-kafka-go/v2 v2.3.0 h1:icCHutJouWlQREayFwCc7lxDAhws08td+W3/gdqgZts= github.com/confluentinc/confluent-kafka-go/v2 v2.3.0/go.mod h1:/VTy8iEpe6mD9pkCH5BhijlUl8ulUXymKv1Qig5Rgb8= github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= @@ -103,17 +132,22 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233 h1:d28BXYi+wUpz1KBmiF9bWrjEMacUEREV6MBi2ODnrfQ= +github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233/go.mod h1:geZJZH3SzKCqnz5VT0q/DyIG/tvu/dZk+VIfXicupJs= github.com/crate-crypto/go-kzg-4844 v0.7.0 h1:C0vgZRk4q4EZ/JgPfzuSoxdCq3C3mOZMBShovmncxvA= github.com/crate-crypto/go-kzg-4844 v0.7.0/go.mod h1:1kMhvPgI0Ky3yIa+9lFySEBUBXkYxeOi8ZF1sYioxhc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/daaku/go.zipexe v1.0.0/go.mod h1:z8IiR6TsVLEYKwXAoE/I+8ys/sDkgTzSL0CLnGVd57E= +github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1m5sE92cU+pd5Mcc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= +github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= github.com/dfuse-io/logging v0.0.0-20201110202154-26697de88c79/go.mod h1:V+ED4kT/t/lKtH99JQmKIb0v9WL3VaYkJ36CfHlVECI= github.com/dfuse-io/logging v0.0.0-20210109005628-b97a57253f70 h1:CuJS05R9jmNlUK8GOxrEELPbfXm0EuGh/30LjkjN5vo= github.com/dfuse-io/logging v0.0.0-20210109005628-b97a57253f70/go.mod h1:EoK/8RFbMEteaCaz89uessDTnCWjbbcr+DXcBh4el5o= @@ -155,6 +189,10 @@ github.com/gagliardetto/treeout v0.1.4 h1:ozeYerrLCmCubo1TcIjFiOWTTGteOOHND1twdF github.com/gagliardetto/treeout v0.1.4/go.mod h1:loUefvXTrlRG5rYmJmExNryyBRh8f89VZhmMOyCyqok= github.com/gagliardetto/utilz v0.1.1 h1:/etW4hl607emKg6R6Lj9jRJ9d6ue2AQOyjhuAwjzs1U= github.com/gagliardetto/utilz v0.1.1/go.mod h1:b+rGFkRHz3HWJD0RYMzat47JyvbTtpE0iEcYTRJTLLA= +github.com/gballet/go-verkle v0.1.1-0.20231031103413-a67434b50f46 h1:BAIP2GihuqhwdILrV+7GJel5lyPV3u1+PgzrWLc0TkE= +github.com/gballet/go-verkle v0.1.1-0.20231031103413-a67434b50f46/go.mod h1:QNpY22eby74jVhqH4WhDLDwxc/vqsern6pW+u2kbkpc= +github.com/getsentry/sentry-go v0.18.0 h1:MtBW5H9QgdcJabtZcuJG80BMOwaBpkRDZkxRkNC1sN0= +github.com/getsentry/sentry-go v0.18.0/go.mod h1:Kgon4Mby+FJ7ZWHFUAZgVaIa8sxHtnRJRLTXZr51aKQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -169,6 +207,8 @@ github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY= +github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= @@ -180,6 +220,8 @@ github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-viper/mapstructure/v2 v2.1.0 h1:gHnMa2Y/pIxElCH2GlZZ1lZSsn6XMtufpGyP1XxdC/w= github.com/go-viper/mapstructure/v2 v2.1.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= +github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -293,6 +335,7 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/invopop/jsonschema v0.12.0 h1:6ovsNSuvn9wEQVOyc72aycBMVQFKz7cPdMJn10CvzRI= github.com/invopop/jsonschema v0.12.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0= +github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jhump/protoreflect v1.15.1 h1:HUMERORf3I3ZdX05WaQ6MIpd/NJ434hTp5YiKgfCL6c= github.com/jhump/protoreflect v1.15.1/go.mod h1:jD/2GMKKE6OqX8qTjhADU1e6DShO+gavG9e0Q693nKo= @@ -301,6 +344,7 @@ github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= @@ -310,6 +354,7 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7yIO+lw= @@ -324,6 +369,10 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/leanovate/gopter v0.2.10-0.20210127095200-9abe2343507a h1:dHCfT5W7gghzPtfsW488uPmEOm85wewI+ypUwibyTdU= +github.com/leanovate/gopter v0.2.10-0.20210127095200-9abe2343507a/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/linkedin/goavro/v2 v2.9.7/go.mod h1:UgQUb2N/pmueQYH9bfqFioWxzYCZXSfF8Jw03O5sjqA= @@ -350,6 +399,8 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= +github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= @@ -394,8 +445,13 @@ github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI github.com/oklog/run v1.0.0 h1:Ru7dDtJNOyC66gQ5dQmaCa0qIsAUFY3sFpK1Xk8igrw= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.24.1 h1:KORJXNNTzJXzu4ScJWssJfJMnJ+2QJqhoQSRwNlze9E= @@ -439,6 +495,8 @@ github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/riferrei/srclient v0.5.4 h1:dfwyR5u23QF7beuVl2WemUY2KXh5+Sc4DHKyPXBNYuc= github.com/riferrei/srclient v0.5.4/go.mod h1:vbkLmWcgYa7JgfPvuy/+K8fTS0p1bApqadxrxi/S1MI= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= @@ -452,6 +510,8 @@ github.com/santhosh-tekuri/jsonschema/v5 v5.0.0/go.mod h1:FKdcjfQW6rpZSnxxUvEA5H github.com/santhosh-tekuri/jsonschema/v5 v5.2.0 h1:WCcC4vZDS1tYNxjWlwRJZQy28r8CMoggKnxNzxsVDMQ= github.com/santhosh-tekuri/jsonschema/v5 v5.2.0/go.mod h1:FKdcjfQW6rpZSnxxUvEA5H/cDPdvJ/SZJQLWWXWGrZ0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1:Bn1aCHHRnjv4Bl16T8rcaFjYSrGrIZvpiGO6P3Q4GpU= +github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= @@ -501,6 +561,8 @@ github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/supranational/blst v0.3.11 h1:LyU6FolezeWAhvQk0k6O/d49jqgO52MSDDfYgbeoEm4= github.com/supranational/blst v0.3.11/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= +github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= +github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/teris-io/shortid v0.0.0-20171029131806-771a37caa5cf/go.mod h1:M8agBzgqHIhgj7wEn9/0hJUZcrvt9VY+Ln+S1I5Mha0= github.com/teris-io/shortid v0.0.0-20201117134242-e59966efd125 h1:3SNcvBmEPE1YlB1JpVZouslJpI3GBNoiqW7+wb0Rz7w= github.com/teris-io/shortid v0.0.0-20201117134242-e59966efd125/go.mod h1:M8agBzgqHIhgj7wEn9/0hJUZcrvt9VY+Ln+S1I5Mha0= @@ -516,6 +578,10 @@ github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JT github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= @@ -599,12 +665,15 @@ go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= @@ -647,6 +716,7 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0= golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= From 31a2f045b4c16179031a45c366a11549a1525997 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 24 Sep 2024 14:06:21 -0400 Subject: [PATCH 034/174] Fix imports --- go.mod | 13 ----- go.sum | 98 -------------------------------- pkg/solana/client/client.go | 3 +- pkg/solana/client/client_test.go | 2 +- 4 files changed, 2 insertions(+), 114 deletions(-) diff --git a/go.mod b/go.mod index 5c11a226b..cb7a616c0 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,6 @@ toolchain go1.22.3 require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc - github.com/ethereum/go-ethereum v1.13.8 github.com/gagliardetto/binary v0.7.7 github.com/gagliardetto/gofuzz v1.2.2 github.com/gagliardetto/solana-go v1.8.4 @@ -34,21 +33,13 @@ require ( github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/bits-and-blooms/bitset v1.10.0 // indirect github.com/blendle/zapdriver v1.3.1 // indirect - github.com/btcsuite/btcd v0.22.0-beta // indirect - github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect github.com/buger/goterm v0.0.0-20200322175922-2f3e71b85129 // indirect github.com/buger/jsonparser v1.1.1 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/confluentinc/confluent-kafka-go/v2 v2.3.0 // indirect - github.com/consensys/bavard v0.1.13 // indirect - github.com/consensys/gnark-crypto v0.12.1 // indirect - github.com/crate-crypto/go-kzg-4844 v0.7.0 // indirect - github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect github.com/dfuse-io/logging v0.0.0-20210109005628-b97a57253f70 // indirect - github.com/ethereum/c-kzg-4844 v0.4.0 // indirect github.com/fatih/color v1.16.0 // indirect github.com/fxamacker/cbor/v2 v2.5.0 // indirect github.com/go-json-experiment/json v0.0.0-20231102232822-2e55bd4e08b0 // indirect @@ -67,7 +58,6 @@ require ( github.com/hako/durafmt v0.0.0-20200710122514-c0fb7b4da026 // indirect github.com/hashicorp/go-hclog v1.5.0 // indirect github.com/hashicorp/yamux v0.1.1 // indirect - github.com/holiman/uint256 v1.2.4 // indirect github.com/invopop/jsonschema v0.12.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.15.15 // indirect @@ -82,7 +72,6 @@ require ( github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/miekg/dns v1.1.35 // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect - github.com/mmcloughlin/addchain v0.4.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mostynb/zstdpool-freelist v0.0.0-20201229113212-927304c0c3b1 // indirect @@ -102,7 +91,6 @@ require ( github.com/smartcontractkit/grpc-proxy v0.0.0-20240830132753-a7e17fec5ab7 // indirect github.com/streamingfast/logging v0.0.0-20220405224725-2755dab2ce75 // indirect github.com/stretchr/objx v0.5.2 // indirect - github.com/supranational/blst v0.3.11 // indirect github.com/teris-io/shortid v0.0.0-20201117134242-e59966efd125 // indirect github.com/tidwall/gjson v1.14.4 // indirect github.com/tidwall/match v1.1.1 // indirect @@ -140,5 +128,4 @@ require ( google.golang.org/protobuf v1.34.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - rsc.io/tmplfunc v0.0.3 // indirect ) diff --git a/go.sum b/go.sum index e08d28643..d1163d17c 100644 --- a/go.sum +++ b/go.sum @@ -34,8 +34,6 @@ github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOEl github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8= -github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0= github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= @@ -43,11 +41,6 @@ github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5 github.com/Microsoft/hcsshim v0.9.4 h1:mnUj0ivWy6UzbB1uLFqKR6F+ZyiDc7j4iGgHTpO+5+I= github.com/Microsoft/hcsshim v0.9.4/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA= -github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= -github.com/VictoriaMetrics/fastcache v1.12.1 h1:i0mICQuojGDL3KblA7wUNlY5lOK6a4bwt3uRKnkZU40= -github.com/VictoriaMetrics/fastcache v1.12.1/go.mod h1:tX04vaqcNoQeGLD+ra5pU5sWkuxnzWhEzLwhP9w653o= -github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -68,26 +61,9 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bits-and-blooms/bitset v1.10.0 h1:ePXTeiPEazB5+opbv5fr8umg2R/1NlzgDsyepwsSr88= -github.com/bits-and-blooms/bitset v1.10.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/blendle/zapdriver v1.3.1 h1:C3dydBOWYRiOk+B8X9IVZ5IOe+7cl+tGOexN4QqHfpE= github.com/blendle/zapdriver v1.3.1/go.mod h1:mdXfREi6u5MArG4j9fewC+FGnXaBR+T4Ox4J2u4eHCc= -github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= -github.com/btcsuite/btcd v0.22.0-beta h1:LTDpDKUM5EeOFBPM8IXpinEcmZ6FWfNZbE3lfrfdnWo= -github.com/btcsuite/btcd v0.22.0-beta/go.mod h1:9n5ntfhhHQBIhUvlhDvD3Qg6fRUj4jkN0VB8L8svzOA= -github.com/btcsuite/btcd/btcec/v2 v2.2.0 h1:fzn1qaOt32TuLjFlkzYSsBC35Q3KUjT1SwPxiMSCF5k= -github.com/btcsuite/btcd/btcec/v2 v2.2.0/go.mod h1:U7MHm051Al6XmscBQ0BoNydpOTsFAn707034b5nY8zU= -github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= -github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= -github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce/go.mod h1:0DVlHczLPewLcPGEIeUEzfOJhqGPQ0mJJRDBtD307+o= -github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= -github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= -github.com/btcsuite/goleveldb v1.0.0/go.mod h1:QiK9vBlgftBg6rWQIj6wFzbPfRjiykIEhBH4obrXJ/I= -github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= -github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= -github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= -github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZlaQsNA= github.com/bufbuild/protocompile v0.4.0/go.mod h1:3v93+mbWn/v3xzN+31nwkJfrEpAUwp+BagBSZWx+TP8= github.com/buger/goterm v0.0.0-20200322175922-2f3e71b85129 h1:gfAMKE626QEuKG3si0pdTRcr/YEbBoxY+3GOH3gWvl4= @@ -106,22 +82,8 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cockroachdb/errors v1.9.1 h1:yFVvsI0VxmRShfawbt/laCIDy/mtTqqnvoNgiy5bEV8= -github.com/cockroachdb/errors v1.9.1/go.mod h1:2sxOtL2WIc096WSZqZ5h8fa17rdDq9HZOZLBCor4mBk= -github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= -github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= -github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593 h1:aPEJyR4rPBvDmeyi+l/FS/VtA00IWvjeFvjen1m1l1A= -github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593/go.mod h1:6hk1eMY/u5t+Cf18q5lFMUA1Rc+Sm5I6Ra1QuPyxXCo= -github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ= -github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= -github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= -github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= github.com/confluentinc/confluent-kafka-go/v2 v2.3.0 h1:icCHutJouWlQREayFwCc7lxDAhws08td+W3/gdqgZts= github.com/confluentinc/confluent-kafka-go/v2 v2.3.0/go.mod h1:/VTy8iEpe6mD9pkCH5BhijlUl8ulUXymKv1Qig5Rgb8= -github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= -github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= -github.com/consensys/gnark-crypto v0.12.1 h1:lHH39WuuFgVHONRl3J0LRBtuYdQTumFSDtJF7HpyG8M= -github.com/consensys/gnark-crypto v0.12.1/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY= github.com/containerd/cgroups v1.0.4 h1:jN/mbWBEaz+T1pi5OFtnkQ+8qnmEbAr1Oo1FRm5B0dA= github.com/containerd/cgroups v1.0.4/go.mod h1:nLNQtsF7Sl2HxNebu77i1R0oDlhiTG+kO4JTrUzo6IA= github.com/containerd/containerd v1.6.8 h1:h4dOFDwzHmqFEP754PgfgTeVXFnLiRc6kiqC7tplDJs= @@ -132,22 +94,12 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233 h1:d28BXYi+wUpz1KBmiF9bWrjEMacUEREV6MBi2ODnrfQ= -github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233/go.mod h1:geZJZH3SzKCqnz5VT0q/DyIG/tvu/dZk+VIfXicupJs= -github.com/crate-crypto/go-kzg-4844 v0.7.0 h1:C0vgZRk4q4EZ/JgPfzuSoxdCq3C3mOZMBShovmncxvA= -github.com/crate-crypto/go-kzg-4844 v0.7.0/go.mod h1:1kMhvPgI0Ky3yIa+9lFySEBUBXkYxeOi8ZF1sYioxhc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/daaku/go.zipexe v1.0.0/go.mod h1:z8IiR6TsVLEYKwXAoE/I+8ys/sDkgTzSL0CLnGVd57E= -github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= -github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1m5sE92cU+pd5Mcc= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= -github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= github.com/dfuse-io/logging v0.0.0-20201110202154-26697de88c79/go.mod h1:V+ED4kT/t/lKtH99JQmKIb0v9WL3VaYkJ36CfHlVECI= github.com/dfuse-io/logging v0.0.0-20210109005628-b97a57253f70 h1:CuJS05R9jmNlUK8GOxrEELPbfXm0EuGh/30LjkjN5vo= github.com/dfuse-io/logging v0.0.0-20210109005628-b97a57253f70/go.mod h1:EoK/8RFbMEteaCaz89uessDTnCWjbbcr+DXcBh4el5o= @@ -166,10 +118,6 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/ethereum/c-kzg-4844 v0.4.0 h1:3MS1s4JtA868KpJxroZoepdV0ZKBp3u/O5HcZ7R3nlY= -github.com/ethereum/c-kzg-4844 v0.4.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= -github.com/ethereum/go-ethereum v1.13.8 h1:1od+thJel3tM52ZUNQwvpYOeRHlbkVFZ5S8fhi0Lgsg= -github.com/ethereum/go-ethereum v1.13.8/go.mod h1:sc48XYQxCzH3fG9BcrXCOOgQk2JfZzNAmIKnceogzsA= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= @@ -189,10 +137,6 @@ github.com/gagliardetto/treeout v0.1.4 h1:ozeYerrLCmCubo1TcIjFiOWTTGteOOHND1twdF github.com/gagliardetto/treeout v0.1.4/go.mod h1:loUefvXTrlRG5rYmJmExNryyBRh8f89VZhmMOyCyqok= github.com/gagliardetto/utilz v0.1.1 h1:/etW4hl607emKg6R6Lj9jRJ9d6ue2AQOyjhuAwjzs1U= github.com/gagliardetto/utilz v0.1.1/go.mod h1:b+rGFkRHz3HWJD0RYMzat47JyvbTtpE0iEcYTRJTLLA= -github.com/gballet/go-verkle v0.1.1-0.20231031103413-a67434b50f46 h1:BAIP2GihuqhwdILrV+7GJel5lyPV3u1+PgzrWLc0TkE= -github.com/gballet/go-verkle v0.1.1-0.20231031103413-a67434b50f46/go.mod h1:QNpY22eby74jVhqH4WhDLDwxc/vqsern6pW+u2kbkpc= -github.com/getsentry/sentry-go v0.18.0 h1:MtBW5H9QgdcJabtZcuJG80BMOwaBpkRDZkxRkNC1sN0= -github.com/getsentry/sentry-go v0.18.0/go.mod h1:Kgon4Mby+FJ7ZWHFUAZgVaIa8sxHtnRJRLTXZr51aKQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -207,8 +151,6 @@ github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY= -github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= @@ -220,8 +162,6 @@ github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-viper/mapstructure/v2 v2.1.0 h1:gHnMa2Y/pIxElCH2GlZZ1lZSsn6XMtufpGyP1XxdC/w= github.com/go-viper/mapstructure/v2 v2.1.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= -github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= -github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -279,7 +219,6 @@ github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -328,14 +267,11 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hashicorp/yamux v0.1.1 h1:yrQxtgseBDrq9Y652vSRDvsKCJKOUD+GzTS4Y0Y8pvE= github.com/hashicorp/yamux v0.1.1/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ= -github.com/holiman/uint256 v1.2.4 h1:jUc4Nk8fm9jZabQuqr2JzednajVmBpC+oiTiXZJEApU= -github.com/holiman/uint256 v1.2.4/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/invopop/jsonschema v0.12.0 h1:6ovsNSuvn9wEQVOyc72aycBMVQFKz7cPdMJn10CvzRI= github.com/invopop/jsonschema v0.12.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0= -github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jhump/protoreflect v1.15.1 h1:HUMERORf3I3ZdX05WaQ6MIpd/NJ434hTp5YiKgfCL6c= github.com/jhump/protoreflect v1.15.1/go.mod h1:jD/2GMKKE6OqX8qTjhADU1e6DShO+gavG9e0Q693nKo= @@ -344,7 +280,6 @@ github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= @@ -354,7 +289,6 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7yIO+lw= @@ -369,10 +303,6 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= -github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/leanovate/gopter v0.2.10-0.20210127095200-9abe2343507a h1:dHCfT5W7gghzPtfsW488uPmEOm85wewI+ypUwibyTdU= -github.com/leanovate/gopter v0.2.10-0.20210127095200-9abe2343507a/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/linkedin/goavro/v2 v2.9.7/go.mod h1:UgQUb2N/pmueQYH9bfqFioWxzYCZXSfF8Jw03O5sjqA= @@ -399,8 +329,6 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= -github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= @@ -417,9 +345,6 @@ github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS4 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY= -github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU= -github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFVejUS1/tS/qU= github.com/moby/sys/mount v0.3.3 h1:fX1SVkXFJ47XWDoeFW4Sq7PdQJnV2QIDZAqjNqgEjUs= github.com/moby/sys/mount v0.3.3/go.mod h1:PBaEorSNTLG5t/+4EgukEQVlAvVEc6ZjTySwKdqp5K0= github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78= @@ -445,13 +370,8 @@ github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI github.com/oklog/run v1.0.0 h1:Ru7dDtJNOyC66gQ5dQmaCa0qIsAUFY3sFpK1Xk8igrw= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= -github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.24.1 h1:KORJXNNTzJXzu4ScJWssJfJMnJ+2QJqhoQSRwNlze9E= @@ -495,8 +415,6 @@ github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/riferrei/srclient v0.5.4 h1:dfwyR5u23QF7beuVl2WemUY2KXh5+Sc4DHKyPXBNYuc= github.com/riferrei/srclient v0.5.4/go.mod h1:vbkLmWcgYa7JgfPvuy/+K8fTS0p1bApqadxrxi/S1MI= -github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= -github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= @@ -510,8 +428,6 @@ github.com/santhosh-tekuri/jsonschema/v5 v5.0.0/go.mod h1:FKdcjfQW6rpZSnxxUvEA5H github.com/santhosh-tekuri/jsonschema/v5 v5.2.0 h1:WCcC4vZDS1tYNxjWlwRJZQy28r8CMoggKnxNzxsVDMQ= github.com/santhosh-tekuri/jsonschema/v5 v5.2.0/go.mod h1:FKdcjfQW6rpZSnxxUvEA5H/cDPdvJ/SZJQLWWXWGrZ0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1:Bn1aCHHRnjv4Bl16T8rcaFjYSrGrIZvpiGO6P3Q4GpU= -github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= @@ -559,10 +475,6 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/supranational/blst v0.3.11 h1:LyU6FolezeWAhvQk0k6O/d49jqgO52MSDDfYgbeoEm4= -github.com/supranational/blst v0.3.11/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= -github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= -github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/teris-io/shortid v0.0.0-20171029131806-771a37caa5cf/go.mod h1:M8agBzgqHIhgj7wEn9/0hJUZcrvt9VY+Ln+S1I5Mha0= github.com/teris-io/shortid v0.0.0-20201117134242-e59966efd125 h1:3SNcvBmEPE1YlB1JpVZouslJpI3GBNoiqW7+wb0Rz7w= github.com/teris-io/shortid v0.0.0-20201117134242-e59966efd125/go.mod h1:M8agBzgqHIhgj7wEn9/0hJUZcrvt9VY+Ln+S1I5Mha0= @@ -578,10 +490,6 @@ github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JT github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= -github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= -github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= -github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= -github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= @@ -665,15 +573,12 @@ go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= @@ -716,7 +621,6 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0= golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -983,5 +887,3 @@ honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU= -rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA= diff --git a/pkg/solana/client/client.go b/pkg/solana/client/client.go index 2e31f5944..d8792518e 100644 --- a/pkg/solana/client/client.go +++ b/pkg/solana/client/client.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "github.com/ethereum/go-ethereum" "math/big" "sync" "time" @@ -73,7 +72,7 @@ type Client struct { pollInterval time.Duration finalizedBlockPollInterval time.Duration stateMu sync.RWMutex // protects state* fields - subs map[ethereum.Subscription]struct{} + subs map[mn.Subscription]struct{} // chStopInFlight can be closed to immediately cancel all in-flight requests on // this RpcClient. Closing and replacing should be serialized through diff --git a/pkg/solana/client/client_test.go b/pkg/solana/client/client_test.go index cb14f8ddb..f1e1cb3ec 100644 --- a/pkg/solana/client/client_test.go +++ b/pkg/solana/client/client_test.go @@ -3,7 +3,6 @@ package client import ( "context" "fmt" - "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" "math/rand" "net/http" "net/http/httptest" @@ -20,6 +19,7 @@ import ( "github.com/stretchr/testify/require" "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" mn "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" "github.com/smartcontractkit/chainlink-solana/pkg/solana/config" From bb5e47c7db7d02d0139bd555d37dee808c78e234 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 24 Sep 2024 15:10:32 -0400 Subject: [PATCH 035/174] Update chain_test.go --- pkg/solana/chain_test.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/pkg/solana/chain_test.go b/pkg/solana/chain_test.go index 39c3e5e7a..659cf3fa2 100644 --- a/pkg/solana/chain_test.go +++ b/pkg/solana/chain_test.go @@ -194,9 +194,12 @@ func TestSolanaChain_VerifiedClient(t *testing.T) { ch := solcfg.Chain{} ch.SetDefaults() + mn := solcfg.MultiNode{} + mn.SetDefaults(false) cfg := &solcfg.TOMLConfig{ - ChainID: ptr("devnet"), - Chain: ch, + ChainID: ptr("devnet"), + Chain: ch, + MultiNode: mn, } testChain := chain{ cfg: cfg, @@ -241,10 +244,13 @@ func TestSolanaChain_VerifiedClient_ParallelClients(t *testing.T) { ch := solcfg.Chain{} ch.SetDefaults() + mn := solcfg.MultiNode{} + mn.SetDefaults(false) cfg := &solcfg.TOMLConfig{ - ChainID: ptr("devnet"), - Enabled: ptr(true), - Chain: ch, + ChainID: ptr("devnet"), + Enabled: ptr(true), + Chain: ch, + MultiNode: mn, } testChain := chain{ id: "devnet", From 98b0e9d6bac51a1e00127676bdd8d23d8c85b5d0 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 27 Sep 2024 11:36:11 -0400 Subject: [PATCH 036/174] Update multinode.go --- pkg/solana/config/multinode.go | 70 +++++++++++++++++----------------- 1 file changed, 34 insertions(+), 36 deletions(-) diff --git a/pkg/solana/config/multinode.go b/pkg/solana/config/multinode.go index 912e87f77..14421f13c 100644 --- a/pkg/solana/config/multinode.go +++ b/pkg/solana/config/multinode.go @@ -80,6 +80,7 @@ func (c *MultiNode) FinalityTagEnabled() bool { return *c.finalityTagEnabled } func (c *MultiNode) FinalizedBlockOffset() uint32 { return *c.finalizedBlockOffset } func (c *MultiNode) SetDefaults() { + // Default disabled c.multiNodeEnabled = ptr(false) // Node Configs @@ -90,7 +91,6 @@ func (c *MultiNode) SetDefaults() { c.syncThreshold = ptr(uint32(5)) - // Period at which we verify if active node is still highest block number c.leaseDuration = config.MustNewDuration(time.Minute) c.nodeIsSyncingEnabled = ptr(false) @@ -106,58 +106,56 @@ func (c *MultiNode) SetDefaults() { c.finalizedBlockOffset = ptr(uint32(0)) } -func (c *MultiNode) SetFrom(fs *MultiNode) { - if fs.multiNodeEnabled != nil { - c.multiNodeEnabled = fs.multiNodeEnabled +func (c *MultiNode) SetFrom(f *MultiNode) { + if f.multiNodeEnabled != nil { + c.multiNodeEnabled = f.multiNodeEnabled } + // TODO: Try using reflection here to loop through each one + // Node Configs - if fs.pollFailureThreshold != nil { - c.pollFailureThreshold = fs.pollFailureThreshold + if f.pollFailureThreshold != nil { + c.pollFailureThreshold = f.pollFailureThreshold } - if fs.pollInterval != nil { - c.pollInterval = fs.pollInterval + if f.pollInterval != nil { + c.pollInterval = f.pollInterval } - if fs.selectionMode != nil { - c.selectionMode = fs.selectionMode + if f.selectionMode != nil { + c.selectionMode = f.selectionMode } - if fs.syncThreshold != nil { - c.syncThreshold = fs.syncThreshold + if f.syncThreshold != nil { + c.syncThreshold = f.syncThreshold } - if fs.nodeIsSyncingEnabled != nil { - c.nodeIsSyncingEnabled = fs.nodeIsSyncingEnabled + if f.nodeIsSyncingEnabled != nil { + c.nodeIsSyncingEnabled = f.nodeIsSyncingEnabled } - if fs.leaseDuration != nil { - c.leaseDuration = fs.leaseDuration + if f.leaseDuration != nil { + c.leaseDuration = f.leaseDuration } - if fs.finalizedBlockPollInterval != nil { - c.finalizedBlockPollInterval = fs.finalizedBlockPollInterval + if f.finalizedBlockPollInterval != nil { + c.finalizedBlockPollInterval = f.finalizedBlockPollInterval } - if fs.enforceRepeatableRead != nil { - c.enforceRepeatableRead = fs.enforceRepeatableRead + if f.enforceRepeatableRead != nil { + c.enforceRepeatableRead = f.enforceRepeatableRead } - if fs.deathDeclarationDelay != nil { - c.deathDeclarationDelay = fs.deathDeclarationDelay + if f.deathDeclarationDelay != nil { + c.deathDeclarationDelay = f.deathDeclarationDelay } // Chain Configs - if fs.nodeNoNewHeadsThreshold != nil { - c.nodeNoNewHeadsThreshold = fs.nodeNoNewHeadsThreshold + if f.nodeNoNewHeadsThreshold != nil { + c.nodeNoNewHeadsThreshold = f.nodeNoNewHeadsThreshold } - if fs.noNewFinalizedHeadsThreshold != nil { - c.noNewFinalizedHeadsThreshold = fs.noNewFinalizedHeadsThreshold + if f.noNewFinalizedHeadsThreshold != nil { + c.noNewFinalizedHeadsThreshold = f.noNewFinalizedHeadsThreshold } - if fs.finalityDepth != nil { - c.finalityDepth = fs.finalityDepth + if f.finalityDepth != nil { + c.finalityDepth = f.finalityDepth } - if fs.finalityTagEnabled != nil { - c.finalityTagEnabled = fs.finalityTagEnabled + if f.finalityTagEnabled != nil { + c.finalityTagEnabled = f.finalityTagEnabled } - if fs.finalizedBlockOffset != nil { - c.finalizedBlockOffset = fs.finalizedBlockOffset + if f.finalizedBlockOffset != nil { + c.finalizedBlockOffset = f.finalizedBlockOffset } } - -func ptr[T any](v T) *T { - return &v -} From fd0cc1638f457baf374801196cce34491ec1b948 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 1 Oct 2024 09:51:28 -0400 Subject: [PATCH 037/174] Add comments --- pkg/solana/config/multinode.go | 84 +++++++++++++++++++++++++--------- 1 file changed, 62 insertions(+), 22 deletions(-) diff --git a/pkg/solana/config/multinode.go b/pkg/solana/config/multinode.go index 14421f13c..4e498872f 100644 --- a/pkg/solana/config/multinode.go +++ b/pkg/solana/config/multinode.go @@ -80,30 +80,70 @@ func (c *MultiNode) FinalityTagEnabled() bool { return *c.finalityTagEnabled } func (c *MultiNode) FinalizedBlockOffset() uint32 { return *c.finalizedBlockOffset } func (c *MultiNode) SetDefaults() { - // Default disabled - c.multiNodeEnabled = ptr(false) - - // Node Configs - c.pollFailureThreshold = ptr(uint32(5)) - c.pollInterval = config.MustNewDuration(10 * time.Second) - - c.selectionMode = ptr(client.NodeSelectionModePriorityLevel) - - c.syncThreshold = ptr(uint32(5)) - - c.leaseDuration = config.MustNewDuration(time.Minute) + // MultiNode is disabled as it's not fully implemented yet: BCFR-122 + if c.multiNodeEnabled == nil { + c.multiNodeEnabled = ptr(false) + } - c.nodeIsSyncingEnabled = ptr(false) - c.finalizedBlockPollInterval = config.MustNewDuration(5 * time.Second) - c.enforceRepeatableRead = ptr(true) - c.deathDeclarationDelay = config.MustNewDuration(10 * time.Second) + /* Node Configs */ + // Failure threshold for polling set to 5 to tolerate some polling failures before taking action. + if c.pollFailureThreshold == nil { + c.pollFailureThreshold = ptr(uint32(5)) + } + // Poll interval is set to 10 seconds to ensure timely updates while minimizing resource usage. + if c.pollInterval == nil { + c.pollInterval = config.MustNewDuration(10 * time.Second) + } + // Selection mode defaults to priority level to enable using node priorities + if c.selectionMode == nil { + c.selectionMode = ptr(client.NodeSelectionModePriorityLevel) + } + // The sync threshold is set to 5 to allow for some flexibility in node synchronization before considering it out of sync. + if c.syncThreshold == nil { + c.syncThreshold = ptr(uint32(5)) + } + // Lease duration is set to 1 minute by default to allow node locks for a reasonable amount of time. + if c.leaseDuration == nil { + c.leaseDuration = config.MustNewDuration(time.Minute) + } + // Node syncing is not relevant for Solana and is disabled by default. + if c.nodeIsSyncingEnabled == nil { + c.nodeIsSyncingEnabled = ptr(false) + } + // The finalized block polling interval is set to 5 seconds to ensure timely updates while minimizing resource usage. + if c.finalizedBlockPollInterval == nil { + c.finalizedBlockPollInterval = config.MustNewDuration(5 * time.Second) + } + // Repeatable read guarantee should be enforced by default. + if c.enforceRepeatableRead == nil { + c.enforceRepeatableRead = ptr(true) + } + // The delay before declaring a node dead is set to 10 seconds to give nodes time to recover from temporary issues. + if c.deathDeclarationDelay == nil { + c.deathDeclarationDelay = config.MustNewDuration(10 * time.Second) + } - // Chain Configs - c.nodeNoNewHeadsThreshold = config.MustNewDuration(10 * time.Second) - c.noNewFinalizedHeadsThreshold = config.MustNewDuration(10 * time.Second) - c.finalityDepth = ptr(uint32(0)) - c.finalityTagEnabled = ptr(true) - c.finalizedBlockOffset = ptr(uint32(0)) + /* Chain Configs */ + // Threshold for no new heads is set to 10 seconds, assuming that heads should update at a reasonable pace. + if c.nodeNoNewHeadsThreshold == nil { + c.nodeNoNewHeadsThreshold = config.MustNewDuration(10 * time.Second) + } + // Similar to heads, finalized heads should be updated within 10 seconds. + if c.noNewFinalizedHeadsThreshold == nil { + c.noNewFinalizedHeadsThreshold = config.MustNewDuration(10 * time.Second) + } + // Finality tags are used in Solana and enabled by default. + if c.finalityTagEnabled == nil { + c.finalityTagEnabled = ptr(true) + } + // Finality depth will not be used since finality tags are enabled. + if c.finalityDepth == nil { + c.finalityDepth = ptr(uint32(0)) + } + // Finalized block offset will not be used since finality tags are enabled. + if c.finalizedBlockOffset == nil { + c.finalizedBlockOffset = ptr(uint32(0)) + } } func (c *MultiNode) SetFrom(f *MultiNode) { From 456be6c3283303b1bf66689e28326122cf2213f5 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 1 Oct 2024 10:41:06 -0400 Subject: [PATCH 038/174] Update multinode.go --- pkg/solana/config/multinode.go | 202 ++++++++++++++++----------------- 1 file changed, 101 insertions(+), 101 deletions(-) diff --git a/pkg/solana/config/multinode.go b/pkg/solana/config/multinode.go index 4e498872f..237df5e22 100644 --- a/pkg/solana/config/multinode.go +++ b/pkg/solana/config/multinode.go @@ -10,192 +10,192 @@ import ( type MultiNode struct { // Feature flag - multiNodeEnabled *bool + Enabled *bool // Node Configs - pollFailureThreshold *uint32 - pollInterval *config.Duration - selectionMode *string - syncThreshold *uint32 - nodeIsSyncingEnabled *bool - leaseDuration *config.Duration - finalizedBlockPollInterval *config.Duration - enforceRepeatableRead *bool - deathDeclarationDelay *config.Duration + PollFailureThreshold *uint32 + PollInterval *config.Duration + SelectionMode *string + SyncThreshold *uint32 + NodeIsSyncingEnabled *bool + LeaseDuration *config.Duration + FinalizedBlockPollInterval *config.Duration + EnforceRepeatableRead *bool + DeathDeclarationDelay *config.Duration // Chain Configs - nodeNoNewHeadsThreshold *config.Duration - noNewFinalizedHeadsThreshold *config.Duration - finalityDepth *uint32 - finalityTagEnabled *bool - finalizedBlockOffset *uint32 + NoNewHeadsThreshold *config.Duration + NoNewFinalizedHeadsThreshold *config.Duration + FinalityDepth *uint32 + FinalityTagEnabled *bool + FinalizedBlockOffset *uint32 } -func (c *MultiNode) MultiNodeEnabled() bool { - return c.multiNodeEnabled != nil && *c.multiNodeEnabled +func (c *MultiNode) GetEnabled() bool { + return c.Enabled != nil && *c.Enabled } -func (c *MultiNode) PollFailureThreshold() uint32 { - return *c.pollFailureThreshold +func (c *MultiNode) GetPollFailureThreshold() uint32 { + return *c.PollFailureThreshold } -func (c *MultiNode) PollInterval() time.Duration { - return c.pollInterval.Duration() +func (c *MultiNode) GetPollInterval() time.Duration { + return c.PollInterval.Duration() } -func (c *MultiNode) SelectionMode() string { - return *c.selectionMode +func (c *MultiNode) GetSelectionMode() string { + return *c.SelectionMode } -func (c *MultiNode) SyncThreshold() uint32 { - return *c.syncThreshold +func (c *MultiNode) GetSyncThreshold() uint32 { + return *c.SyncThreshold } -func (c *MultiNode) NodeIsSyncingEnabled() bool { - return *c.nodeIsSyncingEnabled +func (c *MultiNode) GetNodeIsSyncingEnabled() bool { + return *c.NodeIsSyncingEnabled } -func (c *MultiNode) LeaseDuration() time.Duration { return c.leaseDuration.Duration() } +func (c *MultiNode) GetLeaseDuration() time.Duration { return c.LeaseDuration.Duration() } -func (c *MultiNode) FinalizedBlockPollInterval() time.Duration { - return c.finalizedBlockPollInterval.Duration() +func (c *MultiNode) GetFinalizedBlockPollInterval() time.Duration { + return c.FinalizedBlockPollInterval.Duration() } -func (c *MultiNode) EnforceRepeatableRead() bool { return *c.enforceRepeatableRead } +func (c *MultiNode) GetEnforceRepeatableRead() bool { return *c.EnforceRepeatableRead } -func (c *MultiNode) DeathDeclarationDelay() time.Duration { return c.deathDeclarationDelay.Duration() } +func (c *MultiNode) GetDeathDeclarationDelay() time.Duration { + return c.DeathDeclarationDelay.Duration() +} -func (c *MultiNode) NodeNoNewHeadsThreshold() time.Duration { - return c.nodeNoNewHeadsThreshold.Duration() +func (c *MultiNode) GetNoNewHeadsThreshold() time.Duration { + return c.NoNewHeadsThreshold.Duration() } -func (c *MultiNode) NoNewFinalizedHeadsThreshold() time.Duration { - return c.noNewFinalizedHeadsThreshold.Duration() +func (c *MultiNode) GetNoNewFinalizedHeadsThreshold() time.Duration { + return c.NoNewFinalizedHeadsThreshold.Duration() } -func (c *MultiNode) FinalityDepth() uint32 { return *c.finalityDepth } +func (c *MultiNode) GetFinalityDepth() uint32 { return *c.FinalityDepth } -func (c *MultiNode) FinalityTagEnabled() bool { return *c.finalityTagEnabled } +func (c *MultiNode) GetFinalityTagEnabled() bool { return *c.FinalityTagEnabled } -func (c *MultiNode) FinalizedBlockOffset() uint32 { return *c.finalizedBlockOffset } +func (c *MultiNode) GetFinalizedBlockOffset() uint32 { return *c.FinalizedBlockOffset } func (c *MultiNode) SetDefaults() { // MultiNode is disabled as it's not fully implemented yet: BCFR-122 - if c.multiNodeEnabled == nil { - c.multiNodeEnabled = ptr(false) + if c.Enabled == nil { + c.Enabled = ptr(false) } /* Node Configs */ // Failure threshold for polling set to 5 to tolerate some polling failures before taking action. - if c.pollFailureThreshold == nil { - c.pollFailureThreshold = ptr(uint32(5)) + if c.PollFailureThreshold == nil { + c.PollFailureThreshold = ptr(uint32(5)) } // Poll interval is set to 10 seconds to ensure timely updates while minimizing resource usage. - if c.pollInterval == nil { - c.pollInterval = config.MustNewDuration(10 * time.Second) + if c.PollInterval == nil { + c.PollInterval = config.MustNewDuration(10 * time.Second) } // Selection mode defaults to priority level to enable using node priorities - if c.selectionMode == nil { - c.selectionMode = ptr(client.NodeSelectionModePriorityLevel) + if c.SelectionMode == nil { + c.SelectionMode = ptr(client.NodeSelectionModePriorityLevel) } // The sync threshold is set to 5 to allow for some flexibility in node synchronization before considering it out of sync. - if c.syncThreshold == nil { - c.syncThreshold = ptr(uint32(5)) + if c.SyncThreshold == nil { + c.SyncThreshold = ptr(uint32(5)) } // Lease duration is set to 1 minute by default to allow node locks for a reasonable amount of time. - if c.leaseDuration == nil { - c.leaseDuration = config.MustNewDuration(time.Minute) + if c.LeaseDuration == nil { + c.LeaseDuration = config.MustNewDuration(time.Minute) } // Node syncing is not relevant for Solana and is disabled by default. - if c.nodeIsSyncingEnabled == nil { - c.nodeIsSyncingEnabled = ptr(false) + if c.NodeIsSyncingEnabled == nil { + c.NodeIsSyncingEnabled = ptr(false) } // The finalized block polling interval is set to 5 seconds to ensure timely updates while minimizing resource usage. - if c.finalizedBlockPollInterval == nil { - c.finalizedBlockPollInterval = config.MustNewDuration(5 * time.Second) + if c.FinalizedBlockPollInterval == nil { + c.FinalizedBlockPollInterval = config.MustNewDuration(5 * time.Second) } // Repeatable read guarantee should be enforced by default. - if c.enforceRepeatableRead == nil { - c.enforceRepeatableRead = ptr(true) + if c.EnforceRepeatableRead == nil { + c.EnforceRepeatableRead = ptr(true) } // The delay before declaring a node dead is set to 10 seconds to give nodes time to recover from temporary issues. - if c.deathDeclarationDelay == nil { - c.deathDeclarationDelay = config.MustNewDuration(10 * time.Second) + if c.DeathDeclarationDelay == nil { + c.DeathDeclarationDelay = config.MustNewDuration(10 * time.Second) } /* Chain Configs */ // Threshold for no new heads is set to 10 seconds, assuming that heads should update at a reasonable pace. - if c.nodeNoNewHeadsThreshold == nil { - c.nodeNoNewHeadsThreshold = config.MustNewDuration(10 * time.Second) + if c.NoNewHeadsThreshold == nil { + c.NoNewHeadsThreshold = config.MustNewDuration(10 * time.Second) } // Similar to heads, finalized heads should be updated within 10 seconds. - if c.noNewFinalizedHeadsThreshold == nil { - c.noNewFinalizedHeadsThreshold = config.MustNewDuration(10 * time.Second) + if c.NoNewFinalizedHeadsThreshold == nil { + c.NoNewFinalizedHeadsThreshold = config.MustNewDuration(10 * time.Second) } // Finality tags are used in Solana and enabled by default. - if c.finalityTagEnabled == nil { - c.finalityTagEnabled = ptr(true) + if c.FinalityTagEnabled == nil { + c.FinalityTagEnabled = ptr(true) } // Finality depth will not be used since finality tags are enabled. - if c.finalityDepth == nil { - c.finalityDepth = ptr(uint32(0)) + if c.FinalityDepth == nil { + c.FinalityDepth = ptr(uint32(0)) } // Finalized block offset will not be used since finality tags are enabled. - if c.finalizedBlockOffset == nil { - c.finalizedBlockOffset = ptr(uint32(0)) + if c.FinalizedBlockOffset == nil { + c.FinalizedBlockOffset = ptr(uint32(0)) } } func (c *MultiNode) SetFrom(f *MultiNode) { - if f.multiNodeEnabled != nil { - c.multiNodeEnabled = f.multiNodeEnabled + if f.Enabled != nil { + c.Enabled = f.Enabled } - // TODO: Try using reflection here to loop through each one - // Node Configs - if f.pollFailureThreshold != nil { - c.pollFailureThreshold = f.pollFailureThreshold + if f.PollFailureThreshold != nil { + c.PollFailureThreshold = f.PollFailureThreshold } - if f.pollInterval != nil { - c.pollInterval = f.pollInterval + if f.PollInterval != nil { + c.PollInterval = f.PollInterval } - if f.selectionMode != nil { - c.selectionMode = f.selectionMode + if f.SelectionMode != nil { + c.SelectionMode = f.SelectionMode } - if f.syncThreshold != nil { - c.syncThreshold = f.syncThreshold + if f.SyncThreshold != nil { + c.SyncThreshold = f.SyncThreshold } - if f.nodeIsSyncingEnabled != nil { - c.nodeIsSyncingEnabled = f.nodeIsSyncingEnabled + if f.NodeIsSyncingEnabled != nil { + c.NodeIsSyncingEnabled = f.NodeIsSyncingEnabled } - if f.leaseDuration != nil { - c.leaseDuration = f.leaseDuration + if f.LeaseDuration != nil { + c.LeaseDuration = f.LeaseDuration } - if f.finalizedBlockPollInterval != nil { - c.finalizedBlockPollInterval = f.finalizedBlockPollInterval + if f.FinalizedBlockPollInterval != nil { + c.FinalizedBlockPollInterval = f.FinalizedBlockPollInterval } - if f.enforceRepeatableRead != nil { - c.enforceRepeatableRead = f.enforceRepeatableRead + if f.EnforceRepeatableRead != nil { + c.EnforceRepeatableRead = f.EnforceRepeatableRead } - if f.deathDeclarationDelay != nil { - c.deathDeclarationDelay = f.deathDeclarationDelay + if f.DeathDeclarationDelay != nil { + c.DeathDeclarationDelay = f.DeathDeclarationDelay } // Chain Configs - if f.nodeNoNewHeadsThreshold != nil { - c.nodeNoNewHeadsThreshold = f.nodeNoNewHeadsThreshold + if f.NoNewHeadsThreshold != nil { + c.NoNewHeadsThreshold = f.NoNewHeadsThreshold } - if f.noNewFinalizedHeadsThreshold != nil { - c.noNewFinalizedHeadsThreshold = f.noNewFinalizedHeadsThreshold + if f.NoNewFinalizedHeadsThreshold != nil { + c.NoNewFinalizedHeadsThreshold = f.NoNewFinalizedHeadsThreshold } - if f.finalityDepth != nil { - c.finalityDepth = f.finalityDepth + if f.FinalityDepth != nil { + c.FinalityDepth = f.FinalityDepth } - if f.finalityTagEnabled != nil { - c.finalityTagEnabled = f.finalityTagEnabled + if f.FinalityTagEnabled != nil { + c.FinalityTagEnabled = f.FinalityTagEnabled } - if f.finalizedBlockOffset != nil { - c.finalizedBlockOffset = f.finalizedBlockOffset + if f.FinalizedBlockOffset != nil { + c.FinalizedBlockOffset = f.FinalizedBlockOffset } } From ad70b8a1e6399cb230d9712ea2ddb0e24b034dd8 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 1 Oct 2024 11:22:44 -0400 Subject: [PATCH 039/174] Wrap multinode config --- pkg/solana/chain.go | 8 +- pkg/solana/config/multinode.go | 185 +++++++++++++++++---------------- pkg/solana/config/toml.go | 4 +- 3 files changed, 100 insertions(+), 97 deletions(-) diff --git a/pkg/solana/chain.go b/pkg/solana/chain.go index 212cfa7fc..6fae0af7d 100644 --- a/pkg/solana/chain.go +++ b/pkg/solana/chain.go @@ -230,7 +230,7 @@ func newChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr logger.L clientCache: map[string]*verifiedCachedClient{}, } - if cfg.MultiNode.MultiNodeEnabled() { + if cfg.MultiNodeConfig().Enabled() { chainFamily := "solana" mnCfg := cfg.MultiNodeConfig() @@ -398,7 +398,7 @@ func (c *chain) ChainID() string { // getClient returns a client, randomly selecting one from available and valid nodes func (c *chain) getClient() (client.ReaderWriter, error) { - if c.cfg.MultiNode.MultiNodeEnabled() { + if c.cfg.MultiNode.Enabled() { return c.multiNode.SelectRPC() } @@ -482,7 +482,7 @@ func (c *chain) Start(ctx context.Context) error { c.lggr.Debug("Starting balance monitor") var ms services.MultiStart startAll := []services.StartClose{c.txm, c.balanceMonitor} - if c.cfg.MultiNode.MultiNodeEnabled() { + if c.cfg.MultiNode.Enabled() { c.lggr.Debug("Starting multinode") startAll = append(startAll, c.multiNode, c.txSender) } @@ -496,7 +496,7 @@ func (c *chain) Close() error { c.lggr.Debug("Stopping txm") c.lggr.Debug("Stopping balance monitor") closeAll := []io.Closer{c.txm, c.balanceMonitor} - if c.cfg.MultiNode.MultiNodeEnabled() { + if c.cfg.MultiNode.Enabled() { c.lggr.Debug("Stopping multinode") closeAll = append(closeAll, c.multiNode, c.txSender) } diff --git a/pkg/solana/config/multinode.go b/pkg/solana/config/multinode.go index 237df5e22..8c83d6a0f 100644 --- a/pkg/solana/config/multinode.go +++ b/pkg/solana/config/multinode.go @@ -1,13 +1,16 @@ package config import ( - "time" - "github.com/smartcontractkit/chainlink-common/pkg/config" - client "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" + "time" ) +// MultiNodeConfig is a wrapper to provide required functions while keeping configs Public +type MultiNodeConfig struct { + MultiNode +} + type MultiNode struct { // Feature flag Enabled *bool @@ -24,178 +27,178 @@ type MultiNode struct { DeathDeclarationDelay *config.Duration // Chain Configs - NoNewHeadsThreshold *config.Duration + NodeNoNewHeadsThreshold *config.Duration NoNewFinalizedHeadsThreshold *config.Duration FinalityDepth *uint32 FinalityTagEnabled *bool FinalizedBlockOffset *uint32 } -func (c *MultiNode) GetEnabled() bool { - return c.Enabled != nil && *c.Enabled +func (c *MultiNodeConfig) Enabled() bool { + return c.MultiNode.Enabled != nil && *c.MultiNode.Enabled } -func (c *MultiNode) GetPollFailureThreshold() uint32 { - return *c.PollFailureThreshold +func (c *MultiNodeConfig) PollFailureThreshold() uint32 { + return *c.MultiNode.PollFailureThreshold } -func (c *MultiNode) GetPollInterval() time.Duration { - return c.PollInterval.Duration() +func (c *MultiNodeConfig) PollInterval() time.Duration { + return c.MultiNode.PollInterval.Duration() } -func (c *MultiNode) GetSelectionMode() string { - return *c.SelectionMode +func (c *MultiNodeConfig) SelectionMode() string { + return *c.MultiNode.SelectionMode } -func (c *MultiNode) GetSyncThreshold() uint32 { - return *c.SyncThreshold +func (c *MultiNodeConfig) SyncThreshold() uint32 { + return *c.MultiNode.SyncThreshold } -func (c *MultiNode) GetNodeIsSyncingEnabled() bool { - return *c.NodeIsSyncingEnabled +func (c *MultiNodeConfig) NodeIsSyncingEnabled() bool { + return *c.MultiNode.NodeIsSyncingEnabled } -func (c *MultiNode) GetLeaseDuration() time.Duration { return c.LeaseDuration.Duration() } +func (c *MultiNodeConfig) LeaseDuration() time.Duration { return c.MultiNode.LeaseDuration.Duration() } -func (c *MultiNode) GetFinalizedBlockPollInterval() time.Duration { - return c.FinalizedBlockPollInterval.Duration() +func (c *MultiNodeConfig) FinalizedBlockPollInterval() time.Duration { + return c.MultiNode.FinalizedBlockPollInterval.Duration() } -func (c *MultiNode) GetEnforceRepeatableRead() bool { return *c.EnforceRepeatableRead } +func (c *MultiNodeConfig) EnforceRepeatableRead() bool { return *c.MultiNode.EnforceRepeatableRead } -func (c *MultiNode) GetDeathDeclarationDelay() time.Duration { - return c.DeathDeclarationDelay.Duration() +func (c *MultiNodeConfig) DeathDeclarationDelay() time.Duration { + return c.MultiNode.DeathDeclarationDelay.Duration() } -func (c *MultiNode) GetNoNewHeadsThreshold() time.Duration { - return c.NoNewHeadsThreshold.Duration() +func (c *MultiNodeConfig) NodeNoNewHeadsThreshold() time.Duration { + return c.MultiNode.NodeNoNewHeadsThreshold.Duration() } -func (c *MultiNode) GetNoNewFinalizedHeadsThreshold() time.Duration { - return c.NoNewFinalizedHeadsThreshold.Duration() +func (c *MultiNodeConfig) NoNewFinalizedHeadsThreshold() time.Duration { + return c.MultiNode.NoNewFinalizedHeadsThreshold.Duration() } -func (c *MultiNode) GetFinalityDepth() uint32 { return *c.FinalityDepth } +func (c *MultiNodeConfig) FinalityDepth() uint32 { return *c.MultiNode.FinalityDepth } -func (c *MultiNode) GetFinalityTagEnabled() bool { return *c.FinalityTagEnabled } +func (c *MultiNodeConfig) FinalityTagEnabled() bool { return *c.MultiNode.FinalityTagEnabled } -func (c *MultiNode) GetFinalizedBlockOffset() uint32 { return *c.FinalizedBlockOffset } +func (c *MultiNodeConfig) FinalizedBlockOffset() uint32 { return *c.MultiNode.FinalizedBlockOffset } -func (c *MultiNode) SetDefaults() { +func (c *MultiNodeConfig) SetDefaults() { // MultiNode is disabled as it's not fully implemented yet: BCFR-122 - if c.Enabled == nil { - c.Enabled = ptr(false) + if c.MultiNode.Enabled == nil { + c.MultiNode.Enabled = ptr(false) } /* Node Configs */ // Failure threshold for polling set to 5 to tolerate some polling failures before taking action. - if c.PollFailureThreshold == nil { - c.PollFailureThreshold = ptr(uint32(5)) + if c.MultiNode.PollFailureThreshold == nil { + c.MultiNode.PollFailureThreshold = ptr(uint32(5)) } // Poll interval is set to 10 seconds to ensure timely updates while minimizing resource usage. - if c.PollInterval == nil { - c.PollInterval = config.MustNewDuration(10 * time.Second) + if c.MultiNode.PollInterval == nil { + c.MultiNode.PollInterval = config.MustNewDuration(10 * time.Second) } // Selection mode defaults to priority level to enable using node priorities - if c.SelectionMode == nil { - c.SelectionMode = ptr(client.NodeSelectionModePriorityLevel) + if c.MultiNode.SelectionMode == nil { + c.MultiNode.SelectionMode = ptr(client.NodeSelectionModePriorityLevel) } // The sync threshold is set to 5 to allow for some flexibility in node synchronization before considering it out of sync. - if c.SyncThreshold == nil { - c.SyncThreshold = ptr(uint32(5)) + if c.MultiNode.SyncThreshold == nil { + c.MultiNode.SyncThreshold = ptr(uint32(5)) } // Lease duration is set to 1 minute by default to allow node locks for a reasonable amount of time. - if c.LeaseDuration == nil { - c.LeaseDuration = config.MustNewDuration(time.Minute) + if c.MultiNode.LeaseDuration == nil { + c.MultiNode.LeaseDuration = config.MustNewDuration(time.Minute) } // Node syncing is not relevant for Solana and is disabled by default. - if c.NodeIsSyncingEnabled == nil { - c.NodeIsSyncingEnabled = ptr(false) + if c.MultiNode.NodeIsSyncingEnabled == nil { + c.MultiNode.NodeIsSyncingEnabled = ptr(false) } // The finalized block polling interval is set to 5 seconds to ensure timely updates while minimizing resource usage. - if c.FinalizedBlockPollInterval == nil { - c.FinalizedBlockPollInterval = config.MustNewDuration(5 * time.Second) + if c.MultiNode.FinalizedBlockPollInterval == nil { + c.MultiNode.FinalizedBlockPollInterval = config.MustNewDuration(5 * time.Second) } // Repeatable read guarantee should be enforced by default. - if c.EnforceRepeatableRead == nil { - c.EnforceRepeatableRead = ptr(true) + if c.MultiNode.EnforceRepeatableRead == nil { + c.MultiNode.EnforceRepeatableRead = ptr(true) } // The delay before declaring a node dead is set to 10 seconds to give nodes time to recover from temporary issues. - if c.DeathDeclarationDelay == nil { - c.DeathDeclarationDelay = config.MustNewDuration(10 * time.Second) + if c.MultiNode.DeathDeclarationDelay == nil { + c.MultiNode.DeathDeclarationDelay = config.MustNewDuration(10 * time.Second) } /* Chain Configs */ // Threshold for no new heads is set to 10 seconds, assuming that heads should update at a reasonable pace. - if c.NoNewHeadsThreshold == nil { - c.NoNewHeadsThreshold = config.MustNewDuration(10 * time.Second) + if c.MultiNode.NodeNoNewHeadsThreshold == nil { + c.MultiNode.NodeNoNewHeadsThreshold = config.MustNewDuration(10 * time.Second) } // Similar to heads, finalized heads should be updated within 10 seconds. - if c.NoNewFinalizedHeadsThreshold == nil { - c.NoNewFinalizedHeadsThreshold = config.MustNewDuration(10 * time.Second) + if c.MultiNode.NoNewFinalizedHeadsThreshold == nil { + c.MultiNode.NoNewFinalizedHeadsThreshold = config.MustNewDuration(10 * time.Second) } // Finality tags are used in Solana and enabled by default. - if c.FinalityTagEnabled == nil { - c.FinalityTagEnabled = ptr(true) + if c.MultiNode.FinalityTagEnabled == nil { + c.MultiNode.FinalityTagEnabled = ptr(true) } // Finality depth will not be used since finality tags are enabled. - if c.FinalityDepth == nil { - c.FinalityDepth = ptr(uint32(0)) + if c.MultiNode.FinalityDepth == nil { + c.MultiNode.FinalityDepth = ptr(uint32(0)) } // Finalized block offset will not be used since finality tags are enabled. - if c.FinalizedBlockOffset == nil { - c.FinalizedBlockOffset = ptr(uint32(0)) + if c.MultiNode.FinalizedBlockOffset == nil { + c.MultiNode.FinalizedBlockOffset = ptr(uint32(0)) } } -func (c *MultiNode) SetFrom(f *MultiNode) { - if f.Enabled != nil { - c.Enabled = f.Enabled +func (c *MultiNodeConfig) SetFrom(f *MultiNodeConfig) { + if f.MultiNode.Enabled != nil { + c.MultiNode.Enabled = f.MultiNode.Enabled } // Node Configs - if f.PollFailureThreshold != nil { - c.PollFailureThreshold = f.PollFailureThreshold + if f.MultiNode.PollFailureThreshold != nil { + c.MultiNode.PollFailureThreshold = f.MultiNode.PollFailureThreshold } - if f.PollInterval != nil { - c.PollInterval = f.PollInterval + if f.MultiNode.PollInterval != nil { + c.MultiNode.PollInterval = f.MultiNode.PollInterval } - if f.SelectionMode != nil { - c.SelectionMode = f.SelectionMode + if f.MultiNode.SelectionMode != nil { + c.MultiNode.SelectionMode = f.MultiNode.SelectionMode } - if f.SyncThreshold != nil { - c.SyncThreshold = f.SyncThreshold + if f.MultiNode.SyncThreshold != nil { + c.MultiNode.SyncThreshold = f.MultiNode.SyncThreshold } - if f.NodeIsSyncingEnabled != nil { - c.NodeIsSyncingEnabled = f.NodeIsSyncingEnabled + if f.MultiNode.NodeIsSyncingEnabled != nil { + c.MultiNode.NodeIsSyncingEnabled = f.MultiNode.NodeIsSyncingEnabled } - if f.LeaseDuration != nil { - c.LeaseDuration = f.LeaseDuration + if f.MultiNode.LeaseDuration != nil { + c.MultiNode.LeaseDuration = f.MultiNode.LeaseDuration } - if f.FinalizedBlockPollInterval != nil { - c.FinalizedBlockPollInterval = f.FinalizedBlockPollInterval + if f.MultiNode.FinalizedBlockPollInterval != nil { + c.MultiNode.FinalizedBlockPollInterval = f.MultiNode.FinalizedBlockPollInterval } - if f.EnforceRepeatableRead != nil { - c.EnforceRepeatableRead = f.EnforceRepeatableRead + if f.MultiNode.EnforceRepeatableRead != nil { + c.MultiNode.EnforceRepeatableRead = f.MultiNode.EnforceRepeatableRead } - if f.DeathDeclarationDelay != nil { - c.DeathDeclarationDelay = f.DeathDeclarationDelay + if f.MultiNode.DeathDeclarationDelay != nil { + c.MultiNode.DeathDeclarationDelay = f.MultiNode.DeathDeclarationDelay } // Chain Configs - if f.NoNewHeadsThreshold != nil { - c.NoNewHeadsThreshold = f.NoNewHeadsThreshold + if f.MultiNode.NodeNoNewHeadsThreshold != nil { + c.MultiNode.NodeNoNewHeadsThreshold = f.MultiNode.NodeNoNewHeadsThreshold } - if f.NoNewFinalizedHeadsThreshold != nil { - c.NoNewFinalizedHeadsThreshold = f.NoNewFinalizedHeadsThreshold + if f.MultiNode.NoNewFinalizedHeadsThreshold != nil { + c.MultiNode.NoNewFinalizedHeadsThreshold = f.MultiNode.NoNewFinalizedHeadsThreshold } - if f.FinalityDepth != nil { - c.FinalityDepth = f.FinalityDepth + if f.MultiNode.FinalityDepth != nil { + c.MultiNode.FinalityDepth = f.MultiNode.FinalityDepth } - if f.FinalityTagEnabled != nil { - c.FinalityTagEnabled = f.FinalityTagEnabled + if f.MultiNode.FinalityTagEnabled != nil { + c.MultiNode.FinalityTagEnabled = f.MultiNode.FinalityTagEnabled } - if f.FinalizedBlockOffset != nil { - c.FinalizedBlockOffset = f.FinalizedBlockOffset + if f.MultiNode.FinalizedBlockOffset != nil { + c.MultiNode.FinalizedBlockOffset = f.MultiNode.FinalizedBlockOffset } } diff --git a/pkg/solana/config/toml.go b/pkg/solana/config/toml.go index 4875fc702..84e797c00 100644 --- a/pkg/solana/config/toml.go +++ b/pkg/solana/config/toml.go @@ -113,7 +113,7 @@ type TOMLConfig struct { // Do not access directly, use [IsEnabled] Enabled *bool Chain - MultiNode MultiNode + MultiNode MultiNodeConfig Nodes Nodes } @@ -286,7 +286,7 @@ func (c *TOMLConfig) ListNodes() Nodes { return c.Nodes } -func (c *TOMLConfig) MultiNodeConfig() *MultiNode { +func (c *TOMLConfig) MultiNodeConfig() *MultiNodeConfig { return &c.MultiNode } From e628945cd39574e32bc18af51a67d91565b80b71 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 1 Oct 2024 13:58:27 -0400 Subject: [PATCH 040/174] Fix imports --- .golangci.yml | 2 +- pkg/solana/config/multinode.go | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index d0542f400..6744196f9 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -3,7 +3,7 @@ run: linters: enable: - exhaustive - - exportloopref + - copyloopvar - revive - goimports - gosec diff --git a/pkg/solana/config/multinode.go b/pkg/solana/config/multinode.go index 8c83d6a0f..0c49d8b22 100644 --- a/pkg/solana/config/multinode.go +++ b/pkg/solana/config/multinode.go @@ -1,9 +1,11 @@ package config import ( - "github.com/smartcontractkit/chainlink-common/pkg/config" - client "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" "time" + + "github.com/smartcontractkit/chainlink-common/pkg/config" + + mn "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" ) // MultiNodeConfig is a wrapper to provide required functions while keeping configs Public @@ -101,7 +103,7 @@ func (c *MultiNodeConfig) SetDefaults() { } // Selection mode defaults to priority level to enable using node priorities if c.MultiNode.SelectionMode == nil { - c.MultiNode.SelectionMode = ptr(client.NodeSelectionModePriorityLevel) + c.MultiNode.SelectionMode = ptr(mn.NodeSelectionModePriorityLevel) } // The sync threshold is set to 5 to allow for some flexibility in node synchronization before considering it out of sync. if c.MultiNode.SyncThreshold == nil { From 3b497efa2d3dd5e45c607693720d0e455f8b3d05 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 1 Oct 2024 14:09:11 -0400 Subject: [PATCH 041/174] Update .golangci.yml --- .golangci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.golangci.yml b/.golangci.yml index 6744196f9..d0542f400 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -3,7 +3,7 @@ run: linters: enable: - exhaustive - - copyloopvar + - exportloopref - revive - goimports - gosec From 83dced8e188351c36e62d1c3f9ae23e36e803f46 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 2 Oct 2024 10:18:14 -0400 Subject: [PATCH 042/174] Use MultiNode --- pkg/solana/chain.go | 5 +++- pkg/solana/chain_test.go | 16 +++++------ pkg/solana/client/client.go | 4 +-- pkg/solana/client/client_test.go | 4 ++- pkg/solana/config/toml.go | 2 +- pkg/solana/monitor/balance.go | 20 +++++++++----- pkg/solana/txm/txm.go | 46 ++++++++++++++++++++++---------- 7 files changed, 63 insertions(+), 34 deletions(-) diff --git a/pkg/solana/chain.go b/pkg/solana/chain.go index 6fae0af7d..63cfb76ee 100644 --- a/pkg/solana/chain.go +++ b/pkg/solana/chain.go @@ -256,6 +256,7 @@ func newChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr logger.L } } + // TODO: Should this be *clinet.ReaderWriter instead of Client? And Client IS a ReaderWriter multiNode := mn.NewMultiNode[mn.StringID, *client.Client]( lggr, mnCfg.SelectionMode(), @@ -292,7 +293,8 @@ func newChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr logger.L tc := func() (client.ReaderWriter, error) { return ch.getClient() } - ch.txm = txm.NewTxm(ch.id, tc, cfg, ks, lggr) + // TODO: Pass MultiNode config as txm needs to know if multinode is enabled + ch.txm = txm.NewTxm(ch.id, tc, cfg, ch.multiNode, ks, lggr) bc := func() (monitor.BalanceClient, error) { return ch.getClient() } @@ -397,6 +399,7 @@ func (c *chain) ChainID() string { } // getClient returns a client, randomly selecting one from available and valid nodes +// If multinode is enabled, it will return a client using the multinode selection instead. func (c *chain) getClient() (client.ReaderWriter, error) { if c.cfg.MultiNode.Enabled() { return c.multiNode.SelectRPC() diff --git a/pkg/solana/chain_test.go b/pkg/solana/chain_test.go index 1d2b85a5c..9b028f2d1 100644 --- a/pkg/solana/chain_test.go +++ b/pkg/solana/chain_test.go @@ -55,8 +55,8 @@ func TestSolanaChain_GetClient(t *testing.T) { ch := solcfg.Chain{} ch.SetDefaults() - mn := solcfg.MultiNode{} - mn.SetDefaults(false) + mn := solcfg.MultiNodeConfig{} + mn.SetDefaults() cfg := &solcfg.TOMLConfig{ ChainID: ptr("devnet"), Chain: ch, @@ -142,8 +142,8 @@ func TestSolanaChain_MultiNode_GetClient(t *testing.T) { ch := solcfg.Chain{} ch.SetDefaults() - mn := solcfg.MultiNode{} - mn.SetDefaults(true) + mn := solcfg.MultiNodeConfig{} + mn.SetDefaults() cfg := &solcfg.TOMLConfig{ ChainID: ptr("devnet"), @@ -202,8 +202,8 @@ func TestSolanaChain_VerifiedClient(t *testing.T) { ch := solcfg.Chain{} ch.SetDefaults() - mn := solcfg.MultiNode{} - mn.SetDefaults(false) + mn := solcfg.MultiNodeConfig{} + mn.SetDefaults() cfg := &solcfg.TOMLConfig{ ChainID: ptr("devnet"), Chain: ch, @@ -252,8 +252,8 @@ func TestSolanaChain_VerifiedClient_ParallelClients(t *testing.T) { ch := solcfg.Chain{} ch.SetDefaults() - mn := solcfg.MultiNode{} - mn.SetDefaults(false) + mn := solcfg.MultiNodeConfig{} + mn.SetDefaults() cfg := &solcfg.TOMLConfig{ ChainID: ptr("devnet"), Enabled: ptr(true), diff --git a/pkg/solana/client/client.go b/pkg/solana/client/client.go index d8792518e..b98332d79 100644 --- a/pkg/solana/client/client.go +++ b/pkg/solana/client/client.go @@ -238,7 +238,7 @@ func (c *Client) Ping(ctx context.Context) error { } func (c *Client) IsSyncing(ctx context.Context) (bool, error) { - // Not relevant for Solana + // Not in use for Solana return false, nil } @@ -288,7 +288,7 @@ func (c *Client) GetInterceptedChainInfo() (latest, highestUserObservations mn.C } func (c *Client) SendTransaction(ctx context.Context, tx *solana.Transaction) error { - // TODO: Is this all we need to do here? + // TODO: Use Transaction Sender _, err := c.SendTx(ctx, tx) return err } diff --git a/pkg/solana/client/client_test.go b/pkg/solana/client/client_test.go index f1e1cb3ec..b392353c6 100644 --- a/pkg/solana/client/client_test.go +++ b/pkg/solana/client/client_test.go @@ -309,7 +309,9 @@ func TestClient_Subscriptions_Integration(t *testing.T) { lggr := logger.Test(t) cfg := config.NewDefault() // Enable MultiNode - cfg.MultiNode.SetDefaults(true) + enabled := true + cfg.MultiNode.SetDefaults() + cfg.Enabled = &enabled c, err := NewClient(url, cfg, requestTimeout, lggr) require.NoError(t, err) diff --git a/pkg/solana/config/toml.go b/pkg/solana/config/toml.go index 35f870257..84e797c00 100644 --- a/pkg/solana/config/toml.go +++ b/pkg/solana/config/toml.go @@ -298,6 +298,6 @@ func (c *TOMLConfig) SetDefaults() { func NewDefault() *TOMLConfig { cfg := &TOMLConfig{} cfg.Chain.SetDefaults() - cfg.MultiNode.SetDefaults(false) + cfg.MultiNode.SetDefaults() return cfg } diff --git a/pkg/solana/monitor/balance.go b/pkg/solana/monitor/balance.go index 00f873488..b43aa87eb 100644 --- a/pkg/solana/monitor/balance.go +++ b/pkg/solana/monitor/balance.go @@ -99,16 +99,22 @@ func (b *balanceMonitor) monitor() { } } +// TODO: Use MultiNode for selection if enabled // getReader returns the cached solanaClient.Reader, or creates a new one if nil. func (b *balanceMonitor) getReader() (BalanceClient, error) { - if b.reader == nil { - var err error - b.reader, err = b.newReader() - if err != nil { - return nil, err + // TODO: Use MultiNode for selection if enabled + return b.newReader() + + /* + if b.reader == nil { + var err error + b.reader, err = b.newReader() + if err != nil { + return nil, err + } } - } - return b.reader, nil + return b.reader, nil + */ } func (b *balanceMonitor) updateBalances(ctx context.Context) { diff --git a/pkg/solana/txm/txm.go b/pkg/solana/txm/txm.go index 87861fd83..4c6dfd7db 100644 --- a/pkg/solana/txm/txm.go +++ b/pkg/solana/txm/txm.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + mn "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" "strings" "sync" "time" @@ -52,6 +53,9 @@ type Txm struct { ks SimpleKeystore client *utils.LazyLoad[client.ReaderWriter] fee fees.Estimator + + // Use multiNode for client selection if not nil + multiNode *mn.MultiNode[mn.StringID, *client.Client] } type TxConfig struct { @@ -74,16 +78,17 @@ type pendingTx struct { } // NewTxm creates a txm. Uses simulation so should only be used to send txes to trusted contracts i.e. OCR. -func NewTxm(chainID string, tc func() (client.ReaderWriter, error), cfg config.Config, ks SimpleKeystore, lggr logger.Logger) *Txm { +func NewTxm(chainID string, tc func() (client.ReaderWriter, error), cfg config.Config, multiNode *mn.MultiNode[mn.StringID, *client.Client], ks SimpleKeystore, lggr logger.Logger) *Txm { return &Txm{ - lggr: lggr, - chSend: make(chan pendingTx, MaxQueueLen), // queue can support 1000 pending txs - chSim: make(chan pendingTx, MaxQueueLen), // queue can support 1000 pending txs - chStop: make(chan struct{}), - cfg: cfg, - txs: newPendingTxContextWithProm(chainID), - ks: ks, - client: utils.NewLazyLoad(tc), + lggr: lggr, + chSend: make(chan pendingTx, MaxQueueLen), // queue can support 1000 pending txs + chSim: make(chan pendingTx, MaxQueueLen), // queue can support 1000 pending txs + chStop: make(chan struct{}), + cfg: cfg, + txs: newPendingTxContextWithProm(chainID), + ks: ks, + multiNode: multiNode, + client: utils.NewLazyLoad(tc), } } @@ -115,6 +120,17 @@ func (txm *Txm) Start(ctx context.Context) error { }) } +func (txm *Txm) multiNodeEnabled() bool { + return txm.multiNode != nil +} + +func (txm *Txm) getClient() (client.ReaderWriter, error) { + if txm.multiNodeEnabled() { + return txm.multiNode.SelectRPC() + } + return txm.client.Get() +} + func (txm *Txm) run() { defer txm.done.Done() ctx, cancel := txm.chStop.NewCtx() @@ -131,8 +147,10 @@ func (txm *Txm) run() { tx, id, sig, err := txm.sendWithRetry(ctx, *msg.tx, msg.cfg) if err != nil { txm.lggr.Errorw("failed to send transaction", "error", err) - txm.client.Reset() // clear client if tx fails immediately (potentially bad RPC) - continue // skip remainining + if !txm.multiNodeEnabled() { + txm.client.Reset() // clear client if tx fails immediately (potentially bad RPC) + } + continue // skip remainining } // send tx + signature to simulation queue @@ -154,7 +172,7 @@ func (txm *Txm) run() { func (txm *Txm) sendWithRetry(chanCtx context.Context, baseTx solanaGo.Transaction, txcfg TxConfig) (solanaGo.Transaction, uuid.UUID, solanaGo.Signature, error) { // fetch client - client, clientErr := txm.client.Get() + client, clientErr := txm.getClient() if clientErr != nil { return solanaGo.Transaction{}, uuid.Nil, solanaGo.Signature{}, fmt.Errorf("failed to get client in soltxm.sendWithRetry: %w", clientErr) } @@ -363,7 +381,7 @@ func (txm *Txm) confirm(ctx context.Context) { } // get client - client, err := txm.client.Get() + client, err := txm.getClient() if err != nil { txm.lggr.Errorw("failed to get client in soltxm.confirm", "error", err) break // exit switch @@ -477,7 +495,7 @@ func (txm *Txm) simulate(ctx context.Context) { return case msg := <-txm.chSim: // get client - client, err := txm.client.Get() + client, err := txm.getClient() if err != nil { txm.lggr.Errorw("failed to get client in soltxm.simulate", "error", err) continue From b5c3891d7ba1baa953ae0c47d92a11ae9b7015c6 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 2 Oct 2024 10:25:25 -0400 Subject: [PATCH 043/174] Add multinode to txm --- pkg/solana/txm/txm_internal_test.go | 4 ++-- pkg/solana/txm/txm_race_test.go | 2 +- pkg/solana/txm/txm_test.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/solana/txm/txm_internal_test.go b/pkg/solana/txm/txm_internal_test.go index adfa273f4..0d2645ec0 100644 --- a/pkg/solana/txm/txm_internal_test.go +++ b/pkg/solana/txm/txm_internal_test.go @@ -112,7 +112,7 @@ func TestTxm(t *testing.T) { txm := NewTxm(id, func() (client.ReaderWriter, error) { return mc, nil - }, cfg, mkey, lggr) + }, cfg, nil, mkey, lggr) require.NoError(t, txm.Start(ctx)) // tracking prom metrics @@ -718,7 +718,7 @@ func TestTxm_Enqueue(t *testing.T) { txm := NewTxm("enqueue_test", func() (client.ReaderWriter, error) { return mc, nil - }, cfg, mkey, lggr) + }, cfg, nil, mkey, lggr) require.ErrorContains(t, txm.Enqueue("txmUnstarted", &solana.Transaction{}), "not started") require.NoError(t, txm.Start(ctx)) diff --git a/pkg/solana/txm/txm_race_test.go b/pkg/solana/txm/txm_race_test.go index aa0a6de6a..acdfa4908 100644 --- a/pkg/solana/txm/txm_race_test.go +++ b/pkg/solana/txm/txm_race_test.go @@ -65,7 +65,7 @@ func TestTxm_SendWithRetry_Race(t *testing.T) { } // build minimal txm - txm := NewTxm("retry_race", getClient, cfg, ks, lggr) + txm := NewTxm("retry_race", getClient, cfg, nil, ks, lggr) txm.fee = fee _, _, _, err := txm.sendWithRetry( diff --git a/pkg/solana/txm/txm_test.go b/pkg/solana/txm/txm_test.go index 851aebf89..84d11df62 100644 --- a/pkg/solana/txm/txm_test.go +++ b/pkg/solana/txm/txm_test.go @@ -72,7 +72,7 @@ func TestTxm_Integration(t *testing.T) { getClient := func() (solanaClient.ReaderWriter, error) { return client, nil } - txm := txm.NewTxm("localnet", getClient, cfg, mkey, lggr) + txm := txm.NewTxm("localnet", getClient, cfg, nil, mkey, lggr) // track initial balance initBal, err := client.Balance(pubKey) From 8a2c117de08e14651b9745933abfd92d9d7eeb54 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 2 Oct 2024 10:49:30 -0400 Subject: [PATCH 044/174] Use MultiNode --- pkg/solana/chain.go | 3 +-- pkg/solana/monitor/balance.go | 50 ++++++++++++++++++----------------- pkg/solana/txm/txm.go | 29 +++++++++++--------- 3 files changed, 43 insertions(+), 39 deletions(-) diff --git a/pkg/solana/chain.go b/pkg/solana/chain.go index 63cfb76ee..76dc8761a 100644 --- a/pkg/solana/chain.go +++ b/pkg/solana/chain.go @@ -293,12 +293,11 @@ func newChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr logger.L tc := func() (client.ReaderWriter, error) { return ch.getClient() } - // TODO: Pass MultiNode config as txm needs to know if multinode is enabled ch.txm = txm.NewTxm(ch.id, tc, cfg, ch.multiNode, ks, lggr) bc := func() (monitor.BalanceClient, error) { return ch.getClient() } - ch.balanceMonitor = monitor.NewBalanceMonitor(ch.id, cfg, lggr, ks, bc) + ch.balanceMonitor = monitor.NewBalanceMonitor(ch.id, cfg, cfg.MultiNode.Enabled(), lggr, ks, bc) return &ch, nil } diff --git a/pkg/solana/monitor/balance.go b/pkg/solana/monitor/balance.go index b43aa87eb..d1f3b6de5 100644 --- a/pkg/solana/monitor/balance.go +++ b/pkg/solana/monitor/balance.go @@ -26,19 +26,20 @@ type BalanceClient interface { } // NewBalanceMonitor returns a balance monitoring services.Service which reports the SOL balance of all ks keys to prometheus. -func NewBalanceMonitor(chainID string, cfg Config, lggr logger.Logger, ks Keystore, newReader func() (BalanceClient, error)) services.Service { - return newBalanceMonitor(chainID, cfg, lggr, ks, newReader) +func NewBalanceMonitor(chainID string, cfg Config, multiNodeEnabled bool, lggr logger.Logger, ks Keystore, newReader func() (BalanceClient, error)) services.Service { + return newBalanceMonitor(chainID, cfg, multiNodeEnabled, lggr, ks, newReader) } -func newBalanceMonitor(chainID string, cfg Config, lggr logger.Logger, ks Keystore, newReader func() (BalanceClient, error)) *balanceMonitor { +func newBalanceMonitor(chainID string, cfg Config, multiNodeEnabled bool, lggr logger.Logger, ks Keystore, newReader func() (BalanceClient, error)) *balanceMonitor { b := balanceMonitor{ - chainID: chainID, - cfg: cfg, - lggr: logger.Named(lggr, "BalanceMonitor"), - ks: ks, - newReader: newReader, - stop: make(chan struct{}), - done: make(chan struct{}), + chainID: chainID, + cfg: cfg, + lggr: logger.Named(lggr, "BalanceMonitor"), + ks: ks, + newReader: newReader, + multiNodeEnabled: multiNodeEnabled, + stop: make(chan struct{}), + done: make(chan struct{}), } b.updateFn = b.updateProm return &b @@ -53,7 +54,8 @@ type balanceMonitor struct { newReader func() (BalanceClient, error) updateFn func(acc solana.PublicKey, lamports uint64) // overridable for testing - reader BalanceClient + multiNodeEnabled bool + reader BalanceClient stop services.StopChan done chan struct{} @@ -99,22 +101,22 @@ func (b *balanceMonitor) monitor() { } } -// TODO: Use MultiNode for selection if enabled // getReader returns the cached solanaClient.Reader, or creates a new one if nil. func (b *balanceMonitor) getReader() (BalanceClient, error) { - // TODO: Use MultiNode for selection if enabled - return b.newReader() - - /* - if b.reader == nil { - var err error - b.reader, err = b.newReader() - if err != nil { - return nil, err - } + if b.multiNodeEnabled { + // Allow MultiNode to select the reader + return b.newReader() + } + + // Self leasing wth cached reader + if b.reader == nil { + var err error + b.reader, err = b.newReader() + if err != nil { + return nil, err } - return b.reader, nil - */ + } + return b.reader, nil } func (b *balanceMonitor) updateBalances(ctx context.Context) { diff --git a/pkg/solana/txm/txm.go b/pkg/solana/txm/txm.go index 4c6dfd7db..27e8988d3 100644 --- a/pkg/solana/txm/txm.go +++ b/pkg/solana/txm/txm.go @@ -51,11 +51,13 @@ type Txm struct { cfg config.Config txs PendingTxContext ks SimpleKeystore - client *utils.LazyLoad[client.ReaderWriter] fee fees.Estimator - // Use multiNode for client selection if not nil + // Use multiNode for client selection if set multiNode *mn.MultiNode[mn.StringID, *client.Client] + + // If multiNode is disabled, use lazy load to fetch client + client *utils.LazyLoad[client.ReaderWriter] } type TxConfig struct { @@ -92,6 +94,18 @@ func NewTxm(chainID string, tc func() (client.ReaderWriter, error), cfg config.C } } +func (txm *Txm) multiNodeEnabled() bool { + return txm.multiNode != nil +} + +// getClient returns a client selected by multiNode if enabled, otherwise returns a client from the lazy load +func (txm *Txm) getClient() (client.ReaderWriter, error) { + if txm.multiNodeEnabled() { + return txm.multiNode.SelectRPC() + } + return txm.client.Get() +} + // Start subscribes to queuing channel and processes them. func (txm *Txm) Start(ctx context.Context) error { return txm.starter.StartOnce("solana_txm", func() error { @@ -120,17 +134,6 @@ func (txm *Txm) Start(ctx context.Context) error { }) } -func (txm *Txm) multiNodeEnabled() bool { - return txm.multiNode != nil -} - -func (txm *Txm) getClient() (client.ReaderWriter, error) { - if txm.multiNodeEnabled() { - return txm.multiNode.SelectRPC() - } - return txm.client.Get() -} - func (txm *Txm) run() { defer txm.done.Done() ctx, cancel := txm.chStop.NewCtx() From 284e90ba316d3dfa3e70454f85742a0a275ec738 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 2 Oct 2024 10:50:36 -0400 Subject: [PATCH 045/174] Update chain.go --- pkg/solana/chain.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/solana/chain.go b/pkg/solana/chain.go index 76dc8761a..4d90351be 100644 --- a/pkg/solana/chain.go +++ b/pkg/solana/chain.go @@ -256,7 +256,6 @@ func newChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr logger.L } } - // TODO: Should this be *clinet.ReaderWriter instead of Client? And Client IS a ReaderWriter multiNode := mn.NewMultiNode[mn.StringID, *client.Client]( lggr, mnCfg.SelectionMode(), From 4d72403f60edbf0f247f7a9c4bb1ce5740c509ff Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 2 Oct 2024 11:00:31 -0400 Subject: [PATCH 046/174] Update balance_test.go --- pkg/solana/monitor/balance_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/solana/monitor/balance_test.go b/pkg/solana/monitor/balance_test.go index ff98d0508..a28855534 100644 --- a/pkg/solana/monitor/balance_test.go +++ b/pkg/solana/monitor/balance_test.go @@ -46,7 +46,7 @@ func TestBalanceMonitor(t *testing.T) { exp = append(exp, update{acc.String(), expBals[i]}) } cfg := &config{balancePollPeriod: time.Second} - b := newBalanceMonitor(chainID, cfg, logger.Test(t), ks, nil) + b := newBalanceMonitor(chainID, cfg, false, logger.Test(t), ks, nil) var got []update done := make(chan struct{}) b.updateFn = func(acc solana.PublicKey, lamports uint64) { From df8a3563f4a29d304982b8c24adbfbdc68de76b9 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 2 Oct 2024 12:02:06 -0400 Subject: [PATCH 047/174] Add retries --- pkg/solana/chain_test.go | 6 +++++- pkg/solana/client/client.go | 41 +++++++++++++++++++++++++++++++------ 2 files changed, 40 insertions(+), 7 deletions(-) diff --git a/pkg/solana/chain_test.go b/pkg/solana/chain_test.go index 9b028f2d1..6e333c608 100644 --- a/pkg/solana/chain_test.go +++ b/pkg/solana/chain_test.go @@ -142,7 +142,11 @@ func TestSolanaChain_MultiNode_GetClient(t *testing.T) { ch := solcfg.Chain{} ch.SetDefaults() - mn := solcfg.MultiNodeConfig{} + mn := solcfg.MultiNodeConfig{ + MultiNode: solcfg.MultiNode{ + Enabled: ptr(true), + }, + } mn.SetDefaults() cfg := &solcfg.TOMLConfig{ diff --git a/pkg/solana/client/client.go b/pkg/solana/client/client.go index b98332d79..1195b86fc 100644 --- a/pkg/solana/client/client.go +++ b/pkg/solana/client/client.go @@ -156,19 +156,48 @@ func (c *Client) SubscribeToFinalizedHeads(ctx context.Context) (<-chan *Head, m } func (c *Client) LatestBlock(ctx context.Context) (*Head, error) { + // capture chStopInFlight to ensure we are not updating chainInfo with observations related to previous life cycle + //ctx, cancel, chStopInFlight, _, _ := c.acquireQueryCtx(ctx, c.rpcTimeout) + latestBlockHeight, err := c.rpc.GetBlockHeight(ctx, rpc.CommitmentConfirmed) if err != nil { return nil, err } - block, err := c.rpc.GetBlock(ctx, latestBlockHeight) - if err != nil { - return nil, err + // TODO: Trying to see if retries will fix testing issue + retries := 5 // Number of retries + for i := 0; i < retries; i++ { + block, err := c.rpc.GetBlock(ctx, latestBlockHeight) + if err == nil { + head := &Head{GetBlockResult: *block} + c.onNewHead(ctx, c.chStopInFlight, head) + return head, nil + } + + // Log the error or handle as needed + fmt.Printf("Error fetching block: %v\n", err) + + // Retry after a short delay + time.Sleep(2 * time.Second) } - head := &Head{GetBlockResult: *block} - c.onNewHead(ctx, c.chStopInFlight, head) - return head, nil + return nil, fmt.Errorf("failed to fetch block after %d retries", retries) + + /* + latestBlockHeight, err := c.rpc.GetBlockHeight(ctx, rpc.CommitmentConfirmed) + if err != nil { + return nil, err + } + + block, err := c.rpc.GetBlock(ctx, latestBlockHeight) + if err != nil { + return nil, err + } + + head := &Head{GetBlockResult: *block} + c.onNewHead(ctx, c.chStopInFlight, head) + return head, nil + */ } func (c *Client) LatestFinalizedBlock(ctx context.Context) (*Head, error) { From fe2f29197aec5d3dfcbcdd78d82e7d939bd8b0a2 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 2 Oct 2024 12:35:46 -0400 Subject: [PATCH 048/174] Fix head --- pkg/solana/client/client.go | 56 ++++++++------------------------ pkg/solana/client/client_test.go | 4 +-- 2 files changed, 15 insertions(+), 45 deletions(-) diff --git a/pkg/solana/client/client.go b/pkg/solana/client/client.go index 1195b86fc..32454a82c 100644 --- a/pkg/solana/client/client.go +++ b/pkg/solana/client/client.go @@ -103,7 +103,8 @@ func NewClient(endpoint string, cfg *config.TOMLConfig, requestTimeout time.Dura } type Head struct { - rpc.GetBlockResult + BlockHeight *uint64 + BlockHash *solana.Hash } func (h *Head) BlockNumber() int64 { @@ -121,7 +122,7 @@ func (h *Head) BlockDifficulty() *big.Int { } func (h *Head) IsValid() bool { - return h.BlockHeight != nil + return h.BlockHeight != nil && h.BlockHash != nil } var _ mn.RPCClient[mn.StringID, *Head] = (*Client)(nil) @@ -158,46 +159,17 @@ func (c *Client) SubscribeToFinalizedHeads(ctx context.Context) (<-chan *Head, m func (c *Client) LatestBlock(ctx context.Context) (*Head, error) { // capture chStopInFlight to ensure we are not updating chainInfo with observations related to previous life cycle //ctx, cancel, chStopInFlight, _, _ := c.acquireQueryCtx(ctx, c.rpcTimeout) - - latestBlockHeight, err := c.rpc.GetBlockHeight(ctx, rpc.CommitmentConfirmed) + result, err := c.rpc.GetLatestBlockhash(ctx, rpc.CommitmentConfirmed) if err != nil { return nil, err } - // TODO: Trying to see if retries will fix testing issue - retries := 5 // Number of retries - for i := 0; i < retries; i++ { - block, err := c.rpc.GetBlock(ctx, latestBlockHeight) - if err == nil { - head := &Head{GetBlockResult: *block} - c.onNewHead(ctx, c.chStopInFlight, head) - return head, nil - } - - // Log the error or handle as needed - fmt.Printf("Error fetching block: %v\n", err) - - // Retry after a short delay - time.Sleep(2 * time.Second) + head := &Head{ + BlockHeight: &result.Value.LastValidBlockHeight, + BlockHash: &result.Value.Blockhash, } - - return nil, fmt.Errorf("failed to fetch block after %d retries", retries) - - /* - latestBlockHeight, err := c.rpc.GetBlockHeight(ctx, rpc.CommitmentConfirmed) - if err != nil { - return nil, err - } - - block, err := c.rpc.GetBlock(ctx, latestBlockHeight) - if err != nil { - return nil, err - } - - head := &Head{GetBlockResult: *block} - c.onNewHead(ctx, c.chStopInFlight, head) - return head, nil - */ + c.onNewHead(ctx, c.chStopInFlight, head) + return head, nil } func (c *Client) LatestFinalizedBlock(ctx context.Context) (*Head, error) { @@ -205,17 +177,15 @@ func (c *Client) LatestFinalizedBlock(ctx context.Context) (*Head, error) { // capture chStopInFlight to ensure we are not updating chainInfo with observations related to previous life cycle //ctx, cancel, chStopInFlight, _, _ := c.acquireQueryCtx(ctx, c.rpcTimeout) - finalizedBlockHeight, err := c.rpc.GetBlockHeight(ctx, rpc.CommitmentFinalized) + result, err := c.rpc.GetLatestBlockhash(ctx, rpc.CommitmentFinalized) if err != nil { return nil, err } - block, err := c.rpc.GetBlock(ctx, finalizedBlockHeight) - if err != nil { - return nil, err + head := &Head{ + BlockHeight: &result.Value.LastValidBlockHeight, + BlockHash: &result.Value.Blockhash, } - - head := &Head{GetBlockResult: *block} c.onNewFinalizedHead(ctx, c.chStopInFlight, head) return head, nil } diff --git a/pkg/solana/client/client_test.go b/pkg/solana/client/client_test.go index b392353c6..5d0c441e4 100644 --- a/pkg/solana/client/client_test.go +++ b/pkg/solana/client/client_test.go @@ -333,7 +333,7 @@ func TestClient_Subscriptions_Integration(t *testing.T) { select { case head := <-ch: - require.NotEqual(t, solana.Hash{}, head.Blockhash) + require.NotEqual(t, solana.Hash{}, head.BlockHash) latest, _ := c.GetInterceptedChainInfo() require.Equal(t, head.BlockNumber(), latest.BlockNumber) case <-ctx.Done(): @@ -342,7 +342,7 @@ func TestClient_Subscriptions_Integration(t *testing.T) { select { case finalizedHead := <-finalizedCh: - require.NotEqual(t, solana.Hash{}, finalizedHead.Blockhash) + require.NotEqual(t, solana.Hash{}, finalizedHead.BlockHash) latest, _ := c.GetInterceptedChainInfo() require.Equal(t, finalizedHead.BlockNumber(), latest.FinalizedBlockNumber) case <-ctx.Done(): From 7bfb7000b525f1103970df160befd6d40aca0067 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 2 Oct 2024 12:50:17 -0400 Subject: [PATCH 049/174] Update client.go --- pkg/solana/client/client.go | 48 +++++++++++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/pkg/solana/client/client.go b/pkg/solana/client/client.go index 32454a82c..7a5e0d04f 100644 --- a/pkg/solana/client/client.go +++ b/pkg/solana/client/client.go @@ -8,14 +8,14 @@ import ( "sync" "time" - mn "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" - "github.com/gagliardetto/solana-go" "github.com/gagliardetto/solana-go/rpc" "golang.org/x/sync/singleflight" "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/services" + mn "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" "github.com/smartcontractkit/chainlink-solana/pkg/solana/config" "github.com/smartcontractkit/chainlink-solana/pkg/solana/monitor" ) @@ -158,8 +158,10 @@ func (c *Client) SubscribeToFinalizedHeads(ctx context.Context) (<-chan *Head, m func (c *Client) LatestBlock(ctx context.Context) (*Head, error) { // capture chStopInFlight to ensure we are not updating chainInfo with observations related to previous life cycle - //ctx, cancel, chStopInFlight, _, _ := c.acquireQueryCtx(ctx, c.rpcTimeout) - result, err := c.rpc.GetLatestBlockhash(ctx, rpc.CommitmentConfirmed) + ctx, cancel, chStopInFlight, rawRPC := c.acquireQueryCtx(ctx, c.contextDuration) + defer cancel() + + result, err := rawRPC.GetLatestBlockhash(ctx, rpc.CommitmentConfirmed) if err != nil { return nil, err } @@ -168,16 +170,15 @@ func (c *Client) LatestBlock(ctx context.Context) (*Head, error) { BlockHeight: &result.Value.LastValidBlockHeight, BlockHash: &result.Value.Blockhash, } - c.onNewHead(ctx, c.chStopInFlight, head) + c.onNewHead(ctx, chStopInFlight, head) return head, nil } func (c *Client) LatestFinalizedBlock(ctx context.Context) (*Head, error) { - // TODO: Do we need this? - // capture chStopInFlight to ensure we are not updating chainInfo with observations related to previous life cycle - //ctx, cancel, chStopInFlight, _, _ := c.acquireQueryCtx(ctx, c.rpcTimeout) + ctx, cancel, chStopInFlight, rawRPC := c.acquireQueryCtx(ctx, c.contextDuration) + defer cancel() - result, err := c.rpc.GetLatestBlockhash(ctx, rpc.CommitmentFinalized) + result, err := rawRPC.GetLatestBlockhash(ctx, rpc.CommitmentFinalized) if err != nil { return nil, err } @@ -186,7 +187,7 @@ func (c *Client) LatestFinalizedBlock(ctx context.Context) (*Head, error) { BlockHeight: &result.Value.LastValidBlockHeight, BlockHash: &result.Value.Blockhash, } - c.onNewFinalizedHead(ctx, c.chStopInFlight, head) + c.onNewFinalizedHead(ctx, chStopInFlight, head) return head, nil } @@ -227,6 +228,33 @@ func (c *Client) onNewFinalizedHead(ctx context.Context, requestCh <-chan struct } } +// makeQueryCtx returns a context that cancels if: +// 1. Passed in ctx cancels +// 2. Passed in channel is closed +// 3. Default timeout is reached (queryTimeout) +func makeQueryCtx(ctx context.Context, ch services.StopChan, timeout time.Duration) (context.Context, context.CancelFunc) { + var chCancel, timeoutCancel context.CancelFunc + ctx, chCancel = ch.Ctx(ctx) + ctx, timeoutCancel = context.WithTimeout(ctx, timeout) + cancel := func() { + chCancel() + timeoutCancel() + } + return ctx, cancel +} + +func (c *Client) acquireQueryCtx(parentCtx context.Context, timeout time.Duration) (ctx context.Context, cancel context.CancelFunc, + chStopInFlight chan struct{}, raw *rpc.Client) { + // Need to wrap in mutex because state transition can cancel and replace context + c.stateMu.RLock() + chStopInFlight = c.chStopInFlight + cp := *c.rpc + raw = &cp + c.stateMu.RUnlock() + ctx, cancel = makeQueryCtx(parentCtx, chStopInFlight, timeout) + return +} + func (c *Client) Ping(ctx context.Context) error { version, err := c.rpc.GetVersion(ctx) if err != nil { From 523947ee037b2109f710523e1267655eb17b46af Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 2 Oct 2024 14:41:49 -0400 Subject: [PATCH 050/174] lint --- pkg/solana/chain_test.go | 8 ++++++-- pkg/solana/client/client.go | 1 + pkg/solana/txm/txm.go | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/pkg/solana/chain_test.go b/pkg/solana/chain_test.go index 6e333c608..1b4cc4e71 100644 --- a/pkg/solana/chain_test.go +++ b/pkg/solana/chain_test.go @@ -168,8 +168,12 @@ func TestSolanaChain_MultiNode_GetClient(t *testing.T) { testChain, err := newChain("devnet", cfg, nil, logger.Test(t)) require.NoError(t, err) - err = testChain.multiNode.Start(tests.Context(t)) - assert.NoError(t, err) + err = testChain.Start(tests.Context(t)) + require.NoError(t, err) + defer func() { + err := testChain.Close() + require.NoError(t, err) + }() selectedClient, err := testChain.getClient() assert.NoError(t, err) diff --git a/pkg/solana/client/client.go b/pkg/solana/client/client.go index 7a5e0d04f..aab12942b 100644 --- a/pkg/solana/client/client.go +++ b/pkg/solana/client/client.go @@ -99,6 +99,7 @@ func NewClient(endpoint string, cfg *config.TOMLConfig, requestTimeout time.Dura requestGroup: &singleflight.Group{}, pollInterval: cfg.MultiNode.PollInterval(), finalizedBlockPollInterval: cfg.MultiNode.FinalizedBlockPollInterval(), + chStopInFlight: make(chan struct{}), }, nil } diff --git a/pkg/solana/txm/txm.go b/pkg/solana/txm/txm.go index 27e8988d3..2a427ffed 100644 --- a/pkg/solana/txm/txm.go +++ b/pkg/solana/txm/txm.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - mn "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" "strings" "sync" "time" @@ -19,6 +18,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/utils" "github.com/smartcontractkit/chainlink-solana/pkg/solana/client" + mn "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" "github.com/smartcontractkit/chainlink-solana/pkg/solana/config" "github.com/smartcontractkit/chainlink-solana/pkg/solana/fees" ) From 05625a80a98d338848ac785b4be96f9af4fd0be5 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 2 Oct 2024 14:54:00 -0400 Subject: [PATCH 051/174] lint --- pkg/solana/chain_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/solana/chain_test.go b/pkg/solana/chain_test.go index 1b4cc4e71..06390e0c3 100644 --- a/pkg/solana/chain_test.go +++ b/pkg/solana/chain_test.go @@ -171,8 +171,8 @@ func TestSolanaChain_MultiNode_GetClient(t *testing.T) { err = testChain.Start(tests.Context(t)) require.NoError(t, err) defer func() { - err := testChain.Close() - require.NoError(t, err) + closeErr := testChain.Close() + require.NoError(t, closeErr) }() selectedClient, err := testChain.getClient() From 0a7e9b64d6c9ba5b17ee6c3e324ef0ff3e17ba3e Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 2 Oct 2024 16:12:36 -0400 Subject: [PATCH 052/174] Use MultiNode TxSender --- pkg/solana/chain.go | 14 +-- pkg/solana/classify_errors.go | 100 ++++++++++++++++++ pkg/solana/client/client.go | 12 ++- .../client/multinode/transaction_sender.go | 69 ++++++------ pkg/solana/txm/txm.go | 37 +++++-- pkg/solana/txm/txm_internal_test.go | 4 +- pkg/solana/txm/txm_race_test.go | 2 +- pkg/solana/txm/txm_test.go | 2 +- 8 files changed, 177 insertions(+), 63 deletions(-) create mode 100644 pkg/solana/classify_errors.go diff --git a/pkg/solana/chain.go b/pkg/solana/chain.go index 4d90351be..fb87b1416 100644 --- a/pkg/solana/chain.go +++ b/pkg/solana/chain.go @@ -90,7 +90,7 @@ type chain struct { // if multiNode is enabled, the clientCache will not be used multiNode *mn.MultiNode[mn.StringID, *client.Client] - txSender *mn.TransactionSender[*solanago.Transaction, mn.StringID, *client.Client] + txSender *mn.TransactionSender[*solanago.Transaction, solanago.Signature, mn.StringID, *client.Client] // tracking node chain id for verification clientCache map[string]*verifiedCachedClient // map URL -> {client, chainId} [mainnet/testnet/devnet/localnet] @@ -267,18 +267,12 @@ func newChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr logger.L mnCfg.DeathDeclarationDelay(), ) - // TODO: implement error classification; move logic to separate file if large - // TODO: might be useful to reference anza-xyz/agave@master/sdk/src/transaction/error.rs - classifySendError := func(tx *solanago.Transaction, err error) mn.SendTxReturnCode { - return 0 // TODO ClassifySendError(err, clientErrors, logger.Sugared(logger.Nop()), tx, common.Address{}, false) - } - - txSender := mn.NewTransactionSender[*solanago.Transaction, mn.StringID, *client.Client]( + txSender := mn.NewTransactionSender[*solanago.Transaction, solanago.Signature, mn.StringID, *client.Client]( lggr, mn.StringID(id), chainFamily, multiNode, - classifySendError, + ClassifySendError, 0, // use the default value provided by the implementation ) @@ -292,7 +286,7 @@ func newChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr logger.L tc := func() (client.ReaderWriter, error) { return ch.getClient() } - ch.txm = txm.NewTxm(ch.id, tc, cfg, ch.multiNode, ks, lggr) + ch.txm = txm.NewTxm(ch.id, tc, cfg, ch.multiNode, ch.txSender, ks, lggr) bc := func() (monitor.BalanceClient, error) { return ch.getClient() } diff --git a/pkg/solana/classify_errors.go b/pkg/solana/classify_errors.go new file mode 100644 index 000000000..6c051a603 --- /dev/null +++ b/pkg/solana/classify_errors.go @@ -0,0 +1,100 @@ +package solana + +import ( + "github.com/gagliardetto/solana-go" + mn "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" + "strings" +) + +// ClassifySendError returns the corresponding SendTxReturnCode based on the error. +// Errors derived from anza-xyz/agave@master/sdk/src/transaction/error.rs +func ClassifySendError(tx *solana.Transaction, err error) mn.SendTxReturnCode { + if err == nil { + return mn.Successful + } + + errMsg := err.Error() + + // TODO: Ensure correct error classification for each error message. + // TODO: is strings.Contains good enough for error classification? + switch { + case strings.Contains(errMsg, "Account in use"): + return mn.TransactionAlreadyKnown + case strings.Contains(errMsg, "Account loaded twice"): + return mn.Retryable + case strings.Contains(errMsg, "Attempt to debit an account but found no record of a prior credit"): + return mn.Retryable + case strings.Contains(errMsg, "Attempt to load a program that does not exist"): + return mn.Fatal + case strings.Contains(errMsg, "Insufficient funds for fee"): + return mn.InsufficientFunds + case strings.Contains(errMsg, "This account may not be used to pay transaction fees"): + return mn.Unsupported + case strings.Contains(errMsg, "This transaction has already been processed"): + return mn.TransactionAlreadyKnown + case strings.Contains(errMsg, "Blockhash not found"): + return mn.Retryable + case strings.Contains(errMsg, "Error processing Instruction"): + return mn.Retryable + case strings.Contains(errMsg, "Loader call chain is too deep"): + return mn.Retryable + case strings.Contains(errMsg, "Transaction requires a fee but has no signature present"): + return mn.Retryable + case strings.Contains(errMsg, "Transaction contains an invalid account reference"): + return mn.Unsupported + case strings.Contains(errMsg, "Transaction did not pass signature verification"): + return mn.Fatal + case strings.Contains(errMsg, "This program may not be used for executing instructions"): + return mn.Unsupported + case strings.Contains(errMsg, "Transaction failed to sanitize accounts offsets correctly"): + return mn.Fatal + case strings.Contains(errMsg, "Transactions are currently disabled due to cluster maintenance"): + return mn.Retryable + case strings.Contains(errMsg, "Transaction processing left an account with an outstanding borrowed reference"): + return mn.Fatal + case strings.Contains(errMsg, "Transaction would exceed max Block Cost Limit"): + return mn.ExceedsMaxFee + case strings.Contains(errMsg, "Transaction version is unsupported"): + return mn.Unsupported + case strings.Contains(errMsg, "Transaction loads a writable account that cannot be written"): + return mn.Unsupported + case strings.Contains(errMsg, "Transaction would exceed max account limit within the block"): + return mn.ExceedsMaxFee + case strings.Contains(errMsg, "Transaction would exceed account data limit within the block"): + return mn.ExceedsMaxFee + case strings.Contains(errMsg, "Transaction locked too many accounts"): + return mn.Fatal + case strings.Contains(errMsg, "Transaction loads an address table account that doesn't exist"): + return mn.Unsupported + case strings.Contains(errMsg, "Transaction loads an address table account with an invalid owner"): + return mn.Unsupported + case strings.Contains(errMsg, "Transaction loads an address table account with invalid data"): + return mn.Unsupported + case strings.Contains(errMsg, "Transaction address table lookup uses an invalid index"): + return mn.Unsupported + case strings.Contains(errMsg, "Transaction leaves an account with a lower balance than rent-exempt minimum"): + return mn.Fatal + case strings.Contains(errMsg, "Transaction would exceed max Vote Cost Limit"): + return mn.ExceedsMaxFee + case strings.Contains(errMsg, "Transaction would exceed total account data limit"): + return mn.ExceedsMaxFee + case strings.Contains(errMsg, "Transaction contains a duplicate instruction"): + return mn.Fatal + case strings.Contains(errMsg, "Transaction results in an account with insufficient funds for rent"): + return mn.InsufficientFunds + case strings.Contains(errMsg, "Transaction exceeded max loaded accounts data size cap"): + return mn.Unsupported + case strings.Contains(errMsg, "LoadedAccountsDataSizeLimit set for transaction must be greater than 0"): + return mn.Fatal + case strings.Contains(errMsg, "Sanitized transaction differed before/after feature activation"): + return mn.Fatal + case strings.Contains(errMsg, "Execution of the program referenced by account at index is temporarily restricted"): + return mn.Unsupported + case strings.Contains(errMsg, "Sum of account balances before and after transaction do not match"): + return mn.Fatal + case strings.Contains(errMsg, "Program cache hit max limit"): + return mn.Retryable + default: + return mn.Retryable + } +} diff --git a/pkg/solana/client/client.go b/pkg/solana/client/client.go index aab12942b..f73f0fc6a 100644 --- a/pkg/solana/client/client.go +++ b/pkg/solana/client/client.go @@ -127,7 +127,7 @@ func (h *Head) IsValid() bool { } var _ mn.RPCClient[mn.StringID, *Head] = (*Client)(nil) -var _ mn.SendTxRPCClient[*solana.Transaction] = (*Client)(nil) +var _ mn.SendTxRPCClient[*solana.Transaction, solana.Signature] = (*Client)(nil) func (c *Client) Dial(ctx context.Context) error { return nil @@ -315,10 +315,12 @@ func (c *Client) GetInterceptedChainInfo() (latest, highestUserObservations mn.C return c.latestChainInfo, c.highestUserObservations } -func (c *Client) SendTransaction(ctx context.Context, tx *solana.Transaction) error { - // TODO: Use Transaction Sender - _, err := c.SendTx(ctx, tx) - return err +func (c *Client) SendTransaction(ctx context.Context, tx *solana.Transaction) (*solana.Signature, error) { + sig, err := c.SendTx(ctx, tx) + if err != nil { + return nil, err + } + return &sig, err } func (c *Client) latency(name string) func() { diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go index fbd5acca5..78b75cd10 100644 --- a/pkg/solana/client/multinode/transaction_sender.go +++ b/pkg/solana/client/multinode/transaction_sender.go @@ -28,31 +28,32 @@ var ( // (e.g. Successful, Fatal, Retryable, etc.) type TxErrorClassifier[TX any] func(tx TX, err error) SendTxReturnCode -type sendTxResult struct { +type sendTxResult[RESULT any] struct { Err error ResultCode SendTxReturnCode + Result *RESULT } const sendTxQuorum = 0.7 // SendTxRPCClient - defines interface of an RPC used by TransactionSender to broadcast transaction -type SendTxRPCClient[TX any] interface { +type SendTxRPCClient[TX any, RESULT any] interface { // SendTransaction errors returned should include name or other unique identifier of the RPC - SendTransaction(ctx context.Context, tx TX) error + SendTransaction(ctx context.Context, tx TX) (*RESULT, error) } -func NewTransactionSender[TX any, CHAIN_ID ID, RPC SendTxRPCClient[TX]]( +func NewTransactionSender[TX any, RESULT any, CHAIN_ID ID, RPC SendTxRPCClient[TX, RESULT]]( lggr logger.Logger, chainID CHAIN_ID, chainFamily string, multiNode *MultiNode[CHAIN_ID, RPC], txErrorClassifier TxErrorClassifier[TX], sendTxSoftTimeout time.Duration, -) *TransactionSender[TX, CHAIN_ID, RPC] { +) *TransactionSender[TX, RESULT, CHAIN_ID, RPC] { if sendTxSoftTimeout == 0 { sendTxSoftTimeout = QueryTimeout / 2 } - return &TransactionSender[TX, CHAIN_ID, RPC]{ + return &TransactionSender[TX, RESULT, CHAIN_ID, RPC]{ chainID: chainID, chainFamily: chainFamily, lggr: logger.Sugared(lggr).Named("TransactionSender").With("chainID", chainID.String()), @@ -63,7 +64,7 @@ func NewTransactionSender[TX any, CHAIN_ID ID, RPC SendTxRPCClient[TX]]( } } -type TransactionSender[TX any, CHAIN_ID ID, RPC SendTxRPCClient[TX]] struct { +type TransactionSender[TX any, RESULT any, CHAIN_ID ID, RPC SendTxRPCClient[TX, RESULT]] struct { services.StateMachine chainID CHAIN_ID chainFamily string @@ -94,9 +95,9 @@ type TransactionSender[TX any, CHAIN_ID ID, RPC SendTxRPCClient[TX]] struct { // * If there is at least one terminal error - returns terminal error // * If there is both success and terminal error - returns success and reports invariant violation // * Otherwise, returns any (effectively random) of the errors. -func (txSender *TransactionSender[TX, CHAIN_ID, RPC]) SendTransaction(ctx context.Context, tx TX) (SendTxReturnCode, error) { - txResults := make(chan sendTxResult) - txResultsToReport := make(chan sendTxResult) +func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ctx context.Context, tx TX) (*RESULT, SendTxReturnCode, error) { + txResults := make(chan sendTxResult[RESULT]) + txResultsToReport := make(chan sendTxResult[RESULT]) primaryNodeWg := sync.WaitGroup{} ctx, cancel := txSender.chStop.Ctx(ctx) @@ -145,7 +146,7 @@ func (txSender *TransactionSender[TX, CHAIN_ID, RPC]) SendTransaction(ctx contex }() if err != nil { - return Retryable, err + return nil, Retryable, err } txSender.wg.Add(1) @@ -154,34 +155,34 @@ func (txSender *TransactionSender[TX, CHAIN_ID, RPC]) SendTransaction(ctx contex return txSender.collectTxResults(ctx, tx, healthyNodesNum, txResults) } -func (txSender *TransactionSender[TX, CHAIN_ID, RPC]) broadcastTxAsync(ctx context.Context, rpc RPC, tx TX) sendTxResult { - txErr := rpc.SendTransaction(ctx, tx) +func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) broadcastTxAsync(ctx context.Context, rpc RPC, tx TX) sendTxResult[RESULT] { + result, txErr := rpc.SendTransaction(ctx, tx) txSender.lggr.Debugw("Node sent transaction", "tx", tx, "err", txErr) resultCode := txSender.txErrorClassifier(tx, txErr) if !slices.Contains(sendTxSuccessfulCodes, resultCode) { txSender.lggr.Warnw("RPC returned error", "tx", tx, "err", txErr) } - return sendTxResult{Err: txErr, ResultCode: resultCode} + return sendTxResult[RESULT]{Err: txErr, ResultCode: resultCode, Result: result} } -func (txSender *TransactionSender[TX, CHAIN_ID, RPC]) reportSendTxAnomalies(tx TX, txResults <-chan sendTxResult) { +func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) reportSendTxAnomalies(tx TX, txResults <-chan sendTxResult[RESULT]) { defer txSender.wg.Done() - resultsByCode := sendTxResults{} + resultsByCode := sendTxResults[RESULT]{} // txResults eventually will be closed for txResult := range txResults { - resultsByCode[txResult.ResultCode] = append(resultsByCode[txResult.ResultCode], txResult.Err) + resultsByCode[txResult.ResultCode] = append(resultsByCode[txResult.ResultCode], txResult) } - _, _, criticalErr := aggregateTxResults(resultsByCode) + _, _, _, criticalErr := aggregateTxResults[RESULT](resultsByCode) if criticalErr != nil { txSender.lggr.Criticalw("observed invariant violation on SendTransaction", "tx", tx, "resultsByCode", resultsByCode, "err", criticalErr) PromMultiNodeInvariantViolations.WithLabelValues(txSender.chainFamily, txSender.chainID.String(), criticalErr.Error()).Inc() } } -type sendTxResults map[SendTxReturnCode][]error +type sendTxResults[RESULT any] map[SendTxReturnCode][]sendTxResult[RESULT] -func aggregateTxResults(resultsByCode sendTxResults) (returnCode SendTxReturnCode, txResult error, err error) { +func aggregateTxResults[RESULT any](resultsByCode sendTxResults[RESULT]) (result *RESULT, returnCode SendTxReturnCode, txResult error, err error) { severeCode, severeErrors, hasSevereErrors := findFirstIn(resultsByCode, sendTxSevereErrors) successCode, successResults, hasSuccess := findFirstIn(resultsByCode, sendTxSuccessfulCodes) if hasSuccess { @@ -190,32 +191,32 @@ func aggregateTxResults(resultsByCode sendTxResults) (returnCode SendTxReturnCod if hasSevereErrors { const errMsg = "found contradictions in nodes replies on SendTransaction: got success and severe error" // return success, since at least 1 node has accepted our broadcasted Tx, and thus it can now be included onchain - return successCode, successResults[0], errors.New(errMsg) + return successResults[0].Result, successCode, successResults[0].Err, errors.New(errMsg) } // other errors are temporary - we are safe to return success - return successCode, successResults[0], nil + return successResults[0].Result, successCode, successResults[0].Err, nil } if hasSevereErrors { - return severeCode, severeErrors[0], nil + return nil, severeCode, severeErrors[0].Err, nil } // return temporary error for code, result := range resultsByCode { - return code, result[0], nil + return nil, code, result[0].Err, nil } err = fmt.Errorf("expected at least one response on SendTransaction") - return Retryable, err, err + return nil, Retryable, err, err } -func (txSender *TransactionSender[TX, CHAIN_ID, RPC]) collectTxResults(ctx context.Context, tx TX, healthyNodesNum int, txResults <-chan sendTxResult) (SendTxReturnCode, error) { +func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) collectTxResults(ctx context.Context, tx TX, healthyNodesNum int, txResults <-chan sendTxResult[RESULT]) (*RESULT, SendTxReturnCode, error) { if healthyNodesNum == 0 { - return Retryable, ErroringNodeError + return nil, Retryable, ErroringNodeError } requiredResults := int(math.Ceil(float64(healthyNodesNum) * sendTxQuorum)) - errorsByCode := sendTxResults{} + errorsByCode := sendTxResults[RESULT]{} var softTimeoutChan <-chan time.Time var resultsCount int loop: @@ -223,9 +224,9 @@ loop: select { case <-ctx.Done(): txSender.lggr.Debugw("Failed to collect of the results before context was done", "tx", tx, "errorsByCode", errorsByCode) - return Retryable, ctx.Err() + return nil, Retryable, ctx.Err() case result := <-txResults: - errorsByCode[result.ResultCode] = append(errorsByCode[result.ResultCode], result.Err) + errorsByCode[result.ResultCode] = append(errorsByCode[result.ResultCode], result) resultsCount++ if slices.Contains(sendTxSuccessfulCodes, result.ResultCode) || resultsCount >= requiredResults { break loop @@ -245,17 +246,17 @@ loop: } // ignore critical error as it's reported in reportSendTxAnomalies - returnCode, result, _ := aggregateTxResults(errorsByCode) - return returnCode, result + result, returnCode, resultErr, _ := aggregateTxResults(errorsByCode) + return result, returnCode, resultErr } -func (txSender *TransactionSender[TX, CHAIN_ID, RPC]) Start(ctx context.Context) error { +func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) Start(ctx context.Context) error { return txSender.StartOnce("TransactionSender", func() error { return nil }) } -func (txSender *TransactionSender[TX, CHAIN_ID, RPC]) Close() error { +func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) Close() error { return txSender.StopOnce("TransactionSender", func() error { close(txSender.chStop) txSender.wg.Wait() diff --git a/pkg/solana/txm/txm.go b/pkg/solana/txm/txm.go index 2a427ffed..9e2243f3f 100644 --- a/pkg/solana/txm/txm.go +++ b/pkg/solana/txm/txm.go @@ -53,8 +53,9 @@ type Txm struct { ks SimpleKeystore fee fees.Estimator - // Use multiNode for client selection if set + // Use multiNode for client selection if set, and txSender for sending transactions to all RPCs. multiNode *mn.MultiNode[mn.StringID, *client.Client] + txSender *mn.TransactionSender[*solanaGo.Transaction, solanaGo.Signature, mn.StringID, *client.Client] // If multiNode is disabled, use lazy load to fetch client client *utils.LazyLoad[client.ReaderWriter] @@ -80,7 +81,10 @@ type pendingTx struct { } // NewTxm creates a txm. Uses simulation so should only be used to send txes to trusted contracts i.e. OCR. -func NewTxm(chainID string, tc func() (client.ReaderWriter, error), cfg config.Config, multiNode *mn.MultiNode[mn.StringID, *client.Client], ks SimpleKeystore, lggr logger.Logger) *Txm { +func NewTxm(chainID string, tc func() (client.ReaderWriter, error), cfg config.Config, + multiNode *mn.MultiNode[mn.StringID, *client.Client], + txSender *mn.TransactionSender[*solanaGo.Transaction, solanaGo.Signature, mn.StringID, *client.Client], + ks SimpleKeystore, lggr logger.Logger) *Txm { return &Txm{ lggr: lggr, chSend: make(chan pendingTx, MaxQueueLen), // queue can support 1000 pending txs @@ -90,6 +94,7 @@ func NewTxm(chainID string, tc func() (client.ReaderWriter, error), cfg config.C txs: newPendingTxContextWithProm(chainID), ks: ks, multiNode: multiNode, + txSender: txSender, client: utils.NewLazyLoad(tc), } } @@ -106,6 +111,24 @@ func (txm *Txm) getClient() (client.ReaderWriter, error) { return txm.client.Get() } +// sendTx uses the transaction sender if MultiNode is enabled, otherwise sends the +// transaction using a single client +func (txm *Txm) sendTx(ctx context.Context, tx *solanaGo.Transaction) (solanaGo.Signature, error) { + if txm.multiNodeEnabled() { + result, _, err := txm.txSender.SendTransaction(ctx, tx) + if err != nil { + return solanaGo.Signature{}, err + } + return *result, err + } + + client, err := txm.getClient() + if err != nil { + return solanaGo.Signature{}, err + } + return client.SendTx(ctx, tx) +} + // Start subscribes to queuing channel and processes them. func (txm *Txm) Start(ctx context.Context) error { return txm.starter.StartOnce("solana_txm", func() error { @@ -174,12 +197,6 @@ func (txm *Txm) run() { } func (txm *Txm) sendWithRetry(chanCtx context.Context, baseTx solanaGo.Transaction, txcfg TxConfig) (solanaGo.Transaction, uuid.UUID, solanaGo.Signature, error) { - // fetch client - client, clientErr := txm.getClient() - if clientErr != nil { - return solanaGo.Transaction{}, uuid.Nil, solanaGo.Signature{}, fmt.Errorf("failed to get client in soltxm.sendWithRetry: %w", clientErr) - } - // get key // fee payer account is index 0 account // https://github.com/gagliardetto/solana-go/blob/main/transaction.go#L252 @@ -239,7 +256,7 @@ func (txm *Txm) sendWithRetry(chanCtx context.Context, baseTx solanaGo.Transacti ctx, cancel := context.WithTimeout(chanCtx, txcfg.Timeout) // send initial tx (do not retry and exit early if fails) - sig, initSendErr := client.SendTx(ctx, &initTx) + sig, initSendErr := txm.sendTx(ctx, &initTx) if initSendErr != nil { cancel() // cancel context when exiting early txm.txs.OnError(sig, TxFailReject) // increment failed metric @@ -308,7 +325,7 @@ func (txm *Txm) sendWithRetry(chanCtx context.Context, baseTx solanaGo.Transacti go func(bump bool, count int, retryTx solanaGo.Transaction) { defer wg.Done() - retrySig, retrySendErr := client.SendTx(ctx, &retryTx) + retrySig, retrySendErr := txm.sendTx(ctx, &retryTx) // this could occur if endpoint goes down or if ctx cancelled if retrySendErr != nil { if strings.Contains(retrySendErr.Error(), "context canceled") || strings.Contains(retrySendErr.Error(), "context deadline exceeded") { diff --git a/pkg/solana/txm/txm_internal_test.go b/pkg/solana/txm/txm_internal_test.go index 0d2645ec0..5f1abea21 100644 --- a/pkg/solana/txm/txm_internal_test.go +++ b/pkg/solana/txm/txm_internal_test.go @@ -112,7 +112,7 @@ func TestTxm(t *testing.T) { txm := NewTxm(id, func() (client.ReaderWriter, error) { return mc, nil - }, cfg, nil, mkey, lggr) + }, cfg, nil, nil, mkey, lggr) require.NoError(t, txm.Start(ctx)) // tracking prom metrics @@ -718,7 +718,7 @@ func TestTxm_Enqueue(t *testing.T) { txm := NewTxm("enqueue_test", func() (client.ReaderWriter, error) { return mc, nil - }, cfg, nil, mkey, lggr) + }, cfg, nil, nil, mkey, lggr) require.ErrorContains(t, txm.Enqueue("txmUnstarted", &solana.Transaction{}), "not started") require.NoError(t, txm.Start(ctx)) diff --git a/pkg/solana/txm/txm_race_test.go b/pkg/solana/txm/txm_race_test.go index acdfa4908..68bd7c03b 100644 --- a/pkg/solana/txm/txm_race_test.go +++ b/pkg/solana/txm/txm_race_test.go @@ -65,7 +65,7 @@ func TestTxm_SendWithRetry_Race(t *testing.T) { } // build minimal txm - txm := NewTxm("retry_race", getClient, cfg, nil, ks, lggr) + txm := NewTxm("retry_race", getClient, cfg, nil, nil, ks, lggr) txm.fee = fee _, _, _, err := txm.sendWithRetry( diff --git a/pkg/solana/txm/txm_test.go b/pkg/solana/txm/txm_test.go index 84d11df62..949f7da66 100644 --- a/pkg/solana/txm/txm_test.go +++ b/pkg/solana/txm/txm_test.go @@ -72,7 +72,7 @@ func TestTxm_Integration(t *testing.T) { getClient := func() (solanaClient.ReaderWriter, error) { return client, nil } - txm := txm.NewTxm("localnet", getClient, cfg, nil, mkey, lggr) + txm := txm.NewTxm("localnet", getClient, cfg, nil, nil, mkey, lggr) // track initial balance initBal, err := client.Balance(pubKey) From eee4b501d845c27443b4c81c8c17faccaefd3522 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 2 Oct 2024 16:55:40 -0400 Subject: [PATCH 053/174] Update txm_internal_test.go --- pkg/solana/txm/txm_internal_test.go | 617 ++++++++++++++++++++++++++++ 1 file changed, 617 insertions(+) diff --git a/pkg/solana/txm/txm_internal_test.go b/pkg/solana/txm/txm_internal_test.go index 5f1abea21..a48900beb 100644 --- a/pkg/solana/txm/txm_internal_test.go +++ b/pkg/solana/txm/txm_internal_test.go @@ -5,6 +5,8 @@ package txm import ( "context" "errors" + solana2 "github.com/smartcontractkit/chainlink-solana/pkg/solana" + mn "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" "math/rand" "sync" "testing" @@ -674,6 +676,621 @@ func TestTxm(t *testing.T) { } } +func TestTxm_MultiNode(t *testing.T) { + for _, eName := range []string{"fixed", "blockhistory"} { + estimator := eName + t.Run("estimator-"+estimator, func(t *testing.T) { + t.Parallel() // run estimator tests in parallel + + // set up configs needed in txm + id := "mocknet-" + estimator + "-" + uuid.NewString() + t.Logf("Starting new iteration: %s", id) + + ctx := tests.Context(t) + lggr := logger.Test(t) + + // Enable MultiNode + cfg := config.TOMLConfig{MultiNode: config.MultiNodeConfig{Enabled: true}} + cfg.SetDefaults() + + cfg.Chain.FeeEstimatorMode = &estimator + mc := mocks.NewReaderWriter(t) + mc.On("GetLatestBlock").Return(&rpc.GetBlockResult{}, nil).Maybe() + + nodes := []mn.Node[mn.StringID, mocks.ReaderWriter]{ + mn.NewNode(cfg, mn.StringID("node1"), mc, "chainID", 0), + } + + // TODO: Is this needed to test things? + // TODO: Need to mock more of the ReaderWriter methods for TxSender?? + multiNode := mn.NewMultiNode(cfg, + mn.NodeSelectionModeHighestHead, + cfg.MultiNode.LeaseDuration(), + nodes, + []mn.SendOnlyNode{}, + "chainID", + "Solana", + 0, + ) + + txSender := mn.NewTransactionSender[*solanago.Transaction, solanago.Signature, mn.StringID, *client.Client]( + lggr, + mn.StringID(id), + chainFamily, + multiNode, + solana2.ClassifySendError, + 0, // use the default value provided by the implementation + ) + txSender := mn.NewTransactionSender(multiNode, lggr) + + // mock solana keystore + mkey := keyMocks.NewSimpleKeystore(t) + mkey.On("Sign", mock.Anything, mock.Anything, mock.Anything).Return([]byte{}, nil) + + txm := NewTxm(id, func() (client.ReaderWriter, error) { + return mc, nil + }, cfg, nil, nil, mkey, lggr) + require.NoError(t, txm.Start(ctx)) + + // tracking prom metrics + prom := soltxmProm{id: id} + + // create random signature + getSig := func() solana.Signature { + sig := make([]byte, 64) + rand.Read(sig) + return solana.SignatureFromBytes(sig) + } + + // check if cached transaction is cleared + empty := func() bool { + count := txm.InflightTxs() + assert.Equal(t, float64(count), prom.getInflight()) // validate prom metric and txs length + return count == 0 + } + + // adjust wait time based on config + waitDuration := cfg.TxConfirmTimeout() + waitFor := func(f func() bool) { + for i := 0; i < int(waitDuration.Seconds()*1.5); i++ { + if f() { + return + } + time.Sleep(time.Second) + } + assert.NoError(t, errors.New("unable to confirm inflight txs is empty")) + } + + // handle signature statuses calls + statuses := map[solana.Signature]func() *rpc.SignatureStatusesResult{} + mc.On("SignatureStatuses", mock.Anything, mock.AnythingOfType("[]solana.Signature")).Return( + func(_ context.Context, sigs []solana.Signature) (out []*rpc.SignatureStatusesResult) { + for i := range sigs { + get, exists := statuses[sigs[i]] + if !exists { + out = append(out, nil) + continue + } + out = append(out, get()) + } + return out + }, nil, + ) + + // happy path (send => simulate success => tx: nil => tx: processed => tx: confirmed => done) + t.Run("happyPath", func(t *testing.T) { + sig := getSig() + tx, signed := getTx(t, 0, mkey, 0) + var wg sync.WaitGroup + wg.Add(3) + + sendCount := 0 + var countRW sync.RWMutex + mc.On("SendTx", mock.Anything, signed(0, true)).Run(func(mock.Arguments) { + countRW.Lock() + sendCount++ + countRW.Unlock() + }).After(500*time.Millisecond).Return(sig, nil) + mc.On("SimulateTx", mock.Anything, signed(0, true), mock.Anything).Return(&rpc.SimulateTransactionResult{}, nil).Once() + + // handle signature status calls + count := 0 + statuses[sig] = func() (out *rpc.SignatureStatusesResult) { + defer func() { count++ }() + defer wg.Done() + + out = &rpc.SignatureStatusesResult{} + if count == 1 { + out.ConfirmationStatus = rpc.ConfirmationStatusProcessed + return + } + + if count == 2 { + out.ConfirmationStatus = rpc.ConfirmationStatusConfirmed + return + } + return nil + } + + // send tx + assert.NoError(t, txm.Enqueue(t.Name(), tx)) + wg.Wait() + + // no transactions stored inflight txs list + waitFor(empty) + // transaction should be sent more than twice + countRW.RLock() + t.Logf("sendTx received %d calls", sendCount) + assert.Greater(t, sendCount, 2) + countRW.RUnlock() + + // panic if sendTx called after context cancelled + mc.On("SendTx", mock.Anything, tx).Panic("SendTx should not be called anymore").Maybe() + + // check prom metric + prom.success++ + prom.assertEqual(t) + }) + + // fail on initial transmit (RPC immediate rejects) + t.Run("fail_initialTx", func(t *testing.T) { + tx, signed := getTx(t, 1, mkey, 0) + var wg sync.WaitGroup + wg.Add(1) + + // should only be called once (tx does not start retry, confirming, or simulation) + mc.On("SendTx", mock.Anything, signed(0, true)).Run(func(mock.Arguments) { + wg.Done() + }).Return(solana.Signature{}, errors.New("FAIL")).Once() + + // tx should be able to queue + assert.NoError(t, txm.Enqueue(t.Name(), tx)) + wg.Wait() // wait to be picked up and processed + + // no transactions stored inflight txs list + waitFor(empty) + + // check prom metric + prom.error++ + prom.reject++ + prom.assertEqual(t) + }) + + // tx fails simulation (simulation error) + t.Run("fail_simulation", func(t *testing.T) { + tx, signed := getTx(t, 2, mkey, 0) + sig := getSig() + var wg sync.WaitGroup + wg.Add(1) + + mc.On("SendTx", mock.Anything, signed(0, true)).Return(sig, nil) + mc.On("SimulateTx", mock.Anything, signed(0, true), mock.Anything).Run(func(mock.Arguments) { + wg.Done() + }).Return(&rpc.SimulateTransactionResult{ + Err: "FAIL", + }, nil).Once() + // signature status is nil (handled automatically) + + // tx should be able to queue + assert.NoError(t, txm.Enqueue(t.Name(), tx)) + wg.Wait() // wait to be picked up and processed + waitFor(empty) // txs cleared quickly + + // check prom metric + prom.error++ + prom.simOther++ + prom.assertEqual(t) + }) + + // tx fails simulation (rpc error, timeout should clean up b/c sig status will be nil) + t.Run("fail_simulation_confirmNil", func(t *testing.T) { + tx, signed := getTx(t, 3, mkey, 0) + sig := getSig() + retry0 := getSig() + retry1 := getSig() + retry2 := getSig() + retry3 := getSig() + var wg sync.WaitGroup + wg.Add(1) + + mc.On("SendTx", mock.Anything, signed(0, true)).Return(sig, nil) + mc.On("SendTx", mock.Anything, signed(1, true)).Return(retry0, nil) + mc.On("SendTx", mock.Anything, signed(2, true)).Return(retry1, nil) + mc.On("SendTx", mock.Anything, signed(3, true)).Return(retry2, nil).Maybe() + mc.On("SendTx", mock.Anything, signed(4, true)).Return(retry3, nil).Maybe() + mc.On("SimulateTx", mock.Anything, signed(0, true), mock.Anything).Run(func(mock.Arguments) { + wg.Done() + }).Return(&rpc.SimulateTransactionResult{}, errors.New("FAIL")).Once() + // all signature statuses are nil, handled automatically + + // tx should be able to queue + assert.NoError(t, txm.Enqueue(t.Name(), tx)) + wg.Wait() // wait to be picked up and processed + waitFor(empty) // txs cleared after timeout + + // check prom metric + prom.error++ + prom.drop++ + prom.assertEqual(t) + + // panic if sendTx called after context cancelled + mc.On("SendTx", mock.Anything, tx).Panic("SendTx should not be called anymore").Maybe() + }) + + // tx fails simulation with an InstructionError (indicates reverted execution) + // manager should cancel sending retry immediately + increment reverted prom metric + t.Run("fail_simulation_instructionError", func(t *testing.T) { + tx, signed := getTx(t, 4, mkey, 0) + sig := getSig() + var wg sync.WaitGroup + wg.Add(1) + + // {"InstructionError":[0,{"Custom":6003}]} + tempErr := map[string][]interface{}{ + "InstructionError": { + 0, map[string]int{"Custom": 6003}, + }, + } + mc.On("SendTx", mock.Anything, signed(0, true)).Return(sig, nil) + mc.On("SimulateTx", mock.Anything, signed(0, true), mock.Anything).Run(func(mock.Arguments) { + wg.Done() + }).Return(&rpc.SimulateTransactionResult{ + Err: tempErr, + }, nil).Once() + // all signature statuses are nil, handled automatically + + // tx should be able to queue + assert.NoError(t, txm.Enqueue(t.Name(), tx)) + wg.Wait() // wait to be picked up and processed + waitFor(empty) // txs cleared after timeout + + // check prom metric + prom.error++ + prom.simRevert++ + prom.assertEqual(t) + + // panic if sendTx called after context cancelled + mc.On("SendTx", mock.Anything, tx).Panic("SendTx should not be called anymore").Maybe() + }) + + // tx fails simulation with BlockHashNotFound error + // txm should continue to confirm tx (in this case it will succeed) + t.Run("fail_simulation_blockhashNotFound", func(t *testing.T) { + tx, signed := getTx(t, 5, mkey, 0) + sig := getSig() + var wg sync.WaitGroup + wg.Add(3) + + mc.On("SendTx", mock.Anything, signed(0, true)).Return(sig, nil) + mc.On("SimulateTx", mock.Anything, signed(0, true), mock.Anything).Run(func(mock.Arguments) { + wg.Done() + }).Return(&rpc.SimulateTransactionResult{ + Err: "BlockhashNotFound", + }, nil).Once() + + // handle signature status calls + count := 0 + statuses[sig] = func() (out *rpc.SignatureStatusesResult) { + defer func() { count++ }() + defer wg.Done() + + out = &rpc.SignatureStatusesResult{} + if count == 1 { + out.ConfirmationStatus = rpc.ConfirmationStatusConfirmed + return + } + return nil + } + + // tx should be able to queue + assert.NoError(t, txm.Enqueue(t.Name(), tx)) + wg.Wait() // wait to be picked up and processed + waitFor(empty) // txs cleared after timeout + + // check prom metric + prom.success++ + prom.assertEqual(t) + + // panic if sendTx called after context cancelled + mc.On("SendTx", mock.Anything, tx).Panic("SendTx should not be called anymore").Maybe() + }) + + // tx fails simulation with AlreadyProcessed error + // txm should continue to confirm tx (in this case it will revert) + t.Run("fail_simulation_alreadyProcessed", func(t *testing.T) { + tx, signed := getTx(t, 6, mkey, 0) + sig := getSig() + var wg sync.WaitGroup + wg.Add(2) + + mc.On("SendTx", mock.Anything, signed(0, true)).Return(sig, nil) + mc.On("SimulateTx", mock.Anything, signed(0, true), mock.Anything).Run(func(mock.Arguments) { + wg.Done() + }).Return(&rpc.SimulateTransactionResult{ + Err: "AlreadyProcessed", + }, nil).Once() + + // handle signature status calls + statuses[sig] = func() (out *rpc.SignatureStatusesResult) { + wg.Done() + return &rpc.SignatureStatusesResult{ + Err: "ERROR", + ConfirmationStatus: rpc.ConfirmationStatusConfirmed, + } + } + + // tx should be able to queue + assert.NoError(t, txm.Enqueue(t.Name(), tx)) + wg.Wait() // wait to be picked up and processed + waitFor(empty) // txs cleared after timeout + + // check prom metric + prom.revert++ + prom.error++ + prom.assertEqual(t) + + // panic if sendTx called after context cancelled + mc.On("SendTx", mock.Anything, tx).Panic("SendTx should not be called anymore").Maybe() + }) + + // tx passes sim, never passes processed (timeout should cleanup) + t.Run("fail_confirm_processed", func(t *testing.T) { + tx, signed := getTx(t, 7, mkey, 0) + sig := getSig() + retry0 := getSig() + retry1 := getSig() + retry2 := getSig() + retry3 := getSig() + var wg sync.WaitGroup + wg.Add(1) + + mc.On("SendTx", mock.Anything, signed(0, true)).Return(sig, nil) + mc.On("SendTx", mock.Anything, signed(1, true)).Return(retry0, nil) + mc.On("SendTx", mock.Anything, signed(2, true)).Return(retry1, nil) + mc.On("SendTx", mock.Anything, signed(3, true)).Return(retry2, nil).Maybe() + mc.On("SendTx", mock.Anything, signed(4, true)).Return(retry3, nil).Maybe() + mc.On("SimulateTx", mock.Anything, signed(0, true), mock.Anything).Run(func(mock.Arguments) { + wg.Done() + }).Return(&rpc.SimulateTransactionResult{}, nil).Once() + + // handle signature status calls (initial stays processed, others don't exist) + statuses[sig] = func() (out *rpc.SignatureStatusesResult) { + return &rpc.SignatureStatusesResult{ + ConfirmationStatus: rpc.ConfirmationStatusProcessed, + } + } + + // tx should be able to queue + assert.NoError(t, txm.Enqueue(t.Name(), tx)) + wg.Wait() // wait to be picked up and processed + waitFor(empty) // inflight txs cleared after timeout + + // check prom metric + prom.error++ + prom.drop++ + prom.assertEqual(t) + + // panic if sendTx called after context cancelled + mc.On("SendTx", mock.Anything, tx).Panic("SendTx should not be called anymore").Maybe() + }) + + // tx passes sim, shows processed, moves to nil (timeout should cleanup) + t.Run("fail_confirm_processedToNil", func(t *testing.T) { + tx, signed := getTx(t, 8, mkey, 0) + sig := getSig() + retry0 := getSig() + retry1 := getSig() + retry2 := getSig() + retry3 := getSig() + var wg sync.WaitGroup + wg.Add(1) + + mc.On("SendTx", mock.Anything, signed(0, true)).Return(sig, nil) + mc.On("SendTx", mock.Anything, signed(1, true)).Return(retry0, nil) + mc.On("SendTx", mock.Anything, signed(2, true)).Return(retry1, nil) + mc.On("SendTx", mock.Anything, signed(3, true)).Return(retry2, nil).Maybe() + mc.On("SendTx", mock.Anything, signed(4, true)).Return(retry3, nil).Maybe() + mc.On("SimulateTx", mock.Anything, signed(0, true), mock.Anything).Run(func(mock.Arguments) { + wg.Done() + }).Return(&rpc.SimulateTransactionResult{}, nil).Once() + + // handle signature status calls (initial stays processed => nil, others don't exist) + count := 0 + statuses[sig] = func() (out *rpc.SignatureStatusesResult) { + defer func() { count++ }() + + if count > 2 { + return nil + } + + return &rpc.SignatureStatusesResult{ + ConfirmationStatus: rpc.ConfirmationStatusProcessed, + } + } + + // tx should be able to queue + assert.NoError(t, txm.Enqueue(t.Name(), tx)) + wg.Wait() // wait to be picked up and processed + waitFor(empty) // inflight txs cleared after timeout + + // check prom metric + prom.error++ + prom.drop++ + prom.assertEqual(t) + + // panic if sendTx called after context cancelled + mc.On("SendTx", mock.Anything, tx).Panic("SendTx should not be called anymore").Maybe() + }) + + // tx passes sim, errors on confirm + t.Run("fail_confirm_revert", func(t *testing.T) { + tx, signed := getTx(t, 9, mkey, 0) + sig := getSig() + var wg sync.WaitGroup + wg.Add(1) + + mc.On("SendTx", mock.Anything, signed(0, true)).Return(sig, nil) + mc.On("SimulateTx", mock.Anything, signed(0, true), mock.Anything).Run(func(mock.Arguments) { + wg.Done() + }).Return(&rpc.SimulateTransactionResult{}, nil).Once() + + // handle signature status calls + statuses[sig] = func() (out *rpc.SignatureStatusesResult) { + return &rpc.SignatureStatusesResult{ + ConfirmationStatus: rpc.ConfirmationStatusProcessed, + Err: "ERROR", + } + } + + // tx should be able to queue + assert.NoError(t, txm.Enqueue(t.Name(), tx)) + wg.Wait() // wait to be picked up and processed + waitFor(empty) // inflight txs cleared after timeout + + // check prom metric + prom.error++ + prom.revert++ + prom.assertEqual(t) + + // panic if sendTx called after context cancelled + mc.On("SendTx", mock.Anything, tx).Panic("SendTx should not be called anymore").Maybe() + }) + + // tx passes sim, first retried TXs get dropped + t.Run("success_retryTx", func(t *testing.T) { + tx, signed := getTx(t, 10, mkey, 0) + sig := getSig() + retry0 := getSig() + retry1 := getSig() + retry2 := getSig() + retry3 := getSig() + var wg sync.WaitGroup + wg.Add(2) + + mc.On("SendTx", mock.Anything, signed(0, true)).Return(sig, nil) + mc.On("SendTx", mock.Anything, signed(1, true)).Return(retry0, nil) + mc.On("SendTx", mock.Anything, signed(2, true)).Return(retry1, nil) + mc.On("SendTx", mock.Anything, signed(3, true)).Return(retry2, nil).Maybe() + mc.On("SendTx", mock.Anything, signed(4, true)).Return(retry3, nil).Maybe() + mc.On("SimulateTx", mock.Anything, signed(0, true), mock.Anything).Run(func(mock.Arguments) { + wg.Done() + }).Return(&rpc.SimulateTransactionResult{}, nil).Once() + + // handle signature status calls + statuses[retry1] = func() (out *rpc.SignatureStatusesResult) { + defer wg.Done() + return &rpc.SignatureStatusesResult{ + ConfirmationStatus: rpc.ConfirmationStatusConfirmed, + } + } + + // send tx + assert.NoError(t, txm.Enqueue(t.Name(), tx)) + wg.Wait() + + // no transactions stored inflight txs list + waitFor(empty) + + // panic if sendTx called after context cancelled + mc.On("SendTx", mock.Anything, tx).Panic("SendTx should not be called anymore").Maybe() + + // check prom metric + prom.success++ + prom.assertEqual(t) + }) + + // fee bumping disabled + t.Run("feeBumpingDisabled", func(t *testing.T) { + sig := getSig() + tx, signed := getTx(t, 11, mkey, 0) + + defaultFeeBumpPeriod := cfg.FeeBumpPeriod() + + sendCount := 0 + var countRW sync.RWMutex + mc.On("SendTx", mock.Anything, signed(0, true)).Run(func(mock.Arguments) { + countRW.Lock() + sendCount++ + countRW.Unlock() + }).Return(sig, nil) // only sends one transaction type (no bumping) + mc.On("SimulateTx", mock.Anything, signed(0, true), mock.Anything).Return(&rpc.SimulateTransactionResult{}, nil).Once() + + // handle signature status calls + var wg sync.WaitGroup + wg.Add(1) + count := 0 + start := time.Now() + statuses[sig] = func() (out *rpc.SignatureStatusesResult) { + defer func() { count++ }() + + out = &rpc.SignatureStatusesResult{} + if time.Since(start) > 2*defaultFeeBumpPeriod { + out.ConfirmationStatus = rpc.ConfirmationStatusConfirmed + wg.Done() + return + } + out.ConfirmationStatus = rpc.ConfirmationStatusProcessed + return + } + + // send tx - with disabled fee bumping + assert.NoError(t, txm.Enqueue(t.Name(), tx, SetFeeBumpPeriod(0))) + wg.Wait() + + // no transactions stored inflight txs list + waitFor(empty) + // transaction should be sent more than twice + countRW.RLock() + t.Logf("sendTx received %d calls", sendCount) + assert.Greater(t, sendCount, 2) + countRW.RUnlock() + + // panic if sendTx called after context cancelled + mc.On("SendTx", mock.Anything, tx).Panic("SendTx should not be called anymore").Maybe() + + // check prom metric + prom.success++ + prom.assertEqual(t) + }) + + // compute unit limit disabled + t.Run("computeUnitLimitDisabled", func(t *testing.T) { + sig := getSig() + tx, signed := getTx(t, 12, mkey, 0) + + // should only match transaction without compute unit limit + assert.Len(t, signed(0, false).Message.Instructions, 2) + mc.On("SendTx", mock.Anything, signed(0, false)).Return(sig, nil) // only sends one transaction type (no bumping) + mc.On("SimulateTx", mock.Anything, signed(0, false), mock.Anything).Return(&rpc.SimulateTransactionResult{}, nil).Once() + + // handle signature status calls + var wg sync.WaitGroup + wg.Add(1) + statuses[sig] = func() *rpc.SignatureStatusesResult { + defer wg.Done() + return &rpc.SignatureStatusesResult{ + ConfirmationStatus: rpc.ConfirmationStatusConfirmed, + } + } + + // send tx - with disabled fee bumping and disabled compute unit limit + assert.NoError(t, txm.Enqueue(t.Name(), tx, SetFeeBumpPeriod(0), SetComputeUnitLimit(0))) + wg.Wait() + + // no transactions stored inflight txs list + waitFor(empty) + + // panic if sendTx called after context cancelled + mc.On("SendTx", mock.Anything, tx).Panic("SendTx should not be called anymore").Maybe() + + // check prom metric + prom.success++ + prom.assertEqual(t) + }) + }) + } +} + func TestTxm_Enqueue(t *testing.T) { // set up configs needed in txm lggr := logger.Test(t) From 1eaf4a5014fa32a2048e4628b196d69f557f0890 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 4 Oct 2024 10:41:07 -0400 Subject: [PATCH 054/174] Address comments --- pkg/solana/chain.go | 4 ++-- pkg/solana/chain_test.go | 27 ++++++++++------------- pkg/solana/monitor/balance.go | 9 ++++---- pkg/solana/txm/txm.go | 40 +++++++++++++++-------------------- 4 files changed, 35 insertions(+), 45 deletions(-) diff --git a/pkg/solana/chain.go b/pkg/solana/chain.go index 4d90351be..4e9c08330 100644 --- a/pkg/solana/chain.go +++ b/pkg/solana/chain.go @@ -292,11 +292,11 @@ func newChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr logger.L tc := func() (client.ReaderWriter, error) { return ch.getClient() } - ch.txm = txm.NewTxm(ch.id, tc, cfg, ch.multiNode, ks, lggr) + ch.txm = txm.NewTxm(ch.id, tc, cfg, ks, lggr) bc := func() (monitor.BalanceClient, error) { return ch.getClient() } - ch.balanceMonitor = monitor.NewBalanceMonitor(ch.id, cfg, cfg.MultiNode.Enabled(), lggr, ks, bc) + ch.balanceMonitor = monitor.NewBalanceMonitor(ch.id, cfg, lggr, ks, bc) return &ch, nil } diff --git a/pkg/solana/chain_test.go b/pkg/solana/chain_test.go index 06390e0c3..73afcee11 100644 --- a/pkg/solana/chain_test.go +++ b/pkg/solana/chain_test.go @@ -55,13 +55,11 @@ func TestSolanaChain_GetClient(t *testing.T) { ch := solcfg.Chain{} ch.SetDefaults() - mn := solcfg.MultiNodeConfig{} - mn.SetDefaults() cfg := &solcfg.TOMLConfig{ - ChainID: ptr("devnet"), - Chain: ch, - MultiNode: mn, + ChainID: ptr("devnet"), + Chain: ch, } + cfg.SetDefaults() testChain := chain{ id: "devnet", cfg: cfg, @@ -210,13 +208,12 @@ func TestSolanaChain_VerifiedClient(t *testing.T) { ch := solcfg.Chain{} ch.SetDefaults() - mn := solcfg.MultiNodeConfig{} - mn.SetDefaults() cfg := &solcfg.TOMLConfig{ - ChainID: ptr("devnet"), - Chain: ch, - MultiNode: mn, + ChainID: ptr("devnet"), + Chain: ch, } + cfg.SetDefaults() + testChain := chain{ cfg: cfg, lggr: logger.Test(t), @@ -260,14 +257,12 @@ func TestSolanaChain_VerifiedClient_ParallelClients(t *testing.T) { ch := solcfg.Chain{} ch.SetDefaults() - mn := solcfg.MultiNodeConfig{} - mn.SetDefaults() cfg := &solcfg.TOMLConfig{ - ChainID: ptr("devnet"), - Enabled: ptr(true), - Chain: ch, - MultiNode: mn, + ChainID: ptr("devnet"), + Enabled: ptr(true), + Chain: ch, } + cfg.SetDefaults() testChain := chain{ id: "devnet", cfg: cfg, diff --git a/pkg/solana/monitor/balance.go b/pkg/solana/monitor/balance.go index d1f3b6de5..80f0cdda3 100644 --- a/pkg/solana/monitor/balance.go +++ b/pkg/solana/monitor/balance.go @@ -2,6 +2,7 @@ package monitor import ( "context" + "github.com/smartcontractkit/chainlink-solana/pkg/solana/config" "time" "github.com/gagliardetto/solana-go" @@ -26,18 +27,18 @@ type BalanceClient interface { } // NewBalanceMonitor returns a balance monitoring services.Service which reports the SOL balance of all ks keys to prometheus. -func NewBalanceMonitor(chainID string, cfg Config, multiNodeEnabled bool, lggr logger.Logger, ks Keystore, newReader func() (BalanceClient, error)) services.Service { - return newBalanceMonitor(chainID, cfg, multiNodeEnabled, lggr, ks, newReader) +func NewBalanceMonitor(chainID string, cfg *config.TOMLConfig, lggr logger.Logger, ks Keystore, newReader func() (BalanceClient, error)) services.Service { + return newBalanceMonitor(chainID, cfg, lggr, ks, newReader) } -func newBalanceMonitor(chainID string, cfg Config, multiNodeEnabled bool, lggr logger.Logger, ks Keystore, newReader func() (BalanceClient, error)) *balanceMonitor { +func newBalanceMonitor(chainID string, cfg *config.TOMLConfig, lggr logger.Logger, ks Keystore, newReader func() (BalanceClient, error)) *balanceMonitor { b := balanceMonitor{ chainID: chainID, cfg: cfg, lggr: logger.Named(lggr, "BalanceMonitor"), ks: ks, newReader: newReader, - multiNodeEnabled: multiNodeEnabled, + multiNodeEnabled: cfg.MultiNode.Enabled(), stop: make(chan struct{}), done: make(chan struct{}), } diff --git a/pkg/solana/txm/txm.go b/pkg/solana/txm/txm.go index 2a427ffed..3f4535ddc 100644 --- a/pkg/solana/txm/txm.go +++ b/pkg/solana/txm/txm.go @@ -18,7 +18,6 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/utils" "github.com/smartcontractkit/chainlink-solana/pkg/solana/client" - mn "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" "github.com/smartcontractkit/chainlink-solana/pkg/solana/config" "github.com/smartcontractkit/chainlink-solana/pkg/solana/fees" ) @@ -53,8 +52,8 @@ type Txm struct { ks SimpleKeystore fee fees.Estimator - // Use multiNode for client selection if set - multiNode *mn.MultiNode[mn.StringID, *client.Client] + multiNodeEnabled bool + tc func() (client.ReaderWriter, error) // If multiNode is disabled, use lazy load to fetch client client *utils.LazyLoad[client.ReaderWriter] @@ -80,28 +79,25 @@ type pendingTx struct { } // NewTxm creates a txm. Uses simulation so should only be used to send txes to trusted contracts i.e. OCR. -func NewTxm(chainID string, tc func() (client.ReaderWriter, error), cfg config.Config, multiNode *mn.MultiNode[mn.StringID, *client.Client], ks SimpleKeystore, lggr logger.Logger) *Txm { +func NewTxm(chainID string, tc func() (client.ReaderWriter, error), cfg *config.TOMLConfig, ks SimpleKeystore, lggr logger.Logger) *Txm { return &Txm{ - lggr: lggr, - chSend: make(chan pendingTx, MaxQueueLen), // queue can support 1000 pending txs - chSim: make(chan pendingTx, MaxQueueLen), // queue can support 1000 pending txs - chStop: make(chan struct{}), - cfg: cfg, - txs: newPendingTxContextWithProm(chainID), - ks: ks, - multiNode: multiNode, - client: utils.NewLazyLoad(tc), + lggr: lggr, + chSend: make(chan pendingTx, MaxQueueLen), // queue can support 1000 pending txs + chSim: make(chan pendingTx, MaxQueueLen), // queue can support 1000 pending txs + chStop: make(chan struct{}), + cfg: cfg, + txs: newPendingTxContextWithProm(chainID), + ks: ks, + multiNodeEnabled: cfg.MultiNode.Enabled(), + tc: tc, + client: utils.NewLazyLoad(tc), } } -func (txm *Txm) multiNodeEnabled() bool { - return txm.multiNode != nil -} - // getClient returns a client selected by multiNode if enabled, otherwise returns a client from the lazy load func (txm *Txm) getClient() (client.ReaderWriter, error) { - if txm.multiNodeEnabled() { - return txm.multiNode.SelectRPC() + if txm.multiNodeEnabled { + return txm.tc() } return txm.client.Get() } @@ -150,10 +146,8 @@ func (txm *Txm) run() { tx, id, sig, err := txm.sendWithRetry(ctx, *msg.tx, msg.cfg) if err != nil { txm.lggr.Errorw("failed to send transaction", "error", err) - if !txm.multiNodeEnabled() { - txm.client.Reset() // clear client if tx fails immediately (potentially bad RPC) - } - continue // skip remainining + txm.client.Reset() // clear client if tx fails immediately (potentially bad RPC) + continue // skip remainining } // send tx + signature to simulation queue From be646edef253b742e8d47d06fa67efb18515f92d Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 4 Oct 2024 10:44:04 -0400 Subject: [PATCH 055/174] Remove total difficulty --- pkg/solana/client/client.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/solana/client/client.go b/pkg/solana/client/client.go index aab12942b..642405102 100644 --- a/pkg/solana/client/client.go +++ b/pkg/solana/client/client.go @@ -201,14 +201,12 @@ func (c *Client) onNewHead(ctx context.Context, requestCh <-chan struct{}, head defer c.chainInfoLock.Unlock() if !mn.CtxIsHeathCheckRequest(ctx) { c.highestUserObservations.BlockNumber = max(c.highestUserObservations.BlockNumber, head.BlockNumber()) - c.highestUserObservations.TotalDifficulty = mn.MaxTotalDifficulty(c.highestUserObservations.TotalDifficulty, head.BlockDifficulty()) } select { case <-requestCh: // no need to update latestChainInfo, as rpcClient already started new life cycle return default: c.latestChainInfo.BlockNumber = head.BlockNumber() - c.latestChainInfo.TotalDifficulty = head.BlockDifficulty() } } From ce0c39ce9ca731e6ae0ab827bc3c0ccb05b55d34 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 4 Oct 2024 11:37:59 -0400 Subject: [PATCH 056/174] Register polling subs --- pkg/solana/client/client.go | 43 +++++++++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/pkg/solana/client/client.go b/pkg/solana/client/client.go index 642405102..9c001be8f 100644 --- a/pkg/solana/client/client.go +++ b/pkg/solana/client/client.go @@ -72,6 +72,7 @@ type Client struct { pollInterval time.Duration finalizedBlockPollInterval time.Duration stateMu sync.RWMutex // protects state* fields + subsSliceMu sync.RWMutex subs map[mn.Subscription]struct{} // chStopInFlight can be closed to immediately cancel all in-flight requests on @@ -100,6 +101,7 @@ func NewClient(endpoint string, cfg *config.TOMLConfig, requestTimeout time.Dura pollInterval: cfg.MultiNode.PollInterval(), finalizedBlockPollInterval: cfg.MultiNode.FinalizedBlockPollInterval(), chStopInFlight: make(chan struct{}), + subs: make(map[mn.Subscription]struct{}), }, nil } @@ -129,11 +131,31 @@ func (h *Head) IsValid() bool { var _ mn.RPCClient[mn.StringID, *Head] = (*Client)(nil) var _ mn.SendTxRPCClient[*solana.Transaction] = (*Client)(nil) +// registerSub adds the sub to the rpcClient list +func (c *Client) registerSub(sub mn.Subscription, stopInFLightCh chan struct{}) error { + c.subsSliceMu.Lock() + defer c.subsSliceMu.Unlock() + // ensure that the `sub` belongs to current life cycle of the `rpcClient` and it should not be killed due to + // previous `DisconnectAll` call. + select { + case <-stopInFLightCh: + sub.Unsubscribe() + return fmt.Errorf("failed to register subscription - all in-flight requests were canceled") + default: + } + // TODO: BCI-3358 - delete sub when caller unsubscribes. + c.subs[sub] = struct{}{} + return nil +} + func (c *Client) Dial(ctx context.Context) error { return nil } func (c *Client) SubscribeToHeads(ctx context.Context) (<-chan *Head, mn.Subscription, error) { + ctx, cancel, chStopInFlight, _ := c.acquireQueryCtx(ctx, c.contextDuration) + defer cancel() + if c.pollInterval == 0 { return nil, nil, errors.New("PollInterval is 0") } @@ -142,10 +164,20 @@ func (c *Client) SubscribeToHeads(ctx context.Context) (<-chan *Head, mn.Subscri if err := poller.Start(ctx); err != nil { return nil, nil, err } + + err := c.registerSub(&poller, chStopInFlight) + if err != nil { + poller.Unsubscribe() + return nil, nil, err + } + return channel, &poller, nil } func (c *Client) SubscribeToFinalizedHeads(ctx context.Context) (<-chan *Head, mn.Subscription, error) { + ctx, cancel, chStopInFlight, _ := c.acquireQueryCtx(ctx, c.contextDuration) + defer cancel() + if c.finalizedBlockPollInterval == 0 { return nil, nil, errors.New("FinalizedBlockPollInterval is 0") } @@ -154,6 +186,13 @@ func (c *Client) SubscribeToFinalizedHeads(ctx context.Context) (<-chan *Head, m if err := poller.Start(ctx); err != nil { return nil, nil, err } + + err := c.registerSub(&poller, chStopInFlight) + if err != nil { + poller.Unsubscribe() + return nil, nil, err + } + return channel, &poller, nil } @@ -269,8 +308,8 @@ func (c *Client) IsSyncing(ctx context.Context) (bool, error) { } func (c *Client) UnsubscribeAllExcept(subs ...mn.Subscription) { - c.stateMu.Lock() - defer c.stateMu.Unlock() + c.subsSliceMu.Lock() + defer c.subsSliceMu.Unlock() keepSubs := map[mn.Subscription]struct{}{} for _, sub := range subs { From 1d1fa978b963e1cc40e3104ed5b890cc92f7fd12 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 4 Oct 2024 15:23:27 -0400 Subject: [PATCH 057/174] Extract MultiNodeClient --- pkg/solana/chain.go | 30 +- pkg/solana/client/client.go | 299 +------------------- pkg/solana/client/client_test.go | 54 ---- pkg/solana/client/multinode_client.go | 310 +++++++++++++++++++++ pkg/solana/client/multinode_client_test.go | 66 +++++ pkg/solana/monitor/balance.go | 34 +-- pkg/solana/monitor/balance_test.go | 2 +- pkg/solana/txm/txm.go | 34 +-- 8 files changed, 438 insertions(+), 391 deletions(-) create mode 100644 pkg/solana/client/multinode_client.go create mode 100644 pkg/solana/client/multinode_client_test.go diff --git a/pkg/solana/chain.go b/pkg/solana/chain.go index 11e3cc36f..0b811a140 100644 --- a/pkg/solana/chain.go +++ b/pkg/solana/chain.go @@ -89,8 +89,8 @@ type chain struct { lggr logger.Logger // if multiNode is enabled, the clientCache will not be used - multiNode *mn.MultiNode[mn.StringID, *client.Client] - txSender *mn.TransactionSender[*solanago.Transaction, mn.StringID, *client.Client] + multiNode *mn.MultiNode[mn.StringID, *client.MultiNodeClient] + txSender *mn.TransactionSender[*solanago.Transaction, mn.StringID, *client.MultiNodeClient] // tracking node chain id for verification clientCache map[string]*verifiedCachedClient // map URL -> {client, chainId} [mainnet/testnet/devnet/localnet] @@ -235,28 +235,29 @@ func newChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr logger.L mnCfg := &cfg.MultiNode - var nodes []mn.Node[mn.StringID, *client.Client] - var sendOnlyNodes []mn.SendOnlyNode[mn.StringID, *client.Client] + var nodes []mn.Node[mn.StringID, *client.MultiNodeClient] + var sendOnlyNodes []mn.SendOnlyNode[mn.StringID, *client.MultiNodeClient] for i, nodeInfo := range cfg.ListNodes() { - rpcClient, err := client.NewClient(nodeInfo.URL.String(), cfg, DefaultRequestTimeout, logger.Named(lggr, "Client."+*nodeInfo.Name)) + rpcClient, err := client.NewMultiNodeClient(nodeInfo.URL.String(), cfg, DefaultRequestTimeout, logger.Named(lggr, "Client."+*nodeInfo.Name)) if err != nil { lggr.Warnw("failed to create client", "name", *nodeInfo.Name, "solana-url", nodeInfo.URL.String(), "err", err.Error()) return nil, fmt.Errorf("failed to create client: %w", err) } - newNode := mn.NewNode[mn.StringID, *client.Head, *client.Client]( - mnCfg, mnCfg, lggr, *nodeInfo.URL.URL(), nil, *nodeInfo.Name, - i, mn.StringID(id), 0, rpcClient, chainFamily) - if nodeInfo.SendOnly { - sendOnlyNodes = append(sendOnlyNodes, newNode) + newSendOnly := mn.NewSendOnlyNode[mn.StringID, *client.MultiNodeClient]( + lggr, *nodeInfo.URL.URL(), *nodeInfo.Name, mn.StringID(id), rpcClient) + sendOnlyNodes = append(sendOnlyNodes, newSendOnly) } else { + newNode := mn.NewNode[mn.StringID, *client.Head, *client.MultiNodeClient]( + mnCfg, mnCfg, lggr, *nodeInfo.URL.URL(), nil, *nodeInfo.Name, + i, mn.StringID(id), 0, rpcClient, chainFamily) nodes = append(nodes, newNode) } } - multiNode := mn.NewMultiNode[mn.StringID, *client.Client]( + multiNode := mn.NewMultiNode[mn.StringID, *client.MultiNodeClient]( lggr, mnCfg.SelectionMode(), mnCfg.LeaseDuration(), @@ -273,7 +274,7 @@ func newChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr logger.L return 0 // TODO ClassifySendError(err, clientErrors, logger.Sugared(logger.Nop()), tx, common.Address{}, false) } - txSender := mn.NewTransactionSender[*solanago.Transaction, mn.StringID, *client.Client]( + txSender := mn.NewTransactionSender[*solanago.Transaction, mn.StringID, *client.MultiNodeClient]( lggr, mn.StringID(id), chainFamily, @@ -296,7 +297,10 @@ func newChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr logger.L bc := func() (monitor.BalanceClient, error) { return ch.getClient() } - ch.balanceMonitor = monitor.NewBalanceMonitor(ch.id, cfg, lggr, ks, bc) + + // disable caching reader for MultiNode to always get a healthy client + cacheReader := !cfg.MultiNode.Enabled() + ch.balanceMonitor = monitor.NewBalanceMonitor(ch.id, cfg, lggr, ks, bc, cacheReader) return &ch, nil } diff --git a/pkg/solana/client/client.go b/pkg/solana/client/client.go index 9c001be8f..0b069e4de 100644 --- a/pkg/solana/client/client.go +++ b/pkg/solana/client/client.go @@ -4,8 +4,6 @@ import ( "context" "errors" "fmt" - "math/big" - "sync" "time" "github.com/gagliardetto/solana-go" @@ -13,8 +11,6 @@ import ( "golang.org/x/sync/singleflight" "github.com/smartcontractkit/chainlink-common/pkg/logger" - "github.com/smartcontractkit/chainlink-common/pkg/services" - mn "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" "github.com/smartcontractkit/chainlink-solana/pkg/solana/config" "github.com/smartcontractkit/chainlink-solana/pkg/solana/monitor" @@ -67,297 +63,22 @@ type Client struct { // provides a duplicate function call suppression mechanism requestGroup *singleflight.Group - - // MultiNode - pollInterval time.Duration - finalizedBlockPollInterval time.Duration - stateMu sync.RWMutex // protects state* fields - subsSliceMu sync.RWMutex - subs map[mn.Subscription]struct{} - - // chStopInFlight can be closed to immediately cancel all in-flight requests on - // this RpcClient. Closing and replacing should be serialized through - // stateMu since it can happen on state transitions as well as RpcClient Close. - chStopInFlight chan struct{} - - chainInfoLock sync.RWMutex - // intercepted values seen by callers of the rpcClient excluding health check calls. Need to ensure MultiNode provides repeatable read guarantee - highestUserObservations mn.ChainInfo - // most recent chain info observed during current lifecycle (reseted on DisconnectAll) - latestChainInfo mn.ChainInfo } -func NewClient(endpoint string, cfg *config.TOMLConfig, requestTimeout time.Duration, log logger.Logger) (*Client, error) { +func NewClient(endpoint string, cfg config.Config, requestTimeout time.Duration, log logger.Logger) (*Client, error) { return &Client{ - url: endpoint, - rpc: rpc.New(endpoint), - skipPreflight: cfg.SkipPreflight(), - commitment: cfg.Commitment(), - maxRetries: cfg.MaxRetries(), - txTimeout: cfg.TxTimeout(), - contextDuration: requestTimeout, - log: log, - requestGroup: &singleflight.Group{}, - pollInterval: cfg.MultiNode.PollInterval(), - finalizedBlockPollInterval: cfg.MultiNode.FinalizedBlockPollInterval(), - chStopInFlight: make(chan struct{}), - subs: make(map[mn.Subscription]struct{}), + url: endpoint, + rpc: rpc.New(endpoint), + skipPreflight: cfg.SkipPreflight(), + commitment: cfg.Commitment(), + maxRetries: cfg.MaxRetries(), + txTimeout: cfg.TxTimeout(), + contextDuration: requestTimeout, + log: log, + requestGroup: &singleflight.Group{}, }, nil } -type Head struct { - BlockHeight *uint64 - BlockHash *solana.Hash -} - -func (h *Head) BlockNumber() int64 { - if !h.IsValid() { - return 0 - } - // nolint:gosec - // G115: integer overflow conversion uint64 -> int64 - return int64(*h.BlockHeight) -} - -func (h *Head) BlockDifficulty() *big.Int { - // Not relevant for Solana - return nil -} - -func (h *Head) IsValid() bool { - return h.BlockHeight != nil && h.BlockHash != nil -} - -var _ mn.RPCClient[mn.StringID, *Head] = (*Client)(nil) -var _ mn.SendTxRPCClient[*solana.Transaction] = (*Client)(nil) - -// registerSub adds the sub to the rpcClient list -func (c *Client) registerSub(sub mn.Subscription, stopInFLightCh chan struct{}) error { - c.subsSliceMu.Lock() - defer c.subsSliceMu.Unlock() - // ensure that the `sub` belongs to current life cycle of the `rpcClient` and it should not be killed due to - // previous `DisconnectAll` call. - select { - case <-stopInFLightCh: - sub.Unsubscribe() - return fmt.Errorf("failed to register subscription - all in-flight requests were canceled") - default: - } - // TODO: BCI-3358 - delete sub when caller unsubscribes. - c.subs[sub] = struct{}{} - return nil -} - -func (c *Client) Dial(ctx context.Context) error { - return nil -} - -func (c *Client) SubscribeToHeads(ctx context.Context) (<-chan *Head, mn.Subscription, error) { - ctx, cancel, chStopInFlight, _ := c.acquireQueryCtx(ctx, c.contextDuration) - defer cancel() - - if c.pollInterval == 0 { - return nil, nil, errors.New("PollInterval is 0") - } - timeout := c.pollInterval - poller, channel := mn.NewPoller[*Head](c.pollInterval, c.LatestBlock, timeout, c.log) - if err := poller.Start(ctx); err != nil { - return nil, nil, err - } - - err := c.registerSub(&poller, chStopInFlight) - if err != nil { - poller.Unsubscribe() - return nil, nil, err - } - - return channel, &poller, nil -} - -func (c *Client) SubscribeToFinalizedHeads(ctx context.Context) (<-chan *Head, mn.Subscription, error) { - ctx, cancel, chStopInFlight, _ := c.acquireQueryCtx(ctx, c.contextDuration) - defer cancel() - - if c.finalizedBlockPollInterval == 0 { - return nil, nil, errors.New("FinalizedBlockPollInterval is 0") - } - timeout := c.finalizedBlockPollInterval - poller, channel := mn.NewPoller[*Head](c.finalizedBlockPollInterval, c.LatestFinalizedBlock, timeout, c.log) - if err := poller.Start(ctx); err != nil { - return nil, nil, err - } - - err := c.registerSub(&poller, chStopInFlight) - if err != nil { - poller.Unsubscribe() - return nil, nil, err - } - - return channel, &poller, nil -} - -func (c *Client) LatestBlock(ctx context.Context) (*Head, error) { - // capture chStopInFlight to ensure we are not updating chainInfo with observations related to previous life cycle - ctx, cancel, chStopInFlight, rawRPC := c.acquireQueryCtx(ctx, c.contextDuration) - defer cancel() - - result, err := rawRPC.GetLatestBlockhash(ctx, rpc.CommitmentConfirmed) - if err != nil { - return nil, err - } - - head := &Head{ - BlockHeight: &result.Value.LastValidBlockHeight, - BlockHash: &result.Value.Blockhash, - } - c.onNewHead(ctx, chStopInFlight, head) - return head, nil -} - -func (c *Client) LatestFinalizedBlock(ctx context.Context) (*Head, error) { - ctx, cancel, chStopInFlight, rawRPC := c.acquireQueryCtx(ctx, c.contextDuration) - defer cancel() - - result, err := rawRPC.GetLatestBlockhash(ctx, rpc.CommitmentFinalized) - if err != nil { - return nil, err - } - - head := &Head{ - BlockHeight: &result.Value.LastValidBlockHeight, - BlockHash: &result.Value.Blockhash, - } - c.onNewFinalizedHead(ctx, chStopInFlight, head) - return head, nil -} - -func (c *Client) onNewHead(ctx context.Context, requestCh <-chan struct{}, head *Head) { - if head == nil { - return - } - - c.chainInfoLock.Lock() - defer c.chainInfoLock.Unlock() - if !mn.CtxIsHeathCheckRequest(ctx) { - c.highestUserObservations.BlockNumber = max(c.highestUserObservations.BlockNumber, head.BlockNumber()) - } - select { - case <-requestCh: // no need to update latestChainInfo, as rpcClient already started new life cycle - return - default: - c.latestChainInfo.BlockNumber = head.BlockNumber() - } -} - -func (c *Client) onNewFinalizedHead(ctx context.Context, requestCh <-chan struct{}, head *Head) { - if head == nil { - return - } - c.chainInfoLock.Lock() - defer c.chainInfoLock.Unlock() - if !mn.CtxIsHeathCheckRequest(ctx) { - c.highestUserObservations.FinalizedBlockNumber = max(c.highestUserObservations.FinalizedBlockNumber, head.BlockNumber()) - } - select { - case <-requestCh: // no need to update latestChainInfo, as rpcClient already started new life cycle - return - default: - c.latestChainInfo.FinalizedBlockNumber = head.BlockNumber() - } -} - -// makeQueryCtx returns a context that cancels if: -// 1. Passed in ctx cancels -// 2. Passed in channel is closed -// 3. Default timeout is reached (queryTimeout) -func makeQueryCtx(ctx context.Context, ch services.StopChan, timeout time.Duration) (context.Context, context.CancelFunc) { - var chCancel, timeoutCancel context.CancelFunc - ctx, chCancel = ch.Ctx(ctx) - ctx, timeoutCancel = context.WithTimeout(ctx, timeout) - cancel := func() { - chCancel() - timeoutCancel() - } - return ctx, cancel -} - -func (c *Client) acquireQueryCtx(parentCtx context.Context, timeout time.Duration) (ctx context.Context, cancel context.CancelFunc, - chStopInFlight chan struct{}, raw *rpc.Client) { - // Need to wrap in mutex because state transition can cancel and replace context - c.stateMu.RLock() - chStopInFlight = c.chStopInFlight - cp := *c.rpc - raw = &cp - c.stateMu.RUnlock() - ctx, cancel = makeQueryCtx(parentCtx, chStopInFlight, timeout) - return -} - -func (c *Client) Ping(ctx context.Context) error { - version, err := c.rpc.GetVersion(ctx) - if err != nil { - return fmt.Errorf("ping failed: %v", err) - } - c.log.Debugf("ping client version: %s", version.SolanaCore) - return err -} - -func (c *Client) IsSyncing(ctx context.Context) (bool, error) { - // Not in use for Solana - return false, nil -} - -func (c *Client) UnsubscribeAllExcept(subs ...mn.Subscription) { - c.subsSliceMu.Lock() - defer c.subsSliceMu.Unlock() - - keepSubs := map[mn.Subscription]struct{}{} - for _, sub := range subs { - keepSubs[sub] = struct{}{} - } - - for sub := range c.subs { - if _, keep := keepSubs[sub]; !keep { - sub.Unsubscribe() - delete(c.subs, sub) - } - } -} - -// cancelInflightRequests closes and replaces the chStopInFlight -func (c *Client) cancelInflightRequests() { - c.stateMu.Lock() - defer c.stateMu.Unlock() - close(c.chStopInFlight) - c.chStopInFlight = make(chan struct{}) -} - -func (c *Client) Close() { - defer func() { - err := c.rpc.Close() - if err != nil { - c.log.Errorf("error closing rpc: %v", err) - } - }() - c.cancelInflightRequests() - c.UnsubscribeAllExcept() - c.chainInfoLock.Lock() - c.latestChainInfo = mn.ChainInfo{} - c.chainInfoLock.Unlock() -} - -func (c *Client) GetInterceptedChainInfo() (latest, highestUserObservations mn.ChainInfo) { - c.chainInfoLock.Lock() - defer c.chainInfoLock.Unlock() - return c.latestChainInfo, c.highestUserObservations -} - -func (c *Client) SendTransaction(ctx context.Context, tx *solana.Transaction) error { - // TODO: Use Transaction Sender - _, err := c.SendTx(ctx, tx) - return err -} - func (c *Client) latency(name string) func() { start := time.Now() return func() { diff --git a/pkg/solana/client/client_test.go b/pkg/solana/client/client_test.go index 5d0c441e4..740e0297c 100644 --- a/pkg/solana/client/client_test.go +++ b/pkg/solana/client/client_test.go @@ -19,8 +19,6 @@ import ( "github.com/stretchr/testify/require" "github.com/smartcontractkit/chainlink-common/pkg/logger" - "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" - mn "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" "github.com/smartcontractkit/chainlink-solana/pkg/solana/config" "github.com/smartcontractkit/chainlink-solana/pkg/solana/monitor" @@ -298,58 +296,6 @@ func TestClient_SendTxDuplicates_Integration(t *testing.T) { assert.Equal(t, uint64(5_000), initBal-endBal) } -func TestClient_Subscriptions_Integration(t *testing.T) { - url := SetupLocalSolNode(t) - privKey, err := solana.NewRandomPrivateKey() - require.NoError(t, err) - pubKey := privKey.PublicKey() - FundTestAccounts(t, []solana.PublicKey{pubKey}, url) - - requestTimeout := 5 * time.Second - lggr := logger.Test(t) - cfg := config.NewDefault() - // Enable MultiNode - enabled := true - cfg.MultiNode.SetDefaults() - cfg.Enabled = &enabled - - c, err := NewClient(url, cfg, requestTimeout, lggr) - require.NoError(t, err) - - err = c.Ping(tests.Context(t)) - require.NoError(t, err) - - ch, sub, err := c.SubscribeToHeads(tests.Context(t)) - require.NoError(t, err) - defer sub.Unsubscribe() - - finalizedCh, finalizedSub, err := c.SubscribeToFinalizedHeads(tests.Context(t)) - require.NoError(t, err) - defer finalizedSub.Unsubscribe() - - require.NoError(t, err) - ctx, cancel := context.WithTimeout(tests.Context(t), time.Minute) - defer cancel() - - select { - case head := <-ch: - require.NotEqual(t, solana.Hash{}, head.BlockHash) - latest, _ := c.GetInterceptedChainInfo() - require.Equal(t, head.BlockNumber(), latest.BlockNumber) - case <-ctx.Done(): - t.Fatal("failed to receive head: ", ctx.Err()) - } - - select { - case finalizedHead := <-finalizedCh: - require.NotEqual(t, solana.Hash{}, finalizedHead.BlockHash) - latest, _ := c.GetInterceptedChainInfo() - require.Equal(t, finalizedHead.BlockNumber(), latest.FinalizedBlockNumber) - case <-ctx.Done(): - t.Fatal("failed to receive finalized head: ", ctx.Err()) - } -} - func TestClientLatency(t *testing.T) { c := Client{} v := 100 diff --git a/pkg/solana/client/multinode_client.go b/pkg/solana/client/multinode_client.go new file mode 100644 index 000000000..60efaab6d --- /dev/null +++ b/pkg/solana/client/multinode_client.go @@ -0,0 +1,310 @@ +package client + +import ( + "context" + "errors" + "fmt" + "math/big" + "sync" + "time" + + "github.com/gagliardetto/solana-go" + "github.com/gagliardetto/solana-go/rpc" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/services" + + mn "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" + "github.com/smartcontractkit/chainlink-solana/pkg/solana/config" +) + +type Head struct { + BlockHeight *uint64 + BlockHash *solana.Hash +} + +func (h *Head) BlockNumber() int64 { + if !h.IsValid() { + return 0 + } + // nolint:gosec + // G115: integer overflow conversion uint64 -> int64 + return int64(*h.BlockHeight) +} + +func (h *Head) BlockDifficulty() *big.Int { + // Not relevant for Solana + return nil +} + +func (h *Head) IsValid() bool { + return h.BlockHeight != nil && h.BlockHash != nil +} + +var _ mn.RPCClient[mn.StringID, *Head] = (*MultiNodeClient)(nil) +var _ mn.SendTxRPCClient[*solana.Transaction] = (*MultiNodeClient)(nil) + +type MultiNodeClient struct { + Client + cfg *config.TOMLConfig + log logger.Logger + rpc *rpc.Client + stateMu sync.RWMutex // protects state* fields + subsSliceMu sync.RWMutex + subs map[mn.Subscription]struct{} + + // chStopInFlight can be closed to immediately cancel all in-flight requests on + // this RpcClient. Closing and replacing should be serialized through + // stateMu since it can happen on state transitions as well as RpcClient Close. + chStopInFlight chan struct{} + + chainInfoLock sync.RWMutex + // intercepted values seen by callers of the rpcClient excluding health check calls. Need to ensure MultiNode provides repeatable read guarantee + highestUserObservations mn.ChainInfo + // most recent chain info observed during current lifecycle (reseted on DisconnectAll) + latestChainInfo mn.ChainInfo +} + +func NewMultiNodeClient(endpoint string, cfg *config.TOMLConfig, requestTimeout time.Duration, log logger.Logger) (*MultiNodeClient, error) { + client, err := NewClient(endpoint, cfg, requestTimeout, log) + if err != nil { + return nil, err + } + + return &MultiNodeClient{ + Client: *client, + cfg: cfg, + log: log, + rpc: rpc.New(endpoint), + subs: make(map[mn.Subscription]struct{}), + chStopInFlight: make(chan struct{}), + }, nil +} + +// registerSub adds the sub to the rpcClient list +func (m *MultiNodeClient) registerSub(sub mn.Subscription, stopInFLightCh chan struct{}) error { + m.subsSliceMu.Lock() + defer m.subsSliceMu.Unlock() + // ensure that the `sub` belongs to current life cycle of the `rpcClient` and it should not be killed due to + // previous `DisconnectAll` call. + select { + case <-stopInFLightCh: + sub.Unsubscribe() + return fmt.Errorf("failed to register subscription - all in-flight requests were canceled") + default: + } + // TODO: BCI-3358 - delete sub when caller unsubscribes. + m.subs[sub] = struct{}{} + return nil +} + +func (m *MultiNodeClient) Dial(ctx context.Context) error { + return nil +} + +func (m *MultiNodeClient) SubscribeToHeads(ctx context.Context) (<-chan *Head, mn.Subscription, error) { + ctx, cancel, chStopInFlight, _ := m.acquireQueryCtx(ctx, m.cfg.TxTimeout()) + defer cancel() + + pollInterval := m.cfg.MultiNode.PollInterval() + if pollInterval == 0 { + return nil, nil, errors.New("PollInterval is 0") + } + timeout := pollInterval + poller, channel := mn.NewPoller[*Head](pollInterval, m.LatestBlock, timeout, m.log) + if err := poller.Start(ctx); err != nil { + return nil, nil, err + } + + err := m.registerSub(&poller, chStopInFlight) + if err != nil { + poller.Unsubscribe() + return nil, nil, err + } + + return channel, &poller, nil +} + +func (m *MultiNodeClient) SubscribeToFinalizedHeads(ctx context.Context) (<-chan *Head, mn.Subscription, error) { + ctx, cancel, chStopInFlight, _ := m.acquireQueryCtx(ctx, m.contextDuration) + defer cancel() + + finalizedBlockPollInterval := m.cfg.MultiNode.FinalizedBlockPollInterval() + if finalizedBlockPollInterval == 0 { + return nil, nil, errors.New("FinalizedBlockPollInterval is 0") + } + timeout := finalizedBlockPollInterval + poller, channel := mn.NewPoller[*Head](finalizedBlockPollInterval, m.LatestFinalizedBlock, timeout, m.log) + if err := poller.Start(ctx); err != nil { + return nil, nil, err + } + + err := m.registerSub(&poller, chStopInFlight) + if err != nil { + poller.Unsubscribe() + return nil, nil, err + } + + return channel, &poller, nil +} + +func (m *MultiNodeClient) LatestBlock(ctx context.Context) (*Head, error) { + // capture chStopInFlight to ensure we are not updating chainInfo with observations related to previous life cycle + ctx, cancel, chStopInFlight, rawRPC := m.acquireQueryCtx(ctx, m.contextDuration) + defer cancel() + + result, err := rawRPC.GetLatestBlockhash(ctx, rpc.CommitmentConfirmed) + if err != nil { + return nil, err + } + + head := &Head{ + BlockHeight: &result.Value.LastValidBlockHeight, + BlockHash: &result.Value.Blockhash, + } + m.onNewHead(ctx, chStopInFlight, head) + return head, nil +} + +func (m *MultiNodeClient) LatestFinalizedBlock(ctx context.Context) (*Head, error) { + ctx, cancel, chStopInFlight, rawRPC := m.acquireQueryCtx(ctx, m.contextDuration) + defer cancel() + + result, err := rawRPC.GetLatestBlockhash(ctx, rpc.CommitmentFinalized) + if err != nil { + return nil, err + } + + head := &Head{ + BlockHeight: &result.Value.LastValidBlockHeight, + BlockHash: &result.Value.Blockhash, + } + m.onNewFinalizedHead(ctx, chStopInFlight, head) + return head, nil +} + +func (m *MultiNodeClient) onNewHead(ctx context.Context, requestCh <-chan struct{}, head *Head) { + if head == nil { + return + } + + m.chainInfoLock.Lock() + defer m.chainInfoLock.Unlock() + if !mn.CtxIsHeathCheckRequest(ctx) { + m.highestUserObservations.BlockNumber = max(m.highestUserObservations.BlockNumber, head.BlockNumber()) + } + select { + case <-requestCh: // no need to update latestChainInfo, as rpcClient already started new life cycle + return + default: + m.latestChainInfo.BlockNumber = head.BlockNumber() + } +} + +func (m *MultiNodeClient) onNewFinalizedHead(ctx context.Context, requestCh <-chan struct{}, head *Head) { + if head == nil { + return + } + m.chainInfoLock.Lock() + defer m.chainInfoLock.Unlock() + if !mn.CtxIsHeathCheckRequest(ctx) { + m.highestUserObservations.FinalizedBlockNumber = max(m.highestUserObservations.FinalizedBlockNumber, head.BlockNumber()) + } + select { + case <-requestCh: // no need to update latestChainInfo, as rpcClient already started new life cycle + return + default: + m.latestChainInfo.FinalizedBlockNumber = head.BlockNumber() + } +} + +// makeQueryCtx returns a context that cancels if: +// 1. Passed in ctx cancels +// 2. Passed in channel is closed +// 3. Default timeout is reached (queryTimeout) +func makeQueryCtx(ctx context.Context, ch services.StopChan, timeout time.Duration) (context.Context, context.CancelFunc) { + var chCancel, timeoutCancel context.CancelFunc + ctx, chCancel = ch.Ctx(ctx) + ctx, timeoutCancel = context.WithTimeout(ctx, timeout) + cancel := func() { + chCancel() + timeoutCancel() + } + return ctx, cancel +} + +func (m *MultiNodeClient) acquireQueryCtx(parentCtx context.Context, timeout time.Duration) (ctx context.Context, cancel context.CancelFunc, + chStopInFlight chan struct{}, raw *rpc.Client) { + // Need to wrap in mutex because state transition can cancel and replace context + m.stateMu.RLock() + chStopInFlight = m.chStopInFlight + cp := *m.rpc + raw = &cp + m.stateMu.RUnlock() + ctx, cancel = makeQueryCtx(parentCtx, chStopInFlight, timeout) + return +} + +func (m *MultiNodeClient) Ping(ctx context.Context) error { + version, err := m.rpc.GetVersion(ctx) + if err != nil { + return fmt.Errorf("ping failed: %v", err) + } + m.log.Debugf("ping client version: %s", version.SolanaCore) + return err +} + +func (m *MultiNodeClient) IsSyncing(ctx context.Context) (bool, error) { + // Not in use for Solana + return false, nil +} + +func (m *MultiNodeClient) UnsubscribeAllExcept(subs ...mn.Subscription) { + m.subsSliceMu.Lock() + defer m.subsSliceMu.Unlock() + + keepSubs := map[mn.Subscription]struct{}{} + for _, sub := range subs { + keepSubs[sub] = struct{}{} + } + + for sub := range m.subs { + if _, keep := keepSubs[sub]; !keep { + sub.Unsubscribe() + delete(m.subs, sub) + } + } +} + +// cancelInflightRequests closes and replaces the chStopInFlight +func (m *MultiNodeClient) cancelInflightRequests() { + m.stateMu.Lock() + defer m.stateMu.Unlock() + close(m.chStopInFlight) + m.chStopInFlight = make(chan struct{}) +} + +func (m *MultiNodeClient) Close() { + defer func() { + err := m.rpc.Close() + if err != nil { + m.log.Errorf("error closing rpc: %v", err) + } + }() + m.cancelInflightRequests() + m.UnsubscribeAllExcept() + m.chainInfoLock.Lock() + m.latestChainInfo = mn.ChainInfo{} + m.chainInfoLock.Unlock() +} + +func (m *MultiNodeClient) GetInterceptedChainInfo() (latest, highestUserObservations mn.ChainInfo) { + m.chainInfoLock.Lock() + defer m.chainInfoLock.Unlock() + return m.latestChainInfo, m.highestUserObservations +} + +func (m *MultiNodeClient) SendTransaction(ctx context.Context, tx *solana.Transaction) error { + // TODO: Use Transaction Sender + _, err := m.SendTx(ctx, tx) + return err +} diff --git a/pkg/solana/client/multinode_client_test.go b/pkg/solana/client/multinode_client_test.go new file mode 100644 index 000000000..f1ac65430 --- /dev/null +++ b/pkg/solana/client/multinode_client_test.go @@ -0,0 +1,66 @@ +package client + +import ( + "context" + "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" + "testing" + "time" + + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-solana/pkg/solana/config" +) + +func TestMultiNodeClient_Subscriptions_Integration(t *testing.T) { + url := SetupLocalSolNode(t) + privKey, err := solana.NewRandomPrivateKey() + require.NoError(t, err) + pubKey := privKey.PublicKey() + FundTestAccounts(t, []solana.PublicKey{pubKey}, url) + + requestTimeout := 5 * time.Second + lggr := logger.Test(t) + cfg := config.NewDefault() + // Enable MultiNode + enabled := true + cfg.MultiNode.SetDefaults() + cfg.Enabled = &enabled + + c, err := NewMultiNodeClient(url, cfg, requestTimeout, lggr) + require.NoError(t, err) + + err = c.Ping(tests.Context(t)) + require.NoError(t, err) + + ch, sub, err := c.SubscribeToHeads(tests.Context(t)) + require.NoError(t, err) + defer sub.Unsubscribe() + + finalizedCh, finalizedSub, err := c.SubscribeToFinalizedHeads(tests.Context(t)) + require.NoError(t, err) + defer finalizedSub.Unsubscribe() + + require.NoError(t, err) + ctx, cancel := context.WithTimeout(tests.Context(t), time.Minute) + defer cancel() + + select { + case head := <-ch: + require.NotEqual(t, solana.Hash{}, head.BlockHash) + latest, _ := c.GetInterceptedChainInfo() + require.Equal(t, head.BlockNumber(), latest.BlockNumber) + case <-ctx.Done(): + t.Fatal("failed to receive head: ", ctx.Err()) + } + + select { + case finalizedHead := <-finalizedCh: + require.NotEqual(t, solana.Hash{}, finalizedHead.BlockHash) + latest, _ := c.GetInterceptedChainInfo() + require.Equal(t, finalizedHead.BlockNumber(), latest.FinalizedBlockNumber) + case <-ctx.Done(): + t.Fatal("failed to receive finalized head: ", ctx.Err()) + } +} diff --git a/pkg/solana/monitor/balance.go b/pkg/solana/monitor/balance.go index 80f0cdda3..09e153ba0 100644 --- a/pkg/solana/monitor/balance.go +++ b/pkg/solana/monitor/balance.go @@ -2,7 +2,6 @@ package monitor import ( "context" - "github.com/smartcontractkit/chainlink-solana/pkg/solana/config" "time" "github.com/gagliardetto/solana-go" @@ -27,20 +26,20 @@ type BalanceClient interface { } // NewBalanceMonitor returns a balance monitoring services.Service which reports the SOL balance of all ks keys to prometheus. -func NewBalanceMonitor(chainID string, cfg *config.TOMLConfig, lggr logger.Logger, ks Keystore, newReader func() (BalanceClient, error)) services.Service { - return newBalanceMonitor(chainID, cfg, lggr, ks, newReader) +func NewBalanceMonitor(chainID string, cfg Config, lggr logger.Logger, ks Keystore, newReader func() (BalanceClient, error), cacheReader bool) services.Service { + return newBalanceMonitor(chainID, cfg, lggr, ks, newReader, cacheReader) } -func newBalanceMonitor(chainID string, cfg *config.TOMLConfig, lggr logger.Logger, ks Keystore, newReader func() (BalanceClient, error)) *balanceMonitor { +func newBalanceMonitor(chainID string, cfg Config, lggr logger.Logger, ks Keystore, newReader func() (BalanceClient, error), cacheReader bool) *balanceMonitor { b := balanceMonitor{ - chainID: chainID, - cfg: cfg, - lggr: logger.Named(lggr, "BalanceMonitor"), - ks: ks, - newReader: newReader, - multiNodeEnabled: cfg.MultiNode.Enabled(), - stop: make(chan struct{}), - done: make(chan struct{}), + chainID: chainID, + cfg: cfg, + lggr: logger.Named(lggr, "BalanceMonitor"), + ks: ks, + cacheReader: cacheReader, + newReader: newReader, + stop: make(chan struct{}), + done: make(chan struct{}), } b.updateFn = b.updateProm return &b @@ -55,8 +54,10 @@ type balanceMonitor struct { newReader func() (BalanceClient, error) updateFn func(acc solana.PublicKey, lamports uint64) // overridable for testing - multiNodeEnabled bool - reader BalanceClient + // cacheReader will use a single reader until encountering an error. + // Disabled when using MultiNode to always get a healthy client. + cacheReader bool + reader BalanceClient stop services.StopChan done chan struct{} @@ -104,12 +105,11 @@ func (b *balanceMonitor) monitor() { // getReader returns the cached solanaClient.Reader, or creates a new one if nil. func (b *balanceMonitor) getReader() (BalanceClient, error) { - if b.multiNodeEnabled { - // Allow MultiNode to select the reader + if !b.cacheReader { return b.newReader() } - // Self leasing wth cached reader + // Use cached reader if available if b.reader == nil { var err error b.reader, err = b.newReader() diff --git a/pkg/solana/monitor/balance_test.go b/pkg/solana/monitor/balance_test.go index a28855534..170e45abe 100644 --- a/pkg/solana/monitor/balance_test.go +++ b/pkg/solana/monitor/balance_test.go @@ -46,7 +46,7 @@ func TestBalanceMonitor(t *testing.T) { exp = append(exp, update{acc.String(), expBals[i]}) } cfg := &config{balancePollPeriod: time.Second} - b := newBalanceMonitor(chainID, cfg, false, logger.Test(t), ks, nil) + b := newBalanceMonitor(chainID, cfg, logger.Test(t), ks, nil, true) var got []update done := make(chan struct{}) b.updateFn = func(acc solana.PublicKey, lamports uint64) { diff --git a/pkg/solana/txm/txm.go b/pkg/solana/txm/txm.go index 3f4535ddc..ef62ba9e3 100644 --- a/pkg/solana/txm/txm.go +++ b/pkg/solana/txm/txm.go @@ -51,10 +51,10 @@ type Txm struct { txs PendingTxContext ks SimpleKeystore fee fees.Estimator - - multiNodeEnabled bool - tc func() (client.ReaderWriter, error) - + tc func() (client.ReaderWriter, error) + // lazyLoadClient uses a single client until encountering an error. + // Disabled when using MultiNode to always get a healthy client. + lazyLoadClient bool // If multiNode is disabled, use lazy load to fetch client client *utils.LazyLoad[client.ReaderWriter] } @@ -81,25 +81,25 @@ type pendingTx struct { // NewTxm creates a txm. Uses simulation so should only be used to send txes to trusted contracts i.e. OCR. func NewTxm(chainID string, tc func() (client.ReaderWriter, error), cfg *config.TOMLConfig, ks SimpleKeystore, lggr logger.Logger) *Txm { return &Txm{ - lggr: lggr, - chSend: make(chan pendingTx, MaxQueueLen), // queue can support 1000 pending txs - chSim: make(chan pendingTx, MaxQueueLen), // queue can support 1000 pending txs - chStop: make(chan struct{}), - cfg: cfg, - txs: newPendingTxContextWithProm(chainID), - ks: ks, - multiNodeEnabled: cfg.MultiNode.Enabled(), - tc: tc, - client: utils.NewLazyLoad(tc), + lggr: lggr, + chSend: make(chan pendingTx, MaxQueueLen), // queue can support 1000 pending txs + chSim: make(chan pendingTx, MaxQueueLen), // queue can support 1000 pending txs + chStop: make(chan struct{}), + cfg: cfg, + txs: newPendingTxContextWithProm(chainID), + ks: ks, + tc: tc, + lazyLoadClient: !cfg.MultiNode.Enabled(), + client: utils.NewLazyLoad(tc), } } // getClient returns a client selected by multiNode if enabled, otherwise returns a client from the lazy load func (txm *Txm) getClient() (client.ReaderWriter, error) { - if txm.multiNodeEnabled { - return txm.tc() + if txm.lazyLoadClient { + return txm.client.Get() } - return txm.client.Get() + return txm.tc() } // Start subscribes to queuing channel and processes them. From 7cefbac0490855a70f3293257a8774a219efcf70 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 4 Oct 2024 15:45:59 -0400 Subject: [PATCH 058/174] Remove caching changes --- pkg/solana/monitor/balance.go | 31 ++++++++++------------------ pkg/solana/txm/txm.go | 38 +++++++++++------------------------ 2 files changed, 23 insertions(+), 46 deletions(-) diff --git a/pkg/solana/monitor/balance.go b/pkg/solana/monitor/balance.go index 09e153ba0..00f873488 100644 --- a/pkg/solana/monitor/balance.go +++ b/pkg/solana/monitor/balance.go @@ -26,20 +26,19 @@ type BalanceClient interface { } // NewBalanceMonitor returns a balance monitoring services.Service which reports the SOL balance of all ks keys to prometheus. -func NewBalanceMonitor(chainID string, cfg Config, lggr logger.Logger, ks Keystore, newReader func() (BalanceClient, error), cacheReader bool) services.Service { - return newBalanceMonitor(chainID, cfg, lggr, ks, newReader, cacheReader) +func NewBalanceMonitor(chainID string, cfg Config, lggr logger.Logger, ks Keystore, newReader func() (BalanceClient, error)) services.Service { + return newBalanceMonitor(chainID, cfg, lggr, ks, newReader) } -func newBalanceMonitor(chainID string, cfg Config, lggr logger.Logger, ks Keystore, newReader func() (BalanceClient, error), cacheReader bool) *balanceMonitor { +func newBalanceMonitor(chainID string, cfg Config, lggr logger.Logger, ks Keystore, newReader func() (BalanceClient, error)) *balanceMonitor { b := balanceMonitor{ - chainID: chainID, - cfg: cfg, - lggr: logger.Named(lggr, "BalanceMonitor"), - ks: ks, - cacheReader: cacheReader, - newReader: newReader, - stop: make(chan struct{}), - done: make(chan struct{}), + chainID: chainID, + cfg: cfg, + lggr: logger.Named(lggr, "BalanceMonitor"), + ks: ks, + newReader: newReader, + stop: make(chan struct{}), + done: make(chan struct{}), } b.updateFn = b.updateProm return &b @@ -54,10 +53,7 @@ type balanceMonitor struct { newReader func() (BalanceClient, error) updateFn func(acc solana.PublicKey, lamports uint64) // overridable for testing - // cacheReader will use a single reader until encountering an error. - // Disabled when using MultiNode to always get a healthy client. - cacheReader bool - reader BalanceClient + reader BalanceClient stop services.StopChan done chan struct{} @@ -105,11 +101,6 @@ func (b *balanceMonitor) monitor() { // getReader returns the cached solanaClient.Reader, or creates a new one if nil. func (b *balanceMonitor) getReader() (BalanceClient, error) { - if !b.cacheReader { - return b.newReader() - } - - // Use cached reader if available if b.reader == nil { var err error b.reader, err = b.newReader() diff --git a/pkg/solana/txm/txm.go b/pkg/solana/txm/txm.go index ef62ba9e3..a03c2fbe4 100644 --- a/pkg/solana/txm/txm.go +++ b/pkg/solana/txm/txm.go @@ -51,10 +51,6 @@ type Txm struct { txs PendingTxContext ks SimpleKeystore fee fees.Estimator - tc func() (client.ReaderWriter, error) - // lazyLoadClient uses a single client until encountering an error. - // Disabled when using MultiNode to always get a healthy client. - lazyLoadClient bool // If multiNode is disabled, use lazy load to fetch client client *utils.LazyLoad[client.ReaderWriter] } @@ -79,29 +75,19 @@ type pendingTx struct { } // NewTxm creates a txm. Uses simulation so should only be used to send txes to trusted contracts i.e. OCR. -func NewTxm(chainID string, tc func() (client.ReaderWriter, error), cfg *config.TOMLConfig, ks SimpleKeystore, lggr logger.Logger) *Txm { +func NewTxm(chainID string, tc func() (client.ReaderWriter, error), cfg config.Config, ks SimpleKeystore, lggr logger.Logger) *Txm { return &Txm{ - lggr: lggr, - chSend: make(chan pendingTx, MaxQueueLen), // queue can support 1000 pending txs - chSim: make(chan pendingTx, MaxQueueLen), // queue can support 1000 pending txs - chStop: make(chan struct{}), - cfg: cfg, - txs: newPendingTxContextWithProm(chainID), - ks: ks, - tc: tc, - lazyLoadClient: !cfg.MultiNode.Enabled(), - client: utils.NewLazyLoad(tc), + lggr: lggr, + chSend: make(chan pendingTx, MaxQueueLen), // queue can support 1000 pending txs + chSim: make(chan pendingTx, MaxQueueLen), // queue can support 1000 pending txs + chStop: make(chan struct{}), + cfg: cfg, + txs: newPendingTxContextWithProm(chainID), + ks: ks, + client: utils.NewLazyLoad(tc), } } -// getClient returns a client selected by multiNode if enabled, otherwise returns a client from the lazy load -func (txm *Txm) getClient() (client.ReaderWriter, error) { - if txm.lazyLoadClient { - return txm.client.Get() - } - return txm.tc() -} - // Start subscribes to queuing channel and processes them. func (txm *Txm) Start(ctx context.Context) error { return txm.starter.StartOnce("solana_txm", func() error { @@ -169,7 +155,7 @@ func (txm *Txm) run() { func (txm *Txm) sendWithRetry(chanCtx context.Context, baseTx solanaGo.Transaction, txcfg TxConfig) (solanaGo.Transaction, uuid.UUID, solanaGo.Signature, error) { // fetch client - client, clientErr := txm.getClient() + client, clientErr := txm.client.Get() if clientErr != nil { return solanaGo.Transaction{}, uuid.Nil, solanaGo.Signature{}, fmt.Errorf("failed to get client in soltxm.sendWithRetry: %w", clientErr) } @@ -378,7 +364,7 @@ func (txm *Txm) confirm(ctx context.Context) { } // get client - client, err := txm.getClient() + client, err := txm.client.Get() if err != nil { txm.lggr.Errorw("failed to get client in soltxm.confirm", "error", err) break // exit switch @@ -492,7 +478,7 @@ func (txm *Txm) simulate(ctx context.Context) { return case msg := <-txm.chSim: // get client - client, err := txm.getClient() + client, err := txm.client.Get() if err != nil { txm.lggr.Errorw("failed to get client in soltxm.simulate", "error", err) continue From 50c4f8e0110098b3626b23cd842ce2a7de5b3800 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 4 Oct 2024 15:47:51 -0400 Subject: [PATCH 059/174] Undo cache changes --- pkg/solana/monitor/balance_test.go | 2 +- pkg/solana/txm/txm.go | 3 +-- pkg/solana/txm/txm_race_test.go | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pkg/solana/monitor/balance_test.go b/pkg/solana/monitor/balance_test.go index 170e45abe..ff98d0508 100644 --- a/pkg/solana/monitor/balance_test.go +++ b/pkg/solana/monitor/balance_test.go @@ -46,7 +46,7 @@ func TestBalanceMonitor(t *testing.T) { exp = append(exp, update{acc.String(), expBals[i]}) } cfg := &config{balancePollPeriod: time.Second} - b := newBalanceMonitor(chainID, cfg, logger.Test(t), ks, nil, true) + b := newBalanceMonitor(chainID, cfg, logger.Test(t), ks, nil) var got []update done := make(chan struct{}) b.updateFn = func(acc solana.PublicKey, lamports uint64) { diff --git a/pkg/solana/txm/txm.go b/pkg/solana/txm/txm.go index a03c2fbe4..63a96bf8b 100644 --- a/pkg/solana/txm/txm.go +++ b/pkg/solana/txm/txm.go @@ -51,8 +51,7 @@ type Txm struct { txs PendingTxContext ks SimpleKeystore fee fees.Estimator - // If multiNode is disabled, use lazy load to fetch client - client *utils.LazyLoad[client.ReaderWriter] + client *utils.LazyLoad[client.ReaderWriter] } type TxConfig struct { diff --git a/pkg/solana/txm/txm_race_test.go b/pkg/solana/txm/txm_race_test.go index acdfa4908..aa0a6de6a 100644 --- a/pkg/solana/txm/txm_race_test.go +++ b/pkg/solana/txm/txm_race_test.go @@ -65,7 +65,7 @@ func TestTxm_SendWithRetry_Race(t *testing.T) { } // build minimal txm - txm := NewTxm("retry_race", getClient, cfg, nil, ks, lggr) + txm := NewTxm("retry_race", getClient, cfg, ks, lggr) txm.fee = fee _, _, _, err := txm.sendWithRetry( From 8324639b33b90750ff285714b80f57958341c385 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 4 Oct 2024 15:50:35 -0400 Subject: [PATCH 060/174] Fix tests --- pkg/solana/txm/txm.go | 2 +- pkg/solana/txm/txm_internal_test.go | 2 +- pkg/solana/txm/txm_test.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/solana/txm/txm.go b/pkg/solana/txm/txm.go index 63a96bf8b..87861fd83 100644 --- a/pkg/solana/txm/txm.go +++ b/pkg/solana/txm/txm.go @@ -50,8 +50,8 @@ type Txm struct { cfg config.Config txs PendingTxContext ks SimpleKeystore - fee fees.Estimator client *utils.LazyLoad[client.ReaderWriter] + fee fees.Estimator } type TxConfig struct { diff --git a/pkg/solana/txm/txm_internal_test.go b/pkg/solana/txm/txm_internal_test.go index 0d2645ec0..61b59c066 100644 --- a/pkg/solana/txm/txm_internal_test.go +++ b/pkg/solana/txm/txm_internal_test.go @@ -718,7 +718,7 @@ func TestTxm_Enqueue(t *testing.T) { txm := NewTxm("enqueue_test", func() (client.ReaderWriter, error) { return mc, nil - }, cfg, nil, mkey, lggr) + }, cfg, mkey, lggr) require.ErrorContains(t, txm.Enqueue("txmUnstarted", &solana.Transaction{}), "not started") require.NoError(t, txm.Start(ctx)) diff --git a/pkg/solana/txm/txm_test.go b/pkg/solana/txm/txm_test.go index 84d11df62..851aebf89 100644 --- a/pkg/solana/txm/txm_test.go +++ b/pkg/solana/txm/txm_test.go @@ -72,7 +72,7 @@ func TestTxm_Integration(t *testing.T) { getClient := func() (solanaClient.ReaderWriter, error) { return client, nil } - txm := txm.NewTxm("localnet", getClient, cfg, nil, mkey, lggr) + txm := txm.NewTxm("localnet", getClient, cfg, mkey, lggr) // track initial balance initBal, err := client.Balance(pubKey) From 316ea4b206df96225f18884cca186ec825a42f38 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 4 Oct 2024 16:00:53 -0400 Subject: [PATCH 061/174] Update chain.go --- pkg/solana/chain.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pkg/solana/chain.go b/pkg/solana/chain.go index dd201467a..6cfdf9e88 100644 --- a/pkg/solana/chain.go +++ b/pkg/solana/chain.go @@ -301,15 +301,11 @@ func newChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr logger.L tc := func() (client.ReaderWriter, error) { return ch.getClient() } - cacheReader := !cfg.MultiNode.Enabled() - - ch.txm = txm.NewTxm(ch.id, tc, sendTx, cfg, cacheReader, ks, lggr) + ch.txm = txm.NewTxm(ch.id, tc, sendTx, cfg, ks, lggr) bc := func() (monitor.BalanceClient, error) { return ch.getClient() } - - // disable caching reader for MultiNode to always get a healthy client - ch.balanceMonitor = monitor.NewBalanceMonitor(ch.id, cfg, lggr, ks, bc, cacheReader) + ch.balanceMonitor = monitor.NewBalanceMonitor(ch.id, cfg, lggr, ks, bc) return &ch, nil } From eb50e6a0fde529d5d5b5b91f731d427a7416eb47 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 4 Oct 2024 17:41:49 -0400 Subject: [PATCH 062/174] Fix variables --- pkg/solana/chain.go | 5 +---- pkg/solana/client/multinode_client_test.go | 3 +-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/pkg/solana/chain.go b/pkg/solana/chain.go index 0b811a140..6fe2c1216 100644 --- a/pkg/solana/chain.go +++ b/pkg/solana/chain.go @@ -297,10 +297,7 @@ func newChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr logger.L bc := func() (monitor.BalanceClient, error) { return ch.getClient() } - - // disable caching reader for MultiNode to always get a healthy client - cacheReader := !cfg.MultiNode.Enabled() - ch.balanceMonitor = monitor.NewBalanceMonitor(ch.id, cfg, lggr, ks, bc, cacheReader) + ch.balanceMonitor = monitor.NewBalanceMonitor(ch.id, cfg, lggr, ks, bc) return &ch, nil } diff --git a/pkg/solana/client/multinode_client_test.go b/pkg/solana/client/multinode_client_test.go index f1ac65430..29661d4ba 100644 --- a/pkg/solana/client/multinode_client_test.go +++ b/pkg/solana/client/multinode_client_test.go @@ -13,7 +13,7 @@ import ( "github.com/smartcontractkit/chainlink-solana/pkg/solana/config" ) -func TestMultiNodeClient_Subscriptions_Integration(t *testing.T) { +func TestMultiNodeClient_Subscriptions(t *testing.T) { url := SetupLocalSolNode(t) privKey, err := solana.NewRandomPrivateKey() require.NoError(t, err) @@ -25,7 +25,6 @@ func TestMultiNodeClient_Subscriptions_Integration(t *testing.T) { cfg := config.NewDefault() // Enable MultiNode enabled := true - cfg.MultiNode.SetDefaults() cfg.Enabled = &enabled c, err := NewMultiNodeClient(url, cfg, requestTimeout, lggr) From f9f0f452ceefc541a62fa475530b5f0768ed9675 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 4 Oct 2024 17:50:36 -0400 Subject: [PATCH 063/174] Move classify errors --- pkg/solana/chain.go | 2 +- pkg/solana/{ => client}/classify_errors.go | 2 +- pkg/solana/txm/txm_internal_test.go | 15 +++++++-------- 3 files changed, 9 insertions(+), 10 deletions(-) rename pkg/solana/{ => client}/classify_errors.go (99%) diff --git a/pkg/solana/chain.go b/pkg/solana/chain.go index 6cfdf9e88..42a80901c 100644 --- a/pkg/solana/chain.go +++ b/pkg/solana/chain.go @@ -275,7 +275,7 @@ func newChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr logger.L mn.StringID(id), chainFamily, multiNode, - ClassifySendError, + client.ClassifySendError, 0, // use the default value provided by the implementation ) diff --git a/pkg/solana/classify_errors.go b/pkg/solana/client/classify_errors.go similarity index 99% rename from pkg/solana/classify_errors.go rename to pkg/solana/client/classify_errors.go index 6c051a603..bee7f6664 100644 --- a/pkg/solana/classify_errors.go +++ b/pkg/solana/client/classify_errors.go @@ -1,4 +1,4 @@ -package solana +package client import ( "github.com/gagliardetto/solana-go" diff --git a/pkg/solana/txm/txm_internal_test.go b/pkg/solana/txm/txm_internal_test.go index 4e5da7a7e..0f8ee204a 100644 --- a/pkg/solana/txm/txm_internal_test.go +++ b/pkg/solana/txm/txm_internal_test.go @@ -5,8 +5,6 @@ package txm import ( "context" "errors" - solana2 "github.com/smartcontractkit/chainlink-solana/pkg/solana" - mn "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" "math/rand" "sync" "testing" @@ -114,7 +112,7 @@ func TestTxm(t *testing.T) { txm := NewTxm(id, func() (client.ReaderWriter, error) { return mc, nil - }, cfg, nil, nil, mkey, lggr) + }, nil, cfg, mkey, lggr) require.NoError(t, txm.Start(ctx)) // tracking prom metrics @@ -676,6 +674,7 @@ func TestTxm(t *testing.T) { } } +/* TODO: Test MultiNode with txm func TestTxm_MultiNode(t *testing.T) { for _, eName := range []string{"fixed", "blockhistory"} { estimator := eName @@ -716,12 +715,11 @@ func TestTxm_MultiNode(t *testing.T) { txSender := mn.NewTransactionSender[*solanago.Transaction, solanago.Signature, mn.StringID, *client.Client]( lggr, mn.StringID(id), - chainFamily, + "Solana", multiNode, - solana2.ClassifySendError, + client.ClassifySendError, 0, // use the default value provided by the implementation ) - txSender := mn.NewTransactionSender(multiNode, lggr) // mock solana keystore mkey := keyMocks.NewSimpleKeystore(t) @@ -729,7 +727,7 @@ func TestTxm_MultiNode(t *testing.T) { txm := NewTxm(id, func() (client.ReaderWriter, error) { return mc, nil - }, cfg, nil, nil, mkey, lggr) + }, nil, cfg, mkey, lggr) require.NoError(t, txm.Start(ctx)) // tracking prom metrics @@ -1290,6 +1288,7 @@ func TestTxm_MultiNode(t *testing.T) { }) } } +*/ func TestTxm_Enqueue(t *testing.T) { // set up configs needed in txm @@ -1335,7 +1334,7 @@ func TestTxm_Enqueue(t *testing.T) { txm := NewTxm("enqueue_test", func() (client.ReaderWriter, error) { return mc, nil - }, cfg, mkey, lggr) + }, nil, cfg, mkey, lggr) require.ErrorContains(t, txm.Enqueue("txmUnstarted", &solana.Transaction{}), "not started") require.NoError(t, txm.Start(ctx)) From 9bf92d385d9ee921c2a63eb3b15993f6883e0609 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 4 Oct 2024 18:01:01 -0400 Subject: [PATCH 064/174] Fix imports --- pkg/solana/client/multinode_client_test.go | 3 ++- pkg/solana/txm/txm_internal_test.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/solana/client/multinode_client_test.go b/pkg/solana/client/multinode_client_test.go index 29661d4ba..451091e52 100644 --- a/pkg/solana/client/multinode_client_test.go +++ b/pkg/solana/client/multinode_client_test.go @@ -2,10 +2,11 @@ package client import ( "context" - "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" "testing" "time" + "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" + "github.com/gagliardetto/solana-go" "github.com/stretchr/testify/require" diff --git a/pkg/solana/txm/txm_internal_test.go b/pkg/solana/txm/txm_internal_test.go index 61b59c066..4a9a477a4 100644 --- a/pkg/solana/txm/txm_internal_test.go +++ b/pkg/solana/txm/txm_internal_test.go @@ -112,7 +112,7 @@ func TestTxm(t *testing.T) { txm := NewTxm(id, func() (client.ReaderWriter, error) { return mc, nil - }, cfg, nil, mkey, lggr) + }, nil, cfg, mkey, lggr) require.NoError(t, txm.Start(ctx)) // tracking prom metrics From 38166ee0de6803633c408500b5b4eb787f6ee3d3 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 4 Oct 2024 18:02:29 -0400 Subject: [PATCH 065/174] lint --- pkg/solana/client/client_test.go | 1 + pkg/solana/client/multinode_client_test.go | 5 ++--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/solana/client/client_test.go b/pkg/solana/client/client_test.go index 740e0297c..6a4feb61f 100644 --- a/pkg/solana/client/client_test.go +++ b/pkg/solana/client/client_test.go @@ -19,6 +19,7 @@ import ( "github.com/stretchr/testify/require" "github.com/smartcontractkit/chainlink-common/pkg/logger" + mn "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" "github.com/smartcontractkit/chainlink-solana/pkg/solana/config" "github.com/smartcontractkit/chainlink-solana/pkg/solana/monitor" diff --git a/pkg/solana/client/multinode_client_test.go b/pkg/solana/client/multinode_client_test.go index 451091e52..a29a6cc49 100644 --- a/pkg/solana/client/multinode_client_test.go +++ b/pkg/solana/client/multinode_client_test.go @@ -5,12 +5,11 @@ import ( "testing" "time" - "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" - "github.com/gagliardetto/solana-go" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" "github.com/stretchr/testify/require" - "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-solana/pkg/solana/config" ) From 95789d516ebffe0264606155ed1ca9fca08f9e44 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 4 Oct 2024 18:07:56 -0400 Subject: [PATCH 066/174] Update txm_internal_test.go --- pkg/solana/txm/txm_internal_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/solana/txm/txm_internal_test.go b/pkg/solana/txm/txm_internal_test.go index 4a9a477a4..adfa273f4 100644 --- a/pkg/solana/txm/txm_internal_test.go +++ b/pkg/solana/txm/txm_internal_test.go @@ -112,7 +112,7 @@ func TestTxm(t *testing.T) { txm := NewTxm(id, func() (client.ReaderWriter, error) { return mc, nil - }, nil, cfg, mkey, lggr) + }, cfg, mkey, lggr) require.NoError(t, txm.Start(ctx)) // tracking prom metrics From 179c95a340f4ff2ce417c7baff36cd6bf1172061 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 4 Oct 2024 18:09:14 -0400 Subject: [PATCH 067/174] Update txm_internal_test.go --- pkg/solana/txm/txm_internal_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/solana/txm/txm_internal_test.go b/pkg/solana/txm/txm_internal_test.go index 5773995f4..0f8ee204a 100644 --- a/pkg/solana/txm/txm_internal_test.go +++ b/pkg/solana/txm/txm_internal_test.go @@ -112,7 +112,7 @@ func TestTxm(t *testing.T) { txm := NewTxm(id, func() (client.ReaderWriter, error) { return mc, nil - }, cfg, mkey, lggr) + }, nil, cfg, mkey, lggr) require.NoError(t, txm.Start(ctx)) // tracking prom metrics From ab8fe88a1cc33f9e146b4866b2f7b3d9476f4b8a Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 4 Oct 2024 18:11:25 -0400 Subject: [PATCH 068/174] lint --- pkg/solana/client/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/solana/client/client.go b/pkg/solana/client/client.go index 0b069e4de..3da26340b 100644 --- a/pkg/solana/client/client.go +++ b/pkg/solana/client/client.go @@ -8,9 +8,9 @@ import ( "github.com/gagliardetto/solana-go" "github.com/gagliardetto/solana-go/rpc" + "github.com/smartcontractkit/chainlink-common/pkg/logger" "golang.org/x/sync/singleflight" - "github.com/smartcontractkit/chainlink-common/pkg/logger" mn "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" "github.com/smartcontractkit/chainlink-solana/pkg/solana/config" "github.com/smartcontractkit/chainlink-solana/pkg/solana/monitor" From 2bd9fdc414d1e8c228d16b327cda11899b5641ef Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 8 Oct 2024 14:10:23 -0400 Subject: [PATCH 069/174] Fix error classification --- pkg/solana/chain_test.go | 125 +++++ pkg/solana/client/classify_errors.go | 180 ++++--- pkg/solana/client/classify_errors_test.go | 72 +++ pkg/solana/txm/txm_internal_test.go | 618 ---------------------- 4 files changed, 292 insertions(+), 703 deletions(-) create mode 100644 pkg/solana/client/classify_errors_test.go diff --git a/pkg/solana/chain_test.go b/pkg/solana/chain_test.go index 73afcee11..a93250012 100644 --- a/pkg/solana/chain_test.go +++ b/pkg/solana/chain_test.go @@ -4,6 +4,8 @@ import ( "context" "errors" "fmt" + "github.com/gagliardetto/solana-go/programs/system" + mn "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" "io" "math/big" "net/http" @@ -370,3 +372,126 @@ func TestChain_Transact(t *testing.T) { require.NoError(t, err) assert.Equal(t, fees.ComputeUnitLimit(500), limit) } + +func TestChain_MultiNode_TransactionSender(t *testing.T) { + ctx := tests.Context(t) + url := client.SetupLocalSolNode(t) + lgr, _ := logger.TestObserved(t, zapcore.DebugLevel) + + // transaction parameters + sender, err := solana.NewRandomPrivateKey() + require.NoError(t, err) + receiver, err := solana.NewRandomPrivateKey() + require.NoError(t, err) + client.FundTestAccounts(t, solana.PublicKeySlice{sender.PublicKey()}, url) + + // configuration + cfg := solcfg.NewDefault() + cfg.MultiNode.MultiNode.Enabled = ptr(true) + cfg.Nodes = append(cfg.Nodes, + &solcfg.Node{ + Name: ptr("localnet-" + t.Name() + "-primary-1"), + URL: config.MustParseURL(url), + SendOnly: false, + }, + &solcfg.Node{ + Name: ptr("localnet-" + t.Name() + "-primary-2"), + URL: config.MustParseURL(url), + SendOnly: false, + }, + &solcfg.Node{ + Name: ptr("localnet-" + t.Name() + "-sendonly-1"), + URL: config.MustParseURL(url), + SendOnly: true, + }, + &solcfg.Node{ + Name: ptr("localnet-" + t.Name() + "-sendonly-1"), + URL: config.MustParseURL(url), + SendOnly: true, + }, + ) + + // mocked keystore + mkey := mocks.NewSimpleKeystore(t) + c, err := newChain("localnet", cfg, mkey, lgr) + require.NoError(t, err) + require.NoError(t, c.Start(ctx)) + + t.Run("successful transaction", func(t *testing.T) { + // create + sign transaction + createTx := func(to solana.PublicKey) *solana.Transaction { + cl, err := c.getClient() + require.NoError(t, err) + + hash, hashErr := cl.LatestBlockhash() + assert.NoError(t, hashErr) + + tx, txErr := solana.NewTransaction( + []solana.Instruction{ + system.NewTransferInstruction( + 1, + sender.PublicKey(), + to, + ).Build(), + }, + hash.Value.Blockhash, + solana.TransactionPayer(sender.PublicKey()), + ) + assert.NoError(t, txErr) + _, signErr := tx.Sign( + func(key solana.PublicKey) *solana.PrivateKey { + if sender.PublicKey().Equals(key) { + return &sender + } + return nil + }, + ) + assert.NoError(t, signErr) + return tx + } + + // Send tx using transaction sender + sig, code, err := c.txSender.SendTransaction(ctx, createTx(receiver.PublicKey())) + require.NoError(t, err) + require.Equal(t, mn.Successful, code) + require.NotEmpty(t, sig) + }) + + t.Run("unsigned transaction error", func(t *testing.T) { + // create + sign transaction + unsignedTx := func(to solana.PublicKey) *solana.Transaction { + cl, err := c.getClient() + require.NoError(t, err) + + hash, hashErr := cl.LatestBlockhash() + assert.NoError(t, hashErr) + + tx, txErr := solana.NewTransaction( + []solana.Instruction{ + system.NewTransferInstruction( + 1, + sender.PublicKey(), + to, + ).Build(), + }, + hash.Value.Blockhash, + solana.TransactionPayer(sender.PublicKey()), + ) + assert.NoError(t, txErr) + return tx + } + + // Send tx using transaction sender + sig, code, err := c.txSender.SendTransaction(ctx, unsignedTx(receiver.PublicKey())) + require.Error(t, err) + require.Equal(t, mn.Fatal, code) + require.Empty(t, sig) + }) + + t.Run("empty transaction", func(t *testing.T) { + sig, code, err := c.txSender.SendTransaction(ctx, &solana.Transaction{}) + require.Error(t, err) + require.Equal(t, mn.Fatal, code) + require.Empty(t, sig) + }) +} diff --git a/pkg/solana/client/classify_errors.go b/pkg/solana/client/classify_errors.go index bee7f6664..fed92e6b9 100644 --- a/pkg/solana/client/classify_errors.go +++ b/pkg/solana/client/classify_errors.go @@ -1,100 +1,110 @@ package client import ( + "regexp" + "github.com/gagliardetto/solana-go" + mn "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" - "strings" ) -// ClassifySendError returns the corresponding SendTxReturnCode based on the error. -// Errors derived from anza-xyz/agave@master/sdk/src/transaction/error.rs +var ( + ErrAccountInUse = regexp.MustCompile(`Account in use`) + ErrAccountLoadedTwice = regexp.MustCompile(`Account loaded twice`) + ErrAccountNotFound = regexp.MustCompile(`Attempt to debit an account but found no record of a prior credit\.`) + ErrProgramAccountNotFound = regexp.MustCompile(`Attempt to load a program that does not exist`) + ErrInsufficientFundsForFee = regexp.MustCompile(`Insufficient funds for fee`) + ErrInvalidAccountForFee = regexp.MustCompile(`This account may not be used to pay transaction fees`) + ErrAlreadyProcessed = regexp.MustCompile(`This transaction has already been processed`) + ErrBlockhashNotFound = regexp.MustCompile(`Blockhash not found`) + ErrInstructionError = regexp.MustCompile(`Error processing Instruction \d+: .+`) + ErrCallChainTooDeep = regexp.MustCompile(`Loader call chain is too deep`) + ErrMissingSignatureForFee = regexp.MustCompile(`Transaction requires a fee but has no signature present`) + ErrInvalidAccountIndex = regexp.MustCompile(`Transaction contains an invalid account reference`) + ErrSignatureFailure = regexp.MustCompile(`Transaction did not pass signature verification`) + ErrInvalidProgramForExecution = regexp.MustCompile(`This program may not be used for executing instructions`) + ErrSanitizeFailure = regexp.MustCompile(`Transaction failed to sanitize accounts offsets correctly`) + ErrClusterMaintenance = regexp.MustCompile(`Transactions are currently disabled due to cluster maintenance`) + ErrAccountBorrowOutstanding = regexp.MustCompile(`Transaction processing left an account with an outstanding borrowed reference`) + ErrWouldExceedMaxBlockCostLimit = regexp.MustCompile(`Transaction would exceed max Block Cost Limit`) + ErrUnsupportedVersion = regexp.MustCompile(`Transaction version is unsupported`) + ErrInvalidWritableAccount = regexp.MustCompile(`Transaction loads a writable account that cannot be written`) + ErrWouldExceedMaxAccountCostLimit = regexp.MustCompile(`Transaction would exceed max account limit within the block`) + ErrWouldExceedAccountDataBlockLimit = regexp.MustCompile(`Transaction would exceed account data limit within the block`) + ErrTooManyAccountLocks = regexp.MustCompile(`Transaction locked too many accounts`) + ErrAddressLookupTableNotFound = regexp.MustCompile(`Transaction loads an address table account that doesn't exist`) + ErrInvalidAddressLookupTableOwner = regexp.MustCompile(`Transaction loads an address table account with an invalid owner`) + ErrInvalidAddressLookupTableData = regexp.MustCompile(`Transaction loads an address table account with invalid data`) + ErrInvalidAddressLookupTableIndex = regexp.MustCompile(`Transaction address table lookup uses an invalid index`) + ErrInvalidRentPayingAccount = regexp.MustCompile(`Transaction leaves an account with a lower balance than rent-exempt minimum`) + ErrWouldExceedMaxVoteCostLimit = regexp.MustCompile(`Transaction would exceed max Vote Cost Limit`) + ErrWouldExceedAccountDataTotalLimit = regexp.MustCompile(`Transaction would exceed total account data limit`) + ErrDuplicateInstruction = regexp.MustCompile(`Transaction contains a duplicate instruction \(\d+\) that is not allowed`) + ErrInsufficientFundsForRent = regexp.MustCompile(`Transaction results in an account \(\d+\) with insufficient funds for rent`) + ErrMaxLoadedAccountsDataSizeExceeded = regexp.MustCompile(`Transaction exceeded max loaded accounts data size cap`) + ErrInvalidLoadedAccountsDataSizeLimit = regexp.MustCompile(`LoadedAccountsDataSizeLimit set for transaction must be greater than 0\.`) + ErrResanitizationNeeded = regexp.MustCompile(`Sanitized transaction differed before/after feature activation\. Needs to be resanitized\.`) + ErrProgramExecutionTemporarilyRestricted = regexp.MustCompile(`Execution of the program referenced by account at index \d+ is temporarily restricted\.`) + ErrUnbalancedTransaction = regexp.MustCompile(`Sum of account balances before and after transaction do not match`) + ErrProgramCacheHitMaxLimit = regexp.MustCompile(`Program cache hit max limit`) +) + +// Define a map to associate regex patterns with SendTxReturnCode +var errorPatterns = map[*regexp.Regexp]mn.SendTxReturnCode{ + ErrAccountInUse: mn.Retryable, + ErrAccountLoadedTwice: mn.Retryable, + ErrAccountNotFound: mn.Retryable, + ErrProgramAccountNotFound: mn.Fatal, + ErrInsufficientFundsForFee: mn.InsufficientFunds, + ErrInvalidAccountForFee: mn.Unsupported, + ErrAlreadyProcessed: mn.TransactionAlreadyKnown, + ErrBlockhashNotFound: mn.Retryable, + ErrInstructionError: mn.Retryable, + ErrCallChainTooDeep: mn.Retryable, + ErrMissingSignatureForFee: mn.Retryable, + ErrInvalidAccountIndex: mn.Retryable, + ErrSignatureFailure: mn.Fatal, + ErrInvalidProgramForExecution: mn.Retryable, + ErrSanitizeFailure: mn.Fatal, + ErrClusterMaintenance: mn.Retryable, + ErrAccountBorrowOutstanding: mn.Retryable, + ErrWouldExceedMaxBlockCostLimit: mn.ExceedsMaxFee, + ErrUnsupportedVersion: mn.Unsupported, + ErrInvalidWritableAccount: mn.Retryable, + ErrWouldExceedMaxAccountCostLimit: mn.ExceedsMaxFee, + ErrWouldExceedAccountDataBlockLimit: mn.ExceedsMaxFee, + ErrTooManyAccountLocks: mn.Retryable, + ErrAddressLookupTableNotFound: mn.Retryable, + ErrInvalidAddressLookupTableOwner: mn.Retryable, + ErrInvalidAddressLookupTableData: mn.Retryable, + ErrInvalidAddressLookupTableIndex: mn.Retryable, + ErrInvalidRentPayingAccount: mn.Retryable, + ErrWouldExceedMaxVoteCostLimit: mn.Retryable, + ErrWouldExceedAccountDataTotalLimit: mn.Retryable, + ErrMaxLoadedAccountsDataSizeExceeded: mn.Retryable, + ErrInvalidLoadedAccountsDataSizeLimit: mn.Retryable, + ErrResanitizationNeeded: mn.Retryable, + ErrUnbalancedTransaction: mn.Retryable, + ErrProgramCacheHitMaxLimit: mn.Retryable, + ErrInsufficientFundsForRent: mn.InsufficientFunds, + ErrDuplicateInstruction: mn.Fatal, + ErrProgramExecutionTemporarilyRestricted: mn.Retryable, +} + +// ClassifySendError implements TxErrorClassifier required for MultiNode TransactionSender +var _ mn.TxErrorClassifier[*solana.Transaction] = ClassifySendError + +// ClassifySendError returns the corresponding return code based on the error. func ClassifySendError(tx *solana.Transaction, err error) mn.SendTxReturnCode { if err == nil { return mn.Successful } errMsg := err.Error() - - // TODO: Ensure correct error classification for each error message. - // TODO: is strings.Contains good enough for error classification? - switch { - case strings.Contains(errMsg, "Account in use"): - return mn.TransactionAlreadyKnown - case strings.Contains(errMsg, "Account loaded twice"): - return mn.Retryable - case strings.Contains(errMsg, "Attempt to debit an account but found no record of a prior credit"): - return mn.Retryable - case strings.Contains(errMsg, "Attempt to load a program that does not exist"): - return mn.Fatal - case strings.Contains(errMsg, "Insufficient funds for fee"): - return mn.InsufficientFunds - case strings.Contains(errMsg, "This account may not be used to pay transaction fees"): - return mn.Unsupported - case strings.Contains(errMsg, "This transaction has already been processed"): - return mn.TransactionAlreadyKnown - case strings.Contains(errMsg, "Blockhash not found"): - return mn.Retryable - case strings.Contains(errMsg, "Error processing Instruction"): - return mn.Retryable - case strings.Contains(errMsg, "Loader call chain is too deep"): - return mn.Retryable - case strings.Contains(errMsg, "Transaction requires a fee but has no signature present"): - return mn.Retryable - case strings.Contains(errMsg, "Transaction contains an invalid account reference"): - return mn.Unsupported - case strings.Contains(errMsg, "Transaction did not pass signature verification"): - return mn.Fatal - case strings.Contains(errMsg, "This program may not be used for executing instructions"): - return mn.Unsupported - case strings.Contains(errMsg, "Transaction failed to sanitize accounts offsets correctly"): - return mn.Fatal - case strings.Contains(errMsg, "Transactions are currently disabled due to cluster maintenance"): - return mn.Retryable - case strings.Contains(errMsg, "Transaction processing left an account with an outstanding borrowed reference"): - return mn.Fatal - case strings.Contains(errMsg, "Transaction would exceed max Block Cost Limit"): - return mn.ExceedsMaxFee - case strings.Contains(errMsg, "Transaction version is unsupported"): - return mn.Unsupported - case strings.Contains(errMsg, "Transaction loads a writable account that cannot be written"): - return mn.Unsupported - case strings.Contains(errMsg, "Transaction would exceed max account limit within the block"): - return mn.ExceedsMaxFee - case strings.Contains(errMsg, "Transaction would exceed account data limit within the block"): - return mn.ExceedsMaxFee - case strings.Contains(errMsg, "Transaction locked too many accounts"): - return mn.Fatal - case strings.Contains(errMsg, "Transaction loads an address table account that doesn't exist"): - return mn.Unsupported - case strings.Contains(errMsg, "Transaction loads an address table account with an invalid owner"): - return mn.Unsupported - case strings.Contains(errMsg, "Transaction loads an address table account with invalid data"): - return mn.Unsupported - case strings.Contains(errMsg, "Transaction address table lookup uses an invalid index"): - return mn.Unsupported - case strings.Contains(errMsg, "Transaction leaves an account with a lower balance than rent-exempt minimum"): - return mn.Fatal - case strings.Contains(errMsg, "Transaction would exceed max Vote Cost Limit"): - return mn.ExceedsMaxFee - case strings.Contains(errMsg, "Transaction would exceed total account data limit"): - return mn.ExceedsMaxFee - case strings.Contains(errMsg, "Transaction contains a duplicate instruction"): - return mn.Fatal - case strings.Contains(errMsg, "Transaction results in an account with insufficient funds for rent"): - return mn.InsufficientFunds - case strings.Contains(errMsg, "Transaction exceeded max loaded accounts data size cap"): - return mn.Unsupported - case strings.Contains(errMsg, "LoadedAccountsDataSizeLimit set for transaction must be greater than 0"): - return mn.Fatal - case strings.Contains(errMsg, "Sanitized transaction differed before/after feature activation"): - return mn.Fatal - case strings.Contains(errMsg, "Execution of the program referenced by account at index is temporarily restricted"): - return mn.Unsupported - case strings.Contains(errMsg, "Sum of account balances before and after transaction do not match"): - return mn.Fatal - case strings.Contains(errMsg, "Program cache hit max limit"): - return mn.Retryable - default: - return mn.Retryable + for pattern, code := range errorPatterns { + if pattern.MatchString(errMsg) { + return code + } } + return mn.Retryable } diff --git a/pkg/solana/client/classify_errors_test.go b/pkg/solana/client/classify_errors_test.go new file mode 100644 index 000000000..cd7f5ddb3 --- /dev/null +++ b/pkg/solana/client/classify_errors_test.go @@ -0,0 +1,72 @@ +package client + +import ( + "errors" + "testing" + + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/assert" + + mn "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" +) + +func TestClassifySendError(t *testing.T) { + tests := []struct { + errMsg string + expectedCode mn.SendTxReturnCode + }{ + // Static error cases + {"Account in use", mn.Retryable}, + {"Account loaded twice", mn.Retryable}, + {"Attempt to debit an account but found no record of a prior credit.", mn.Retryable}, + {"Attempt to load a program that does not exist", mn.Fatal}, + {"Insufficient funds for fee", mn.InsufficientFunds}, + {"This account may not be used to pay transaction fees", mn.Unsupported}, + {"This transaction has already been processed", mn.TransactionAlreadyKnown}, + {"Blockhash not found", mn.Retryable}, + {"Loader call chain is too deep", mn.Retryable}, + {"Transaction requires a fee but has no signature present", mn.Retryable}, + {"Transaction contains an invalid account reference", mn.Retryable}, + {"Transaction did not pass signature verification", mn.Fatal}, + {"This program may not be used for executing instructions", mn.Retryable}, + {"Transaction failed to sanitize accounts offsets correctly", mn.Fatal}, + {"Transactions are currently disabled due to cluster maintenance", mn.Retryable}, + {"Transaction processing left an account with an outstanding borrowed reference", mn.Retryable}, + {"Transaction would exceed max Block Cost Limit", mn.ExceedsMaxFee}, + {"Transaction version is unsupported", mn.Unsupported}, + {"Transaction loads a writable account that cannot be written", mn.Retryable}, + {"Transaction would exceed max account limit within the block", mn.ExceedsMaxFee}, + {"Transaction would exceed account data limit within the block", mn.ExceedsMaxFee}, + {"Transaction locked too many accounts", mn.Retryable}, + {"Address lookup table not found", mn.Retryable}, + {"Attempted to lookup addresses from an account owned by the wrong program", mn.Retryable}, + {"Attempted to lookup addresses from an invalid account", mn.Retryable}, + {"Address table lookup uses an invalid index", mn.Retryable}, + {"Transaction leaves an account with a lower balance than rent-exempt minimum", mn.Retryable}, + {"Transaction would exceed max Vote Cost Limit", mn.Retryable}, + {"Transaction would exceed total account data limit", mn.Retryable}, + {"Transaction contains a duplicate instruction", mn.Retryable}, + {"Transaction exceeded max loaded accounts data size cap", mn.Retryable}, + {"LoadedAccountsDataSizeLimit set for transaction must be greater than 0.", mn.Retryable}, + {"Sanitized transaction differed before/after feature activation. Needs to be resanitized.", mn.Retryable}, + {"Program cache hit max limit", mn.Retryable}, + + // Dynamic error cases + {"Transaction results in an account (123) with insufficient funds for rent", mn.InsufficientFunds}, + {"Error processing Instruction 2: Some error details", mn.Retryable}, + {"Execution of the program referenced by account at index 3 is temporarily restricted.", mn.Retryable}, + + // Edge cases + {"Unknown error message", mn.Retryable}, + {"", mn.Retryable}, // Empty message + } + + for _, tt := range tests { + t.Run(tt.errMsg, func(t *testing.T) { + tx := &solana.Transaction{} // Dummy transaction + err := errors.New(tt.errMsg) // Create a standard Go error with the message + result := ClassifySendError(tx, err) + assert.Equal(t, tt.expectedCode, result, "Expected %v but got %v for error message: %s", tt.expectedCode, result, tt.errMsg) + }) + } +} diff --git a/pkg/solana/txm/txm_internal_test.go b/pkg/solana/txm/txm_internal_test.go index 0f8ee204a..8fb24cb02 100644 --- a/pkg/solana/txm/txm_internal_test.go +++ b/pkg/solana/txm/txm_internal_test.go @@ -1,5 +1,3 @@ -//go:build integration - package txm import ( @@ -674,622 +672,6 @@ func TestTxm(t *testing.T) { } } -/* TODO: Test MultiNode with txm -func TestTxm_MultiNode(t *testing.T) { - for _, eName := range []string{"fixed", "blockhistory"} { - estimator := eName - t.Run("estimator-"+estimator, func(t *testing.T) { - t.Parallel() // run estimator tests in parallel - - // set up configs needed in txm - id := "mocknet-" + estimator + "-" + uuid.NewString() - t.Logf("Starting new iteration: %s", id) - - ctx := tests.Context(t) - lggr := logger.Test(t) - - // Enable MultiNode - cfg := config.TOMLConfig{MultiNode: config.MultiNodeConfig{Enabled: true}} - cfg.SetDefaults() - - cfg.Chain.FeeEstimatorMode = &estimator - mc := mocks.NewReaderWriter(t) - mc.On("GetLatestBlock").Return(&rpc.GetBlockResult{}, nil).Maybe() - - nodes := []mn.Node[mn.StringID, mocks.ReaderWriter]{ - mn.NewNode(cfg, mn.StringID("node1"), mc, "chainID", 0), - } - - // TODO: Is this needed to test things? - // TODO: Need to mock more of the ReaderWriter methods for TxSender?? - multiNode := mn.NewMultiNode(cfg, - mn.NodeSelectionModeHighestHead, - cfg.MultiNode.LeaseDuration(), - nodes, - []mn.SendOnlyNode{}, - "chainID", - "Solana", - 0, - ) - - txSender := mn.NewTransactionSender[*solanago.Transaction, solanago.Signature, mn.StringID, *client.Client]( - lggr, - mn.StringID(id), - "Solana", - multiNode, - client.ClassifySendError, - 0, // use the default value provided by the implementation - ) - - // mock solana keystore - mkey := keyMocks.NewSimpleKeystore(t) - mkey.On("Sign", mock.Anything, mock.Anything, mock.Anything).Return([]byte{}, nil) - - txm := NewTxm(id, func() (client.ReaderWriter, error) { - return mc, nil - }, nil, cfg, mkey, lggr) - require.NoError(t, txm.Start(ctx)) - - // tracking prom metrics - prom := soltxmProm{id: id} - - // create random signature - getSig := func() solana.Signature { - sig := make([]byte, 64) - rand.Read(sig) - return solana.SignatureFromBytes(sig) - } - - // check if cached transaction is cleared - empty := func() bool { - count := txm.InflightTxs() - assert.Equal(t, float64(count), prom.getInflight()) // validate prom metric and txs length - return count == 0 - } - - // adjust wait time based on config - waitDuration := cfg.TxConfirmTimeout() - waitFor := func(f func() bool) { - for i := 0; i < int(waitDuration.Seconds()*1.5); i++ { - if f() { - return - } - time.Sleep(time.Second) - } - assert.NoError(t, errors.New("unable to confirm inflight txs is empty")) - } - - // handle signature statuses calls - statuses := map[solana.Signature]func() *rpc.SignatureStatusesResult{} - mc.On("SignatureStatuses", mock.Anything, mock.AnythingOfType("[]solana.Signature")).Return( - func(_ context.Context, sigs []solana.Signature) (out []*rpc.SignatureStatusesResult) { - for i := range sigs { - get, exists := statuses[sigs[i]] - if !exists { - out = append(out, nil) - continue - } - out = append(out, get()) - } - return out - }, nil, - ) - - // happy path (send => simulate success => tx: nil => tx: processed => tx: confirmed => done) - t.Run("happyPath", func(t *testing.T) { - sig := getSig() - tx, signed := getTx(t, 0, mkey, 0) - var wg sync.WaitGroup - wg.Add(3) - - sendCount := 0 - var countRW sync.RWMutex - mc.On("SendTx", mock.Anything, signed(0, true)).Run(func(mock.Arguments) { - countRW.Lock() - sendCount++ - countRW.Unlock() - }).After(500*time.Millisecond).Return(sig, nil) - mc.On("SimulateTx", mock.Anything, signed(0, true), mock.Anything).Return(&rpc.SimulateTransactionResult{}, nil).Once() - - // handle signature status calls - count := 0 - statuses[sig] = func() (out *rpc.SignatureStatusesResult) { - defer func() { count++ }() - defer wg.Done() - - out = &rpc.SignatureStatusesResult{} - if count == 1 { - out.ConfirmationStatus = rpc.ConfirmationStatusProcessed - return - } - - if count == 2 { - out.ConfirmationStatus = rpc.ConfirmationStatusConfirmed - return - } - return nil - } - - // send tx - assert.NoError(t, txm.Enqueue(t.Name(), tx)) - wg.Wait() - - // no transactions stored inflight txs list - waitFor(empty) - // transaction should be sent more than twice - countRW.RLock() - t.Logf("sendTx received %d calls", sendCount) - assert.Greater(t, sendCount, 2) - countRW.RUnlock() - - // panic if sendTx called after context cancelled - mc.On("SendTx", mock.Anything, tx).Panic("SendTx should not be called anymore").Maybe() - - // check prom metric - prom.success++ - prom.assertEqual(t) - }) - - // fail on initial transmit (RPC immediate rejects) - t.Run("fail_initialTx", func(t *testing.T) { - tx, signed := getTx(t, 1, mkey, 0) - var wg sync.WaitGroup - wg.Add(1) - - // should only be called once (tx does not start retry, confirming, or simulation) - mc.On("SendTx", mock.Anything, signed(0, true)).Run(func(mock.Arguments) { - wg.Done() - }).Return(solana.Signature{}, errors.New("FAIL")).Once() - - // tx should be able to queue - assert.NoError(t, txm.Enqueue(t.Name(), tx)) - wg.Wait() // wait to be picked up and processed - - // no transactions stored inflight txs list - waitFor(empty) - - // check prom metric - prom.error++ - prom.reject++ - prom.assertEqual(t) - }) - - // tx fails simulation (simulation error) - t.Run("fail_simulation", func(t *testing.T) { - tx, signed := getTx(t, 2, mkey, 0) - sig := getSig() - var wg sync.WaitGroup - wg.Add(1) - - mc.On("SendTx", mock.Anything, signed(0, true)).Return(sig, nil) - mc.On("SimulateTx", mock.Anything, signed(0, true), mock.Anything).Run(func(mock.Arguments) { - wg.Done() - }).Return(&rpc.SimulateTransactionResult{ - Err: "FAIL", - }, nil).Once() - // signature status is nil (handled automatically) - - // tx should be able to queue - assert.NoError(t, txm.Enqueue(t.Name(), tx)) - wg.Wait() // wait to be picked up and processed - waitFor(empty) // txs cleared quickly - - // check prom metric - prom.error++ - prom.simOther++ - prom.assertEqual(t) - }) - - // tx fails simulation (rpc error, timeout should clean up b/c sig status will be nil) - t.Run("fail_simulation_confirmNil", func(t *testing.T) { - tx, signed := getTx(t, 3, mkey, 0) - sig := getSig() - retry0 := getSig() - retry1 := getSig() - retry2 := getSig() - retry3 := getSig() - var wg sync.WaitGroup - wg.Add(1) - - mc.On("SendTx", mock.Anything, signed(0, true)).Return(sig, nil) - mc.On("SendTx", mock.Anything, signed(1, true)).Return(retry0, nil) - mc.On("SendTx", mock.Anything, signed(2, true)).Return(retry1, nil) - mc.On("SendTx", mock.Anything, signed(3, true)).Return(retry2, nil).Maybe() - mc.On("SendTx", mock.Anything, signed(4, true)).Return(retry3, nil).Maybe() - mc.On("SimulateTx", mock.Anything, signed(0, true), mock.Anything).Run(func(mock.Arguments) { - wg.Done() - }).Return(&rpc.SimulateTransactionResult{}, errors.New("FAIL")).Once() - // all signature statuses are nil, handled automatically - - // tx should be able to queue - assert.NoError(t, txm.Enqueue(t.Name(), tx)) - wg.Wait() // wait to be picked up and processed - waitFor(empty) // txs cleared after timeout - - // check prom metric - prom.error++ - prom.drop++ - prom.assertEqual(t) - - // panic if sendTx called after context cancelled - mc.On("SendTx", mock.Anything, tx).Panic("SendTx should not be called anymore").Maybe() - }) - - // tx fails simulation with an InstructionError (indicates reverted execution) - // manager should cancel sending retry immediately + increment reverted prom metric - t.Run("fail_simulation_instructionError", func(t *testing.T) { - tx, signed := getTx(t, 4, mkey, 0) - sig := getSig() - var wg sync.WaitGroup - wg.Add(1) - - // {"InstructionError":[0,{"Custom":6003}]} - tempErr := map[string][]interface{}{ - "InstructionError": { - 0, map[string]int{"Custom": 6003}, - }, - } - mc.On("SendTx", mock.Anything, signed(0, true)).Return(sig, nil) - mc.On("SimulateTx", mock.Anything, signed(0, true), mock.Anything).Run(func(mock.Arguments) { - wg.Done() - }).Return(&rpc.SimulateTransactionResult{ - Err: tempErr, - }, nil).Once() - // all signature statuses are nil, handled automatically - - // tx should be able to queue - assert.NoError(t, txm.Enqueue(t.Name(), tx)) - wg.Wait() // wait to be picked up and processed - waitFor(empty) // txs cleared after timeout - - // check prom metric - prom.error++ - prom.simRevert++ - prom.assertEqual(t) - - // panic if sendTx called after context cancelled - mc.On("SendTx", mock.Anything, tx).Panic("SendTx should not be called anymore").Maybe() - }) - - // tx fails simulation with BlockHashNotFound error - // txm should continue to confirm tx (in this case it will succeed) - t.Run("fail_simulation_blockhashNotFound", func(t *testing.T) { - tx, signed := getTx(t, 5, mkey, 0) - sig := getSig() - var wg sync.WaitGroup - wg.Add(3) - - mc.On("SendTx", mock.Anything, signed(0, true)).Return(sig, nil) - mc.On("SimulateTx", mock.Anything, signed(0, true), mock.Anything).Run(func(mock.Arguments) { - wg.Done() - }).Return(&rpc.SimulateTransactionResult{ - Err: "BlockhashNotFound", - }, nil).Once() - - // handle signature status calls - count := 0 - statuses[sig] = func() (out *rpc.SignatureStatusesResult) { - defer func() { count++ }() - defer wg.Done() - - out = &rpc.SignatureStatusesResult{} - if count == 1 { - out.ConfirmationStatus = rpc.ConfirmationStatusConfirmed - return - } - return nil - } - - // tx should be able to queue - assert.NoError(t, txm.Enqueue(t.Name(), tx)) - wg.Wait() // wait to be picked up and processed - waitFor(empty) // txs cleared after timeout - - // check prom metric - prom.success++ - prom.assertEqual(t) - - // panic if sendTx called after context cancelled - mc.On("SendTx", mock.Anything, tx).Panic("SendTx should not be called anymore").Maybe() - }) - - // tx fails simulation with AlreadyProcessed error - // txm should continue to confirm tx (in this case it will revert) - t.Run("fail_simulation_alreadyProcessed", func(t *testing.T) { - tx, signed := getTx(t, 6, mkey, 0) - sig := getSig() - var wg sync.WaitGroup - wg.Add(2) - - mc.On("SendTx", mock.Anything, signed(0, true)).Return(sig, nil) - mc.On("SimulateTx", mock.Anything, signed(0, true), mock.Anything).Run(func(mock.Arguments) { - wg.Done() - }).Return(&rpc.SimulateTransactionResult{ - Err: "AlreadyProcessed", - }, nil).Once() - - // handle signature status calls - statuses[sig] = func() (out *rpc.SignatureStatusesResult) { - wg.Done() - return &rpc.SignatureStatusesResult{ - Err: "ERROR", - ConfirmationStatus: rpc.ConfirmationStatusConfirmed, - } - } - - // tx should be able to queue - assert.NoError(t, txm.Enqueue(t.Name(), tx)) - wg.Wait() // wait to be picked up and processed - waitFor(empty) // txs cleared after timeout - - // check prom metric - prom.revert++ - prom.error++ - prom.assertEqual(t) - - // panic if sendTx called after context cancelled - mc.On("SendTx", mock.Anything, tx).Panic("SendTx should not be called anymore").Maybe() - }) - - // tx passes sim, never passes processed (timeout should cleanup) - t.Run("fail_confirm_processed", func(t *testing.T) { - tx, signed := getTx(t, 7, mkey, 0) - sig := getSig() - retry0 := getSig() - retry1 := getSig() - retry2 := getSig() - retry3 := getSig() - var wg sync.WaitGroup - wg.Add(1) - - mc.On("SendTx", mock.Anything, signed(0, true)).Return(sig, nil) - mc.On("SendTx", mock.Anything, signed(1, true)).Return(retry0, nil) - mc.On("SendTx", mock.Anything, signed(2, true)).Return(retry1, nil) - mc.On("SendTx", mock.Anything, signed(3, true)).Return(retry2, nil).Maybe() - mc.On("SendTx", mock.Anything, signed(4, true)).Return(retry3, nil).Maybe() - mc.On("SimulateTx", mock.Anything, signed(0, true), mock.Anything).Run(func(mock.Arguments) { - wg.Done() - }).Return(&rpc.SimulateTransactionResult{}, nil).Once() - - // handle signature status calls (initial stays processed, others don't exist) - statuses[sig] = func() (out *rpc.SignatureStatusesResult) { - return &rpc.SignatureStatusesResult{ - ConfirmationStatus: rpc.ConfirmationStatusProcessed, - } - } - - // tx should be able to queue - assert.NoError(t, txm.Enqueue(t.Name(), tx)) - wg.Wait() // wait to be picked up and processed - waitFor(empty) // inflight txs cleared after timeout - - // check prom metric - prom.error++ - prom.drop++ - prom.assertEqual(t) - - // panic if sendTx called after context cancelled - mc.On("SendTx", mock.Anything, tx).Panic("SendTx should not be called anymore").Maybe() - }) - - // tx passes sim, shows processed, moves to nil (timeout should cleanup) - t.Run("fail_confirm_processedToNil", func(t *testing.T) { - tx, signed := getTx(t, 8, mkey, 0) - sig := getSig() - retry0 := getSig() - retry1 := getSig() - retry2 := getSig() - retry3 := getSig() - var wg sync.WaitGroup - wg.Add(1) - - mc.On("SendTx", mock.Anything, signed(0, true)).Return(sig, nil) - mc.On("SendTx", mock.Anything, signed(1, true)).Return(retry0, nil) - mc.On("SendTx", mock.Anything, signed(2, true)).Return(retry1, nil) - mc.On("SendTx", mock.Anything, signed(3, true)).Return(retry2, nil).Maybe() - mc.On("SendTx", mock.Anything, signed(4, true)).Return(retry3, nil).Maybe() - mc.On("SimulateTx", mock.Anything, signed(0, true), mock.Anything).Run(func(mock.Arguments) { - wg.Done() - }).Return(&rpc.SimulateTransactionResult{}, nil).Once() - - // handle signature status calls (initial stays processed => nil, others don't exist) - count := 0 - statuses[sig] = func() (out *rpc.SignatureStatusesResult) { - defer func() { count++ }() - - if count > 2 { - return nil - } - - return &rpc.SignatureStatusesResult{ - ConfirmationStatus: rpc.ConfirmationStatusProcessed, - } - } - - // tx should be able to queue - assert.NoError(t, txm.Enqueue(t.Name(), tx)) - wg.Wait() // wait to be picked up and processed - waitFor(empty) // inflight txs cleared after timeout - - // check prom metric - prom.error++ - prom.drop++ - prom.assertEqual(t) - - // panic if sendTx called after context cancelled - mc.On("SendTx", mock.Anything, tx).Panic("SendTx should not be called anymore").Maybe() - }) - - // tx passes sim, errors on confirm - t.Run("fail_confirm_revert", func(t *testing.T) { - tx, signed := getTx(t, 9, mkey, 0) - sig := getSig() - var wg sync.WaitGroup - wg.Add(1) - - mc.On("SendTx", mock.Anything, signed(0, true)).Return(sig, nil) - mc.On("SimulateTx", mock.Anything, signed(0, true), mock.Anything).Run(func(mock.Arguments) { - wg.Done() - }).Return(&rpc.SimulateTransactionResult{}, nil).Once() - - // handle signature status calls - statuses[sig] = func() (out *rpc.SignatureStatusesResult) { - return &rpc.SignatureStatusesResult{ - ConfirmationStatus: rpc.ConfirmationStatusProcessed, - Err: "ERROR", - } - } - - // tx should be able to queue - assert.NoError(t, txm.Enqueue(t.Name(), tx)) - wg.Wait() // wait to be picked up and processed - waitFor(empty) // inflight txs cleared after timeout - - // check prom metric - prom.error++ - prom.revert++ - prom.assertEqual(t) - - // panic if sendTx called after context cancelled - mc.On("SendTx", mock.Anything, tx).Panic("SendTx should not be called anymore").Maybe() - }) - - // tx passes sim, first retried TXs get dropped - t.Run("success_retryTx", func(t *testing.T) { - tx, signed := getTx(t, 10, mkey, 0) - sig := getSig() - retry0 := getSig() - retry1 := getSig() - retry2 := getSig() - retry3 := getSig() - var wg sync.WaitGroup - wg.Add(2) - - mc.On("SendTx", mock.Anything, signed(0, true)).Return(sig, nil) - mc.On("SendTx", mock.Anything, signed(1, true)).Return(retry0, nil) - mc.On("SendTx", mock.Anything, signed(2, true)).Return(retry1, nil) - mc.On("SendTx", mock.Anything, signed(3, true)).Return(retry2, nil).Maybe() - mc.On("SendTx", mock.Anything, signed(4, true)).Return(retry3, nil).Maybe() - mc.On("SimulateTx", mock.Anything, signed(0, true), mock.Anything).Run(func(mock.Arguments) { - wg.Done() - }).Return(&rpc.SimulateTransactionResult{}, nil).Once() - - // handle signature status calls - statuses[retry1] = func() (out *rpc.SignatureStatusesResult) { - defer wg.Done() - return &rpc.SignatureStatusesResult{ - ConfirmationStatus: rpc.ConfirmationStatusConfirmed, - } - } - - // send tx - assert.NoError(t, txm.Enqueue(t.Name(), tx)) - wg.Wait() - - // no transactions stored inflight txs list - waitFor(empty) - - // panic if sendTx called after context cancelled - mc.On("SendTx", mock.Anything, tx).Panic("SendTx should not be called anymore").Maybe() - - // check prom metric - prom.success++ - prom.assertEqual(t) - }) - - // fee bumping disabled - t.Run("feeBumpingDisabled", func(t *testing.T) { - sig := getSig() - tx, signed := getTx(t, 11, mkey, 0) - - defaultFeeBumpPeriod := cfg.FeeBumpPeriod() - - sendCount := 0 - var countRW sync.RWMutex - mc.On("SendTx", mock.Anything, signed(0, true)).Run(func(mock.Arguments) { - countRW.Lock() - sendCount++ - countRW.Unlock() - }).Return(sig, nil) // only sends one transaction type (no bumping) - mc.On("SimulateTx", mock.Anything, signed(0, true), mock.Anything).Return(&rpc.SimulateTransactionResult{}, nil).Once() - - // handle signature status calls - var wg sync.WaitGroup - wg.Add(1) - count := 0 - start := time.Now() - statuses[sig] = func() (out *rpc.SignatureStatusesResult) { - defer func() { count++ }() - - out = &rpc.SignatureStatusesResult{} - if time.Since(start) > 2*defaultFeeBumpPeriod { - out.ConfirmationStatus = rpc.ConfirmationStatusConfirmed - wg.Done() - return - } - out.ConfirmationStatus = rpc.ConfirmationStatusProcessed - return - } - - // send tx - with disabled fee bumping - assert.NoError(t, txm.Enqueue(t.Name(), tx, SetFeeBumpPeriod(0))) - wg.Wait() - - // no transactions stored inflight txs list - waitFor(empty) - // transaction should be sent more than twice - countRW.RLock() - t.Logf("sendTx received %d calls", sendCount) - assert.Greater(t, sendCount, 2) - countRW.RUnlock() - - // panic if sendTx called after context cancelled - mc.On("SendTx", mock.Anything, tx).Panic("SendTx should not be called anymore").Maybe() - - // check prom metric - prom.success++ - prom.assertEqual(t) - }) - - // compute unit limit disabled - t.Run("computeUnitLimitDisabled", func(t *testing.T) { - sig := getSig() - tx, signed := getTx(t, 12, mkey, 0) - - // should only match transaction without compute unit limit - assert.Len(t, signed(0, false).Message.Instructions, 2) - mc.On("SendTx", mock.Anything, signed(0, false)).Return(sig, nil) // only sends one transaction type (no bumping) - mc.On("SimulateTx", mock.Anything, signed(0, false), mock.Anything).Return(&rpc.SimulateTransactionResult{}, nil).Once() - - // handle signature status calls - var wg sync.WaitGroup - wg.Add(1) - statuses[sig] = func() *rpc.SignatureStatusesResult { - defer wg.Done() - return &rpc.SignatureStatusesResult{ - ConfirmationStatus: rpc.ConfirmationStatusConfirmed, - } - } - - // send tx - with disabled fee bumping and disabled compute unit limit - assert.NoError(t, txm.Enqueue(t.Name(), tx, SetFeeBumpPeriod(0), SetComputeUnitLimit(0))) - wg.Wait() - - // no transactions stored inflight txs list - waitFor(empty) - - // panic if sendTx called after context cancelled - mc.On("SendTx", mock.Anything, tx).Panic("SendTx should not be called anymore").Maybe() - - // check prom metric - prom.success++ - prom.assertEqual(t) - }) - }) - } -} -*/ - func TestTxm_Enqueue(t *testing.T) { // set up configs needed in txm lggr := logger.Test(t) From 40a4d5db29ee6b0a9b2802b3117f94af5deb6061 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 8 Oct 2024 14:31:58 -0400 Subject: [PATCH 070/174] Update txm_internal_test.go --- pkg/solana/txm/txm_internal_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/solana/txm/txm_internal_test.go b/pkg/solana/txm/txm_internal_test.go index 8fb24cb02..ba0b2548c 100644 --- a/pkg/solana/txm/txm_internal_test.go +++ b/pkg/solana/txm/txm_internal_test.go @@ -1,3 +1,5 @@ +//go:build integration + package txm import ( From 324dfdeb5dccb32ef2086650158307fa93589ae0 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 8 Oct 2024 14:35:38 -0400 Subject: [PATCH 071/174] Update multinode_client.go --- pkg/solana/client/multinode_client.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pkg/solana/client/multinode_client.go b/pkg/solana/client/multinode_client.go index 60efaab6d..65827fdfa 100644 --- a/pkg/solana/client/multinode_client.go +++ b/pkg/solana/client/multinode_client.go @@ -46,8 +46,6 @@ var _ mn.SendTxRPCClient[*solana.Transaction] = (*MultiNodeClient)(nil) type MultiNodeClient struct { Client cfg *config.TOMLConfig - log logger.Logger - rpc *rpc.Client stateMu sync.RWMutex // protects state* fields subsSliceMu sync.RWMutex subs map[mn.Subscription]struct{} @@ -73,8 +71,6 @@ func NewMultiNodeClient(endpoint string, cfg *config.TOMLConfig, requestTimeout return &MultiNodeClient{ Client: *client, cfg: cfg, - log: log, - rpc: rpc.New(endpoint), subs: make(map[mn.Subscription]struct{}), chStopInFlight: make(chan struct{}), }, nil From 83cf95093b62a1d39d2fac5dd960fe3d65416214 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 8 Oct 2024 14:41:25 -0400 Subject: [PATCH 072/174] lint --- pkg/solana/chain_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/solana/chain_test.go b/pkg/solana/chain_test.go index a93250012..9053d312f 100644 --- a/pkg/solana/chain_test.go +++ b/pkg/solana/chain_test.go @@ -4,8 +4,6 @@ import ( "context" "errors" "fmt" - "github.com/gagliardetto/solana-go/programs/system" - mn "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" "io" "math/big" "net/http" @@ -15,6 +13,7 @@ import ( "testing" "github.com/gagliardetto/solana-go" + "github.com/gagliardetto/solana-go/programs/system" "github.com/gagliardetto/solana-go/rpc" "github.com/google/uuid" "github.com/stretchr/testify/assert" @@ -27,6 +26,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" "github.com/smartcontractkit/chainlink-solana/pkg/solana/client" + mn "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" solcfg "github.com/smartcontractkit/chainlink-solana/pkg/solana/config" "github.com/smartcontractkit/chainlink-solana/pkg/solana/fees" "github.com/smartcontractkit/chainlink-solana/pkg/solana/txm/mocks" From 489040ff72f1f6a6c1be2cb4bfc3729e2df42896 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 8 Oct 2024 14:53:19 -0400 Subject: [PATCH 073/174] Update classify_errors.go --- pkg/solana/client/classify_errors.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/solana/client/classify_errors.go b/pkg/solana/client/classify_errors.go index fed92e6b9..89a9a8cd3 100644 --- a/pkg/solana/client/classify_errors.go +++ b/pkg/solana/client/classify_errors.go @@ -8,6 +8,8 @@ import ( mn "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" ) +// Solana error patters +// https://github.com/anza-xyz/agave/blob/master/sdk/src/transaction/error.rs var ( ErrAccountInUse = regexp.MustCompile(`Account in use`) ErrAccountLoadedTwice = regexp.MustCompile(`Account loaded twice`) @@ -50,7 +52,7 @@ var ( ) // Define a map to associate regex patterns with SendTxReturnCode -var errorPatterns = map[*regexp.Regexp]mn.SendTxReturnCode{ +var errCodes = map[*regexp.Regexp]mn.SendTxReturnCode{ ErrAccountInUse: mn.Retryable, ErrAccountLoadedTwice: mn.Retryable, ErrAccountNotFound: mn.Retryable, @@ -101,7 +103,7 @@ func ClassifySendError(tx *solana.Transaction, err error) mn.SendTxReturnCode { } errMsg := err.Error() - for pattern, code := range errorPatterns { + for pattern, code := range errCodes { if pattern.MatchString(errMsg) { return code } From 03da4d4d34def33f733c1fdad0731e67c1079204 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 8 Oct 2024 14:57:01 -0400 Subject: [PATCH 074/174] Update classify_errors.go --- pkg/solana/client/classify_errors.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/solana/client/classify_errors.go b/pkg/solana/client/classify_errors.go index 89a9a8cd3..2f275649a 100644 --- a/pkg/solana/client/classify_errors.go +++ b/pkg/solana/client/classify_errors.go @@ -51,7 +51,7 @@ var ( ErrProgramCacheHitMaxLimit = regexp.MustCompile(`Program cache hit max limit`) ) -// Define a map to associate regex patterns with SendTxReturnCode +// errCodes maps regex patterns to corresponding return code var errCodes = map[*regexp.Regexp]mn.SendTxReturnCode{ ErrAccountInUse: mn.Retryable, ErrAccountLoadedTwice: mn.Retryable, From 47bba5deea9796f4a27e41a8e687fa6103d6299e Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 8 Oct 2024 15:30:30 -0400 Subject: [PATCH 075/174] Add tests --- pkg/solana/client/multinode_client_test.go | 145 ++++++++++++++++----- 1 file changed, 114 insertions(+), 31 deletions(-) diff --git a/pkg/solana/client/multinode_client_test.go b/pkg/solana/client/multinode_client_test.go index a29a6cc49..04339b5be 100644 --- a/pkg/solana/client/multinode_client_test.go +++ b/pkg/solana/client/multinode_client_test.go @@ -13,7 +13,7 @@ import ( "github.com/smartcontractkit/chainlink-solana/pkg/solana/config" ) -func TestMultiNodeClient_Subscriptions(t *testing.T) { +func initializeMultiNodeClient(t *testing.T) *MultiNodeClient { url := SetupLocalSolNode(t) privKey, err := solana.NewRandomPrivateKey() require.NoError(t, err) @@ -23,43 +23,126 @@ func TestMultiNodeClient_Subscriptions(t *testing.T) { requestTimeout := 5 * time.Second lggr := logger.Test(t) cfg := config.NewDefault() - // Enable MultiNode enabled := true - cfg.Enabled = &enabled + cfg.MultiNode.MultiNode.Enabled = &enabled c, err := NewMultiNodeClient(url, cfg, requestTimeout, lggr) require.NoError(t, err) + return c +} - err = c.Ping(tests.Context(t)) - require.NoError(t, err) - - ch, sub, err := c.SubscribeToHeads(tests.Context(t)) - require.NoError(t, err) - defer sub.Unsubscribe() - - finalizedCh, finalizedSub, err := c.SubscribeToFinalizedHeads(tests.Context(t)) - require.NoError(t, err) - defer finalizedSub.Unsubscribe() +func TestMultiNodeClient_Ping(t *testing.T) { + c := initializeMultiNodeClient(t) + require.NoError(t, c.Ping(tests.Context(t))) +} - require.NoError(t, err) - ctx, cancel := context.WithTimeout(tests.Context(t), time.Minute) - defer cancel() +func TestMultiNodeClient_LatestBlock(t *testing.T) { + c := initializeMultiNodeClient(t) - select { - case head := <-ch: + t.Run("LatestBlock", func(t *testing.T) { + head, err := c.LatestBlock(tests.Context(t)) + require.NoError(t, err) + require.Equal(t, true, head.IsValid()) require.NotEqual(t, solana.Hash{}, head.BlockHash) - latest, _ := c.GetInterceptedChainInfo() - require.Equal(t, head.BlockNumber(), latest.BlockNumber) - case <-ctx.Done(): - t.Fatal("failed to receive head: ", ctx.Err()) - } - - select { - case finalizedHead := <-finalizedCh: + }) + + t.Run("LatestFinalizedBlock", func(t *testing.T) { + finalizedHead, err := c.LatestFinalizedBlock(tests.Context(t)) + require.NoError(t, err) + require.Equal(t, true, finalizedHead.IsValid()) require.NotEqual(t, solana.Hash{}, finalizedHead.BlockHash) - latest, _ := c.GetInterceptedChainInfo() - require.Equal(t, finalizedHead.BlockNumber(), latest.FinalizedBlockNumber) - case <-ctx.Done(): - t.Fatal("failed to receive finalized head: ", ctx.Err()) - } + }) +} + +func TestMultiNodeClient_HeadSubscriptions(t *testing.T) { + c := initializeMultiNodeClient(t) + + t.Run("SubscribeToHeads", func(t *testing.T) { + ch, sub, err := c.SubscribeToHeads(tests.Context(t)) + require.NoError(t, err) + defer sub.Unsubscribe() + + ctx, cancel := context.WithTimeout(tests.Context(t), time.Minute) + defer cancel() + select { + case head := <-ch: + require.NotEqual(t, solana.Hash{}, head.BlockHash) + latest, _ := c.GetInterceptedChainInfo() + require.Equal(t, head.BlockNumber(), latest.BlockNumber) + case <-ctx.Done(): + t.Fatal("failed to receive head: ", ctx.Err()) + } + }) + + t.Run("SubscribeToFinalizedHeads", func(t *testing.T) { + finalizedCh, finalizedSub, err := c.SubscribeToFinalizedHeads(tests.Context(t)) + require.NoError(t, err) + defer finalizedSub.Unsubscribe() + + ctx, cancel := context.WithTimeout(tests.Context(t), time.Minute) + defer cancel() + select { + case finalizedHead := <-finalizedCh: + require.NotEqual(t, solana.Hash{}, finalizedHead.BlockHash) + latest, _ := c.GetInterceptedChainInfo() + require.Equal(t, finalizedHead.BlockNumber(), latest.FinalizedBlockNumber) + case <-ctx.Done(): + t.Fatal("failed to receive finalized head: ", ctx.Err()) + } + }) +} + +type mockSub struct { + unsubscribed bool +} + +func newMockSub() *mockSub { + return &mockSub{unsubscribed: false} +} + +func (s *mockSub) Unsubscribe() { + s.unsubscribed = true +} +func (s *mockSub) Err() <-chan error { + return nil +} + +func TestMultiNodeClient_RegisterSubs(t *testing.T) { + c := initializeMultiNodeClient(t) + + t.Run("registerSub", func(t *testing.T) { + sub := newMockSub() + err := c.registerSub(sub, make(chan struct{})) + require.NoError(t, err) + require.Len(t, c.subs, 1) + c.UnsubscribeAllExcept() + }) + + t.Run("chStopInFlight returns error and unsubscribes", func(t *testing.T) { + chStopInFlight := make(chan struct{}) + close(chStopInFlight) + sub := newMockSub() + err := c.registerSub(sub, chStopInFlight) + require.Error(t, err) + require.Equal(t, true, sub.unsubscribed) + }) + + t.Run("UnsubscribeAllExcept", func(t *testing.T) { + chStopInFlight := make(chan struct{}) + sub1 := newMockSub() + sub2 := newMockSub() + err := c.registerSub(sub1, chStopInFlight) + require.NoError(t, err) + err = c.registerSub(sub2, chStopInFlight) + require.NoError(t, err) + require.Len(t, c.subs, 2) + + c.UnsubscribeAllExcept(sub1) + require.Len(t, c.subs, 1) + require.Equal(t, true, sub2.unsubscribed) + + c.UnsubscribeAllExcept() + require.Len(t, c.subs, 0) + require.Equal(t, true, sub1.unsubscribed) + }) } From 5b4811f5bbee80389bcd43d5be6ee7b952510dd0 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 8 Oct 2024 17:15:08 -0400 Subject: [PATCH 076/174] Add test coverage --- pkg/solana/chain_test.go | 241 ++++++++++++++++------ pkg/solana/client/classify_errors.go | 2 +- pkg/solana/client/classify_errors_test.go | 5 +- 3 files changed, 180 insertions(+), 68 deletions(-) diff --git a/pkg/solana/chain_test.go b/pkg/solana/chain_test.go index 9053d312f..73d6958ac 100644 --- a/pkg/solana/chain_test.go +++ b/pkg/solana/chain_test.go @@ -11,20 +11,20 @@ import ( "strings" "sync" "testing" + "time" "github.com/gagliardetto/solana-go" "github.com/gagliardetto/solana-go/programs/system" "github.com/gagliardetto/solana-go/rpc" "github.com/google/uuid" + "github.com/smartcontractkit/chainlink-common/pkg/config" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "go.uber.org/zap/zapcore" - "github.com/smartcontractkit/chainlink-common/pkg/config" - "github.com/smartcontractkit/chainlink-common/pkg/logger" - "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" - "github.com/smartcontractkit/chainlink-solana/pkg/solana/client" mn "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" solcfg "github.com/smartcontractkit/chainlink-solana/pkg/solana/config" @@ -128,61 +128,6 @@ func TestSolanaChain_GetClient(t *testing.T) { assert.NoError(t, err) } -func TestSolanaChain_MultiNode_GetClient(t *testing.T) { - mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - out := fmt.Sprintf(TestSolanaGenesisHashTemplate, client.MainnetGenesisHash) // mainnet genesis hash - if !strings.Contains(r.URL.Path, "/mismatch") { - // devnet gensis hash - out = fmt.Sprintf(TestSolanaGenesisHashTemplate, client.DevnetGenesisHash) - } - _, err := w.Write([]byte(out)) - require.NoError(t, err) - })) - defer mockServer.Close() - - ch := solcfg.Chain{} - ch.SetDefaults() - mn := solcfg.MultiNodeConfig{ - MultiNode: solcfg.MultiNode{ - Enabled: ptr(true), - }, - } - mn.SetDefaults() - - cfg := &solcfg.TOMLConfig{ - ChainID: ptr("devnet"), - Chain: ch, - MultiNode: mn, - } - cfg.Nodes = []*solcfg.Node{ - { - Name: ptr("devnet"), - URL: config.MustParseURL(mockServer.URL + "/1"), - }, - { - Name: ptr("devnet"), - URL: config.MustParseURL(mockServer.URL + "/2"), - }, - } - - testChain, err := newChain("devnet", cfg, nil, logger.Test(t)) - require.NoError(t, err) - - err = testChain.Start(tests.Context(t)) - require.NoError(t, err) - defer func() { - closeErr := testChain.Close() - require.NoError(t, closeErr) - }() - - selectedClient, err := testChain.getClient() - assert.NoError(t, err) - - id, err := selectedClient.ChainID(tests.Context(t)) - assert.NoError(t, err) - assert.Equal(t, "devnet", id.String()) -} - func TestSolanaChain_VerifiedClient(t *testing.T) { called := false mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -373,6 +318,61 @@ func TestChain_Transact(t *testing.T) { assert.Equal(t, fees.ComputeUnitLimit(500), limit) } +func TestSolanaChain_MultiNode_GetClient(t *testing.T) { + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + out := fmt.Sprintf(TestSolanaGenesisHashTemplate, client.MainnetGenesisHash) // mainnet genesis hash + if !strings.Contains(r.URL.Path, "/mismatch") { + // devnet gensis hash + out = fmt.Sprintf(TestSolanaGenesisHashTemplate, client.DevnetGenesisHash) + } + _, err := w.Write([]byte(out)) + require.NoError(t, err) + })) + defer mockServer.Close() + + ch := solcfg.Chain{} + ch.SetDefaults() + mn := solcfg.MultiNodeConfig{ + MultiNode: solcfg.MultiNode{ + Enabled: ptr(true), + }, + } + mn.SetDefaults() + + cfg := &solcfg.TOMLConfig{ + ChainID: ptr("devnet"), + Chain: ch, + MultiNode: mn, + } + cfg.Nodes = []*solcfg.Node{ + { + Name: ptr("devnet"), + URL: config.MustParseURL(mockServer.URL + "/1"), + }, + { + Name: ptr("devnet"), + URL: config.MustParseURL(mockServer.URL + "/2"), + }, + } + + testChain, err := newChain("devnet", cfg, nil, logger.Test(t)) + require.NoError(t, err) + + err = testChain.Start(tests.Context(t)) + require.NoError(t, err) + defer func() { + closeErr := testChain.Close() + require.NoError(t, closeErr) + }() + + selectedClient, err := testChain.getClient() + assert.NoError(t, err) + + id, err := selectedClient.ChainID(tests.Context(t)) + assert.NoError(t, err) + assert.Equal(t, "devnet", id.String()) +} + func TestChain_MultiNode_TransactionSender(t *testing.T) { ctx := tests.Context(t) url := client.SetupLocalSolNode(t) @@ -391,22 +391,22 @@ func TestChain_MultiNode_TransactionSender(t *testing.T) { cfg.Nodes = append(cfg.Nodes, &solcfg.Node{ Name: ptr("localnet-" + t.Name() + "-primary-1"), - URL: config.MustParseURL(url), + URL: config.MustParseURL(client.SetupLocalSolNode(t)), SendOnly: false, }, &solcfg.Node{ Name: ptr("localnet-" + t.Name() + "-primary-2"), - URL: config.MustParseURL(url), + URL: config.MustParseURL(client.SetupLocalSolNode(t)), SendOnly: false, }, &solcfg.Node{ Name: ptr("localnet-" + t.Name() + "-sendonly-1"), - URL: config.MustParseURL(url), + URL: config.MustParseURL(client.SetupLocalSolNode(t)), SendOnly: true, }, &solcfg.Node{ Name: ptr("localnet-" + t.Name() + "-sendonly-1"), - URL: config.MustParseURL(url), + URL: config.MustParseURL(client.SetupLocalSolNode(t)), SendOnly: true, }, ) @@ -416,6 +416,9 @@ func TestChain_MultiNode_TransactionSender(t *testing.T) { c, err := newChain("localnet", cfg, mkey, lgr) require.NoError(t, err) require.NoError(t, c.Start(ctx)) + defer func() { + require.NoError(t, c.Close()) + }() t.Run("successful transaction", func(t *testing.T) { // create + sign transaction @@ -495,3 +498,115 @@ func TestChain_MultiNode_TransactionSender(t *testing.T) { require.Empty(t, sig) }) } + +func TestSolanaChain_MultiNode_Txm(t *testing.T) { + cfg := solcfg.NewDefault() + cfg.MultiNode.MultiNode.Enabled = ptr(true) + cfg.Nodes = []*solcfg.Node{ + { + Name: ptr("primary-1"), + URL: config.MustParseURL(client.SetupLocalSolNode(t)), + }, + { + Name: ptr("primary-2"), + URL: config.MustParseURL(client.SetupLocalSolNode(t)), + }, + { + Name: ptr("sendonly-1"), + URL: config.MustParseURL(client.SetupLocalSolNode(t)), + SendOnly: true, + }, + { + Name: ptr("sendonly-2"), + URL: config.MustParseURL(client.SetupLocalSolNode(t)), + SendOnly: true, + }, + } + + // setup keys + key, err := solana.NewRandomPrivateKey() + require.NoError(t, err) + pubKey := key.PublicKey() + + // setup receiver key + privKeyReceiver, err := solana.NewRandomPrivateKey() + pubKeyReceiver := privKeyReceiver.PublicKey() + + // mocked keystore + mkey := mocks.NewSimpleKeystore(t) + mkey.On("Sign", mock.Anything, pubKey.String(), mock.Anything).Return(func(_ context.Context, _ string, data []byte) []byte { + sig, _ := key.Sign(data) + return sig[:] + }, nil) + mkey.On("Sign", mock.Anything, pubKeyReceiver.String(), mock.Anything).Return([]byte{}, config.KeyNotFoundError{ID: pubKeyReceiver.String(), KeyType: "Solana"}) + + testChain, err := newChain("localnet", cfg, mkey, logger.Test(t)) + require.NoError(t, err) + + err = testChain.Start(tests.Context(t)) + require.NoError(t, err) + defer func() { + require.NoError(t, testChain.Close()) + }() + + // fund keys + client.FundTestAccounts(t, []solana.PublicKey{pubKey}, cfg.Nodes[0].URL.String()) + + // track initial balance + selectedClient, err := testChain.getClient() + require.NoError(t, err) + receiverBal, err := selectedClient.Balance(pubKeyReceiver) + assert.NoError(t, err) + assert.Equal(t, uint64(0), receiverBal) + + createTx := func(signer solana.PublicKey, sender solana.PublicKey, receiver solana.PublicKey, amt uint64) *solana.Transaction { + selectedClient, err = testChain.getClient() + assert.NoError(t, err) + hash, err := selectedClient.LatestBlockhash() + assert.NoError(t, err) + tx, err := solana.NewTransaction( + []solana.Instruction{ + system.NewTransferInstruction( + amt, + sender, + receiver, + ).Build(), + }, + hash.Value.Blockhash, + solana.TransactionPayer(signer), + ) + require.NoError(t, err) + return tx + } + + // Send funds twice, along with an invalid transaction + require.NoError(t, testChain.txm.Enqueue("test_success", createTx(pubKey, pubKey, pubKeyReceiver, solana.LAMPORTS_PER_SOL))) + time.Sleep(500 * time.Millisecond) // pause 0.5s for new blockhash + require.NoError(t, testChain.txm.Enqueue("test_success_2", createTx(pubKey, pubKey, pubKeyReceiver, solana.LAMPORTS_PER_SOL))) + require.Error(t, testChain.txm.Enqueue("test_invalidSigner", createTx(pubKeyReceiver, pubKey, pubKeyReceiver, solana.LAMPORTS_PER_SOL))) // cannot sign tx before enqueuing + + // wait for all txes to finish + ctx, cancel := context.WithCancel(tests.Context(t)) + t.Cleanup(cancel) + ticker := time.NewTicker(time.Second) + defer ticker.Stop() +loop: + for { + select { + case <-ctx.Done(): + assert.Equal(t, 0, testChain.txm.InflightTxs()) + break loop + case <-ticker.C: + if testChain.txm.InflightTxs() == 0 { + cancel() // exit for loop + } + } + } + + // verify funds were transferred through transaction sender + selectedClient, err = testChain.getClient() + assert.NoError(t, err) + receiverBal, err = selectedClient.Balance(pubKeyReceiver) + assert.NoError(t, err) + require.Equal(t, 2*solana.LAMPORTS_PER_SOL, receiverBal) +} diff --git a/pkg/solana/client/classify_errors.go b/pkg/solana/client/classify_errors.go index 2f275649a..0e3c1456e 100644 --- a/pkg/solana/client/classify_errors.go +++ b/pkg/solana/client/classify_errors.go @@ -97,7 +97,7 @@ var errCodes = map[*regexp.Regexp]mn.SendTxReturnCode{ var _ mn.TxErrorClassifier[*solana.Transaction] = ClassifySendError // ClassifySendError returns the corresponding return code based on the error. -func ClassifySendError(tx *solana.Transaction, err error) mn.SendTxReturnCode { +func ClassifySendError(_ *solana.Transaction, err error) mn.SendTxReturnCode { if err == nil { return mn.Successful } diff --git a/pkg/solana/client/classify_errors_test.go b/pkg/solana/client/classify_errors_test.go index cd7f5ddb3..5d3e8de68 100644 --- a/pkg/solana/client/classify_errors_test.go +++ b/pkg/solana/client/classify_errors_test.go @@ -4,7 +4,6 @@ import ( "errors" "testing" - "github.com/gagliardetto/solana-go" "github.com/stretchr/testify/assert" mn "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" @@ -63,9 +62,7 @@ func TestClassifySendError(t *testing.T) { for _, tt := range tests { t.Run(tt.errMsg, func(t *testing.T) { - tx := &solana.Transaction{} // Dummy transaction - err := errors.New(tt.errMsg) // Create a standard Go error with the message - result := ClassifySendError(tx, err) + result := ClassifySendError(nil, errors.New(tt.errMsg)) assert.Equal(t, tt.expectedCode, result, "Expected %v but got %v for error message: %s", tt.expectedCode, result, tt.errMsg) }) } From 7b2450162aac259235277643c743384c8472ccbe Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 8 Oct 2024 17:20:47 -0400 Subject: [PATCH 077/174] lint --- pkg/solana/chain_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/solana/chain_test.go b/pkg/solana/chain_test.go index 73d6958ac..64a785a63 100644 --- a/pkg/solana/chain_test.go +++ b/pkg/solana/chain_test.go @@ -530,6 +530,7 @@ func TestSolanaChain_MultiNode_Txm(t *testing.T) { // setup receiver key privKeyReceiver, err := solana.NewRandomPrivateKey() + require.NoError(t, err) pubKeyReceiver := privKeyReceiver.PublicKey() // mocked keystore @@ -562,8 +563,8 @@ func TestSolanaChain_MultiNode_Txm(t *testing.T) { createTx := func(signer solana.PublicKey, sender solana.PublicKey, receiver solana.PublicKey, amt uint64) *solana.Transaction { selectedClient, err = testChain.getClient() assert.NoError(t, err) - hash, err := selectedClient.LatestBlockhash() - assert.NoError(t, err) + hash, hashErr := selectedClient.LatestBlockhash() + assert.NoError(t, hashErr) tx, err := solana.NewTransaction( []solana.Instruction{ system.NewTransferInstruction( From 37dd998a9a120d180e5fcb7e78ef5079843a14ad Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 9 Oct 2024 10:58:34 -0400 Subject: [PATCH 078/174] Add dial comment --- pkg/solana/client/multinode_client.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/solana/client/multinode_client.go b/pkg/solana/client/multinode_client.go index 65827fdfa..09a7002ea 100644 --- a/pkg/solana/client/multinode_client.go +++ b/pkg/solana/client/multinode_client.go @@ -94,6 +94,7 @@ func (m *MultiNodeClient) registerSub(sub mn.Subscription, stopInFLightCh chan s } func (m *MultiNodeClient) Dial(ctx context.Context) error { + // Not relevant for Solana as the RPCs don't need to be dialled. return nil } From 778fbb3af3943f7371f3b25cac8d9ddd9b99af1f Mon Sep 17 00:00:00 2001 From: Damjan Smickovski Date: Tue, 15 Oct 2024 08:52:34 +0200 Subject: [PATCH 079/174] CTF bump for image build --- integration-tests/go.mod | 12 ++++++------ integration-tests/go.sum | 24 ++++++++++++------------ 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/integration-tests/go.mod b/integration-tests/go.mod index 509753a3a..ed59ee223 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -6,7 +6,7 @@ replace github.com/smartcontractkit/chainlink-solana => ../ require ( github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df - github.com/docker/docker v25.0.2+incompatible + github.com/docker/docker v27.3.1+incompatible github.com/gagliardetto/binary v0.7.7 github.com/gagliardetto/solana-go v1.8.4 github.com/go-resty/resty/v2 v2.11.0 @@ -16,13 +16,13 @@ require ( github.com/rs/zerolog v1.33.0 github.com/smartcontractkit/chainlink-common v0.2.3-0.20240926180110-0784a13b2536 github.com/smartcontractkit/chainlink-solana v1.1.1-0.20240911182932-3c609a6ac664 - github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.9 + github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.11 github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.1 github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20240924233109-8b37da54ea01 github.com/smartcontractkit/chainlink/v2 v2.14.0-mercury-20240807.0.20240924233109-8b37da54ea01 github.com/smartcontractkit/libocr v0.0.0-20240717100443-f6226e09bee7 github.com/stretchr/testify v1.9.0 - github.com/testcontainers/testcontainers-go v0.28.0 + github.com/testcontainers/testcontainers-go v0.33.0 golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 golang.org/x/sync v0.8.0 golang.org/x/text v0.18.0 @@ -57,7 +57,6 @@ require ( github.com/Masterminds/semver/v3 v3.2.1 // indirect github.com/Masterminds/sprig/v3 v3.2.3 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect - github.com/Microsoft/hcsshim v0.11.5 // indirect github.com/NethermindEth/juno v0.3.1 // indirect github.com/NethermindEth/starknet.go v0.7.1-0.20240401080518-34a506f3cfdb // indirect github.com/VictoriaMetrics/fastcache v1.12.1 // indirect @@ -116,10 +115,9 @@ require ( github.com/confio/ics23/go v0.9.0 // indirect github.com/consensys/bavard v0.1.13 // indirect github.com/consensys/gnark-crypto v0.12.1 // indirect - github.com/containerd/containerd v1.7.18 // indirect github.com/containerd/continuity v0.4.3 // indirect - github.com/containerd/errdefs v0.1.0 // indirect github.com/containerd/log v0.1.0 // indirect + github.com/containerd/platforms v0.2.1 // indirect github.com/coreos/go-semver v0.3.1 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cosmos/btcutil v1.0.5 // indirect @@ -314,10 +312,12 @@ require ( github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/mmcloughlin/addchain v0.4.0 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/patternmatcher v0.6.0 // indirect github.com/moby/spdystream v0.4.0 // indirect github.com/moby/sys/sequential v0.5.0 // indirect github.com/moby/sys/user v0.1.0 // indirect + github.com/moby/sys/userns v0.1.0 // indirect github.com/moby/term v0.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect diff --git a/integration-tests/go.sum b/integration-tests/go.sum index 55f790540..0afe13dcb 100644 --- a/integration-tests/go.sum +++ b/integration-tests/go.sum @@ -139,8 +139,6 @@ github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= -github.com/Microsoft/hcsshim v0.11.5 h1:haEcLNpj9Ka1gd3B3tAEs9CpE0c+1IhoL59w/exYU38= -github.com/Microsoft/hcsshim v0.11.5/go.mod h1:MV8xMfmECjl5HdO7U/3/hFVnkmSBjAjmA09d4bExKcU= github.com/NethermindEth/juno v0.3.1 h1:AW72LiAm9gqUeCVJWvepnZcTnpU4Vkl0KzPMxS+42FA= github.com/NethermindEth/juno v0.3.1/go.mod h1:SGbTpgGaCsxhFsKOid7Ylnz//WZ8swtILk+NbHGsk/Q= github.com/NethermindEth/starknet.go v0.7.1-0.20240401080518-34a506f3cfdb h1:Mv8SscePPyw2ju4igIJAjFgcq5zCQfjgbz53DwYu5mc= @@ -343,14 +341,12 @@ github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/Yj github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= github.com/consensys/gnark-crypto v0.12.1 h1:lHH39WuuFgVHONRl3J0LRBtuYdQTumFSDtJF7HpyG8M= github.com/consensys/gnark-crypto v0.12.1/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY= -github.com/containerd/containerd v1.7.18 h1:jqjZTQNfXGoEaZdW1WwPU0RqSn1Bm2Ay/KJPUuO8nao= -github.com/containerd/containerd v1.7.18/go.mod h1:IYEk9/IO6wAPUz2bCMVUbsfXjzw5UNP5fLz4PsUygQ4= github.com/containerd/continuity v0.4.3 h1:6HVkalIp+2u1ZLH1J/pYX2oBVXlJZvh1X1A7bEZ9Su8= github.com/containerd/continuity v0.4.3/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ= -github.com/containerd/errdefs v0.1.0 h1:m0wCRBiu1WJT/Fr+iOoQHMQS/eP5myQ8lCv4Dz5ZURM= -github.com/containerd/errdefs v0.1.0/go.mod h1:YgWiiHtLmSeBrvpw+UfPijzbLaB77mEG1WwJTDETIV0= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= +github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -445,8 +441,8 @@ github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v25.0.2+incompatible h1:/OaKeauroa10K4Nqavw4zlhcDq/WBcPMc5DbjOGgozY= -github.com/docker/docker v25.0.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v27.3.1+incompatible h1:KttF0XoteNTicmUtBO0L2tP+J7FGRFTjaEF4k6WdhfI= +github.com/docker/docker v27.3.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -1170,6 +1166,8 @@ github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY= github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU= github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFVejUS1/tS/qU= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= github.com/moby/spdystream v0.4.0 h1:Vy79D6mHeJJjiPdFEL2yku1kl0chZpJfZcPpb16BRl8= @@ -1178,6 +1176,8 @@ github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5 github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= github.com/moby/sys/user v0.1.0 h1:WmZ93f5Ux6het5iituh9x2zAG7NFY9Aqi49jjE1PaQg= github.com/moby/sys/user v0.1.0/go.mod h1:fKJhFOnsCN6xZ5gSfbM6zaHGgDJMrqt9/reuj4T7MmU= +github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= +github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -1404,8 +1404,8 @@ github.com/smartcontractkit/chainlink-feeds v0.0.0-20240910155501-42f20443189f h github.com/smartcontractkit/chainlink-feeds v0.0.0-20240910155501-42f20443189f/go.mod h1:FLlWBt2hwiMVgt9AcSo6wBJYIRd/nsc8ENbV1Wir1bw= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240911194142-506bc469d8ae h1:d+B8y2Nd/PrnPMNoaSPn3eDgUgxcVcIqAxGrvYu/gGw= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240911194142-506bc469d8ae/go.mod h1:ec/a20UZ7YRK4oxJcnTBFzp1+DBcJcwqEaerUMsktMs= -github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.9 h1:/2kAb6y854viKigkdFMWDNNbaz3zD0gAkbZoSHC8Rrg= -github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.9/go.mod h1:7R5wGWWJi0dr5Y5cXbLQ4vSeIj0ElvhBaymcfvqqUmo= +github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.11 h1:JvRVMS6aXMoux9i/xihHo/qZtNwtv4lpbjsxo2O/1gE= +github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.11/go.mod h1:c5Is0W7DUUEeV369pWbAOYqEktlGeIBQXefGyIMCzvE= github.com/smartcontractkit/chainlink-testing-framework/lib/grafana v1.50.0 h1:VIxK8u0Jd0Q/VuhmsNm6Bls6Tb31H/sA3A/rbc5hnhg= github.com/smartcontractkit/chainlink-testing-framework/lib/grafana v1.50.0/go.mod h1:lyAu+oMXdNUzEDScj2DXB2IueY+SDXPPfyl/kb63tMM= github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.1 h1:2OxnPfvjC+zs0ZokSsRTRnJrEGJ4NVJwZgfroS1lPHs= @@ -1497,8 +1497,8 @@ github.com/teris-io/shortid v0.0.0-20201117134242-e59966efd125 h1:3SNcvBmEPE1YlB github.com/teris-io/shortid v0.0.0-20201117134242-e59966efd125/go.mod h1:M8agBzgqHIhgj7wEn9/0hJUZcrvt9VY+Ln+S1I5Mha0= github.com/test-go/testify v1.1.4 h1:Tf9lntrKUMHiXQ07qBScBTSA0dhYQlu83hswqelv1iE= github.com/test-go/testify v1.1.4/go.mod h1:rH7cfJo/47vWGdi4GPj16x3/t1xGOj2YxzmNQzk2ghU= -github.com/testcontainers/testcontainers-go v0.28.0 h1:1HLm9qm+J5VikzFDYhOd+Zw12NtOl+8drH2E8nTY1r8= -github.com/testcontainers/testcontainers-go v0.28.0/go.mod h1:COlDpUXbwW3owtpMkEB1zo9gwb1CoKVKlyrVPejF4AU= +github.com/testcontainers/testcontainers-go v0.33.0 h1:zJS9PfXYT5O0ZFXM2xxXfk4J5UMw/kRiISng037Gxdw= +github.com/testcontainers/testcontainers-go v0.33.0/go.mod h1:W80YpTa8D5C3Yy16icheD01UTDu+LmXIA2Keo+jWtT8= github.com/theodesp/go-heaps v0.0.0-20190520121037-88e35354fe0a h1:YuO+afVc3eqrjiCUizNCxI53bl/BnPiVwXqLzqYTqgU= github.com/theodesp/go-heaps v0.0.0-20190520121037-88e35354fe0a/go.mod h1:/sfW47zCZp9FrtGcWyo1VjbgDaodxX9ovZvgLb/MxaA= github.com/tidwall/btree v1.6.0 h1:LDZfKfQIBHGHWSwckhXI0RPSXzlo+KYdjK7FWSqOzzg= From 56ae7dd4a44b5bb8fa64f956233eb493722a1485 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 15 Oct 2024 10:12:10 -0400 Subject: [PATCH 080/174] Update pkg/solana/client/multinode_client.go Co-authored-by: Dmytro Haidashenko <34754799+dhaidashenko@users.noreply.github.com> --- pkg/solana/client/multinode_client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/solana/client/multinode_client.go b/pkg/solana/client/multinode_client.go index 09a7002ea..086699cef 100644 --- a/pkg/solana/client/multinode_client.go +++ b/pkg/solana/client/multinode_client.go @@ -37,7 +37,7 @@ func (h *Head) BlockDifficulty() *big.Int { } func (h *Head) IsValid() bool { - return h.BlockHeight != nil && h.BlockHash != nil + return h != nil && h.BlockHeight != nil && h.BlockHash != nil } var _ mn.RPCClient[mn.StringID, *Head] = (*MultiNodeClient)(nil) From 4f445c51e1180fe269968e788b166853f45fb655 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 15 Oct 2024 11:43:22 -0400 Subject: [PATCH 081/174] Update txm.go --- pkg/solana/txm/txm.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/pkg/solana/txm/txm.go b/pkg/solana/txm/txm.go index c458791f9..37b7b02cd 100644 --- a/pkg/solana/txm/txm.go +++ b/pkg/solana/txm/txm.go @@ -41,17 +41,17 @@ var _ loop.Keystore = (SimpleKeystore)(nil) // Txm manages transactions for the solana blockchain. // simple implementation with no persistently stored txs type Txm struct { - starter services.StateMachine - lggr logger.Logger - chSend chan pendingTx - chSim chan pendingTx - chStop services.StopChan - done sync.WaitGroup - cfg config.Config - txs PendingTxContext - ks SimpleKeystore - client *utils.LazyLoad[client.ReaderWriter] - fee fees.Estimator + services.StateMachine + lggr logger.Logger + chSend chan pendingTx + chSim chan pendingTx + chStop services.StopChan + done sync.WaitGroup + cfg config.Config + txs PendingTxContext + ks SimpleKeystore + client *utils.LazyLoad[client.ReaderWriter] + fee fees.Estimator // sendTx is an override for sending transactions rather than using a single client // Enabling MultiNode uses this function to send transactions to all RPCs sendTx func(ctx context.Context, tx *solanaGo.Transaction) (solanaGo.Signature, error) From 86c92a849c2c95277f0fdd545eeb59a1eeb40f9f Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 16 Oct 2024 13:51:45 -0400 Subject: [PATCH 082/174] Create loader --- pkg/solana/chain.go | 10 ++-- pkg/solana/chain_test.go | 4 +- .../client/multinode/transaction_sender.go | 13 ++++++ pkg/solana/fees/block_history.go | 7 +-- pkg/solana/internal/loader.go | 41 +++++++++++++++++ pkg/solana/internal/loader_test.go | 46 +++++++++++++++++++ pkg/solana/monitor/balance.go | 20 ++------ pkg/solana/monitor/balance_test.go | 6 ++- pkg/solana/txm/txm.go | 10 ++-- 9 files changed, 129 insertions(+), 28 deletions(-) create mode 100644 pkg/solana/internal/loader.go create mode 100644 pkg/solana/internal/loader_test.go diff --git a/pkg/solana/chain.go b/pkg/solana/chain.go index 42a80901c..0e6aa5aa4 100644 --- a/pkg/solana/chain.go +++ b/pkg/solana/chain.go @@ -23,10 +23,10 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/types" "github.com/smartcontractkit/chainlink-common/pkg/types/core" - mn "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" - "github.com/smartcontractkit/chainlink-solana/pkg/solana/client" + mn "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" "github.com/smartcontractkit/chainlink-solana/pkg/solana/config" + "github.com/smartcontractkit/chainlink-solana/pkg/solana/internal" "github.com/smartcontractkit/chainlink-solana/pkg/solana/monitor" "github.com/smartcontractkit/chainlink-solana/pkg/solana/txm" ) @@ -301,7 +301,11 @@ func newChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr logger.L tc := func() (client.ReaderWriter, error) { return ch.getClient() } - ch.txm = txm.NewTxm(ch.id, tc, sendTx, cfg, ks, lggr) + + // Use lazy loader if MultiNode is disabled + loader := internal.NewLoader(!cfg.MultiNode.Enabled(), tc) + + ch.txm = txm.NewTxm(ch.id, loader, sendTx, cfg, ks, lggr) bc := func() (monitor.BalanceClient, error) { return ch.getClient() } diff --git a/pkg/solana/chain_test.go b/pkg/solana/chain_test.go index 64a785a63..f39152014 100644 --- a/pkg/solana/chain_test.go +++ b/pkg/solana/chain_test.go @@ -565,7 +565,7 @@ func TestSolanaChain_MultiNode_Txm(t *testing.T) { assert.NoError(t, err) hash, hashErr := selectedClient.LatestBlockhash() assert.NoError(t, hashErr) - tx, err := solana.NewTransaction( + tx, txErr := solana.NewTransaction( []solana.Instruction{ system.NewTransferInstruction( amt, @@ -576,7 +576,7 @@ func TestSolanaChain_MultiNode_Txm(t *testing.T) { hash.Value.Blockhash, solana.TransactionPayer(signer), ) - require.NoError(t, err) + require.NoError(t, txErr) return tx } diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go index 78b75cd10..306cd65e7 100644 --- a/pkg/solana/client/multinode/transaction_sender.go +++ b/pkg/solana/client/multinode/transaction_sender.go @@ -9,6 +9,7 @@ import ( "sync" "time" + "github.com/gagliardetto/solana-go" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -34,6 +35,18 @@ type sendTxResult[RESULT any] struct { Result *RESULT } +// TODO: Implement using this instead...? +type SendTxResult interface { + Code() SendTxReturnCode + Error() error +} + +type SolanaTxResult struct { + code SendTxReturnCode + err error + sig solana.Signature +} + const sendTxQuorum = 0.7 // SendTxRPCClient - defines interface of an RPC used by TransactionSender to broadcast transaction diff --git a/pkg/solana/fees/block_history.go b/pkg/solana/fees/block_history.go index 214612e90..bc2db327d 100644 --- a/pkg/solana/fees/block_history.go +++ b/pkg/solana/fees/block_history.go @@ -3,6 +3,8 @@ package fees import ( "context" "fmt" + "github.com/smartcontractkit/chainlink-solana/pkg/solana/client" + "github.com/smartcontractkit/chainlink-solana/pkg/solana/internal" "sync" "time" @@ -11,7 +13,6 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/utils" "github.com/smartcontractkit/chainlink-common/pkg/utils/mathutil" - "github.com/smartcontractkit/chainlink-solana/pkg/solana/client" "github.com/smartcontractkit/chainlink-solana/pkg/solana/config" ) @@ -22,7 +23,7 @@ type blockHistoryEstimator struct { chStop chan struct{} done sync.WaitGroup - client *utils.LazyLoad[client.ReaderWriter] + client internal.Loader[client.ReaderWriter] cfg config.Config lgr logger.Logger @@ -33,7 +34,7 @@ type blockHistoryEstimator struct { // NewBlockHistoryEstimator creates a new fee estimator that parses historical fees from a fetched block // Note: getRecentPrioritizationFees is not used because it provides the lowest prioritization fee for an included tx in the block // which is not effective enough for increasing the chances of block inclusion -func NewBlockHistoryEstimator(c *utils.LazyLoad[client.ReaderWriter], cfg config.Config, lgr logger.Logger) (*blockHistoryEstimator, error) { +func NewBlockHistoryEstimator(c internal.Loader[client.ReaderWriter], cfg config.Config, lgr logger.Logger) (*blockHistoryEstimator, error) { return &blockHistoryEstimator{ chStop: make(chan struct{}), client: c, diff --git a/pkg/solana/internal/loader.go b/pkg/solana/internal/loader.go new file mode 100644 index 000000000..43b847f3b --- /dev/null +++ b/pkg/solana/internal/loader.go @@ -0,0 +1,41 @@ +package internal + +import ( + "github.com/smartcontractkit/chainlink-common/pkg/utils" +) + +type Loader[T any] interface { + Get() (T, error) + Reset() +} + +var _ Loader[any] = (*loader[any])(nil) + +type loader[T any] struct { + getClient func() (T, error) + lazyLoader *utils.LazyLoad[T] +} + +func (c *loader[T]) Get() (T, error) { + if c.lazyLoader != nil { + return c.lazyLoader.Get() + } + return c.getClient() +} + +func (c *loader[T]) Reset() { + if c.lazyLoader != nil { + c.lazyLoader.Reset() + } +} + +func NewLoader[T any](lazyLoad bool, getClient func() (T, error)) *loader[T] { + var lazyLoader *utils.LazyLoad[T] + if lazyLoad { + lazyLoader = utils.NewLazyLoad(getClient) + } + return &loader[T]{ + lazyLoader: lazyLoader, + getClient: getClient, + } +} diff --git a/pkg/solana/internal/loader_test.go b/pkg/solana/internal/loader_test.go new file mode 100644 index 000000000..7fc8e0745 --- /dev/null +++ b/pkg/solana/internal/loader_test.go @@ -0,0 +1,46 @@ +package internal + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +type testLoader struct { + Loader[any] + callCount int +} + +func (t *testLoader) load() (any, error) { + t.callCount++ + return nil, nil +} + +func newTestLoader(lazyLoad bool) *testLoader { + loader := testLoader{} + loader.Loader = NewLoader[any](lazyLoad, loader.load) + return &loader +} + +func TestLoader(t *testing.T) { + t.Run("direct loading", func(t *testing.T) { + loader := newTestLoader(false) + _, _ = loader.Get() + _, _ = loader.Get() + _, _ = loader.Get() + require.Equal(t, 3, loader.callCount) + }) + + t.Run("lazy loading", func(t *testing.T) { + loader := newTestLoader(true) + _, _ = loader.Get() + _, _ = loader.Get() + require.Equal(t, 1, loader.callCount) + + // Calls load function again after Reset() + loader.Reset() + _, _ = loader.Get() + _, _ = loader.Get() + require.Equal(t, 2, loader.callCount) + }) +} diff --git a/pkg/solana/monitor/balance.go b/pkg/solana/monitor/balance.go index 5c7c88f5d..9cf47351c 100644 --- a/pkg/solana/monitor/balance.go +++ b/pkg/solana/monitor/balance.go @@ -9,6 +9,8 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink-common/pkg/utils" + + "github.com/smartcontractkit/chainlink-solana/pkg/solana/internal" ) // Config defines the monitor configuration. @@ -53,7 +55,7 @@ type balanceMonitor struct { newReader func() (BalanceClient, error) updateFn func(acc solana.PublicKey, lamports uint64) // overridable for testing - reader BalanceClient + reader internal.Loader[BalanceClient] stop services.StopChan done chan struct{} @@ -99,18 +101,6 @@ func (b *balanceMonitor) monitor() { } } -// getReader returns the cached solanaClient.Reader, or creates a new one if nil. -func (b *balanceMonitor) getReader() (BalanceClient, error) { - if b.reader == nil { - var err error - b.reader, err = b.newReader() - if err != nil { - return nil, err - } - } - return b.reader, nil -} - func (b *balanceMonitor) updateBalances(ctx context.Context) { keys, err := b.ks.Accounts(ctx) if err != nil { @@ -120,7 +110,7 @@ func (b *balanceMonitor) updateBalances(ctx context.Context) { if len(keys) == 0 { return } - reader, err := b.getReader() + reader, err := b.reader.Get() if err != nil { b.lggr.Errorw("Failed to get client", "err", err) return @@ -148,6 +138,6 @@ func (b *balanceMonitor) updateBalances(ctx context.Context) { } if !gotSomeBals { // Try a new client next time. - b.reader = nil + b.reader.Reset() } } diff --git a/pkg/solana/monitor/balance_test.go b/pkg/solana/monitor/balance_test.go index ff98d0508..978ce03ae 100644 --- a/pkg/solana/monitor/balance_test.go +++ b/pkg/solana/monitor/balance_test.go @@ -61,7 +61,11 @@ func TestBalanceMonitor(t *testing.T) { close(done) } } - b.reader = client + + getClient := func() (BalanceClient, error) { + return client, nil + } + b.reader = internal.NewLoader[BalanceClient](true, getClient) require.NoError(t, b.Start(tests.Context(t))) t.Cleanup(func() { diff --git a/pkg/solana/txm/txm.go b/pkg/solana/txm/txm.go index 37b7b02cd..9c7c08d23 100644 --- a/pkg/solana/txm/txm.go +++ b/pkg/solana/txm/txm.go @@ -4,6 +4,8 @@ import ( "context" "errors" "fmt" + "github.com/smartcontractkit/chainlink-solana/pkg/solana/client" + "github.com/smartcontractkit/chainlink-solana/pkg/solana/internal" "strings" "sync" "time" @@ -17,7 +19,6 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink-common/pkg/utils" - "github.com/smartcontractkit/chainlink-solana/pkg/solana/client" "github.com/smartcontractkit/chainlink-solana/pkg/solana/config" "github.com/smartcontractkit/chainlink-solana/pkg/solana/fees" ) @@ -50,7 +51,7 @@ type Txm struct { cfg config.Config txs PendingTxContext ks SimpleKeystore - client *utils.LazyLoad[client.ReaderWriter] + client internal.Loader[client.ReaderWriter] fee fees.Estimator // sendTx is an override for sending transactions rather than using a single client // Enabling MultiNode uses this function to send transactions to all RPCs @@ -77,9 +78,10 @@ type pendingTx struct { } // NewTxm creates a txm. Uses simulation so should only be used to send txes to trusted contracts i.e. OCR. -func NewTxm(chainID string, tc func() (client.ReaderWriter, error), +func NewTxm(chainID string, client internal.Loader[client.ReaderWriter], sendTx func(ctx context.Context, tx *solanaGo.Transaction) (solanaGo.Signature, error), cfg config.Config, ks SimpleKeystore, lggr logger.Logger) *Txm { + return &Txm{ lggr: logger.Named(lggr, "Txm"), chSend: make(chan pendingTx, MaxQueueLen), // queue can support 1000 pending txs @@ -88,7 +90,7 @@ func NewTxm(chainID string, tc func() (client.ReaderWriter, error), cfg: cfg, txs: newPendingTxContextWithProm(chainID), ks: ks, - client: utils.NewLazyLoad(tc), + client: client, sendTx: sendTx, } } From 8073c46cb1ce692d19b6c1437557f4963b1af8ae Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 16 Oct 2024 14:42:02 -0400 Subject: [PATCH 083/174] Update transaction_sender.go --- pkg/solana/client/multinode/transaction_sender.go | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go index 306cd65e7..78b75cd10 100644 --- a/pkg/solana/client/multinode/transaction_sender.go +++ b/pkg/solana/client/multinode/transaction_sender.go @@ -9,7 +9,6 @@ import ( "sync" "time" - "github.com/gagliardetto/solana-go" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -35,18 +34,6 @@ type sendTxResult[RESULT any] struct { Result *RESULT } -// TODO: Implement using this instead...? -type SendTxResult interface { - Code() SendTxReturnCode - Error() error -} - -type SolanaTxResult struct { - code SendTxReturnCode - err error - sig solana.Signature -} - const sendTxQuorum = 0.7 // SendTxRPCClient - defines interface of an RPC used by TransactionSender to broadcast transaction From 1091173875c5bbac48f7a031247a2c0c29f4b0bf Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 16 Oct 2024 15:38:14 -0400 Subject: [PATCH 084/174] Fix tests --- pkg/solana/txm/txm_race_test.go | 4 +++- pkg/solana/txm/txm_test.go | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/solana/txm/txm_race_test.go b/pkg/solana/txm/txm_race_test.go index 6f21099ad..85f0afeb9 100644 --- a/pkg/solana/txm/txm_race_test.go +++ b/pkg/solana/txm/txm_race_test.go @@ -19,6 +19,7 @@ import ( cfgmocks "github.com/smartcontractkit/chainlink-solana/pkg/solana/config/mocks" "github.com/smartcontractkit/chainlink-solana/pkg/solana/fees" feemocks "github.com/smartcontractkit/chainlink-solana/pkg/solana/fees/mocks" + "github.com/smartcontractkit/chainlink-solana/pkg/solana/internal" ksmocks "github.com/smartcontractkit/chainlink-solana/pkg/solana/txm/mocks" "github.com/stretchr/testify/assert" @@ -65,7 +66,8 @@ func TestTxm_SendWithRetry_Race(t *testing.T) { } // build minimal txm - txm := NewTxm("retry_race", getClient, nil, cfg, ks, lggr) + loader := internal.NewLoader(true, getClient) + txm := NewTxm("retry_race", loader, nil, cfg, ks, lggr) txm.fee = fee _, _, _, err := txm.sendWithRetry( diff --git a/pkg/solana/txm/txm_test.go b/pkg/solana/txm/txm_test.go index 75720b5da..21e3065fd 100644 --- a/pkg/solana/txm/txm_test.go +++ b/pkg/solana/txm/txm_test.go @@ -16,6 +16,7 @@ import ( solanaClient "github.com/smartcontractkit/chainlink-solana/pkg/solana/client" "github.com/smartcontractkit/chainlink-solana/pkg/solana/config" + "github.com/smartcontractkit/chainlink-solana/pkg/solana/internal" "github.com/smartcontractkit/chainlink-solana/pkg/solana/txm" keyMocks "github.com/smartcontractkit/chainlink-solana/pkg/solana/txm/mocks" @@ -72,7 +73,8 @@ func TestTxm_Integration(t *testing.T) { getClient := func() (solanaClient.ReaderWriter, error) { return client, nil } - txm := txm.NewTxm("localnet", getClient, nil, cfg, mkey, lggr) + loader := internal.NewLoader(true, getClient) + txm := txm.NewTxm("localnet", loader, nil, cfg, mkey, lggr) // track initial balance initBal, err := client.Balance(pubKey) From edd480f97175d5b54383e98d0f190fc82f1088fa Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 16 Oct 2024 16:03:59 -0400 Subject: [PATCH 085/174] Update txm_internal_test.go --- pkg/solana/txm/txm_internal_test.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/pkg/solana/txm/txm_internal_test.go b/pkg/solana/txm/txm_internal_test.go index e287e6782..3f8a653fc 100644 --- a/pkg/solana/txm/txm_internal_test.go +++ b/pkg/solana/txm/txm_internal_test.go @@ -5,6 +5,7 @@ package txm import ( "context" "errors" + "github.com/smartcontractkit/chainlink-solana/pkg/solana/internal" "math/rand" "sync" "testing" @@ -110,9 +111,11 @@ func TestTxm(t *testing.T) { mkey := keyMocks.NewSimpleKeystore(t) mkey.On("Sign", mock.Anything, mock.Anything, mock.Anything).Return([]byte{}, nil) - txm := NewTxm(id, func() (client.ReaderWriter, error) { + getClient := func() (client.ReaderWriter, error) { return mc, nil - }, nil, cfg, mkey, lggr) + } + loader := internal.NewLoader(true, getClient) + txm := NewTxm(id, loader, nil, cfg, mkey, lggr) require.NoError(t, txm.Start(ctx)) // tracking prom metrics @@ -717,9 +720,11 @@ func TestTxm_Enqueue(t *testing.T) { ) require.NoError(t, err) - txm := NewTxm("enqueue_test", func() (client.ReaderWriter, error) { + getClient := func() (client.ReaderWriter, error) { return mc, nil - }, nil, cfg, mkey, lggr) + } + loader := internal.NewLoader(true, getClient) + txm := NewTxm("enqueue_test", loader, nil, cfg, mkey, lggr) require.ErrorContains(t, txm.Enqueue("txmUnstarted", &solana.Transaction{}), "not started") require.NoError(t, txm.Start(ctx)) From f374f3eb0340f72d93f0ee15e7edcfa3edbd7030 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 16 Oct 2024 16:05:53 -0400 Subject: [PATCH 086/174] lint --- pkg/solana/fees/block_history.go | 4 ++-- pkg/solana/txm/txm.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/solana/fees/block_history.go b/pkg/solana/fees/block_history.go index bc2db327d..fbaeb2539 100644 --- a/pkg/solana/fees/block_history.go +++ b/pkg/solana/fees/block_history.go @@ -3,8 +3,6 @@ package fees import ( "context" "fmt" - "github.com/smartcontractkit/chainlink-solana/pkg/solana/client" - "github.com/smartcontractkit/chainlink-solana/pkg/solana/internal" "sync" "time" @@ -13,7 +11,9 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/utils" "github.com/smartcontractkit/chainlink-common/pkg/utils/mathutil" + "github.com/smartcontractkit/chainlink-solana/pkg/solana/client" "github.com/smartcontractkit/chainlink-solana/pkg/solana/config" + "github.com/smartcontractkit/chainlink-solana/pkg/solana/internal" ) var _ Estimator = &blockHistoryEstimator{} diff --git a/pkg/solana/txm/txm.go b/pkg/solana/txm/txm.go index 9c7c08d23..ad6872f51 100644 --- a/pkg/solana/txm/txm.go +++ b/pkg/solana/txm/txm.go @@ -4,8 +4,6 @@ import ( "context" "errors" "fmt" - "github.com/smartcontractkit/chainlink-solana/pkg/solana/client" - "github.com/smartcontractkit/chainlink-solana/pkg/solana/internal" "strings" "sync" "time" @@ -19,8 +17,10 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink-common/pkg/utils" + "github.com/smartcontractkit/chainlink-solana/pkg/solana/client" "github.com/smartcontractkit/chainlink-solana/pkg/solana/config" "github.com/smartcontractkit/chainlink-solana/pkg/solana/fees" + "github.com/smartcontractkit/chainlink-solana/pkg/solana/internal" ) const ( From 45170ba6e8ef94f5ba14e5980a2cdbffb5911c5f Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 16 Oct 2024 16:11:50 -0400 Subject: [PATCH 087/174] Update txm.go --- pkg/solana/txm/txm.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/solana/txm/txm.go b/pkg/solana/txm/txm.go index ad6872f51..530ccc834 100644 --- a/pkg/solana/txm/txm.go +++ b/pkg/solana/txm/txm.go @@ -81,7 +81,6 @@ type pendingTx struct { func NewTxm(chainID string, client internal.Loader[client.ReaderWriter], sendTx func(ctx context.Context, tx *solanaGo.Transaction) (solanaGo.Signature, error), cfg config.Config, ks SimpleKeystore, lggr logger.Logger) *Txm { - return &Txm{ lggr: logger.Named(lggr, "Txm"), chSend: make(chan pendingTx, MaxQueueLen), // queue can support 1000 pending txs From de243124b151779006f6852813053aa3c5b00134 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 17 Oct 2024 09:55:44 -0400 Subject: [PATCH 088/174] Add ctx --- pkg/solana/chain_test.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/solana/chain_test.go b/pkg/solana/chain_test.go index b39436630..0be58212b 100644 --- a/pkg/solana/chain_test.go +++ b/pkg/solana/chain_test.go @@ -427,7 +427,7 @@ func TestChain_MultiNode_TransactionSender(t *testing.T) { cl, err := c.getClient() require.NoError(t, err) - hash, hashErr := cl.LatestBlockhash() + hash, hashErr := cl.LatestBlockhash(tests.Context(t)) assert.NoError(t, hashErr) tx, txErr := solana.NewTransaction( @@ -467,7 +467,7 @@ func TestChain_MultiNode_TransactionSender(t *testing.T) { cl, err := c.getClient() require.NoError(t, err) - hash, hashErr := cl.LatestBlockhash() + hash, hashErr := cl.LatestBlockhash(tests.Context(t)) assert.NoError(t, hashErr) tx, txErr := solana.NewTransaction( @@ -557,14 +557,14 @@ func TestSolanaChain_MultiNode_Txm(t *testing.T) { // track initial balance selectedClient, err := testChain.getClient() require.NoError(t, err) - receiverBal, err := selectedClient.Balance(pubKeyReceiver) + receiverBal, err := selectedClient.Balance(tests.Context(t), pubKeyReceiver) assert.NoError(t, err) assert.Equal(t, uint64(0), receiverBal) createTx := func(signer solana.PublicKey, sender solana.PublicKey, receiver solana.PublicKey, amt uint64) *solana.Transaction { selectedClient, err = testChain.getClient() assert.NoError(t, err) - hash, hashErr := selectedClient.LatestBlockhash() + hash, hashErr := selectedClient.LatestBlockhash(tests.Context(t)) assert.NoError(t, hashErr) tx, txErr := solana.NewTransaction( []solana.Instruction{ @@ -582,10 +582,10 @@ func TestSolanaChain_MultiNode_Txm(t *testing.T) { } // Send funds twice, along with an invalid transaction - require.NoError(t, testChain.txm.Enqueue("test_success", createTx(pubKey, pubKey, pubKeyReceiver, solana.LAMPORTS_PER_SOL))) + require.NoError(t, testChain.txm.Enqueue(tests.Context(t), "test_success", createTx(pubKey, pubKey, pubKeyReceiver, solana.LAMPORTS_PER_SOL))) time.Sleep(500 * time.Millisecond) // pause 0.5s for new blockhash - require.NoError(t, testChain.txm.Enqueue("test_success_2", createTx(pubKey, pubKey, pubKeyReceiver, solana.LAMPORTS_PER_SOL))) - require.Error(t, testChain.txm.Enqueue("test_invalidSigner", createTx(pubKeyReceiver, pubKey, pubKeyReceiver, solana.LAMPORTS_PER_SOL))) // cannot sign tx before enqueuing + require.NoError(t, testChain.txm.Enqueue(tests.Context(t), "test_success_2", createTx(pubKey, pubKey, pubKeyReceiver, solana.LAMPORTS_PER_SOL))) + require.Error(t, testChain.txm.Enqueue(tests.Context(t), "test_invalidSigner", createTx(pubKeyReceiver, pubKey, pubKeyReceiver, solana.LAMPORTS_PER_SOL))) // cannot sign tx before enqueuing // wait for all txes to finish ctx, cancel := context.WithCancel(tests.Context(t)) @@ -608,7 +608,7 @@ loop: // verify funds were transferred through transaction sender selectedClient, err = testChain.getClient() assert.NoError(t, err) - receiverBal, err = selectedClient.Balance(pubKeyReceiver) + receiverBal, err = selectedClient.Balance(tests.Context(t), pubKeyReceiver) assert.NoError(t, err) require.Equal(t, 2*solana.LAMPORTS_PER_SOL, receiverBal) } From 4aca05eb2c89978b80458ff97fab641a40f707d4 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 17 Oct 2024 10:02:57 -0400 Subject: [PATCH 089/174] Fix imports --- pkg/solana/fees/block_history.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/solana/fees/block_history.go b/pkg/solana/fees/block_history.go index 912d7913d..d1c346214 100644 --- a/pkg/solana/fees/block_history.go +++ b/pkg/solana/fees/block_history.go @@ -7,7 +7,6 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" - "github.com/smartcontractkit/chainlink-common/pkg/utils" "github.com/smartcontractkit/chainlink-common/pkg/utils/mathutil" "github.com/smartcontractkit/chainlink-solana/pkg/solana/client" From 0bd7741982628f3f774f688a90b4bc1dc72d5421 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 17 Oct 2024 12:07:09 -0400 Subject: [PATCH 090/174] Add SendTxResult to TxSender --- pkg/solana/chain.go | 13 ++-- pkg/solana/chain_test.go | 30 +++++--- .../client/multinode/transaction_sender.go | 77 ++++++++++--------- pkg/solana/client/multinode_client.go | 35 ++++++++- 4 files changed, 97 insertions(+), 58 deletions(-) diff --git a/pkg/solana/chain.go b/pkg/solana/chain.go index 81debc58b..adc7d5669 100644 --- a/pkg/solana/chain.go +++ b/pkg/solana/chain.go @@ -90,7 +90,7 @@ type chain struct { // if multiNode is enabled, the clientCache will not be used multiNode *mn.MultiNode[mn.StringID, *client.MultiNodeClient] - txSender *mn.TransactionSender[*solanago.Transaction, solanago.Signature, mn.StringID, *client.MultiNodeClient] + txSender *mn.TransactionSender[*solanago.Transaction, *client.SendTxResult, mn.StringID, *client.MultiNodeClient] // tracking node chain id for verification clientCache map[string]*verifiedCachedClient // map URL -> {client, chainId} [mainnet/testnet/devnet/localnet] @@ -270,7 +270,7 @@ func newChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr logger.L mnCfg.DeathDeclarationDelay(), ) - txSender := mn.NewTransactionSender[*solanago.Transaction, solanago.Signature, mn.StringID, *client.MultiNodeClient]( + txSender := mn.NewTransactionSender[*solanago.Transaction, *client.SendTxResult, mn.StringID, *client.MultiNodeClient]( lggr, mn.StringID(id), chainFamily, @@ -287,14 +287,17 @@ func newChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr logger.L sendTx = func(ctx context.Context, tx *solanago.Transaction) (solanago.Signature, error) { // Send tx using MultiNode transaction sender - result, _, err := ch.txSender.SendTransaction(ctx, tx) + result, err := ch.txSender.SendTransaction(ctx, tx) if err != nil { return solanago.Signature{}, err } if result == nil { - return solanago.Signature{}, errors.New("tx sender returned nil signature") + return solanago.Signature{}, errors.New("tx sender returned nil result") } - return *result, err + if (*result).Signature().IsZero() { + return solanago.Signature{}, errors.New("tx sender returned empty signature") + } + return (*result).Signature(), nil } } diff --git a/pkg/solana/chain_test.go b/pkg/solana/chain_test.go index 0be58212b..c211d0d27 100644 --- a/pkg/solana/chain_test.go +++ b/pkg/solana/chain_test.go @@ -455,10 +455,12 @@ func TestChain_MultiNode_TransactionSender(t *testing.T) { } // Send tx using transaction sender - sig, code, err := c.txSender.SendTransaction(ctx, createTx(receiver.PublicKey())) + result, err := c.txSender.SendTransaction(ctx, createTx(receiver.PublicKey())) require.NoError(t, err) - require.Equal(t, mn.Successful, code) - require.NotEmpty(t, sig) + require.NotNil(t, result) + require.NotNil(t, *result) + require.Equal(t, mn.Successful, (*result).Code()) + require.NotEmpty(t, (*result).Signature()) }) t.Run("unsigned transaction error", func(t *testing.T) { @@ -486,17 +488,23 @@ func TestChain_MultiNode_TransactionSender(t *testing.T) { } // Send tx using transaction sender - sig, code, err := c.txSender.SendTransaction(ctx, unsignedTx(receiver.PublicKey())) - require.Error(t, err) - require.Equal(t, mn.Fatal, code) - require.Empty(t, sig) + result, err := c.txSender.SendTransaction(ctx, unsignedTx(receiver.PublicKey())) + require.NoError(t, err) + require.NotNil(t, result) + require.NotNil(t, *result) + require.Error(t, (*result).TxError()) + require.Equal(t, mn.Fatal, (*result).Code()) + require.Empty(t, (*result).Signature()) }) t.Run("empty transaction", func(t *testing.T) { - sig, code, err := c.txSender.SendTransaction(ctx, &solana.Transaction{}) - require.Error(t, err) - require.Equal(t, mn.Fatal, code) - require.Empty(t, sig) + result, err := c.txSender.SendTransaction(ctx, &solana.Transaction{}) + require.NoError(t, err) + require.NotNil(t, result) + require.NotNil(t, *result) + require.Error(t, (*result).TxError()) + require.Equal(t, mn.Fatal, (*result).Code()) + require.Empty(t, (*result).Signature()) }) } diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go index 78b75cd10..d174d9634 100644 --- a/pkg/solana/client/multinode/transaction_sender.go +++ b/pkg/solana/client/multinode/transaction_sender.go @@ -28,21 +28,21 @@ var ( // (e.g. Successful, Fatal, Retryable, etc.) type TxErrorClassifier[TX any] func(tx TX, err error) SendTxReturnCode -type sendTxResult[RESULT any] struct { - Err error - ResultCode SendTxReturnCode - Result *RESULT +type SendTxResult interface { + Code() SendTxReturnCode + SetCode(code SendTxReturnCode) + TxError() error } const sendTxQuorum = 0.7 // SendTxRPCClient - defines interface of an RPC used by TransactionSender to broadcast transaction -type SendTxRPCClient[TX any, RESULT any] interface { +type SendTxRPCClient[TX any, RESULT SendTxResult] interface { // SendTransaction errors returned should include name or other unique identifier of the RPC - SendTransaction(ctx context.Context, tx TX) (*RESULT, error) + SendTransaction(ctx context.Context, tx TX) RESULT } -func NewTransactionSender[TX any, RESULT any, CHAIN_ID ID, RPC SendTxRPCClient[TX, RESULT]]( +func NewTransactionSender[TX any, RESULT SendTxResult, CHAIN_ID ID, RPC SendTxRPCClient[TX, RESULT]]( lggr logger.Logger, chainID CHAIN_ID, chainFamily string, @@ -64,7 +64,7 @@ func NewTransactionSender[TX any, RESULT any, CHAIN_ID ID, RPC SendTxRPCClient[T } } -type TransactionSender[TX any, RESULT any, CHAIN_ID ID, RPC SendTxRPCClient[TX, RESULT]] struct { +type TransactionSender[TX any, RESULT SendTxResult, CHAIN_ID ID, RPC SendTxRPCClient[TX, RESULT]] struct { services.StateMachine chainID CHAIN_ID chainFamily string @@ -95,9 +95,9 @@ type TransactionSender[TX any, RESULT any, CHAIN_ID ID, RPC SendTxRPCClient[TX, // * If there is at least one terminal error - returns terminal error // * If there is both success and terminal error - returns success and reports invariant violation // * Otherwise, returns any (effectively random) of the errors. -func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ctx context.Context, tx TX) (*RESULT, SendTxReturnCode, error) { - txResults := make(chan sendTxResult[RESULT]) - txResultsToReport := make(chan sendTxResult[RESULT]) +func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ctx context.Context, tx TX) (*RESULT, error) { + txResults := make(chan RESULT) + txResultsToReport := make(chan RESULT) primaryNodeWg := sync.WaitGroup{} ctx, cancel := txSender.chStop.Ctx(ctx) @@ -146,7 +146,7 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct }() if err != nil { - return nil, Retryable, err + return nil, err } txSender.wg.Add(1) @@ -155,65 +155,66 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct return txSender.collectTxResults(ctx, tx, healthyNodesNum, txResults) } -func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) broadcastTxAsync(ctx context.Context, rpc RPC, tx TX) sendTxResult[RESULT] { - result, txErr := rpc.SendTransaction(ctx, tx) - txSender.lggr.Debugw("Node sent transaction", "tx", tx, "err", txErr) - resultCode := txSender.txErrorClassifier(tx, txErr) +func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) broadcastTxAsync(ctx context.Context, rpc RPC, tx TX) RESULT { + result := rpc.SendTransaction(ctx, tx) + txSender.lggr.Debugw("Node sent transaction", "tx", tx, "err", result.TxError()) + resultCode := txSender.txErrorClassifier(tx, result.TxError()) if !slices.Contains(sendTxSuccessfulCodes, resultCode) { - txSender.lggr.Warnw("RPC returned error", "tx", tx, "err", txErr) + txSender.lggr.Warnw("RPC returned error", "tx", tx, "err", result.TxError()) } - return sendTxResult[RESULT]{Err: txErr, ResultCode: resultCode, Result: result} + result.SetCode(resultCode) + return result } -func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) reportSendTxAnomalies(tx TX, txResults <-chan sendTxResult[RESULT]) { +func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) reportSendTxAnomalies(tx TX, txResults <-chan RESULT) { defer txSender.wg.Done() resultsByCode := sendTxResults[RESULT]{} // txResults eventually will be closed for txResult := range txResults { - resultsByCode[txResult.ResultCode] = append(resultsByCode[txResult.ResultCode], txResult) + resultsByCode[txResult.Code()] = append(resultsByCode[txResult.Code()], txResult) } - _, _, _, criticalErr := aggregateTxResults[RESULT](resultsByCode) + _, criticalErr := aggregateTxResults[RESULT](resultsByCode) if criticalErr != nil { txSender.lggr.Criticalw("observed invariant violation on SendTransaction", "tx", tx, "resultsByCode", resultsByCode, "err", criticalErr) PromMultiNodeInvariantViolations.WithLabelValues(txSender.chainFamily, txSender.chainID.String(), criticalErr.Error()).Inc() } } -type sendTxResults[RESULT any] map[SendTxReturnCode][]sendTxResult[RESULT] +type sendTxResults[RESULT any] map[SendTxReturnCode][]RESULT -func aggregateTxResults[RESULT any](resultsByCode sendTxResults[RESULT]) (result *RESULT, returnCode SendTxReturnCode, txResult error, err error) { - severeCode, severeErrors, hasSevereErrors := findFirstIn(resultsByCode, sendTxSevereErrors) - successCode, successResults, hasSuccess := findFirstIn(resultsByCode, sendTxSuccessfulCodes) +func aggregateTxResults[RESULT any](resultsByCode sendTxResults[RESULT]) (result *RESULT, err error) { + _, severeErrors, hasSevereErrors := findFirstIn(resultsByCode, sendTxSevereErrors) + _, successResults, hasSuccess := findFirstIn(resultsByCode, sendTxSuccessfulCodes) if hasSuccess { // We assume that primary node would never report false positive txResult for a transaction. // Thus, if such case occurs it's probably due to misconfiguration or a bug and requires manual intervention. if hasSevereErrors { const errMsg = "found contradictions in nodes replies on SendTransaction: got success and severe error" // return success, since at least 1 node has accepted our broadcasted Tx, and thus it can now be included onchain - return successResults[0].Result, successCode, successResults[0].Err, errors.New(errMsg) + return &successResults[0], errors.New(errMsg) } // other errors are temporary - we are safe to return success - return successResults[0].Result, successCode, successResults[0].Err, nil + return &successResults[0], nil } if hasSevereErrors { - return nil, severeCode, severeErrors[0].Err, nil + return &severeErrors[0], nil } // return temporary error - for code, result := range resultsByCode { - return nil, code, result[0].Err, nil + for _, result := range resultsByCode { + return &result[0], nil } err = fmt.Errorf("expected at least one response on SendTransaction") - return nil, Retryable, err, err + return nil, err } -func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) collectTxResults(ctx context.Context, tx TX, healthyNodesNum int, txResults <-chan sendTxResult[RESULT]) (*RESULT, SendTxReturnCode, error) { +func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) collectTxResults(ctx context.Context, tx TX, healthyNodesNum int, txResults <-chan RESULT) (*RESULT, error) { if healthyNodesNum == 0 { - return nil, Retryable, ErroringNodeError + return nil, ErroringNodeError } requiredResults := int(math.Ceil(float64(healthyNodesNum) * sendTxQuorum)) errorsByCode := sendTxResults[RESULT]{} @@ -224,11 +225,11 @@ loop: select { case <-ctx.Done(): txSender.lggr.Debugw("Failed to collect of the results before context was done", "tx", tx, "errorsByCode", errorsByCode) - return nil, Retryable, ctx.Err() + return nil, ctx.Err() case result := <-txResults: - errorsByCode[result.ResultCode] = append(errorsByCode[result.ResultCode], result) + errorsByCode[result.Code()] = append(errorsByCode[result.Code()], result) resultsCount++ - if slices.Contains(sendTxSuccessfulCodes, result.ResultCode) || resultsCount >= requiredResults { + if slices.Contains(sendTxSuccessfulCodes, result.Code()) || resultsCount >= requiredResults { break loop } case <-softTimeoutChan: @@ -246,8 +247,8 @@ loop: } // ignore critical error as it's reported in reportSendTxAnomalies - result, returnCode, resultErr, _ := aggregateTxResults(errorsByCode) - return result, returnCode, resultErr + result, _ := aggregateTxResults(errorsByCode) + return result, nil } func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) Start(ctx context.Context) error { diff --git a/pkg/solana/client/multinode_client.go b/pkg/solana/client/multinode_client.go index a462f4a8e..aa2a9dbad 100644 --- a/pkg/solana/client/multinode_client.go +++ b/pkg/solana/client/multinode_client.go @@ -41,7 +41,7 @@ func (h *Head) IsValid() bool { } var _ mn.RPCClient[mn.StringID, *Head] = (*MultiNodeClient)(nil) -var _ mn.SendTxRPCClient[*solana.Transaction, solana.Signature] = (*MultiNodeClient)(nil) +var _ mn.SendTxRPCClient[*solana.Transaction, *SendTxResult] = (*MultiNodeClient)(nil) type MultiNodeClient struct { Client @@ -300,10 +300,37 @@ func (m *MultiNodeClient) GetInterceptedChainInfo() (latest, highestUserObservat return m.latestChainInfo, m.highestUserObservations } -func (m *MultiNodeClient) SendTransaction(ctx context.Context, tx *solana.Transaction) (*solana.Signature, error) { +type SendTxResult struct { + err error + code mn.SendTxReturnCode + sig solana.Signature +} + +var _ mn.SendTxResult = (*SendTxResult)(nil) + +func (r *SendTxResult) TxError() error { + return r.err +} + +func (r *SendTxResult) Code() mn.SendTxReturnCode { + return r.code +} + +func (r *SendTxResult) SetCode(code mn.SendTxReturnCode) { + r.code = code +} + +func (r *SendTxResult) Signature() solana.Signature { + return r.sig +} + +func (m *MultiNodeClient) SendTransaction(ctx context.Context, tx *solana.Transaction) *SendTxResult { + var sendTxResult = &SendTxResult{} sig, err := m.SendTx(ctx, tx) if err != nil { - return nil, err + sendTxResult.err = err + return sendTxResult } - return &sig, nil + sendTxResult.sig = sig + return sendTxResult } From ca8d2c745e50fdb06a63ea1c1046e8e5fda51f63 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 17 Oct 2024 12:27:29 -0400 Subject: [PATCH 091/174] Update chain_test.go --- pkg/solana/chain_test.go | 61 ++-------------------------------------- 1 file changed, 3 insertions(+), 58 deletions(-) diff --git a/pkg/solana/chain_test.go b/pkg/solana/chain_test.go index f1d8be15c..ce8f11201 100644 --- a/pkg/solana/chain_test.go +++ b/pkg/solana/chain_test.go @@ -128,61 +128,6 @@ func TestSolanaChain_GetClient(t *testing.T) { assert.NoError(t, err) } -func TestSolanaChain_MultiNode_GetClient(t *testing.T) { - mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - out := fmt.Sprintf(TestSolanaGenesisHashTemplate, client.MainnetGenesisHash) // mainnet genesis hash - if !strings.Contains(r.URL.Path, "/mismatch") { - // devnet gensis hash - out = fmt.Sprintf(TestSolanaGenesisHashTemplate, client.DevnetGenesisHash) - } - _, err := w.Write([]byte(out)) - require.NoError(t, err) - })) - defer mockServer.Close() - - ch := solcfg.Chain{} - ch.SetDefaults() - mn := solcfg.MultiNodeConfig{ - MultiNode: solcfg.MultiNode{ - Enabled: ptr(true), - }, - } - mn.SetDefaults() - - cfg := &solcfg.TOMLConfig{ - ChainID: ptr("devnet"), - Chain: ch, - MultiNode: mn, - } - cfg.Nodes = []*solcfg.Node{ - { - Name: ptr("devnet"), - URL: config.MustParseURL(mockServer.URL + "/1"), - }, - { - Name: ptr("devnet"), - URL: config.MustParseURL(mockServer.URL + "/2"), - }, - } - - testChain, err := newChain("devnet", cfg, nil, logger.Test(t)) - require.NoError(t, err) - - err = testChain.Start(tests.Context(t)) - require.NoError(t, err) - defer func() { - closeErr := testChain.Close() - require.NoError(t, closeErr) - }() - - selectedClient, err := testChain.getClient() - assert.NoError(t, err) - - id, err := selectedClient.ChainID(tests.Context(t)) - assert.NoError(t, err) - assert.Equal(t, "devnet", id.String()) -} - func TestSolanaChain_VerifiedClient(t *testing.T) { ctx := tests.Context(t) called := false @@ -388,17 +333,17 @@ func TestSolanaChain_MultiNode_GetClient(t *testing.T) { ch := solcfg.Chain{} ch.SetDefaults() - mn := solcfg.MultiNodeConfig{ + mnCfg := solcfg.MultiNodeConfig{ MultiNode: solcfg.MultiNode{ Enabled: ptr(true), }, } - mn.SetDefaults() + mnCfg.SetDefaults() cfg := &solcfg.TOMLConfig{ ChainID: ptr("devnet"), Chain: ch, - MultiNode: mn, + MultiNode: mnCfg, } cfg.Nodes = []*solcfg.Node{ { From 49076b6dfc7422ddcbf405bec944c899f562b012 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 17 Oct 2024 12:47:30 -0400 Subject: [PATCH 092/174] Enable MultiNode --- integration-tests/testconfig/testconfig.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/integration-tests/testconfig/testconfig.go b/integration-tests/testconfig/testconfig.go index 394d2bcee..2de02594a 100644 --- a/integration-tests/testconfig/testconfig.go +++ b/integration-tests/testconfig/testconfig.go @@ -273,6 +273,9 @@ func (c *TestConfig) GetNodeConfigTOML() (string, error) { URL: config.MustParseURL(url), }, }, + MultiNode: solcfg.MultiNodeConfig{ + MultiNode: solcfg.MultiNode{Enabled: ptr.Ptr(true)}, + }, } baseConfig := node.NewBaseConfig() baseConfig.Solana = solcfg.TOMLConfigs{ From 75c33ce41c9398c358f060fea6fdc30ab13ee20e Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 17 Oct 2024 13:12:04 -0400 Subject: [PATCH 093/174] Move error classification --- pkg/solana/chain.go | 1 - pkg/solana/client/classify_errors.go | 3 --- pkg/solana/client/multinode/transaction_sender.go | 12 +----------- pkg/solana/client/multinode_client.go | 5 +---- 4 files changed, 2 insertions(+), 19 deletions(-) diff --git a/pkg/solana/chain.go b/pkg/solana/chain.go index adc7d5669..927105ad2 100644 --- a/pkg/solana/chain.go +++ b/pkg/solana/chain.go @@ -275,7 +275,6 @@ func newChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr logger.L mn.StringID(id), chainFamily, multiNode, - client.ClassifySendError, 0, // use the default value provided by the implementation ) diff --git a/pkg/solana/client/classify_errors.go b/pkg/solana/client/classify_errors.go index 0e3c1456e..b6ed1a84b 100644 --- a/pkg/solana/client/classify_errors.go +++ b/pkg/solana/client/classify_errors.go @@ -93,9 +93,6 @@ var errCodes = map[*regexp.Regexp]mn.SendTxReturnCode{ ErrProgramExecutionTemporarilyRestricted: mn.Retryable, } -// ClassifySendError implements TxErrorClassifier required for MultiNode TransactionSender -var _ mn.TxErrorClassifier[*solana.Transaction] = ClassifySendError - // ClassifySendError returns the corresponding return code based on the error. func ClassifySendError(_ *solana.Transaction, err error) mn.SendTxReturnCode { if err == nil { diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go index d174d9634..55e63511b 100644 --- a/pkg/solana/client/multinode/transaction_sender.go +++ b/pkg/solana/client/multinode/transaction_sender.go @@ -24,13 +24,8 @@ var ( }, []string{"network", "chainId", "invariant"}) ) -// TxErrorClassifier - defines interface of a function that transforms raw RPC error into the SendTxReturnCode enum -// (e.g. Successful, Fatal, Retryable, etc.) -type TxErrorClassifier[TX any] func(tx TX, err error) SendTxReturnCode - type SendTxResult interface { Code() SendTxReturnCode - SetCode(code SendTxReturnCode) TxError() error } @@ -47,7 +42,6 @@ func NewTransactionSender[TX any, RESULT SendTxResult, CHAIN_ID ID, RPC SendTxRP chainID CHAIN_ID, chainFamily string, multiNode *MultiNode[CHAIN_ID, RPC], - txErrorClassifier TxErrorClassifier[TX], sendTxSoftTimeout time.Duration, ) *TransactionSender[TX, RESULT, CHAIN_ID, RPC] { if sendTxSoftTimeout == 0 { @@ -58,7 +52,6 @@ func NewTransactionSender[TX any, RESULT SendTxResult, CHAIN_ID ID, RPC SendTxRP chainFamily: chainFamily, lggr: logger.Sugared(lggr).Named("TransactionSender").With("chainID", chainID.String()), multiNode: multiNode, - txErrorClassifier: txErrorClassifier, sendTxSoftTimeout: sendTxSoftTimeout, chStop: make(services.StopChan), } @@ -70,7 +63,6 @@ type TransactionSender[TX any, RESULT SendTxResult, CHAIN_ID ID, RPC SendTxRPCCl chainFamily string lggr logger.SugaredLogger multiNode *MultiNode[CHAIN_ID, RPC] - txErrorClassifier TxErrorClassifier[TX] sendTxSoftTimeout time.Duration // defines max waiting time from first response til responses evaluation wg sync.WaitGroup // waits for all reporting goroutines to finish @@ -158,11 +150,9 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) broadcastTxAsync(ctx context.Context, rpc RPC, tx TX) RESULT { result := rpc.SendTransaction(ctx, tx) txSender.lggr.Debugw("Node sent transaction", "tx", tx, "err", result.TxError()) - resultCode := txSender.txErrorClassifier(tx, result.TxError()) - if !slices.Contains(sendTxSuccessfulCodes, resultCode) { + if !slices.Contains(sendTxSuccessfulCodes, result.Code()) { txSender.lggr.Warnw("RPC returned error", "tx", tx, "err", result.TxError()) } - result.SetCode(resultCode) return result } diff --git a/pkg/solana/client/multinode_client.go b/pkg/solana/client/multinode_client.go index aa2a9dbad..fb29fce8d 100644 --- a/pkg/solana/client/multinode_client.go +++ b/pkg/solana/client/multinode_client.go @@ -316,10 +316,6 @@ func (r *SendTxResult) Code() mn.SendTxReturnCode { return r.code } -func (r *SendTxResult) SetCode(code mn.SendTxReturnCode) { - r.code = code -} - func (r *SendTxResult) Signature() solana.Signature { return r.sig } @@ -327,6 +323,7 @@ func (r *SendTxResult) Signature() solana.Signature { func (m *MultiNodeClient) SendTransaction(ctx context.Context, tx *solana.Transaction) *SendTxResult { var sendTxResult = &SendTxResult{} sig, err := m.SendTx(ctx, tx) + sendTxResult.code = ClassifySendError(tx, err) if err != nil { sendTxResult.err = err return sendTxResult From f67e09c1d7af9ef539984597a9e8ef7b3c6f914d Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 17 Oct 2024 13:37:13 -0400 Subject: [PATCH 094/174] Add MultiNode config --- integration-tests/testconfig/testconfig.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/integration-tests/testconfig/testconfig.go b/integration-tests/testconfig/testconfig.go index 2de02594a..6f7b68598 100644 --- a/integration-tests/testconfig/testconfig.go +++ b/integration-tests/testconfig/testconfig.go @@ -264,6 +264,11 @@ func (c *TestConfig) GetNodeConfigTOML() (string, error) { url = c.GetURL() } + mnConfig := solcfg.MultiNodeConfig{ + MultiNode: solcfg.MultiNode{Enabled: ptr.Ptr(true)}, + } + mnConfig.SetDefaults() + solConfig := solcfg.TOMLConfig{ Enabled: ptr.Ptr(true), ChainID: ptr.Ptr(chainID), @@ -273,9 +278,7 @@ func (c *TestConfig) GetNodeConfigTOML() (string, error) { URL: config.MustParseURL(url), }, }, - MultiNode: solcfg.MultiNodeConfig{ - MultiNode: solcfg.MultiNode{Enabled: ptr.Ptr(true)}, - }, + MultiNode: mnConfig, } baseConfig := node.NewBaseConfig() baseConfig.Solana = solcfg.TOMLConfigs{ From e11de9afa409d9877c208babce40980f173f75e6 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 17 Oct 2024 14:39:15 -0400 Subject: [PATCH 095/174] Use loader --- pkg/solana/chain.go | 14 +++++--------- pkg/solana/monitor/balance.go | 31 +++++++++++++++---------------- 2 files changed, 20 insertions(+), 25 deletions(-) diff --git a/pkg/solana/chain.go b/pkg/solana/chain.go index 927105ad2..236c0e836 100644 --- a/pkg/solana/chain.go +++ b/pkg/solana/chain.go @@ -300,17 +300,13 @@ func newChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr logger.L } } - tc := func() (client.ReaderWriter, error) { - return ch.getClient() - } - // Use lazy loader if MultiNode is disabled - loader := internal.NewLoader(!cfg.MultiNode.Enabled(), tc) + lazyLoad := !cfg.MultiNode.Enabled() - ch.txm = txm.NewTxm(ch.id, loader, sendTx, cfg, ks, lggr) - bc := func() (monitor.BalanceClient, error) { - return ch.getClient() - } + tc := internal.NewLoader(lazyLoad, func() (client.ReaderWriter, error) { return ch.getClient() }) + ch.txm = txm.NewTxm(ch.id, tc, sendTx, cfg, ks, lggr) + + bc := internal.NewLoader(lazyLoad, func() (monitor.BalanceClient, error) { return ch.getClient() }) ch.balanceMonitor = monitor.NewBalanceMonitor(ch.id, cfg, lggr, ks, bc) return &ch, nil } diff --git a/pkg/solana/monitor/balance.go b/pkg/solana/monitor/balance.go index 590cd72ac..10ea487db 100644 --- a/pkg/solana/monitor/balance.go +++ b/pkg/solana/monitor/balance.go @@ -28,19 +28,19 @@ type BalanceClient interface { } // NewBalanceMonitor returns a balance monitoring services.Service which reports the SOL balance of all ks keys to prometheus. -func NewBalanceMonitor(chainID string, cfg Config, lggr logger.Logger, ks Keystore, newReader func() (BalanceClient, error)) services.Service { - return newBalanceMonitor(chainID, cfg, lggr, ks, newReader) +func NewBalanceMonitor(chainID string, cfg Config, lggr logger.Logger, ks Keystore, reader internal.Loader[BalanceClient]) services.Service { + return newBalanceMonitor(chainID, cfg, lggr, ks, reader) } -func newBalanceMonitor(chainID string, cfg Config, lggr logger.Logger, ks Keystore, newReader func() (BalanceClient, error)) *balanceMonitor { +func newBalanceMonitor(chainID string, cfg Config, lggr logger.Logger, ks Keystore, reader internal.Loader[BalanceClient]) *balanceMonitor { b := balanceMonitor{ - chainID: chainID, - cfg: cfg, - lggr: logger.Named(lggr, "BalanceMonitor"), - ks: ks, - newReader: newReader, - stop: make(chan struct{}), - done: make(chan struct{}), + chainID: chainID, + cfg: cfg, + lggr: logger.Named(lggr, "BalanceMonitor"), + ks: ks, + reader: reader, + stop: make(chan struct{}), + done: make(chan struct{}), } b.updateFn = b.updateProm return &b @@ -48,12 +48,11 @@ func newBalanceMonitor(chainID string, cfg Config, lggr logger.Logger, ks Keysto type balanceMonitor struct { services.StateMachine - chainID string - cfg Config - lggr logger.Logger - ks Keystore - newReader func() (BalanceClient, error) - updateFn func(acc solana.PublicKey, lamports uint64) // overridable for testing + chainID string + cfg Config + lggr logger.Logger + ks Keystore + updateFn func(acc solana.PublicKey, lamports uint64) // overridable for testing reader internal.Loader[BalanceClient] From cb8313e9703002b02c0374851f828d86200823dc Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 17 Oct 2024 15:01:47 -0400 Subject: [PATCH 096/174] Update multinode.go --- pkg/solana/config/multinode.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/solana/config/multinode.go b/pkg/solana/config/multinode.go index 0c49d8b22..da20a4a30 100644 --- a/pkg/solana/config/multinode.go +++ b/pkg/solana/config/multinode.go @@ -89,7 +89,7 @@ func (c *MultiNodeConfig) FinalizedBlockOffset() uint32 { return *c.MultiNode.Fi func (c *MultiNodeConfig) SetDefaults() { // MultiNode is disabled as it's not fully implemented yet: BCFR-122 if c.MultiNode.Enabled == nil { - c.MultiNode.Enabled = ptr(false) + c.MultiNode.Enabled = ptr(true) } /* Node Configs */ From 3f8d20b8cba2163b11fbce6a2166f2370aa571bb Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 17 Oct 2024 15:17:55 -0400 Subject: [PATCH 097/174] Update multinode.go --- pkg/solana/config/multinode.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/solana/config/multinode.go b/pkg/solana/config/multinode.go index da20a4a30..0c49d8b22 100644 --- a/pkg/solana/config/multinode.go +++ b/pkg/solana/config/multinode.go @@ -89,7 +89,7 @@ func (c *MultiNodeConfig) FinalizedBlockOffset() uint32 { return *c.MultiNode.Fi func (c *MultiNodeConfig) SetDefaults() { // MultiNode is disabled as it's not fully implemented yet: BCFR-122 if c.MultiNode.Enabled == nil { - c.MultiNode.Enabled = ptr(true) + c.MultiNode.Enabled = ptr(false) } /* Node Configs */ From 9757621e423785d1f7a5cd711ca1043b13f8d267 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 18 Oct 2024 10:51:09 -0400 Subject: [PATCH 098/174] Use loader in txm tests --- pkg/solana/txm/txm_unit_test.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pkg/solana/txm/txm_unit_test.go b/pkg/solana/txm/txm_unit_test.go index e1a7aaf1b..240b445dc 100644 --- a/pkg/solana/txm/txm_unit_test.go +++ b/pkg/solana/txm/txm_unit_test.go @@ -20,6 +20,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/logger" bigmath "github.com/smartcontractkit/chainlink-common/pkg/utils/big_math" "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" + "github.com/smartcontractkit/chainlink-solana/pkg/solana/internal" ) func TestTxm_EstimateComputeUnitLimit(t *testing.T) { @@ -45,10 +46,8 @@ func TestTxm_EstimateComputeUnitLimit(t *testing.T) { cfg := config.NewDefault() client := clientmocks.NewReaderWriter(t) require.NoError(t, err) - getClient := func() (solanaClient.ReaderWriter, error) { - return client, nil - } - txm := solanatxm.NewTxm("localnet", getClient, cfg, mkey, lggr) + loader := internal.NewLoader(true, func() (solanaClient.ReaderWriter, error) { return client, nil }) + txm := solanatxm.NewTxm("localnet", loader, nil, cfg, mkey, lggr) t.Run("successfully sets estimated compute unit limit", func(t *testing.T) { usedCompute := uint64(100) From a8c9a0d904ee7b32fca57c8d403a766cc7223371 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 18 Oct 2024 10:54:28 -0400 Subject: [PATCH 099/174] lint --- pkg/solana/txm/txm_unit_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/solana/txm/txm_unit_test.go b/pkg/solana/txm/txm_unit_test.go index 240b445dc..b5948a61a 100644 --- a/pkg/solana/txm/txm_unit_test.go +++ b/pkg/solana/txm/txm_unit_test.go @@ -14,13 +14,13 @@ import ( solanaClient "github.com/smartcontractkit/chainlink-solana/pkg/solana/client" clientmocks "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/mocks" "github.com/smartcontractkit/chainlink-solana/pkg/solana/config" + "github.com/smartcontractkit/chainlink-solana/pkg/solana/internal" solanatxm "github.com/smartcontractkit/chainlink-solana/pkg/solana/txm" keyMocks "github.com/smartcontractkit/chainlink-solana/pkg/solana/txm/mocks" "github.com/smartcontractkit/chainlink-common/pkg/logger" bigmath "github.com/smartcontractkit/chainlink-common/pkg/utils/big_math" "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" - "github.com/smartcontractkit/chainlink-solana/pkg/solana/internal" ) func TestTxm_EstimateComputeUnitLimit(t *testing.T) { From 43edd923d17984cb586bb8b19918179c96a94572 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 18 Oct 2024 11:42:19 -0400 Subject: [PATCH 100/174] Update testconfig.go --- integration-tests/testconfig/testconfig.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/integration-tests/testconfig/testconfig.go b/integration-tests/testconfig/testconfig.go index 6f7b68598..ad8985d45 100644 --- a/integration-tests/testconfig/testconfig.go +++ b/integration-tests/testconfig/testconfig.go @@ -264,8 +264,14 @@ func (c *TestConfig) GetNodeConfigTOML() (string, error) { url = c.GetURL() } + // TODO: Does the simulated default actually create new heads/ finalized heads? + // TODO: If not, then those tests will fail on CI and only work with actual RPCs. mnConfig := solcfg.MultiNodeConfig{ - MultiNode: solcfg.MultiNode{Enabled: ptr.Ptr(true)}, + MultiNode: solcfg.MultiNode{ + Enabled: ptr.Ptr(true), + NodeNoNewHeadsThreshold: config.MustNewDuration(time.Minute), + NoNewFinalizedHeadsThreshold: config.MustNewDuration(2 * time.Minute), + }, } mnConfig.SetDefaults() From f2e3cb8ceb310061492c88f3fd3f5be38d95e11a Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 18 Oct 2024 12:37:20 -0400 Subject: [PATCH 101/174] Update loader --- pkg/solana/chain.go | 19 ++++++---- pkg/solana/chain_test.go | 35 +++++-------------- .../client/multinode/transaction_sender.go | 28 +++++++-------- pkg/solana/internal/loader.go | 25 +++---------- pkg/solana/txm/txm_internal_test.go | 12 ++----- pkg/solana/txm/txm_load_test.go | 7 ++-- pkg/solana/txm/txm_race_test.go | 10 +++--- pkg/solana/txm/txm_unit_test.go | 4 +-- 8 files changed, 50 insertions(+), 90 deletions(-) diff --git a/pkg/solana/chain.go b/pkg/solana/chain.go index 236c0e836..4228566d3 100644 --- a/pkg/solana/chain.go +++ b/pkg/solana/chain.go @@ -22,6 +22,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink-common/pkg/types" "github.com/smartcontractkit/chainlink-common/pkg/types/core" + "github.com/smartcontractkit/chainlink-common/pkg/utils" "github.com/smartcontractkit/chainlink-solana/pkg/solana/client" mn "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/multinode" @@ -293,20 +294,24 @@ func newChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr logger.L if result == nil { return solanago.Signature{}, errors.New("tx sender returned nil result") } - if (*result).Signature().IsZero() { + if result.Signature().IsZero() { return solanago.Signature{}, errors.New("tx sender returned empty signature") } - return (*result).Signature(), nil + return result.Signature(), nil } } - // Use lazy loader if MultiNode is disabled - lazyLoad := !cfg.MultiNode.Enabled() + var tc internal.Loader[client.ReaderWriter] + var bc internal.Loader[monitor.BalanceClient] + if cfg.MultiNode.Enabled() { + tc = internal.NewLoader[client.ReaderWriter](func() (client.ReaderWriter, error) { return ch.multiNode.SelectRPC() }) + bc = internal.NewLoader[monitor.BalanceClient](func() (monitor.BalanceClient, error) { return ch.multiNode.SelectRPC() }) + } else { + tc = utils.NewLazyLoad(func() (client.ReaderWriter, error) { return ch.getClient() }) + bc = utils.NewLazyLoad(func() (monitor.BalanceClient, error) { return ch.getClient() }) + } - tc := internal.NewLoader(lazyLoad, func() (client.ReaderWriter, error) { return ch.getClient() }) ch.txm = txm.NewTxm(ch.id, tc, sendTx, cfg, ks, lggr) - - bc := internal.NewLoader(lazyLoad, func() (monitor.BalanceClient, error) { return ch.getClient() }) ch.balanceMonitor = monitor.NewBalanceMonitor(ch.id, cfg, lggr, ks, bc) return &ch, nil } diff --git a/pkg/solana/chain_test.go b/pkg/solana/chain_test.go index ce8f11201..81d6441ff 100644 --- a/pkg/solana/chain_test.go +++ b/pkg/solana/chain_test.go @@ -458,9 +458,8 @@ func TestChain_MultiNode_TransactionSender(t *testing.T) { result, err := c.txSender.SendTransaction(ctx, createTx(receiver.PublicKey())) require.NoError(t, err) require.NotNil(t, result) - require.NotNil(t, *result) - require.Equal(t, mn.Successful, (*result).Code()) - require.NotEmpty(t, (*result).Signature()) + require.Equal(t, mn.Successful, result.Code()) + require.NotEmpty(t, result.Signature()) }) t.Run("unsigned transaction error", func(t *testing.T) { @@ -491,20 +490,18 @@ func TestChain_MultiNode_TransactionSender(t *testing.T) { result, err := c.txSender.SendTransaction(ctx, unsignedTx(receiver.PublicKey())) require.NoError(t, err) require.NotNil(t, result) - require.NotNil(t, *result) - require.Error(t, (*result).TxError()) - require.Equal(t, mn.Fatal, (*result).Code()) - require.Empty(t, (*result).Signature()) + require.Error(t, result.TxError()) + require.Equal(t, mn.Fatal, result.Code()) + require.Empty(t, result.Signature()) }) t.Run("empty transaction", func(t *testing.T) { result, err := c.txSender.SendTransaction(ctx, &solana.Transaction{}) require.NoError(t, err) require.NotNil(t, result) - require.NotNil(t, *result) - require.Error(t, (*result).TxError()) - require.Equal(t, mn.Fatal, (*result).Code()) - require.Empty(t, (*result).Signature()) + require.Error(t, result.TxError()) + require.Equal(t, mn.Fatal, result.Code()) + require.Empty(t, result.Signature()) }) } @@ -513,23 +510,9 @@ func TestSolanaChain_MultiNode_Txm(t *testing.T) { cfg.MultiNode.MultiNode.Enabled = ptr(true) cfg.Nodes = []*solcfg.Node{ { - Name: ptr("primary-1"), + Name: ptr("primary"), URL: config.MustParseURL(client.SetupLocalSolNode(t)), }, - { - Name: ptr("primary-2"), - URL: config.MustParseURL(client.SetupLocalSolNode(t)), - }, - { - Name: ptr("sendonly-1"), - URL: config.MustParseURL(client.SetupLocalSolNode(t)), - SendOnly: true, - }, - { - Name: ptr("sendonly-2"), - URL: config.MustParseURL(client.SetupLocalSolNode(t)), - SendOnly: true, - }, } // setup keys diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go index 55e63511b..21c740182 100644 --- a/pkg/solana/client/multinode/transaction_sender.go +++ b/pkg/solana/client/multinode/transaction_sender.go @@ -87,7 +87,7 @@ type TransactionSender[TX any, RESULT SendTxResult, CHAIN_ID ID, RPC SendTxRPCCl // * If there is at least one terminal error - returns terminal error // * If there is both success and terminal error - returns success and reports invariant violation // * Otherwise, returns any (effectively random) of the errors. -func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ctx context.Context, tx TX) (*RESULT, error) { +func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ctx context.Context, tx TX) (result RESULT, err error) { txResults := make(chan RESULT) txResultsToReport := make(chan RESULT) primaryNodeWg := sync.WaitGroup{} @@ -96,7 +96,7 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct defer cancel() healthyNodesNum := 0 - err := txSender.multiNode.DoAll(ctx, func(ctx context.Context, rpc RPC, isSendOnly bool) { + err = txSender.multiNode.DoAll(ctx, func(ctx context.Context, rpc RPC, isSendOnly bool) { if isSendOnly { txSender.wg.Add(1) go func() { @@ -138,7 +138,7 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct }() if err != nil { - return nil, err + return result, err } txSender.wg.Add(1) @@ -173,7 +173,7 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) reportSendTxAnomal type sendTxResults[RESULT any] map[SendTxReturnCode][]RESULT -func aggregateTxResults[RESULT any](resultsByCode sendTxResults[RESULT]) (result *RESULT, err error) { +func aggregateTxResults[RESULT any](resultsByCode sendTxResults[RESULT]) (result RESULT, err error) { _, severeErrors, hasSevereErrors := findFirstIn(resultsByCode, sendTxSevereErrors) _, successResults, hasSuccess := findFirstIn(resultsByCode, sendTxSuccessfulCodes) if hasSuccess { @@ -182,29 +182,29 @@ func aggregateTxResults[RESULT any](resultsByCode sendTxResults[RESULT]) (result if hasSevereErrors { const errMsg = "found contradictions in nodes replies on SendTransaction: got success and severe error" // return success, since at least 1 node has accepted our broadcasted Tx, and thus it can now be included onchain - return &successResults[0], errors.New(errMsg) + return successResults[0], errors.New(errMsg) } // other errors are temporary - we are safe to return success - return &successResults[0], nil + return successResults[0], nil } if hasSevereErrors { - return &severeErrors[0], nil + return severeErrors[0], nil } // return temporary error - for _, result := range resultsByCode { - return &result[0], nil + for _, r := range resultsByCode { + return r[0], nil } err = fmt.Errorf("expected at least one response on SendTransaction") - return nil, err + return result, err } -func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) collectTxResults(ctx context.Context, tx TX, healthyNodesNum int, txResults <-chan RESULT) (*RESULT, error) { +func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) collectTxResults(ctx context.Context, tx TX, healthyNodesNum int, txResults <-chan RESULT) (result RESULT, err error) { if healthyNodesNum == 0 { - return nil, ErroringNodeError + return result, ErroringNodeError } requiredResults := int(math.Ceil(float64(healthyNodesNum) * sendTxQuorum)) errorsByCode := sendTxResults[RESULT]{} @@ -215,7 +215,7 @@ loop: select { case <-ctx.Done(): txSender.lggr.Debugw("Failed to collect of the results before context was done", "tx", tx, "errorsByCode", errorsByCode) - return nil, ctx.Err() + return result, ctx.Err() case result := <-txResults: errorsByCode[result.Code()] = append(errorsByCode[result.Code()], result) resultsCount++ @@ -237,7 +237,7 @@ loop: } // ignore critical error as it's reported in reportSendTxAnomalies - result, _ := aggregateTxResults(errorsByCode) + result, _ = aggregateTxResults(errorsByCode) return result, nil } diff --git a/pkg/solana/internal/loader.go b/pkg/solana/internal/loader.go index 43b847f3b..ba0bc5ee4 100644 --- a/pkg/solana/internal/loader.go +++ b/pkg/solana/internal/loader.go @@ -1,9 +1,5 @@ package internal -import ( - "github.com/smartcontractkit/chainlink-common/pkg/utils" -) - type Loader[T any] interface { Get() (T, error) Reset() @@ -12,30 +8,17 @@ type Loader[T any] interface { var _ Loader[any] = (*loader[any])(nil) type loader[T any] struct { - getClient func() (T, error) - lazyLoader *utils.LazyLoad[T] + getClient func() (T, error) } func (c *loader[T]) Get() (T, error) { - if c.lazyLoader != nil { - return c.lazyLoader.Get() - } return c.getClient() } -func (c *loader[T]) Reset() { - if c.lazyLoader != nil { - c.lazyLoader.Reset() - } -} +func (c *loader[T]) Reset() { /* do nothing */ } -func NewLoader[T any](lazyLoad bool, getClient func() (T, error)) *loader[T] { - var lazyLoader *utils.LazyLoad[T] - if lazyLoad { - lazyLoader = utils.NewLazyLoad(getClient) - } +func NewLoader[T any](getClient func() (T, error)) *loader[T] { return &loader[T]{ - lazyLoader: lazyLoader, - getClient: getClient, + getClient: getClient, } } diff --git a/pkg/solana/txm/txm_internal_test.go b/pkg/solana/txm/txm_internal_test.go index ee749fe29..89ab4bc56 100644 --- a/pkg/solana/txm/txm_internal_test.go +++ b/pkg/solana/txm/txm_internal_test.go @@ -5,7 +5,6 @@ package txm import ( "context" "errors" - "github.com/smartcontractkit/chainlink-solana/pkg/solana/internal" "math/rand" "sync" "testing" @@ -28,6 +27,7 @@ import ( relayconfig "github.com/smartcontractkit/chainlink-common/pkg/config" "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/utils" "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" ) @@ -111,10 +111,7 @@ func TestTxm(t *testing.T) { mkey := keyMocks.NewSimpleKeystore(t) mkey.On("Sign", mock.Anything, mock.Anything, mock.Anything).Return([]byte{}, nil) - getClient := func() (client.ReaderWriter, error) { - return mc, nil - } - loader := internal.NewLoader(true, getClient) + loader := utils.NewLazyLoad(func() (client.ReaderWriter, error) { return mc, nil }) txm := NewTxm(id, loader, nil, cfg, mkey, lggr) require.NoError(t, txm.Start(ctx)) @@ -720,10 +717,7 @@ func TestTxm_Enqueue(t *testing.T) { ) require.NoError(t, err) - getClient := func() (client.ReaderWriter, error) { - return mc, nil - } - loader := internal.NewLoader(true, getClient) + loader := utils.NewLazyLoad(func() (client.ReaderWriter, error) { return mc, nil }) txm := NewTxm("enqueue_test", loader, nil, cfg, mkey, lggr) require.ErrorContains(t, txm.Enqueue(ctx, "txmUnstarted", &solana.Transaction{}), "not started") diff --git a/pkg/solana/txm/txm_load_test.go b/pkg/solana/txm/txm_load_test.go index 9d83d5341..744610e1f 100644 --- a/pkg/solana/txm/txm_load_test.go +++ b/pkg/solana/txm/txm_load_test.go @@ -17,12 +17,12 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/services/servicetest" solanaClient "github.com/smartcontractkit/chainlink-solana/pkg/solana/client" "github.com/smartcontractkit/chainlink-solana/pkg/solana/config" - "github.com/smartcontractkit/chainlink-solana/pkg/solana/internal" "github.com/smartcontractkit/chainlink-solana/pkg/solana/txm" keyMocks "github.com/smartcontractkit/chainlink-solana/pkg/solana/txm/mocks" relayconfig "github.com/smartcontractkit/chainlink-common/pkg/config" "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/utils" "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" ) @@ -71,10 +71,7 @@ func TestTxm_Integration(t *testing.T) { cfg.Chain.FeeEstimatorMode = &estimator client, err := solanaClient.NewClient(url, cfg, 2*time.Second, lggr) require.NoError(t, err) - getClient := func() (solanaClient.ReaderWriter, error) { - return client, nil - } - loader := internal.NewLoader(true, getClient) + loader := utils.NewLazyLoad(func() (solanaClient.ReaderWriter, error) { return client, nil }) txm := txm.NewTxm("localnet", loader, nil, cfg, mkey, lggr) // track initial balance diff --git a/pkg/solana/txm/txm_race_test.go b/pkg/solana/txm/txm_race_test.go index b7ca10f2d..81f2c15f6 100644 --- a/pkg/solana/txm/txm_race_test.go +++ b/pkg/solana/txm/txm_race_test.go @@ -12,6 +12,7 @@ import ( "go.uber.org/zap/zapcore" "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/utils" "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" solanaClient "github.com/smartcontractkit/chainlink-solana/pkg/solana/client" @@ -19,7 +20,6 @@ import ( cfgmocks "github.com/smartcontractkit/chainlink-solana/pkg/solana/config/mocks" "github.com/smartcontractkit/chainlink-solana/pkg/solana/fees" feemocks "github.com/smartcontractkit/chainlink-solana/pkg/solana/fees/mocks" - "github.com/smartcontractkit/chainlink-solana/pkg/solana/internal" ksmocks "github.com/smartcontractkit/chainlink-solana/pkg/solana/txm/mocks" "github.com/stretchr/testify/assert" @@ -62,12 +62,10 @@ func TestTxm_SendWithRetry_Race(t *testing.T) { tx := NewTestTx() testRunner := func(t *testing.T, client solanaClient.ReaderWriter) { - getClient := func() (solanaClient.ReaderWriter, error) { - return client, nil - } - // build minimal txm - loader := internal.NewLoader(true, getClient) + loader := utils.NewLazyLoad(func() (solanaClient.ReaderWriter, error) { + return client, nil + }) txm := NewTxm("retry_race", loader, nil, cfg, ks, lggr) txm.fee = fee diff --git a/pkg/solana/txm/txm_unit_test.go b/pkg/solana/txm/txm_unit_test.go index b5948a61a..bb2108f4e 100644 --- a/pkg/solana/txm/txm_unit_test.go +++ b/pkg/solana/txm/txm_unit_test.go @@ -14,11 +14,11 @@ import ( solanaClient "github.com/smartcontractkit/chainlink-solana/pkg/solana/client" clientmocks "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/mocks" "github.com/smartcontractkit/chainlink-solana/pkg/solana/config" - "github.com/smartcontractkit/chainlink-solana/pkg/solana/internal" solanatxm "github.com/smartcontractkit/chainlink-solana/pkg/solana/txm" keyMocks "github.com/smartcontractkit/chainlink-solana/pkg/solana/txm/mocks" "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/utils" bigmath "github.com/smartcontractkit/chainlink-common/pkg/utils/big_math" "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" ) @@ -46,7 +46,7 @@ func TestTxm_EstimateComputeUnitLimit(t *testing.T) { cfg := config.NewDefault() client := clientmocks.NewReaderWriter(t) require.NoError(t, err) - loader := internal.NewLoader(true, func() (solanaClient.ReaderWriter, error) { return client, nil }) + loader := utils.NewLazyLoad(func() (solanaClient.ReaderWriter, error) { return client, nil }) txm := solanatxm.NewTxm("localnet", loader, nil, cfg, mkey, lggr) t.Run("successfully sets estimated compute unit limit", func(t *testing.T) { From 193143f174990c700112862008f46f76925cb9e1 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 18 Oct 2024 12:39:19 -0400 Subject: [PATCH 102/174] Use single RPC --- pkg/solana/chain_test.go | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/pkg/solana/chain_test.go b/pkg/solana/chain_test.go index 81d6441ff..48a53e885 100644 --- a/pkg/solana/chain_test.go +++ b/pkg/solana/chain_test.go @@ -391,26 +391,10 @@ func TestChain_MultiNode_TransactionSender(t *testing.T) { cfg.MultiNode.MultiNode.Enabled = ptr(true) cfg.Nodes = append(cfg.Nodes, &solcfg.Node{ - Name: ptr("localnet-" + t.Name() + "-primary-1"), + Name: ptr("localnet-" + t.Name() + "-primary"), URL: config.MustParseURL(client.SetupLocalSolNode(t)), SendOnly: false, - }, - &solcfg.Node{ - Name: ptr("localnet-" + t.Name() + "-primary-2"), - URL: config.MustParseURL(client.SetupLocalSolNode(t)), - SendOnly: false, - }, - &solcfg.Node{ - Name: ptr("localnet-" + t.Name() + "-sendonly-1"), - URL: config.MustParseURL(client.SetupLocalSolNode(t)), - SendOnly: true, - }, - &solcfg.Node{ - Name: ptr("localnet-" + t.Name() + "-sendonly-1"), - URL: config.MustParseURL(client.SetupLocalSolNode(t)), - SendOnly: true, - }, - ) + }) // mocked keystore mkey := mocks.NewSimpleKeystore(t) From b22f85f4ee3da9335ab1647aebd6fb54fc9a4af8 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 18 Oct 2024 12:47:17 -0400 Subject: [PATCH 103/174] Fix tests --- pkg/solana/internal/loader_test.go | 19 +++---------------- pkg/solana/monitor/balance_test.go | 5 ++--- 2 files changed, 5 insertions(+), 19 deletions(-) diff --git a/pkg/solana/internal/loader_test.go b/pkg/solana/internal/loader_test.go index 7fc8e0745..8d17a27ea 100644 --- a/pkg/solana/internal/loader_test.go +++ b/pkg/solana/internal/loader_test.go @@ -16,31 +16,18 @@ func (t *testLoader) load() (any, error) { return nil, nil } -func newTestLoader(lazyLoad bool) *testLoader { +func newTestLoader() *testLoader { loader := testLoader{} - loader.Loader = NewLoader[any](lazyLoad, loader.load) + loader.Loader = NewLoader[any](loader.load) return &loader } func TestLoader(t *testing.T) { t.Run("direct loading", func(t *testing.T) { - loader := newTestLoader(false) + loader := newTestLoader() _, _ = loader.Get() _, _ = loader.Get() _, _ = loader.Get() require.Equal(t, 3, loader.callCount) }) - - t.Run("lazy loading", func(t *testing.T) { - loader := newTestLoader(true) - _, _ = loader.Get() - _, _ = loader.Get() - require.Equal(t, 1, loader.callCount) - - // Calls load function again after Reset() - loader.Reset() - _, _ = loader.Get() - _, _ = loader.Get() - require.Equal(t, 2, loader.callCount) - }) } diff --git a/pkg/solana/monitor/balance_test.go b/pkg/solana/monitor/balance_test.go index 6f6bd03b9..a6cc231c9 100644 --- a/pkg/solana/monitor/balance_test.go +++ b/pkg/solana/monitor/balance_test.go @@ -63,10 +63,9 @@ func TestBalanceMonitor(t *testing.T) { } } - getClient := func() (BalanceClient, error) { + b.reader = internal.NewLoader[BalanceClient](func() (BalanceClient, error) { return client, nil - } - b.reader = internal.NewLoader[BalanceClient](true, getClient) + }) servicetest.Run(t, b) select { From 6b87b567ba94647d9ae3a4cf43aef378ccbd8058 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 18 Oct 2024 12:55:19 -0400 Subject: [PATCH 104/174] lint --- pkg/solana/client/multinode/transaction_sender.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go index 21c740182..3e5481bfb 100644 --- a/pkg/solana/client/multinode/transaction_sender.go +++ b/pkg/solana/client/multinode/transaction_sender.go @@ -113,17 +113,17 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct primaryNodeWg.Add(1) go func() { defer primaryNodeWg.Done() - result := txSender.broadcastTxAsync(ctx, rpc, tx) + r := txSender.broadcastTxAsync(ctx, rpc, tx) select { case <-ctx.Done(): return - case txResults <- result: + case txResults <- r: } select { case <-ctx.Done(): return - case txResultsToReport <- result: + case txResultsToReport <- r: } }() }) @@ -216,10 +216,10 @@ loop: case <-ctx.Done(): txSender.lggr.Debugw("Failed to collect of the results before context was done", "tx", tx, "errorsByCode", errorsByCode) return result, ctx.Err() - case result := <-txResults: - errorsByCode[result.Code()] = append(errorsByCode[result.Code()], result) + case r := <-txResults: + errorsByCode[r.Code()] = append(errorsByCode[r.Code()], r) resultsCount++ - if slices.Contains(sendTxSuccessfulCodes, result.Code()) || resultsCount >= requiredResults { + if slices.Contains(sendTxSuccessfulCodes, r.Code()) || resultsCount >= requiredResults { break loop } case <-softTimeoutChan: From ac5fdd7d0a0935f1fd991177a62787c39a7eadf4 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 18 Oct 2024 14:43:37 -0400 Subject: [PATCH 105/174] Use default thresholds --- integration-tests/testconfig/testconfig.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/integration-tests/testconfig/testconfig.go b/integration-tests/testconfig/testconfig.go index ad8985d45..6370c575b 100644 --- a/integration-tests/testconfig/testconfig.go +++ b/integration-tests/testconfig/testconfig.go @@ -268,9 +268,7 @@ func (c *TestConfig) GetNodeConfigTOML() (string, error) { // TODO: If not, then those tests will fail on CI and only work with actual RPCs. mnConfig := solcfg.MultiNodeConfig{ MultiNode: solcfg.MultiNode{ - Enabled: ptr.Ptr(true), - NodeNoNewHeadsThreshold: config.MustNewDuration(time.Minute), - NoNewFinalizedHeadsThreshold: config.MustNewDuration(2 * time.Minute), + Enabled: ptr.Ptr(true), }, } mnConfig.SetDefaults() From d1227009cc106c387e3f33c8fa0cec954ee86097 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Mon, 21 Oct 2024 13:23:33 -0400 Subject: [PATCH 106/174] Address comments --- pkg/solana/chain.go | 17 +++---- pkg/solana/chain_test.go | 33 +++++++++++--- pkg/solana/client/classify_errors.go | 44 +++---------------- pkg/solana/client/classify_errors_test.go | 10 ++--- .../client/multinode/transaction_sender.go | 26 ++++++----- pkg/solana/client/multinode_client.go | 27 ++++++++---- 6 files changed, 78 insertions(+), 79 deletions(-) diff --git a/pkg/solana/chain.go b/pkg/solana/chain.go index 4228566d3..d10f56073 100644 --- a/pkg/solana/chain.go +++ b/pkg/solana/chain.go @@ -233,6 +233,9 @@ func newChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr logger.L var sendTx func(ctx context.Context, tx *solanago.Transaction) (solanago.Signature, error) + var tc internal.Loader[client.ReaderWriter] = utils.NewLazyLoad(func() (client.ReaderWriter, error) { return ch.getClient() }) + var bc internal.Loader[monitor.BalanceClient] = utils.NewLazyLoad(func() (monitor.BalanceClient, error) { return ch.getClient() }) + if cfg.MultiNode.Enabled() { chainFamily := "solana" @@ -276,6 +279,7 @@ func newChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr logger.L mn.StringID(id), chainFamily, multiNode, + client.NewSendTxResult, 0, // use the default value provided by the implementation ) @@ -287,9 +291,9 @@ func newChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr logger.L sendTx = func(ctx context.Context, tx *solanago.Transaction) (solanago.Signature, error) { // Send tx using MultiNode transaction sender - result, err := ch.txSender.SendTransaction(ctx, tx) - if err != nil { - return solanago.Signature{}, err + result := ch.txSender.SendTransaction(ctx, tx) + if result.Error() != nil { + return solanago.Signature{}, result.Error() } if result == nil { return solanago.Signature{}, errors.New("tx sender returned nil result") @@ -299,16 +303,9 @@ func newChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr logger.L } return result.Signature(), nil } - } - var tc internal.Loader[client.ReaderWriter] - var bc internal.Loader[monitor.BalanceClient] - if cfg.MultiNode.Enabled() { tc = internal.NewLoader[client.ReaderWriter](func() (client.ReaderWriter, error) { return ch.multiNode.SelectRPC() }) bc = internal.NewLoader[monitor.BalanceClient](func() (monitor.BalanceClient, error) { return ch.multiNode.SelectRPC() }) - } else { - tc = utils.NewLazyLoad(func() (client.ReaderWriter, error) { return ch.getClient() }) - bc = utils.NewLazyLoad(func() (monitor.BalanceClient, error) { return ch.getClient() }) } ch.txm = txm.NewTxm(ch.id, tc, sendTx, cfg, ks, lggr) diff --git a/pkg/solana/chain_test.go b/pkg/solana/chain_test.go index 48a53e885..895a87149 100644 --- a/pkg/solana/chain_test.go +++ b/pkg/solana/chain_test.go @@ -439,9 +439,9 @@ func TestChain_MultiNode_TransactionSender(t *testing.T) { } // Send tx using transaction sender - result, err := c.txSender.SendTransaction(ctx, createTx(receiver.PublicKey())) - require.NoError(t, err) + result := c.txSender.SendTransaction(ctx, createTx(receiver.PublicKey())) require.NotNil(t, result) + require.NoError(t, result.Error()) require.Equal(t, mn.Successful, result.Code()) require.NotEmpty(t, result.Signature()) }) @@ -471,18 +471,18 @@ func TestChain_MultiNode_TransactionSender(t *testing.T) { } // Send tx using transaction sender - result, err := c.txSender.SendTransaction(ctx, unsignedTx(receiver.PublicKey())) - require.NoError(t, err) + result := c.txSender.SendTransaction(ctx, unsignedTx(receiver.PublicKey())) require.NotNil(t, result) + require.NoError(t, result.Error()) require.Error(t, result.TxError()) require.Equal(t, mn.Fatal, result.Code()) require.Empty(t, result.Signature()) }) t.Run("empty transaction", func(t *testing.T) { - result, err := c.txSender.SendTransaction(ctx, &solana.Transaction{}) - require.NoError(t, err) + result := c.txSender.SendTransaction(ctx, &solana.Transaction{}) require.NotNil(t, result) + require.NoError(t, result.Error()) require.Error(t, result.TxError()) require.Equal(t, mn.Fatal, result.Code()) require.Empty(t, result.Signature()) @@ -558,7 +558,26 @@ func TestSolanaChain_MultiNode_Txm(t *testing.T) { // Send funds twice, along with an invalid transaction require.NoError(t, testChain.txm.Enqueue(tests.Context(t), "test_success", createTx(pubKey, pubKey, pubKeyReceiver, solana.LAMPORTS_PER_SOL))) - time.Sleep(500 * time.Millisecond) // pause 0.5s for new blockhash + + // Wait for new block hash + currentBh, err := selectedClient.LatestBlockhash(tests.Context(t)) + require.NoError(t, err) + timeout := time.After(time.Minute) + +NewBlockHash: + for { + select { + case <-timeout: + t.Fatal("timed out waiting for new block hash") + default: + newBh, err := selectedClient.LatestBlockhash(tests.Context(t)) + require.NoError(t, err) + if newBh.Value.LastValidBlockHeight > currentBh.Value.LastValidBlockHeight { + break NewBlockHash + } + } + } + require.NoError(t, testChain.txm.Enqueue(tests.Context(t), "test_success_2", createTx(pubKey, pubKey, pubKeyReceiver, solana.LAMPORTS_PER_SOL))) require.Error(t, testChain.txm.Enqueue(tests.Context(t), "test_invalidSigner", createTx(pubKeyReceiver, pubKey, pubKeyReceiver, solana.LAMPORTS_PER_SOL))) // cannot sign tx before enqueuing diff --git a/pkg/solana/client/classify_errors.go b/pkg/solana/client/classify_errors.go index b6ed1a84b..1fe711b0b 100644 --- a/pkg/solana/client/classify_errors.go +++ b/pkg/solana/client/classify_errors.go @@ -53,44 +53,12 @@ var ( // errCodes maps regex patterns to corresponding return code var errCodes = map[*regexp.Regexp]mn.SendTxReturnCode{ - ErrAccountInUse: mn.Retryable, - ErrAccountLoadedTwice: mn.Retryable, - ErrAccountNotFound: mn.Retryable, - ErrProgramAccountNotFound: mn.Fatal, - ErrInsufficientFundsForFee: mn.InsufficientFunds, - ErrInvalidAccountForFee: mn.Unsupported, - ErrAlreadyProcessed: mn.TransactionAlreadyKnown, - ErrBlockhashNotFound: mn.Retryable, - ErrInstructionError: mn.Retryable, - ErrCallChainTooDeep: mn.Retryable, - ErrMissingSignatureForFee: mn.Retryable, - ErrInvalidAccountIndex: mn.Retryable, - ErrSignatureFailure: mn.Fatal, - ErrInvalidProgramForExecution: mn.Retryable, - ErrSanitizeFailure: mn.Fatal, - ErrClusterMaintenance: mn.Retryable, - ErrAccountBorrowOutstanding: mn.Retryable, - ErrWouldExceedMaxBlockCostLimit: mn.ExceedsMaxFee, - ErrUnsupportedVersion: mn.Unsupported, - ErrInvalidWritableAccount: mn.Retryable, - ErrWouldExceedMaxAccountCostLimit: mn.ExceedsMaxFee, - ErrWouldExceedAccountDataBlockLimit: mn.ExceedsMaxFee, - ErrTooManyAccountLocks: mn.Retryable, - ErrAddressLookupTableNotFound: mn.Retryable, - ErrInvalidAddressLookupTableOwner: mn.Retryable, - ErrInvalidAddressLookupTableData: mn.Retryable, - ErrInvalidAddressLookupTableIndex: mn.Retryable, - ErrInvalidRentPayingAccount: mn.Retryable, - ErrWouldExceedMaxVoteCostLimit: mn.Retryable, - ErrWouldExceedAccountDataTotalLimit: mn.Retryable, - ErrMaxLoadedAccountsDataSizeExceeded: mn.Retryable, - ErrInvalidLoadedAccountsDataSizeLimit: mn.Retryable, - ErrResanitizationNeeded: mn.Retryable, - ErrUnbalancedTransaction: mn.Retryable, - ErrProgramCacheHitMaxLimit: mn.Retryable, - ErrInsufficientFundsForRent: mn.InsufficientFunds, - ErrDuplicateInstruction: mn.Fatal, - ErrProgramExecutionTemporarilyRestricted: mn.Retryable, + ErrInsufficientFundsForFee: mn.InsufficientFunds, + ErrAlreadyProcessed: mn.TransactionAlreadyKnown, + ErrWouldExceedMaxBlockCostLimit: mn.ExceedsMaxFee, + ErrUnsupportedVersion: mn.Unsupported, + ErrWouldExceedMaxAccountCostLimit: mn.ExceedsMaxFee, + ErrInsufficientFundsForRent: mn.InsufficientFunds, } // ClassifySendError returns the corresponding return code based on the error. diff --git a/pkg/solana/client/classify_errors_test.go b/pkg/solana/client/classify_errors_test.go index 5d3e8de68..fcab3dab4 100644 --- a/pkg/solana/client/classify_errors_test.go +++ b/pkg/solana/client/classify_errors_test.go @@ -18,24 +18,24 @@ func TestClassifySendError(t *testing.T) { {"Account in use", mn.Retryable}, {"Account loaded twice", mn.Retryable}, {"Attempt to debit an account but found no record of a prior credit.", mn.Retryable}, - {"Attempt to load a program that does not exist", mn.Fatal}, + {"Attempt to load a program that does not exist", mn.Retryable}, {"Insufficient funds for fee", mn.InsufficientFunds}, - {"This account may not be used to pay transaction fees", mn.Unsupported}, + {"This account may not be used to pay transaction fees", mn.Retryable}, {"This transaction has already been processed", mn.TransactionAlreadyKnown}, {"Blockhash not found", mn.Retryable}, {"Loader call chain is too deep", mn.Retryable}, {"Transaction requires a fee but has no signature present", mn.Retryable}, {"Transaction contains an invalid account reference", mn.Retryable}, - {"Transaction did not pass signature verification", mn.Fatal}, + {"Transaction did not pass signature verification", mn.Retryable}, {"This program may not be used for executing instructions", mn.Retryable}, - {"Transaction failed to sanitize accounts offsets correctly", mn.Fatal}, + {"Transaction failed to sanitize accounts offsets correctly", mn.Retryable}, {"Transactions are currently disabled due to cluster maintenance", mn.Retryable}, {"Transaction processing left an account with an outstanding borrowed reference", mn.Retryable}, {"Transaction would exceed max Block Cost Limit", mn.ExceedsMaxFee}, {"Transaction version is unsupported", mn.Unsupported}, {"Transaction loads a writable account that cannot be written", mn.Retryable}, {"Transaction would exceed max account limit within the block", mn.ExceedsMaxFee}, - {"Transaction would exceed account data limit within the block", mn.ExceedsMaxFee}, + {"Transaction would exceed account data limit within the block", mn.Retryable}, {"Transaction locked too many accounts", mn.Retryable}, {"Address lookup table not found", mn.Retryable}, {"Attempted to lookup addresses from an account owned by the wrong program", mn.Retryable}, diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go index 3e5481bfb..dcea8b11a 100644 --- a/pkg/solana/client/multinode/transaction_sender.go +++ b/pkg/solana/client/multinode/transaction_sender.go @@ -27,6 +27,7 @@ var ( type SendTxResult interface { Code() SendTxReturnCode TxError() error + Error() error } const sendTxQuorum = 0.7 @@ -42,6 +43,7 @@ func NewTransactionSender[TX any, RESULT SendTxResult, CHAIN_ID ID, RPC SendTxRP chainID CHAIN_ID, chainFamily string, multiNode *MultiNode[CHAIN_ID, RPC], + newResult func(err error) RESULT, sendTxSoftTimeout time.Duration, ) *TransactionSender[TX, RESULT, CHAIN_ID, RPC] { if sendTxSoftTimeout == 0 { @@ -52,6 +54,7 @@ func NewTransactionSender[TX any, RESULT SendTxResult, CHAIN_ID ID, RPC SendTxRP chainFamily: chainFamily, lggr: logger.Sugared(lggr).Named("TransactionSender").With("chainID", chainID.String()), multiNode: multiNode, + newResult: newResult, sendTxSoftTimeout: sendTxSoftTimeout, chStop: make(services.StopChan), } @@ -63,6 +66,7 @@ type TransactionSender[TX any, RESULT SendTxResult, CHAIN_ID ID, RPC SendTxRPCCl chainFamily string lggr logger.SugaredLogger multiNode *MultiNode[CHAIN_ID, RPC] + newResult func(err error) RESULT sendTxSoftTimeout time.Duration // defines max waiting time from first response til responses evaluation wg sync.WaitGroup // waits for all reporting goroutines to finish @@ -87,7 +91,7 @@ type TransactionSender[TX any, RESULT SendTxResult, CHAIN_ID ID, RPC SendTxRPCCl // * If there is at least one terminal error - returns terminal error // * If there is both success and terminal error - returns success and reports invariant violation // * Otherwise, returns any (effectively random) of the errors. -func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ctx context.Context, tx TX) (result RESULT, err error) { +func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ctx context.Context, tx TX) (result RESULT) { txResults := make(chan RESULT) txResultsToReport := make(chan RESULT) primaryNodeWg := sync.WaitGroup{} @@ -96,7 +100,7 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct defer cancel() healthyNodesNum := 0 - err = txSender.multiNode.DoAll(ctx, func(ctx context.Context, rpc RPC, isSendOnly bool) { + err := txSender.multiNode.DoAll(ctx, func(ctx context.Context, rpc RPC, isSendOnly bool) { if isSendOnly { txSender.wg.Add(1) go func() { @@ -138,7 +142,7 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct }() if err != nil { - return result, err + return txSender.newResult(err) } txSender.wg.Add(1) @@ -173,7 +177,7 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) reportSendTxAnomal type sendTxResults[RESULT any] map[SendTxReturnCode][]RESULT -func aggregateTxResults[RESULT any](resultsByCode sendTxResults[RESULT]) (result RESULT, err error) { +func aggregateTxResults[RESULT any](resultsByCode sendTxResults[RESULT]) (result RESULT, criticalErr error) { _, severeErrors, hasSevereErrors := findFirstIn(resultsByCode, sendTxSevereErrors) _, successResults, hasSuccess := findFirstIn(resultsByCode, sendTxSuccessfulCodes) if hasSuccess { @@ -198,13 +202,13 @@ func aggregateTxResults[RESULT any](resultsByCode sendTxResults[RESULT]) (result return r[0], nil } - err = fmt.Errorf("expected at least one response on SendTransaction") - return result, err + criticalErr = fmt.Errorf("expected at least one response on SendTransaction") + return result, criticalErr } -func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) collectTxResults(ctx context.Context, tx TX, healthyNodesNum int, txResults <-chan RESULT) (result RESULT, err error) { +func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) collectTxResults(ctx context.Context, tx TX, healthyNodesNum int, txResults <-chan RESULT) RESULT { if healthyNodesNum == 0 { - return result, ErroringNodeError + return txSender.newResult(ErroringNodeError) } requiredResults := int(math.Ceil(float64(healthyNodesNum) * sendTxQuorum)) errorsByCode := sendTxResults[RESULT]{} @@ -215,7 +219,7 @@ loop: select { case <-ctx.Done(): txSender.lggr.Debugw("Failed to collect of the results before context was done", "tx", tx, "errorsByCode", errorsByCode) - return result, ctx.Err() + return txSender.newResult(ctx.Err()) case r := <-txResults: errorsByCode[r.Code()] = append(errorsByCode[r.Code()], r) resultsCount++ @@ -237,8 +241,8 @@ loop: } // ignore critical error as it's reported in reportSendTxAnomalies - result, _ = aggregateTxResults(errorsByCode) - return result, nil + result, _ := aggregateTxResults(errorsByCode) + return result } func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) Start(ctx context.Context) error { diff --git a/pkg/solana/client/multinode_client.go b/pkg/solana/client/multinode_client.go index fb29fce8d..c083f52c9 100644 --- a/pkg/solana/client/multinode_client.go +++ b/pkg/solana/client/multinode_client.go @@ -301,17 +301,28 @@ func (m *MultiNodeClient) GetInterceptedChainInfo() (latest, highestUserObservat } type SendTxResult struct { - err error - code mn.SendTxReturnCode - sig solana.Signature + err error + txErr error + code mn.SendTxReturnCode + sig solana.Signature } var _ mn.SendTxResult = (*SendTxResult)(nil) -func (r *SendTxResult) TxError() error { +func NewSendTxResult(err error) *SendTxResult { + return &SendTxResult{ + err: err, + } +} + +func (r *SendTxResult) Error() error { return r.err } +func (r *SendTxResult) TxError() error { + return r.txErr +} + func (r *SendTxResult) Code() mn.SendTxReturnCode { return r.code } @@ -322,10 +333,10 @@ func (r *SendTxResult) Signature() solana.Signature { func (m *MultiNodeClient) SendTransaction(ctx context.Context, tx *solana.Transaction) *SendTxResult { var sendTxResult = &SendTxResult{} - sig, err := m.SendTx(ctx, tx) - sendTxResult.code = ClassifySendError(tx, err) - if err != nil { - sendTxResult.err = err + sig, txErr := m.SendTx(ctx, tx) + sendTxResult.code = ClassifySendError(tx, txErr) + if txErr != nil { + sendTxResult.txErr = txErr return sendTxResult } sendTxResult.sig = sig From cacffbf644fb16855bf90df6d22c0e5c62832c59 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Mon, 21 Oct 2024 13:41:04 -0400 Subject: [PATCH 107/174] Update classify_errors.go --- pkg/solana/client/classify_errors.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/pkg/solana/client/classify_errors.go b/pkg/solana/client/classify_errors.go index 1fe711b0b..3bf33d511 100644 --- a/pkg/solana/client/classify_errors.go +++ b/pkg/solana/client/classify_errors.go @@ -51,14 +51,12 @@ var ( ErrProgramCacheHitMaxLimit = regexp.MustCompile(`Program cache hit max limit`) ) -// errCodes maps regex patterns to corresponding return code +// errCodes maps regex patterns to their corresponding return code +// errors are considered Retryable by default if not in this map var errCodes = map[*regexp.Regexp]mn.SendTxReturnCode{ - ErrInsufficientFundsForFee: mn.InsufficientFunds, - ErrAlreadyProcessed: mn.TransactionAlreadyKnown, - ErrWouldExceedMaxBlockCostLimit: mn.ExceedsMaxFee, - ErrUnsupportedVersion: mn.Unsupported, - ErrWouldExceedMaxAccountCostLimit: mn.ExceedsMaxFee, - ErrInsufficientFundsForRent: mn.InsufficientFunds, + ErrAlreadyProcessed: mn.TransactionAlreadyKnown, // Transaction was already processed and thus known by the RPC + ErrInsufficientFundsForFee: mn.InsufficientFunds, + ErrInsufficientFundsForRent: mn.InsufficientFunds, } // ClassifySendError returns the corresponding return code based on the error. From 5ecb5b7ef5613a31a36e84c142edb962fee54572 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Mon, 21 Oct 2024 15:02:39 -0400 Subject: [PATCH 108/174] Update testconfig.go --- integration-tests/testconfig/testconfig.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/integration-tests/testconfig/testconfig.go b/integration-tests/testconfig/testconfig.go index 6370c575b..e6b57412a 100644 --- a/integration-tests/testconfig/testconfig.go +++ b/integration-tests/testconfig/testconfig.go @@ -264,8 +264,6 @@ func (c *TestConfig) GetNodeConfigTOML() (string, error) { url = c.GetURL() } - // TODO: Does the simulated default actually create new heads/ finalized heads? - // TODO: If not, then those tests will fail on CI and only work with actual RPCs. mnConfig := solcfg.MultiNodeConfig{ MultiNode: solcfg.MultiNode{ Enabled: ptr.Ptr(true), From 9ef4e76d95bc276c2a5b033a04dc3f063cf65134 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Mon, 21 Oct 2024 15:26:54 -0400 Subject: [PATCH 109/174] Update errors --- pkg/solana/chain_test.go | 61 +++++++++++------------ pkg/solana/client/classify_errors.go | 6 +-- pkg/solana/client/classify_errors_test.go | 10 ++-- 3 files changed, 38 insertions(+), 39 deletions(-) diff --git a/pkg/solana/chain_test.go b/pkg/solana/chain_test.go index 895a87149..08e779b1b 100644 --- a/pkg/solana/chain_test.go +++ b/pkg/solana/chain_test.go @@ -405,41 +405,40 @@ func TestChain_MultiNode_TransactionSender(t *testing.T) { require.NoError(t, c.Close()) }() - t.Run("successful transaction", func(t *testing.T) { - // create + sign transaction - createTx := func(to solana.PublicKey) *solana.Transaction { - cl, err := c.getClient() - require.NoError(t, err) + createTx := func(from solana.PrivateKey, to solana.PrivateKey) *solana.Transaction { + cl, err := c.getClient() + require.NoError(t, err) - hash, hashErr := cl.LatestBlockhash(tests.Context(t)) - assert.NoError(t, hashErr) + hash, hashErr := cl.LatestBlockhash(tests.Context(t)) + assert.NoError(t, hashErr) - tx, txErr := solana.NewTransaction( - []solana.Instruction{ - system.NewTransferInstruction( - 1, - sender.PublicKey(), - to, - ).Build(), - }, - hash.Value.Blockhash, - solana.TransactionPayer(sender.PublicKey()), - ) - assert.NoError(t, txErr) - _, signErr := tx.Sign( - func(key solana.PublicKey) *solana.PrivateKey { - if sender.PublicKey().Equals(key) { - return &sender - } - return nil - }, - ) - assert.NoError(t, signErr) - return tx - } + tx, txErr := solana.NewTransaction( + []solana.Instruction{ + system.NewTransferInstruction( + 1, + from.PublicKey(), + to.PublicKey(), + ).Build(), + }, + hash.Value.Blockhash, + solana.TransactionPayer(from.PublicKey()), + ) + assert.NoError(t, txErr) + _, signErr := tx.Sign( + func(key solana.PublicKey) *solana.PrivateKey { + if from.PublicKey().Equals(key) { + return &from + } + return nil + }, + ) + assert.NoError(t, signErr) + return tx + } + t.Run("successful transaction", func(t *testing.T) { // Send tx using transaction sender - result := c.txSender.SendTransaction(ctx, createTx(receiver.PublicKey())) + result := c.txSender.SendTransaction(ctx, createTx(sender, receiver)) require.NotNil(t, result) require.NoError(t, result.Error()) require.Equal(t, mn.Successful, result.Code()) diff --git a/pkg/solana/client/classify_errors.go b/pkg/solana/client/classify_errors.go index 3bf33d511..ae3402694 100644 --- a/pkg/solana/client/classify_errors.go +++ b/pkg/solana/client/classify_errors.go @@ -54,9 +54,9 @@ var ( // errCodes maps regex patterns to their corresponding return code // errors are considered Retryable by default if not in this map var errCodes = map[*regexp.Regexp]mn.SendTxReturnCode{ - ErrAlreadyProcessed: mn.TransactionAlreadyKnown, // Transaction was already processed and thus known by the RPC - ErrInsufficientFundsForFee: mn.InsufficientFunds, - ErrInsufficientFundsForRent: mn.InsufficientFunds, + ErrSanitizeFailure: mn.Fatal, // Transaction formatting is invalid and cannot be processed or retried + ErrAlreadyProcessed: mn.TransactionAlreadyKnown, // Transaction was already processed and thus known by the RPC + ErrInsufficientFundsForFee: mn.InsufficientFunds, // Transaction was rejected due to insufficient funds for gas fees } // ClassifySendError returns the corresponding return code based on the error. diff --git a/pkg/solana/client/classify_errors_test.go b/pkg/solana/client/classify_errors_test.go index fcab3dab4..29d6a3bc0 100644 --- a/pkg/solana/client/classify_errors_test.go +++ b/pkg/solana/client/classify_errors_test.go @@ -28,13 +28,13 @@ func TestClassifySendError(t *testing.T) { {"Transaction contains an invalid account reference", mn.Retryable}, {"Transaction did not pass signature verification", mn.Retryable}, {"This program may not be used for executing instructions", mn.Retryable}, - {"Transaction failed to sanitize accounts offsets correctly", mn.Retryable}, + {"Transaction failed to sanitize accounts offsets correctly", mn.Fatal}, {"Transactions are currently disabled due to cluster maintenance", mn.Retryable}, {"Transaction processing left an account with an outstanding borrowed reference", mn.Retryable}, - {"Transaction would exceed max Block Cost Limit", mn.ExceedsMaxFee}, - {"Transaction version is unsupported", mn.Unsupported}, + {"Transaction would exceed max Block Cost Limit", mn.Retryable}, + {"Transaction version is unsupported", mn.Retryable}, {"Transaction loads a writable account that cannot be written", mn.Retryable}, - {"Transaction would exceed max account limit within the block", mn.ExceedsMaxFee}, + {"Transaction would exceed max account limit within the block", mn.Retryable}, {"Transaction would exceed account data limit within the block", mn.Retryable}, {"Transaction locked too many accounts", mn.Retryable}, {"Address lookup table not found", mn.Retryable}, @@ -51,7 +51,7 @@ func TestClassifySendError(t *testing.T) { {"Program cache hit max limit", mn.Retryable}, // Dynamic error cases - {"Transaction results in an account (123) with insufficient funds for rent", mn.InsufficientFunds}, + {"Transaction results in an account (123) with insufficient funds for rent", mn.Retryable}, {"Error processing Instruction 2: Some error details", mn.Retryable}, {"Execution of the program referenced by account at index 3 is temporarily restricted.", mn.Retryable}, From 6647fa13af235352315b501032a502e5e50152fa Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Mon, 21 Oct 2024 15:59:42 -0400 Subject: [PATCH 110/174] lint --- pkg/solana/chain_test.go | 4 ++-- pkg/solana/client/multinode/transaction_sender.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/solana/chain_test.go b/pkg/solana/chain_test.go index 08e779b1b..b705860c9 100644 --- a/pkg/solana/chain_test.go +++ b/pkg/solana/chain_test.go @@ -569,8 +569,8 @@ NewBlockHash: case <-timeout: t.Fatal("timed out waiting for new block hash") default: - newBh, err := selectedClient.LatestBlockhash(tests.Context(t)) - require.NoError(t, err) + newBh, bhErr := selectedClient.LatestBlockhash(tests.Context(t)) + require.NoError(t, bhErr) if newBh.Value.LastValidBlockHeight > currentBh.Value.LastValidBlockHeight { break NewBlockHash } diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go index dcea8b11a..1c2d637fd 100644 --- a/pkg/solana/client/multinode/transaction_sender.go +++ b/pkg/solana/client/multinode/transaction_sender.go @@ -91,7 +91,7 @@ type TransactionSender[TX any, RESULT SendTxResult, CHAIN_ID ID, RPC SendTxRPCCl // * If there is at least one terminal error - returns terminal error // * If there is both success and terminal error - returns success and reports invariant violation // * Otherwise, returns any (effectively random) of the errors. -func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ctx context.Context, tx TX) (result RESULT) { +func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ctx context.Context, tx TX) RESULT { txResults := make(chan RESULT) txResultsToReport := make(chan RESULT) primaryNodeWg := sync.WaitGroup{} From 6c568c6f024bd39cb027273a872e6c58e02109ac Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 22 Oct 2024 11:30:31 -0400 Subject: [PATCH 111/174] Fix SendTransaction --- pkg/solana/client/multinode/transaction_sender.go | 7 +++++-- pkg/solana/client/multinode_client.go | 9 ++------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go index 1c2d637fd..a8d8c3418 100644 --- a/pkg/solana/client/multinode/transaction_sender.go +++ b/pkg/solana/client/multinode/transaction_sender.go @@ -96,8 +96,9 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct txResultsToReport := make(chan RESULT) primaryNodeWg := sync.WaitGroup{} - ctx, cancel := txSender.chStop.Ctx(ctx) - defer cancel() + if txSender.State() != "Started" { + return txSender.newResult(errors.New("TransactionSender not started")) + } healthyNodesNum := 0 err := txSender.multiNode.DoAll(ctx, func(ctx context.Context, rpc RPC, isSendOnly bool) { @@ -210,6 +211,8 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) collectTxResults(c if healthyNodesNum == 0 { return txSender.newResult(ErroringNodeError) } + ctx, cancel := txSender.chStop.Ctx(ctx) + defer cancel() requiredResults := int(math.Ceil(float64(healthyNodesNum) * sendTxQuorum)) errorsByCode := sendTxResults[RESULT]{} var softTimeoutChan <-chan time.Time diff --git a/pkg/solana/client/multinode_client.go b/pkg/solana/client/multinode_client.go index c083f52c9..452b1cf3d 100644 --- a/pkg/solana/client/multinode_client.go +++ b/pkg/solana/client/multinode_client.go @@ -333,12 +333,7 @@ func (r *SendTxResult) Signature() solana.Signature { func (m *MultiNodeClient) SendTransaction(ctx context.Context, tx *solana.Transaction) *SendTxResult { var sendTxResult = &SendTxResult{} - sig, txErr := m.SendTx(ctx, tx) - sendTxResult.code = ClassifySendError(tx, txErr) - if txErr != nil { - sendTxResult.txErr = txErr - return sendTxResult - } - sendTxResult.sig = sig + sendTxResult.sig, sendTxResult.txErr = m.SendTx(ctx, tx) + sendTxResult.code = ClassifySendError(tx, sendTxResult.txErr) return sendTxResult } From f87632f1055ef0c7ea5f9a6510f9a32ce04942c6 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 22 Oct 2024 11:36:49 -0400 Subject: [PATCH 112/174] Update chain.go --- pkg/solana/chain.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/pkg/solana/chain.go b/pkg/solana/chain.go index d10f56073..98ed0c2c2 100644 --- a/pkg/solana/chain.go +++ b/pkg/solana/chain.go @@ -292,16 +292,13 @@ func newChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr logger.L sendTx = func(ctx context.Context, tx *solanago.Transaction) (solanago.Signature, error) { // Send tx using MultiNode transaction sender result := ch.txSender.SendTransaction(ctx, tx) - if result.Error() != nil { - return solanago.Signature{}, result.Error() - } if result == nil { return solanago.Signature{}, errors.New("tx sender returned nil result") } - if result.Signature().IsZero() { - return solanago.Signature{}, errors.New("tx sender returned empty signature") + if result.Error() != nil { + return solanago.Signature{}, result.Error() } - return result.Signature(), nil + return result.Signature(), result.TxError() } tc = internal.NewLoader[client.ReaderWriter](func() (client.ReaderWriter, error) { return ch.multiNode.SelectRPC() }) From 06c0c86bbdd8511212546e29802d81ea1267eaf6 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 22 Oct 2024 12:52:04 -0400 Subject: [PATCH 113/174] Update sendTx --- pkg/solana/chain.go | 7 ++++--- pkg/solana/txm/txm.go | 29 +++++++++++++---------------- 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/pkg/solana/chain.go b/pkg/solana/chain.go index 98ed0c2c2..c47e1cf1b 100644 --- a/pkg/solana/chain.go +++ b/pkg/solana/chain.go @@ -231,11 +231,12 @@ func newChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr logger.L clientCache: map[string]*verifiedCachedClient{}, } - var sendTx func(ctx context.Context, tx *solanago.Transaction) (solanago.Signature, error) - var tc internal.Loader[client.ReaderWriter] = utils.NewLazyLoad(func() (client.ReaderWriter, error) { return ch.getClient() }) var bc internal.Loader[monitor.BalanceClient] = utils.NewLazyLoad(func() (monitor.BalanceClient, error) { return ch.getClient() }) + // txm will default to sending transactions using a single RPC client if sendTx is nil + var sendTx func(ctx context.Context, tx *solanago.Transaction) (solanago.Signature, error) + if cfg.MultiNode.Enabled() { chainFamily := "solana" @@ -289,8 +290,8 @@ func newChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr logger.L // clientCache will not be used if multinode is enabled ch.clientCache = nil + // Send tx using MultiNode transaction sender sendTx = func(ctx context.Context, tx *solanago.Transaction) (solanago.Signature, error) { - // Send tx using MultiNode transaction sender result := ch.txSender.SendTransaction(ctx, tx) if result == nil { return solanago.Signature{}, errors.New("tx sender returned nil result") diff --git a/pkg/solana/txm/txm.go b/pkg/solana/txm/txm.go index daa7e9322..7cd09cf5e 100644 --- a/pkg/solana/txm/txm.go +++ b/pkg/solana/txm/txm.go @@ -86,6 +86,17 @@ type pendingTx struct { func NewTxm(chainID string, client internal.Loader[client.ReaderWriter], sendTx func(ctx context.Context, tx *solanaGo.Transaction) (solanaGo.Signature, error), cfg config.Config, ks SimpleKeystore, lggr logger.Logger) *Txm { + if sendTx == nil { + // default sendTx using a single RPC + sendTx = func(ctx context.Context, tx *solanaGo.Transaction) (solanaGo.Signature, error) { + c, err := client.Get() + if err != nil { + return solanaGo.Signature{}, err + } + return c.SendTx(ctx, tx) + } + } + return &Txm{ lggr: logger.Named(lggr, "Txm"), chSend: make(chan pendingTx, MaxQueueLen), // queue can support 1000 pending txs @@ -99,20 +110,6 @@ func NewTxm(chainID string, client internal.Loader[client.ReaderWriter], } } -// SendTx sends a transaction using a single client or an override sendTx function -func (txm *Txm) SendTx(ctx context.Context, tx *solanaGo.Transaction) (solanaGo.Signature, error) { - if txm.sendTx != nil { - return txm.sendTx(ctx, tx) - } - - // Send tx using a single RPC client - client, err := txm.client.Get() - if err != nil { - return solanaGo.Signature{}, err - } - return client.SendTx(ctx, tx) -} - // Start subscribes to queuing channel and processes them. func (txm *Txm) Start(ctx context.Context) error { return txm.StartOnce("Txm", func() error { @@ -237,7 +234,7 @@ func (txm *Txm) sendWithRetry(ctx context.Context, baseTx solanaGo.Transaction, ctx, cancel := context.WithTimeout(ctx, txcfg.Timeout) // send initial tx (do not retry and exit early if fails) - sig, initSendErr := txm.SendTx(ctx, &initTx) + sig, initSendErr := txm.sendTx(ctx, &initTx) if initSendErr != nil { cancel() // cancel context when exiting early txm.txs.OnError(sig, TxFailReject) // increment failed metric @@ -308,7 +305,7 @@ func (txm *Txm) sendWithRetry(ctx context.Context, baseTx solanaGo.Transaction, go func(bump bool, count int, retryTx solanaGo.Transaction) { defer wg.Done() - retrySig, retrySendErr := txm.SendTx(ctx, &retryTx) + retrySig, retrySendErr := txm.sendTx(ctx, &retryTx) // this could occur if endpoint goes down or if ctx cancelled if retrySendErr != nil { if strings.Contains(retrySendErr.Error(), "context canceled") || strings.Contains(retrySendErr.Error(), "context deadline exceeded") { From c673d2bc354c9d65b8147bfea0dbe6e91343aeb5 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 23 Oct 2024 12:15:25 -0400 Subject: [PATCH 114/174] Fix ctx issues --- pkg/solana/chain.go | 3 ++- pkg/solana/client/multinode/transaction_sender.go | 4 ++-- pkg/solana/client/multinode_client.go | 7 +++++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/pkg/solana/chain.go b/pkg/solana/chain.go index c47e1cf1b..ca635f548 100644 --- a/pkg/solana/chain.go +++ b/pkg/solana/chain.go @@ -292,7 +292,8 @@ func newChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr logger.L // Send tx using MultiNode transaction sender sendTx = func(ctx context.Context, tx *solanago.Transaction) (solanago.Signature, error) { - result := ch.txSender.SendTransaction(ctx, tx) + // Use empty context since individual RPC timeouts are handled by the client + result := ch.txSender.SendTransaction(context.Background(), tx) if result == nil { return solanago.Signature{}, errors.New("tx sender returned nil result") } diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go index a8d8c3418..0fccabd68 100644 --- a/pkg/solana/client/multinode/transaction_sender.go +++ b/pkg/solana/client/multinode/transaction_sender.go @@ -100,6 +100,8 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct return txSender.newResult(errors.New("TransactionSender not started")) } + ctx, _ = txSender.chStop.Ctx(ctx) + healthyNodesNum := 0 err := txSender.multiNode.DoAll(ctx, func(ctx context.Context, rpc RPC, isSendOnly bool) { if isSendOnly { @@ -211,8 +213,6 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) collectTxResults(c if healthyNodesNum == 0 { return txSender.newResult(ErroringNodeError) } - ctx, cancel := txSender.chStop.Ctx(ctx) - defer cancel() requiredResults := int(math.Ceil(float64(healthyNodesNum) * sendTxQuorum)) errorsByCode := sendTxResults[RESULT]{} var softTimeoutChan <-chan time.Time diff --git a/pkg/solana/client/multinode_client.go b/pkg/solana/client/multinode_client.go index 452b1cf3d..0a68b78f6 100644 --- a/pkg/solana/client/multinode_client.go +++ b/pkg/solana/client/multinode_client.go @@ -310,9 +310,12 @@ type SendTxResult struct { var _ mn.SendTxResult = (*SendTxResult)(nil) func NewSendTxResult(err error) *SendTxResult { - return &SendTxResult{ - err: err, + result := &SendTxResult{ + err: err, + txErr: err, } + result.code = ClassifySendError(nil, err) + return result } func (r *SendTxResult) Error() error { From ac74b1c74979e50a4ee87e02f270d070f3a73e15 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 23 Oct 2024 15:42:58 -0400 Subject: [PATCH 115/174] Enable multiple RPCs in soak tests --- integration-tests/common/common.go | 6 +-- integration-tests/common/test_common.go | 10 ++--- integration-tests/config/config.go | 12 ++--- integration-tests/testconfig/testconfig.go | 44 ++++++++++--------- pkg/solana/client/multinode/multi_node.go | 3 ++ .../client/multinode/transaction_sender.go | 3 +- 6 files changed, 43 insertions(+), 35 deletions(-) diff --git a/integration-tests/common/common.go b/integration-tests/common/common.go index c4de45aea..57fb96452 100644 --- a/integration-tests/common/common.go +++ b/integration-tests/common/common.go @@ -51,7 +51,7 @@ type TestEnvDetails struct { type ChainDetails struct { ChainName string ChainID string - RPCUrl string + RPCUrl []string RPCURLExternal string WSURLExternal string ProgramAddresses *chainConfig.ProgramAddresses @@ -116,7 +116,7 @@ func New(testConfig *tc.TestConfig) *Common { config = chainConfig.DevnetConfig() privateKeyString = *testConfig.Common.PrivateKey - if *testConfig.Common.RPCURL != "" { + if len(*testConfig.Common.RPCURL) > 0 { config.RPCUrl = *testConfig.Common.RPCURL config.WSUrl = *testConfig.Common.WsURL config.ProgramAddresses = &chainConfig.ProgramAddresses{ @@ -146,7 +146,7 @@ func New(testConfig *tc.TestConfig) *Common { } // provide getters for TestConfig (pointers to chain details) c.TestConfig.GetChainID = func() string { return c.ChainDetails.ChainID } - c.TestConfig.GetURL = func() string { return c.ChainDetails.RPCUrl } + c.TestConfig.GetURL = func() []string { return c.ChainDetails.RPCUrl } return c } diff --git a/integration-tests/common/test_common.go b/integration-tests/common/test_common.go index a775a5199..c738f632c 100644 --- a/integration-tests/common/test_common.go +++ b/integration-tests/common/test_common.go @@ -119,8 +119,8 @@ func (m *OCRv2TestState) DeployCluster(contractsDir string) { if *m.Config.TestConfig.Common.Network == "devnet" { m.Common.ChainDetails.RPCUrl = *m.Config.TestConfig.Common.RPCURL - m.Common.ChainDetails.RPCURLExternal = *m.Config.TestConfig.Common.RPCURL - m.Common.ChainDetails.WSURLExternal = *m.Config.TestConfig.Common.WsURL + m.Common.ChainDetails.RPCURLExternal = (*m.Config.TestConfig.Common.RPCURL)[0] + m.Common.ChainDetails.WSURLExternal = (*m.Config.TestConfig.Common.WsURL)[0] } m.Common.ChainDetails.MockserverURLInternal = m.Common.Env.URLs["qa_mock_adapter_internal"][0] @@ -133,14 +133,14 @@ func (m *OCRv2TestState) DeployCluster(contractsDir string) { require.NoError(m.Config.T, err) // Setting the External RPC url for Gauntlet - m.Common.ChainDetails.RPCUrl = sol.InternalHTTPURL + m.Common.ChainDetails.RPCUrl = []string{sol.InternalHTTPURL} m.Common.ChainDetails.RPCURLExternal = sol.ExternalHTTPURL m.Common.ChainDetails.WSURLExternal = sol.ExternalWsURL if *m.Config.TestConfig.Common.Network == "devnet" { m.Common.ChainDetails.RPCUrl = *m.Config.TestConfig.Common.RPCURL - m.Common.ChainDetails.RPCURLExternal = *m.Config.TestConfig.Common.RPCURL - m.Common.ChainDetails.WSURLExternal = *m.Config.TestConfig.Common.WsURL + m.Common.ChainDetails.RPCURLExternal = (*m.Config.TestConfig.Common.RPCURL)[0] + m.Common.ChainDetails.WSURLExternal = (*m.Config.TestConfig.Common.WsURL)[0] } b, err := test_env.NewCLTestEnvBuilder(). diff --git a/integration-tests/config/config.go b/integration-tests/config/config.go index 232dfa5d3..303e1a5a0 100644 --- a/integration-tests/config/config.go +++ b/integration-tests/config/config.go @@ -3,8 +3,8 @@ package config type Config struct { ChainName string ChainID string - RPCUrl string - WSUrl string + RPCUrl []string + WSUrl []string ProgramAddresses *ProgramAddresses PrivateKey string } @@ -20,8 +20,8 @@ func DevnetConfig() *Config { ChainName: "solana", ChainID: "devnet", // Will be overridden if set in toml - RPCUrl: "https://api.devnet.solana.com", - WSUrl: "wss://api.devnet.solana.com/", + RPCUrl: []string{"https://api.devnet.solana.com"}, + WSUrl: []string{"wss://api.devnet.solana.com/"}, } } @@ -30,8 +30,8 @@ func LocalNetConfig() *Config { ChainName: "solana", ChainID: "localnet", // Will be overridden if set in toml - RPCUrl: "http://sol:8899", - WSUrl: "ws://sol:8900", + RPCUrl: []string{"http://sol:8899"}, + WSUrl: []string{"ws://sol:8900"}, ProgramAddresses: &ProgramAddresses{ OCR2: "E3j24rx12SyVsG6quKuZPbQqZPkhAUCh8Uek4XrKYD2x", AccessController: "2ckhep7Mvy1dExenBqpcdevhRu7CLuuctMcx7G9mWEvo", diff --git a/integration-tests/testconfig/testconfig.go b/integration-tests/testconfig/testconfig.go index e6b57412a..ea92ebdf5 100644 --- a/integration-tests/testconfig/testconfig.go +++ b/integration-tests/testconfig/testconfig.go @@ -44,7 +44,7 @@ type TestConfig struct { // getter funcs for passing parameters GetChainID func() string - GetURL func() string + GetURL func() []string } const ( @@ -188,8 +188,8 @@ func (c *TestConfig) ReadFromEnvVar() error { c.Network.RpcWsUrls = rpcWsUrls } - commonRPCURL := ctf_config.MustReadEnvVar_String(E2E_TEST_COMMON_RPC_URL_ENV) - if commonRPCURL != "" { + commonRPCURL := ctf_config.MustReadEnvVar_Strings(E2E_TEST_COMMON_RPC_URL_ENV, ",") + if len(commonRPCURL) > 0 { if c.Common == nil { c.Common = &Common{} } @@ -197,8 +197,8 @@ func (c *TestConfig) ReadFromEnvVar() error { c.Common.RPCURL = &commonRPCURL } - commonWSURL := ctf_config.MustReadEnvVar_String(E2E_TEST_COMMON_WS_URL_ENV) - if commonWSURL != "" { + commonWSURL := ctf_config.MustReadEnvVar_Strings(E2E_TEST_COMMON_WS_URL_ENV, ",") + if len(commonWSURL) > 0 { if c.Common == nil { c.Common = &Common{} } @@ -256,7 +256,8 @@ func (c *TestConfig) GetNodeConfig() *ctf_config.NodeConfig { } func (c *TestConfig) GetNodeConfigTOML() (string, error) { - var chainID, url string + var chainID string + var url []string if c.GetChainID != nil { chainID = c.GetChainID() } @@ -271,15 +272,18 @@ func (c *TestConfig) GetNodeConfigTOML() (string, error) { } mnConfig.SetDefaults() + var nodes []*solcfg.Node + for i, u := range url { + nodes = append(nodes, &solcfg.Node{ + Name: ptr.Ptr(fmt.Sprintf("primary-%d", i)), + URL: config.MustParseURL(u), + }) + } + solConfig := solcfg.TOMLConfig{ - Enabled: ptr.Ptr(true), - ChainID: ptr.Ptr(chainID), - Nodes: []*solcfg.Node{ - { - Name: ptr.Ptr("primary"), - URL: config.MustParseURL(url), - }, - }, + Enabled: ptr.Ptr(true), + ChainID: ptr.Ptr(chainID), + Nodes: nodes, MultiNode: mnConfig, } baseConfig := node.NewBaseConfig() @@ -365,12 +369,12 @@ type Common struct { InsideK8s *bool `toml:"inside_k8"` User *string `toml:"user"` // if rpc requires api key to be passed as an HTTP header - RPCURL *string `toml:"-"` - WsURL *string `toml:"-"` - PrivateKey *string `toml:"-"` - Stateful *bool `toml:"stateful_db"` - InternalDockerRepo *string `toml:"internal_docker_repo"` - DevnetImage *string `toml:"devnet_image"` + RPCURL *[]string `toml:"-"` + WsURL *[]string `toml:"-"` + PrivateKey *string `toml:"-"` + Stateful *bool `toml:"stateful_db"` + InternalDockerRepo *string `toml:"internal_docker_repo"` + DevnetImage *string `toml:"devnet_image"` } type SolanaConfig struct { diff --git a/pkg/solana/client/multinode/multi_node.go b/pkg/solana/client/multinode/multi_node.go index 8b7efc46b..59fc9c9f5 100644 --- a/pkg/solana/client/multinode/multi_node.go +++ b/pkg/solana/client/multinode/multi_node.go @@ -97,14 +97,17 @@ func (c *MultiNode[CHAIN_ID, RPC]) DoAll(baseCtx context.Context, do func(ctx co callsCompleted := 0 for _, n := range c.primaryNodes { + c.lggr.Debugw("DoAll", "node", n.Name()) select { case <-ctx.Done(): err = ctx.Err() return default: if n.State() != NodeStateAlive { + c.lggr.Debugw("DoAll: Node is not alive", "node", n.Name()) continue } + c.lggr.Debugw("DoAll: Calling do on primary node", "node", n.Name()) do(ctx, n.RPC(), false) callsCompleted++ } diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go index 0fccabd68..9e5f0e4e7 100644 --- a/pkg/solana/client/multinode/transaction_sender.go +++ b/pkg/solana/client/multinode/transaction_sender.go @@ -100,7 +100,8 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct return txSender.newResult(errors.New("TransactionSender not started")) } - ctx, _ = txSender.chStop.Ctx(ctx) + ctx, cancel := txSender.chStop.Ctx(ctx) + defer cancel() healthyNodesNum := 0 err := txSender.multiNode.DoAll(ctx, func(ctx context.Context, rpc RPC, isSendOnly bool) { From b23a62c53c67ea8146061a0b564dba3338043eef Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 23 Oct 2024 18:20:53 -0400 Subject: [PATCH 116/174] Update defaults for testing --- pkg/solana/client/multinode/multi_node.go | 3 +-- pkg/solana/config/multinode.go | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/pkg/solana/client/multinode/multi_node.go b/pkg/solana/client/multinode/multi_node.go index 59fc9c9f5..596130f93 100644 --- a/pkg/solana/client/multinode/multi_node.go +++ b/pkg/solana/client/multinode/multi_node.go @@ -97,14 +97,13 @@ func (c *MultiNode[CHAIN_ID, RPC]) DoAll(baseCtx context.Context, do func(ctx co callsCompleted := 0 for _, n := range c.primaryNodes { - c.lggr.Debugw("DoAll", "node", n.Name()) select { case <-ctx.Done(): err = ctx.Err() return default: if n.State() != NodeStateAlive { - c.lggr.Debugw("DoAll: Node is not alive", "node", n.Name()) + c.lggr.Warnw("DoAll: Node is not alive", "node", n.Name()) continue } c.lggr.Debugw("DoAll: Calling do on primary node", "node", n.Name()) diff --git a/pkg/solana/config/multinode.go b/pkg/solana/config/multinode.go index 0c49d8b22..4e08f7d42 100644 --- a/pkg/solana/config/multinode.go +++ b/pkg/solana/config/multinode.go @@ -95,11 +95,11 @@ func (c *MultiNodeConfig) SetDefaults() { /* Node Configs */ // Failure threshold for polling set to 5 to tolerate some polling failures before taking action. if c.MultiNode.PollFailureThreshold == nil { - c.MultiNode.PollFailureThreshold = ptr(uint32(5)) + c.MultiNode.PollFailureThreshold = ptr(uint32(8)) } // Poll interval is set to 10 seconds to ensure timely updates while minimizing resource usage. if c.MultiNode.PollInterval == nil { - c.MultiNode.PollInterval = config.MustNewDuration(10 * time.Second) + c.MultiNode.PollInterval = config.MustNewDuration(15 * time.Second) } // Selection mode defaults to priority level to enable using node priorities if c.MultiNode.SelectionMode == nil { @@ -107,7 +107,7 @@ func (c *MultiNodeConfig) SetDefaults() { } // The sync threshold is set to 5 to allow for some flexibility in node synchronization before considering it out of sync. if c.MultiNode.SyncThreshold == nil { - c.MultiNode.SyncThreshold = ptr(uint32(5)) + c.MultiNode.SyncThreshold = ptr(uint32(10)) } // Lease duration is set to 1 minute by default to allow node locks for a reasonable amount of time. if c.MultiNode.LeaseDuration == nil { @@ -119,25 +119,25 @@ func (c *MultiNodeConfig) SetDefaults() { } // The finalized block polling interval is set to 5 seconds to ensure timely updates while minimizing resource usage. if c.MultiNode.FinalizedBlockPollInterval == nil { - c.MultiNode.FinalizedBlockPollInterval = config.MustNewDuration(5 * time.Second) + c.MultiNode.FinalizedBlockPollInterval = config.MustNewDuration(15 * time.Second) } // Repeatable read guarantee should be enforced by default. if c.MultiNode.EnforceRepeatableRead == nil { c.MultiNode.EnforceRepeatableRead = ptr(true) } - // The delay before declaring a node dead is set to 10 seconds to give nodes time to recover from temporary issues. + // The delay before declaring a node dead is set to 20 seconds to give nodes time to recover from temporary issues. if c.MultiNode.DeathDeclarationDelay == nil { - c.MultiNode.DeathDeclarationDelay = config.MustNewDuration(10 * time.Second) + c.MultiNode.DeathDeclarationDelay = config.MustNewDuration(45 * time.Second) } /* Chain Configs */ - // Threshold for no new heads is set to 10 seconds, assuming that heads should update at a reasonable pace. + // Threshold for no new heads is set to 20 seconds, assuming that heads should update at a reasonable pace. if c.MultiNode.NodeNoNewHeadsThreshold == nil { - c.MultiNode.NodeNoNewHeadsThreshold = config.MustNewDuration(10 * time.Second) + c.MultiNode.NodeNoNewHeadsThreshold = config.MustNewDuration(45 * time.Second) } - // Similar to heads, finalized heads should be updated within 10 seconds. + // Similar to heads, finalized heads should be updated within 20 seconds. if c.MultiNode.NoNewFinalizedHeadsThreshold == nil { - c.MultiNode.NoNewFinalizedHeadsThreshold = config.MustNewDuration(10 * time.Second) + c.MultiNode.NoNewFinalizedHeadsThreshold = config.MustNewDuration(45 * time.Second) } // Finality tags are used in Solana and enabled by default. if c.MultiNode.FinalityTagEnabled == nil { From 4b86236de8a5379a106ec2c959149f8440024f21 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 24 Oct 2024 10:39:21 -0400 Subject: [PATCH 117/174] Add health check tags --- pkg/solana/client/multinode/node_fsm.go | 1 + pkg/solana/client/multinode_client.go | 15 +++++++++++++-- pkg/solana/config/multinode.go | 4 ++-- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/pkg/solana/client/multinode/node_fsm.go b/pkg/solana/client/multinode/node_fsm.go index 136910868..e44cdbb19 100644 --- a/pkg/solana/client/multinode/node_fsm.go +++ b/pkg/solana/client/multinode/node_fsm.go @@ -153,6 +153,7 @@ func (n *node[CHAIN_ID, HEAD, RPC]) isFinalizedBlockOutOfSync() bool { return latest.FinalizedBlockNumber < highestObservedByCaller.FinalizedBlockNumber-int64(n.chainCfg.FinalizedBlockOffset()) } + // TODO: Should this be using finality depth instead of non finalized blocks? return latest.BlockNumber < highestObservedByCaller.BlockNumber-int64(n.chainCfg.FinalizedBlockOffset()) } diff --git a/pkg/solana/client/multinode_client.go b/pkg/solana/client/multinode_client.go index 0a68b78f6..7e5de1a4b 100644 --- a/pkg/solana/client/multinode_client.go +++ b/pkg/solana/client/multinode_client.go @@ -107,7 +107,13 @@ func (m *MultiNodeClient) SubscribeToHeads(ctx context.Context) (<-chan *Head, m return nil, nil, errors.New("PollInterval is 0") } timeout := pollInterval - poller, channel := mn.NewPoller[*Head](pollInterval, m.LatestBlock, timeout, m.log) + poller, channel := mn.NewPoller[*Head](pollInterval, func(pollRequestCtx context.Context) (*Head, error) { + if mn.CtxIsHeathCheckRequest(ctx) { + pollRequestCtx = mn.CtxAddHealthCheckFlag(pollRequestCtx) + } + return m.LatestBlock(pollRequestCtx) + }, timeout, m.log) + if err := poller.Start(ctx); err != nil { return nil, nil, err } @@ -130,7 +136,12 @@ func (m *MultiNodeClient) SubscribeToFinalizedHeads(ctx context.Context) (<-chan return nil, nil, errors.New("FinalizedBlockPollInterval is 0") } timeout := finalizedBlockPollInterval - poller, channel := mn.NewPoller[*Head](finalizedBlockPollInterval, m.LatestFinalizedBlock, timeout, m.log) + poller, channel := mn.NewPoller[*Head](finalizedBlockPollInterval, func(pollRequestCtx context.Context) (*Head, error) { + if mn.CtxIsHeathCheckRequest(ctx) { + pollRequestCtx = mn.CtxAddHealthCheckFlag(pollRequestCtx) + } + return m.LatestFinalizedBlock(pollRequestCtx) + }, timeout, m.log) if err := poller.Start(ctx); err != nil { return nil, nil, err } diff --git a/pkg/solana/config/multinode.go b/pkg/solana/config/multinode.go index 4e08f7d42..4685358bb 100644 --- a/pkg/solana/config/multinode.go +++ b/pkg/solana/config/multinode.go @@ -147,9 +147,9 @@ func (c *MultiNodeConfig) SetDefaults() { if c.MultiNode.FinalityDepth == nil { c.MultiNode.FinalityDepth = ptr(uint32(0)) } - // Finalized block offset will not be used since finality tags are enabled. + // Finalized block offset allows for RPCs to be slightly behind the finalized block. if c.MultiNode.FinalizedBlockOffset == nil { - c.MultiNode.FinalizedBlockOffset = ptr(uint32(0)) + c.MultiNode.FinalizedBlockOffset = ptr(uint32(5)) } } From fc4455b499512e550383581e367f624b2e7fee1f Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 24 Oct 2024 12:10:19 -0400 Subject: [PATCH 118/174] Increase sync threshold --- pkg/solana/config/multinode.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/solana/config/multinode.go b/pkg/solana/config/multinode.go index 4685358bb..331da44e7 100644 --- a/pkg/solana/config/multinode.go +++ b/pkg/solana/config/multinode.go @@ -107,7 +107,7 @@ func (c *MultiNodeConfig) SetDefaults() { } // The sync threshold is set to 5 to allow for some flexibility in node synchronization before considering it out of sync. if c.MultiNode.SyncThreshold == nil { - c.MultiNode.SyncThreshold = ptr(uint32(10)) + c.MultiNode.SyncThreshold = ptr(uint32(50)) // TODO: Increased to 50 for slow test environment } // Lease duration is set to 1 minute by default to allow node locks for a reasonable amount of time. if c.MultiNode.LeaseDuration == nil { @@ -149,7 +149,7 @@ func (c *MultiNodeConfig) SetDefaults() { } // Finalized block offset allows for RPCs to be slightly behind the finalized block. if c.MultiNode.FinalizedBlockOffset == nil { - c.MultiNode.FinalizedBlockOffset = ptr(uint32(5)) + c.MultiNode.FinalizedBlockOffset = ptr(uint32(50)) // TODO: Set to 50 for slow test environment } } From 71dd2942312ebd07feb70828198a0752ea522662 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 24 Oct 2024 12:50:36 -0400 Subject: [PATCH 119/174] Validate heads --- pkg/solana/client/multinode/node_lifecycle.go | 3 ++- pkg/solana/client/multinode_client.go | 10 +++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/pkg/solana/client/multinode/node_lifecycle.go b/pkg/solana/client/multinode/node_lifecycle.go index d6b150690..6e3d75af6 100644 --- a/pkg/solana/client/multinode/node_lifecycle.go +++ b/pkg/solana/client/multinode/node_lifecycle.go @@ -167,7 +167,8 @@ func (n *node[CHAIN_ID, HEAD, RPC]) aliveLoop() { _, latestChainInfo := n.StateAndLatest() if outOfSync, liveNodes := n.isOutOfSyncWithPool(latestChainInfo); outOfSync { // note: there must be another live node for us to be out of sync - lggr.Errorw("RPC endpoint has fallen behind", "blockNumber", latestChainInfo.BlockNumber, "totalDifficulty", latestChainInfo.TotalDifficulty, "nodeState", n.getCachedState()) + highest := n.HighestUserObservations() + lggr.Errorw("RPC endpoint has fallen behind", "blockNumber", latestChainInfo.BlockNumber, "highestBlockNumber", highest.BlockNumber, "totalDifficulty", latestChainInfo.TotalDifficulty, "nodeState", n.getCachedState()) if liveNodes < 2 { lggr.Criticalf("RPC endpoint has fallen behind; %s %s", msgCannotDisable, msgDegradedState) continue diff --git a/pkg/solana/client/multinode_client.go b/pkg/solana/client/multinode_client.go index 7e5de1a4b..c677f7547 100644 --- a/pkg/solana/client/multinode_client.go +++ b/pkg/solana/client/multinode_client.go @@ -37,7 +37,7 @@ func (h *Head) BlockDifficulty() *big.Int { } func (h *Head) IsValid() bool { - return h != nil && h.BlockHeight != nil && h.BlockHash != nil + return h != nil && h.BlockHeight != nil && *h.BlockHeight > 0 && h.BlockHash != nil } var _ mn.RPCClient[mn.StringID, *Head] = (*MultiNodeClient)(nil) @@ -169,6 +169,10 @@ func (m *MultiNodeClient) LatestBlock(ctx context.Context) (*Head, error) { BlockHeight: &result.Value.LastValidBlockHeight, BlockHash: &result.Value.Blockhash, } + if !head.IsValid() { + return nil, errors.New("invalid head") + } + m.onNewHead(ctx, chStopInFlight, head) return head, nil } @@ -186,6 +190,10 @@ func (m *MultiNodeClient) LatestFinalizedBlock(ctx context.Context) (*Head, erro BlockHeight: &result.Value.LastValidBlockHeight, BlockHash: &result.Value.Blockhash, } + if !head.IsValid() { + return nil, errors.New("invalid head") + } + m.onNewFinalizedHead(ctx, chStopInFlight, head) return head, nil } From 8c3ba8827bf02bdc8d3b804737c6d08614564420 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 25 Oct 2024 11:40:48 -0400 Subject: [PATCH 120/174] Use latestChainInfo --- integration-tests/smoke/run_soak_test.sh | 58 +++++++++++++++++++ pkg/solana/client/multinode/node_lifecycle.go | 27 +++++---- 2 files changed, 74 insertions(+), 11 deletions(-) create mode 100755 integration-tests/smoke/run_soak_test.sh diff --git a/integration-tests/smoke/run_soak_test.sh b/integration-tests/smoke/run_soak_test.sh new file mode 100755 index 000000000..9485fdf17 --- /dev/null +++ b/integration-tests/smoke/run_soak_test.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +NODE_VERSION=18 + +echo "Switching to required Node.js version $NODE_VERSION..." +export NVM_DIR="$HOME/.nvm" +[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" +nvm use $NODE_VERSION + +echo "Initializing soak test..." +terminated_by_script=false +while IFS= read -r line; do + echo "$line" + # Check if the line contains the target string + if echo "$line" | grep -q "ocr2:inspect:responses"; then + # Send SIGINT (Ctrl+C) to the 'go test' process + sudo pkill -INT -P $$ go 2>/dev/null + terminated_by_script=true + break + fi +done < <(sudo go test -timeout 24h -count=1 -run TestSolanaOCRV2Smoke/embedded -test.timeout 30m 2>&1) + + +# Capture the PID of the background process +READER_PID=$! + +# Start a background timer (sleeps for 15 minutes, then sends SIGALRM to the script) +( sleep 900 && kill -s ALRM $$ ) & +TIMER_PID=$! + +# Set a trap to catch the SIGALRM signal for timeout +trap 'on_timeout' ALRM + +# Function to handle timeout +on_timeout() { + echo "Error: failed to start soak test: timeout exceeded (15 minutes)." + # Send SIGINT to the 'go test' process + pkill -INT -P $$ go 2>/dev/null + # Clean up + kill "$TIMER_PID" 2>/dev/null + kill "$READER_PID" 2>/dev/null + exit 1 +} + +# Wait for the reader process to finish +wait "$READER_PID" +EXIT_STATUS=$? + +# Clean up: kill the timer process if it's still running +kill "$TIMER_PID" 2>/dev/null + +if [ "$terminated_by_script" = true ]; then + echo "Soak test started successfully" + exit 0 +else + echo "Soak test failed to start" + exit 1 +fi diff --git a/pkg/solana/client/multinode/node_lifecycle.go b/pkg/solana/client/multinode/node_lifecycle.go index 6e3d75af6..ede6d1f0a 100644 --- a/pkg/solana/client/multinode/node_lifecycle.go +++ b/pkg/solana/client/multinode/node_lifecycle.go @@ -128,7 +128,6 @@ func (n *node[CHAIN_ID, HEAD, RPC]) aliveLoop() { } } - localHighestChainInfo, _ := n.rpc.GetInterceptedChainInfo() var pollFailures uint32 for { @@ -167,8 +166,8 @@ func (n *node[CHAIN_ID, HEAD, RPC]) aliveLoop() { _, latestChainInfo := n.StateAndLatest() if outOfSync, liveNodes := n.isOutOfSyncWithPool(latestChainInfo); outOfSync { // note: there must be another live node for us to be out of sync - highest := n.HighestUserObservations() - lggr.Errorw("RPC endpoint has fallen behind", "blockNumber", latestChainInfo.BlockNumber, "highestBlockNumber", highest.BlockNumber, "totalDifficulty", latestChainInfo.TotalDifficulty, "nodeState", n.getCachedState()) + _, highest := n.poolInfoProvider.LatestChainInfo() + lggr.Errorw("RPC endpoint has fallen behind", "blockNumber", latestChainInfo.BlockNumber, "bestLatestBlockNumber", highest.BlockNumber, "totalDifficulty", latestChainInfo.TotalDifficulty, "nodeState", n.getCachedState()) if liveNodes < 2 { lggr.Criticalf("RPC endpoint has fallen behind; %s %s", msgCannotDisable, msgDegradedState) continue @@ -182,7 +181,9 @@ func (n *node[CHAIN_ID, HEAD, RPC]) aliveLoop() { n.declareUnreachable() return } - receivedNewHead := n.onNewHead(lggr, &localHighestChainInfo, bh) + + _, latestChainInfo := n.StateAndLatest() + receivedNewHead := n.onNewHead(lggr, &latestChainInfo, bh) if receivedNewHead && noNewHeadsTimeoutThreshold > 0 { headsSub.ResetTimer(noNewHeadsTimeoutThreshold) } @@ -193,7 +194,8 @@ func (n *node[CHAIN_ID, HEAD, RPC]) aliveLoop() { case <-headsSub.NoNewHeads: // We haven't received a head on the channel for at least the // threshold amount of time, mark it broken - lggr.Errorw(fmt.Sprintf("RPC endpoint detected out of sync; no new heads received for %s (last head received was %v)", noNewHeadsTimeoutThreshold, localHighestChainInfo.BlockNumber), "nodeState", n.getCachedState(), "latestReceivedBlockNumber", localHighestChainInfo.BlockNumber, "noNewHeadsTimeoutThreshold", noNewHeadsTimeoutThreshold) + _, latestChainInfo := n.StateAndLatest() + lggr.Errorw(fmt.Sprintf("RPC endpoint detected out of sync; no new heads received for %s (last head received was %v)", noNewHeadsTimeoutThreshold, latestChainInfo.BlockNumber), "nodeState", n.getCachedState(), "latestReceivedBlockNumber", latestChainInfo.BlockNumber, "noNewHeadsTimeoutThreshold", noNewHeadsTimeoutThreshold) if n.poolInfoProvider != nil { if l, _ := n.poolInfoProvider.LatestChainInfo(); l < 2 { lggr.Criticalf("RPC endpoint detected out of sync; %s %s", msgCannotDisable, msgDegradedState) @@ -212,14 +214,16 @@ func (n *node[CHAIN_ID, HEAD, RPC]) aliveLoop() { return } - receivedNewHead := n.onNewFinalizedHead(lggr, &localHighestChainInfo, latestFinalized) + _, latestChainInfo := n.StateAndLatest() + receivedNewHead := n.onNewFinalizedHead(lggr, &latestChainInfo, latestFinalized) if receivedNewHead && noNewFinalizedBlocksTimeoutThreshold > 0 { finalizedHeadsSub.ResetTimer(noNewFinalizedBlocksTimeoutThreshold) } case <-finalizedHeadsSub.NoNewHeads: // We haven't received a finalized head on the channel for at least the // threshold amount of time, mark it broken - lggr.Errorw(fmt.Sprintf("RPC's finalized state is out of sync; no new finalized heads received for %s (last finalized head received was %v)", noNewFinalizedBlocksTimeoutThreshold, localHighestChainInfo.FinalizedBlockNumber), "latestReceivedBlockNumber", localHighestChainInfo.BlockNumber) + _, latestChainInfo := n.StateAndLatest() + lggr.Errorw(fmt.Sprintf("RPC's finalized state is out of sync; no new finalized heads received for %s (last finalized head received was %v)", noNewFinalizedBlocksTimeoutThreshold, latestChainInfo.FinalizedBlockNumber), "latestReceivedBlockNumber", latestChainInfo.BlockNumber) if n.poolInfoProvider != nil { if l, _ := n.poolInfoProvider.LatestChainInfo(); l < 2 { lggr.Criticalf("RPC's finalized state is out of sync; %s %s", msgCannotDisable, msgDegradedState) @@ -436,7 +440,6 @@ func (n *node[CHAIN_ID, HEAD, RPC]) outOfSyncLoop(syncIssues syncStatus) { lggr.Tracew("Successfully subscribed to finalized heads feed on out-of-sync RPC node") } - _, localHighestChainInfo := n.rpc.GetInterceptedChainInfo() for { if syncIssues == syncStatusSynced { // back in-sync! flip back into alive loop @@ -455,13 +458,14 @@ func (n *node[CHAIN_ID, HEAD, RPC]) outOfSyncLoop(syncIssues syncStatus) { return } - if !n.onNewHead(lggr, &localHighestChainInfo, head) { + _, latestChainInfo := n.StateAndLatest() + if !n.onNewHead(lggr, &latestChainInfo, head) { continue } // received a new head - clear NoNewHead flag syncIssues &= ^syncStatusNoNewHead - if outOfSync, _ := n.isOutOfSyncWithPool(localHighestChainInfo); !outOfSync { + if outOfSync, _ := n.isOutOfSyncWithPool(latestChainInfo); !outOfSync { // we caught up with the pool - clear NotInSyncWithPool flag syncIssues &= ^syncStatusNotInSyncWithPool } else { @@ -501,7 +505,8 @@ func (n *node[CHAIN_ID, HEAD, RPC]) outOfSyncLoop(syncIssues syncStatus) { continue } - receivedNewHead := n.onNewFinalizedHead(lggr, &localHighestChainInfo, latestFinalized) + _, latestChainInfo := n.StateAndLatest() + receivedNewHead := n.onNewFinalizedHead(lggr, &latestChainInfo, latestFinalized) if !receivedNewHead { continue } From 45b79ab2d0aefdb5b03008de0542a79c56bd9d9d Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 25 Oct 2024 14:29:18 -0400 Subject: [PATCH 121/174] Fix AliveLoop bug --- pkg/solana/client/multinode/node_lifecycle.go | 48 +++++++++---------- pkg/solana/client/multinode/poller.go | 2 +- pkg/solana/config/multinode.go | 4 +- 3 files changed, 27 insertions(+), 27 deletions(-) diff --git a/pkg/solana/client/multinode/node_lifecycle.go b/pkg/solana/client/multinode/node_lifecycle.go index ede6d1f0a..7050ec78d 100644 --- a/pkg/solana/client/multinode/node_lifecycle.go +++ b/pkg/solana/client/multinode/node_lifecycle.go @@ -128,6 +128,8 @@ func (n *node[CHAIN_ID, HEAD, RPC]) aliveLoop() { } } + // Get the latest chain info to use as local highest + localHighestChainInfo, _ := n.rpc.GetInterceptedChainInfo() var pollFailures uint32 for { @@ -163,6 +165,16 @@ func (n *node[CHAIN_ID, HEAD, RPC]) aliveLoop() { n.declareUnreachable() return } + case bh, open := <-headsSub.Heads: + if !open { + lggr.Errorw("Subscription channel unexpectedly closed", "nodeState", n.getCachedState()) + n.declareUnreachable() + return + } + receivedNewHead := n.onNewHead(lggr, &localHighestChainInfo, bh) + if receivedNewHead && noNewHeadsTimeoutThreshold > 0 { + headsSub.ResetTimer(noNewHeadsTimeoutThreshold) + } _, latestChainInfo := n.StateAndLatest() if outOfSync, liveNodes := n.isOutOfSyncWithPool(latestChainInfo); outOfSync { // note: there must be another live node for us to be out of sync @@ -175,18 +187,6 @@ func (n *node[CHAIN_ID, HEAD, RPC]) aliveLoop() { n.declareOutOfSync(syncStatusNotInSyncWithPool) return } - case bh, open := <-headsSub.Heads: - if !open { - lggr.Errorw("Subscription channel unexpectedly closed", "nodeState", n.getCachedState()) - n.declareUnreachable() - return - } - - _, latestChainInfo := n.StateAndLatest() - receivedNewHead := n.onNewHead(lggr, &latestChainInfo, bh) - if receivedNewHead && noNewHeadsTimeoutThreshold > 0 { - headsSub.ResetTimer(noNewHeadsTimeoutThreshold) - } case err = <-headsSub.Errors: lggr.Errorw("Subscription was terminated", "err", err, "nodeState", n.getCachedState()) n.declareUnreachable() @@ -214,16 +214,14 @@ func (n *node[CHAIN_ID, HEAD, RPC]) aliveLoop() { return } - _, latestChainInfo := n.StateAndLatest() - receivedNewHead := n.onNewFinalizedHead(lggr, &latestChainInfo, latestFinalized) + receivedNewHead := n.onNewFinalizedHead(lggr, &localHighestChainInfo, latestFinalized) if receivedNewHead && noNewFinalizedBlocksTimeoutThreshold > 0 { finalizedHeadsSub.ResetTimer(noNewFinalizedBlocksTimeoutThreshold) } case <-finalizedHeadsSub.NoNewHeads: // We haven't received a finalized head on the channel for at least the // threshold amount of time, mark it broken - _, latestChainInfo := n.StateAndLatest() - lggr.Errorw(fmt.Sprintf("RPC's finalized state is out of sync; no new finalized heads received for %s (last finalized head received was %v)", noNewFinalizedBlocksTimeoutThreshold, latestChainInfo.FinalizedBlockNumber), "latestReceivedBlockNumber", latestChainInfo.BlockNumber) + lggr.Errorw(fmt.Sprintf("RPC's finalized state is out of sync; no new finalized heads received for %s (last finalized head received was %v)", noNewFinalizedBlocksTimeoutThreshold, localHighestChainInfo.FinalizedBlockNumber), "latestReceivedBlockNumber", localHighestChainInfo.BlockNumber) if n.poolInfoProvider != nil { if l, _ := n.poolInfoProvider.LatestChainInfo(); l < 2 { lggr.Criticalf("RPC's finalized state is out of sync; %s %s", msgCannotDisable, msgDegradedState) @@ -311,9 +309,9 @@ func (n *node[CHAIN_ID, HEAD, RPC]) onNewFinalizedHead(lggr logger.SugaredLogger } latestFinalizedBN := latestFinalized.BlockNumber() - lggr.Tracew("Got latest finalized head", "latestFinalized", latestFinalized) + lggr.Debugw("Got latest finalized head", "latestFinalized", latestFinalized) if latestFinalizedBN <= chainInfo.FinalizedBlockNumber { - lggr.Tracew("Ignoring previously seen finalized block number") + lggr.Debugw("Ignoring previously seen finalized block number") return false } @@ -329,10 +327,10 @@ func (n *node[CHAIN_ID, HEAD, RPC]) onNewHead(lggr logger.SugaredLogger, chainIn } promPoolRPCNodeNumSeenBlocks.WithLabelValues(n.chainID.String(), n.name).Inc() - lggr.Tracew("Got head", "head", head) + lggr.Debugw("Got head", "head", head) lggr = lggr.With("latestReceivedBlockNumber", chainInfo.BlockNumber, "blockNumber", head.BlockNumber(), "nodeState", n.getCachedState()) if head.BlockNumber() <= chainInfo.BlockNumber { - lggr.Tracew("Ignoring previously seen block number") + lggr.Debugw("Ignoring previously seen block number") return false } @@ -440,6 +438,7 @@ func (n *node[CHAIN_ID, HEAD, RPC]) outOfSyncLoop(syncIssues syncStatus) { lggr.Tracew("Successfully subscribed to finalized heads feed on out-of-sync RPC node") } + _, localHighestChainInfo := n.rpc.GetInterceptedChainInfo() for { if syncIssues == syncStatusSynced { // back in-sync! flip back into alive loop @@ -458,14 +457,13 @@ func (n *node[CHAIN_ID, HEAD, RPC]) outOfSyncLoop(syncIssues syncStatus) { return } - _, latestChainInfo := n.StateAndLatest() - if !n.onNewHead(lggr, &latestChainInfo, head) { + if !n.onNewHead(lggr, &localHighestChainInfo, head) { continue } // received a new head - clear NoNewHead flag syncIssues &= ^syncStatusNoNewHead - if outOfSync, _ := n.isOutOfSyncWithPool(latestChainInfo); !outOfSync { + if outOfSync, _ := n.isOutOfSyncWithPool(localHighestChainInfo); !outOfSync { // we caught up with the pool - clear NotInSyncWithPool flag syncIssues &= ^syncStatusNotInSyncWithPool } else { @@ -517,7 +515,9 @@ func (n *node[CHAIN_ID, HEAD, RPC]) outOfSyncLoop(syncIssues syncStatus) { finalizedHeadsSub.ResetTimer(noNewFinalizedBlocksTimeoutThreshold) } - lggr.Debugw(msgReceivedFinalizedBlock, "blockNumber", latestFinalized.BlockNumber(), "syncIssues", syncIssues) + _, highestSeen := n.poolInfoProvider.LatestChainInfo() + + lggr.Debugw(msgReceivedFinalizedBlock, "blockNumber", latestFinalized.BlockNumber(), "highestBlockNumber", highestSeen.FinalizedBlockNumber, "syncIssues", syncIssues) case err := <-finalizedHeadsSub.Errors: lggr.Errorw("Finalized head subscription was terminated", "err", err) n.declareUnreachable() diff --git a/pkg/solana/client/multinode/poller.go b/pkg/solana/client/multinode/poller.go index 9ebe1dcfc..ae2dd9640 100644 --- a/pkg/solana/client/multinode/poller.go +++ b/pkg/solana/client/multinode/poller.go @@ -74,7 +74,7 @@ func (p *Poller[T]) pollingLoop(ctx context.Context) { return case <-ticker.C: // Set polling timeout - pollingCtx, cancelPolling := context.WithTimeout(ctx, p.pollingTimeout) + pollingCtx, cancelPolling := context.WithTimeout(context.Background(), p.pollingTimeout) // Execute polling function result, err := p.pollingFunc(pollingCtx) cancelPolling() diff --git a/pkg/solana/config/multinode.go b/pkg/solana/config/multinode.go index 331da44e7..fe231f9f2 100644 --- a/pkg/solana/config/multinode.go +++ b/pkg/solana/config/multinode.go @@ -99,7 +99,7 @@ func (c *MultiNodeConfig) SetDefaults() { } // Poll interval is set to 10 seconds to ensure timely updates while minimizing resource usage. if c.MultiNode.PollInterval == nil { - c.MultiNode.PollInterval = config.MustNewDuration(15 * time.Second) + c.MultiNode.PollInterval = config.MustNewDuration(10 * time.Second) } // Selection mode defaults to priority level to enable using node priorities if c.MultiNode.SelectionMode == nil { @@ -107,7 +107,7 @@ func (c *MultiNodeConfig) SetDefaults() { } // The sync threshold is set to 5 to allow for some flexibility in node synchronization before considering it out of sync. if c.MultiNode.SyncThreshold == nil { - c.MultiNode.SyncThreshold = ptr(uint32(50)) // TODO: Increased to 50 for slow test environment + c.MultiNode.SyncThreshold = ptr(uint32(160)) // TODO: Increased to 200 for slow test environment RPCs that are always behind } // Lease duration is set to 1 minute by default to allow node locks for a reasonable amount of time. if c.MultiNode.LeaseDuration == nil { From f2e68b0d5f128f0eb68a3d2f2c8548ea429cc340 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 25 Oct 2024 15:33:08 -0400 Subject: [PATCH 122/174] Update configurations --- pkg/solana/client/multinode/node_lifecycle.go | 20 +++++++++---------- pkg/solana/client/multinode/poller.go | 3 ++- pkg/solana/client/multinode_client.go | 2 +- pkg/solana/config/multinode.go | 6 +++--- 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/pkg/solana/client/multinode/node_lifecycle.go b/pkg/solana/client/multinode/node_lifecycle.go index 7050ec78d..88c6e83e4 100644 --- a/pkg/solana/client/multinode/node_lifecycle.go +++ b/pkg/solana/client/multinode/node_lifecycle.go @@ -165,16 +165,6 @@ func (n *node[CHAIN_ID, HEAD, RPC]) aliveLoop() { n.declareUnreachable() return } - case bh, open := <-headsSub.Heads: - if !open { - lggr.Errorw("Subscription channel unexpectedly closed", "nodeState", n.getCachedState()) - n.declareUnreachable() - return - } - receivedNewHead := n.onNewHead(lggr, &localHighestChainInfo, bh) - if receivedNewHead && noNewHeadsTimeoutThreshold > 0 { - headsSub.ResetTimer(noNewHeadsTimeoutThreshold) - } _, latestChainInfo := n.StateAndLatest() if outOfSync, liveNodes := n.isOutOfSyncWithPool(latestChainInfo); outOfSync { // note: there must be another live node for us to be out of sync @@ -187,6 +177,16 @@ func (n *node[CHAIN_ID, HEAD, RPC]) aliveLoop() { n.declareOutOfSync(syncStatusNotInSyncWithPool) return } + case bh, open := <-headsSub.Heads: + if !open { + lggr.Errorw("Subscription channel unexpectedly closed", "nodeState", n.getCachedState()) + n.declareUnreachable() + return + } + receivedNewHead := n.onNewHead(lggr, &localHighestChainInfo, bh) + if receivedNewHead && noNewHeadsTimeoutThreshold > 0 { + headsSub.ResetTimer(noNewHeadsTimeoutThreshold) + } case err = <-headsSub.Errors: lggr.Errorw("Subscription was terminated", "err", err, "nodeState", n.getCachedState()) n.declareUnreachable() diff --git a/pkg/solana/client/multinode/poller.go b/pkg/solana/client/multinode/poller.go index ae2dd9640..f17d81458 100644 --- a/pkg/solana/client/multinode/poller.go +++ b/pkg/solana/client/multinode/poller.go @@ -65,7 +65,8 @@ func (p *Poller[T]) Err() <-chan error { } func (p *Poller[T]) pollingLoop(ctx context.Context) { - ticker := time.NewTicker(p.pollingInterval) + tickerCfg := services.TickerConfig{Initial: 0, JitterPct: services.DefaultJitter} + ticker := tickerCfg.NewTicker(p.pollingInterval) defer ticker.Stop() for { diff --git a/pkg/solana/client/multinode_client.go b/pkg/solana/client/multinode_client.go index c677f7547..6280cd65c 100644 --- a/pkg/solana/client/multinode_client.go +++ b/pkg/solana/client/multinode_client.go @@ -102,7 +102,7 @@ func (m *MultiNodeClient) SubscribeToHeads(ctx context.Context) (<-chan *Head, m ctx, cancel, chStopInFlight, _ := m.acquireQueryCtx(ctx, m.cfg.TxTimeout()) defer cancel() - pollInterval := m.cfg.MultiNode.PollInterval() + pollInterval := m.cfg.MultiNode.FinalizedBlockPollInterval() // TODO: Should have HeadPollInterval separate from Version Poll interval if pollInterval == 0 { return nil, nil, errors.New("PollInterval is 0") } diff --git a/pkg/solana/config/multinode.go b/pkg/solana/config/multinode.go index fe231f9f2..00b82f9cd 100644 --- a/pkg/solana/config/multinode.go +++ b/pkg/solana/config/multinode.go @@ -99,7 +99,7 @@ func (c *MultiNodeConfig) SetDefaults() { } // Poll interval is set to 10 seconds to ensure timely updates while minimizing resource usage. if c.MultiNode.PollInterval == nil { - c.MultiNode.PollInterval = config.MustNewDuration(10 * time.Second) + c.MultiNode.PollInterval = config.MustNewDuration(15 * time.Second) } // Selection mode defaults to priority level to enable using node priorities if c.MultiNode.SelectionMode == nil { @@ -107,7 +107,7 @@ func (c *MultiNodeConfig) SetDefaults() { } // The sync threshold is set to 5 to allow for some flexibility in node synchronization before considering it out of sync. if c.MultiNode.SyncThreshold == nil { - c.MultiNode.SyncThreshold = ptr(uint32(160)) // TODO: Increased to 200 for slow test environment RPCs that are always behind + c.MultiNode.SyncThreshold = ptr(uint32(160)) // TODO: Increased to 160 for slow test environment RPCs that are always behind } // Lease duration is set to 1 minute by default to allow node locks for a reasonable amount of time. if c.MultiNode.LeaseDuration == nil { @@ -119,7 +119,7 @@ func (c *MultiNodeConfig) SetDefaults() { } // The finalized block polling interval is set to 5 seconds to ensure timely updates while minimizing resource usage. if c.MultiNode.FinalizedBlockPollInterval == nil { - c.MultiNode.FinalizedBlockPollInterval = config.MustNewDuration(15 * time.Second) + c.MultiNode.FinalizedBlockPollInterval = config.MustNewDuration(5 * time.Second) } // Repeatable read guarantee should be enforced by default. if c.MultiNode.EnforceRepeatableRead == nil { From fe263046e895bfff5ee22381e830e50c3b5620eb Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 25 Oct 2024 16:10:54 -0400 Subject: [PATCH 123/174] Update transaction_sender.go --- pkg/solana/client/multinode/transaction_sender.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go index 9e5f0e4e7..9bff5bda2 100644 --- a/pkg/solana/client/multinode/transaction_sender.go +++ b/pkg/solana/client/multinode/transaction_sender.go @@ -100,9 +100,6 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct return txSender.newResult(errors.New("TransactionSender not started")) } - ctx, cancel := txSender.chStop.Ctx(ctx) - defer cancel() - healthyNodesNum := 0 err := txSender.multiNode.DoAll(ctx, func(ctx context.Context, rpc RPC, isSendOnly bool) { if isSendOnly { From e1138548dcca9aaf809c08ab35e183350d224628 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Mon, 28 Oct 2024 10:30:18 -0400 Subject: [PATCH 124/174] Get chain info --- pkg/solana/client/multinode/node_lifecycle.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/pkg/solana/client/multinode/node_lifecycle.go b/pkg/solana/client/multinode/node_lifecycle.go index 88c6e83e4..2ab3cc8b1 100644 --- a/pkg/solana/client/multinode/node_lifecycle.go +++ b/pkg/solana/client/multinode/node_lifecycle.go @@ -165,10 +165,10 @@ func (n *node[CHAIN_ID, HEAD, RPC]) aliveLoop() { n.declareUnreachable() return } - _, latestChainInfo := n.StateAndLatest() - if outOfSync, liveNodes := n.isOutOfSyncWithPool(latestChainInfo); outOfSync { + if outOfSync, liveNodes := n.isOutOfSyncWithPool(); outOfSync { // note: there must be another live node for us to be out of sync _, highest := n.poolInfoProvider.LatestChainInfo() + _, latestChainInfo := n.StateAndLatest() lggr.Errorw("RPC endpoint has fallen behind", "blockNumber", latestChainInfo.BlockNumber, "bestLatestBlockNumber", highest.BlockNumber, "totalDifficulty", latestChainInfo.TotalDifficulty, "nodeState", n.getCachedState()) if liveNodes < 2 { lggr.Criticalf("RPC endpoint has fallen behind; %s %s", msgCannotDisable, msgDegradedState) @@ -357,7 +357,7 @@ const ( // isOutOfSyncWithPool returns outOfSync true if num or td is more than SyncThresold behind the best node. // Always returns outOfSync false for SyncThreshold 0. // liveNodes is only included when outOfSync is true. -func (n *node[CHAIN_ID, HEAD, RPC]) isOutOfSyncWithPool(localState ChainInfo) (outOfSync bool, liveNodes int) { +func (n *node[CHAIN_ID, HEAD, RPC]) isOutOfSyncWithPool() (outOfSync bool, liveNodes int) { if n.poolInfoProvider == nil { n.lfcLog.Warn("skipping sync state against the pool - should only occur in tests") return // skip for tests @@ -368,13 +368,14 @@ func (n *node[CHAIN_ID, HEAD, RPC]) isOutOfSyncWithPool(localState ChainInfo) (o } // Check against best node ln, ci := n.poolInfoProvider.LatestChainInfo() + _, localChainInfo := n.StateAndLatest() mode := n.nodePoolCfg.SelectionMode() switch mode { case NodeSelectionModeHighestHead, NodeSelectionModeRoundRobin, NodeSelectionModePriorityLevel: - return localState.BlockNumber < ci.BlockNumber-int64(threshold), ln + return localChainInfo.BlockNumber < ci.BlockNumber-int64(threshold), ln case NodeSelectionModeTotalDifficulty: bigThreshold := big.NewInt(int64(threshold)) - return localState.TotalDifficulty.Cmp(bigmath.Sub(ci.TotalDifficulty, bigThreshold)) < 0, ln + return localChainInfo.TotalDifficulty.Cmp(bigmath.Sub(ci.TotalDifficulty, bigThreshold)) < 0, ln default: panic("unrecognized NodeSelectionMode: " + mode) } @@ -463,7 +464,7 @@ func (n *node[CHAIN_ID, HEAD, RPC]) outOfSyncLoop(syncIssues syncStatus) { // received a new head - clear NoNewHead flag syncIssues &= ^syncStatusNoNewHead - if outOfSync, _ := n.isOutOfSyncWithPool(localHighestChainInfo); !outOfSync { + if outOfSync, _ := n.isOutOfSyncWithPool(); !outOfSync { // we caught up with the pool - clear NotInSyncWithPool flag syncIssues &= ^syncStatusNotInSyncWithPool } else { From c3527b417ce9e5a4007a57c09a8ac182623e55d2 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 29 Oct 2024 10:04:50 -0400 Subject: [PATCH 125/174] Update ctx --- pkg/solana/chain.go | 3 +-- .../client/multinode/transaction_sender.go | 17 ++++++++--------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/pkg/solana/chain.go b/pkg/solana/chain.go index ca635f548..c47e1cf1b 100644 --- a/pkg/solana/chain.go +++ b/pkg/solana/chain.go @@ -292,8 +292,7 @@ func newChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr logger.L // Send tx using MultiNode transaction sender sendTx = func(ctx context.Context, tx *solanago.Transaction) (solanago.Signature, error) { - // Use empty context since individual RPC timeouts are handled by the client - result := ch.txSender.SendTransaction(context.Background(), tx) + result := ch.txSender.SendTransaction(ctx, tx) if result == nil { return solanago.Signature{}, errors.New("tx sender returned nil result") } diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go index 0fccabd68..36219af2b 100644 --- a/pkg/solana/client/multinode/transaction_sender.go +++ b/pkg/solana/client/multinode/transaction_sender.go @@ -100,8 +100,6 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct return txSender.newResult(errors.New("TransactionSender not started")) } - ctx, _ = txSender.chStop.Ctx(ctx) - healthyNodesNum := 0 err := txSender.multiNode.DoAll(ctx, func(ctx context.Context, rpc RPC, isSendOnly bool) { if isSendOnly { @@ -155,7 +153,7 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct } func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) broadcastTxAsync(ctx context.Context, rpc RPC, tx TX) RESULT { - result := rpc.SendTransaction(ctx, tx) + result := rpc.SendTransaction(context.WithoutCancel(ctx), tx) // let rpc handle tx timeouts txSender.lggr.Debugw("Node sent transaction", "tx", tx, "err", result.TxError()) if !slices.Contains(sendTxSuccessfulCodes, result.Code()) { txSender.lggr.Warnw("RPC returned error", "tx", tx, "err", result.TxError()) @@ -181,8 +179,8 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) reportSendTxAnomal type sendTxResults[RESULT any] map[SendTxReturnCode][]RESULT func aggregateTxResults[RESULT any](resultsByCode sendTxResults[RESULT]) (result RESULT, criticalErr error) { - _, severeErrors, hasSevereErrors := findFirstIn(resultsByCode, sendTxSevereErrors) - _, successResults, hasSuccess := findFirstIn(resultsByCode, sendTxSuccessfulCodes) + severeErrors, hasSevereErrors := findFirstIn(resultsByCode, sendTxSevereErrors) + successResults, hasSuccess := findFirstIn(resultsByCode, sendTxSuccessfulCodes) if hasSuccess { // We assume that primary node would never report false positive txResult for a transaction. // Thus, if such case occurs it's probably due to misconfiguration or a bug and requires manual intervention. @@ -213,6 +211,8 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) collectTxResults(c if healthyNodesNum == 0 { return txSender.newResult(ErroringNodeError) } + ctx, cancel := txSender.chStop.Ctx(ctx) + defer cancel() requiredResults := int(math.Ceil(float64(healthyNodesNum) * sendTxQuorum)) errorsByCode := sendTxResults[RESULT]{} var softTimeoutChan <-chan time.Time @@ -263,13 +263,12 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) Close() error { } // findFirstIn - returns the first existing key and value for the slice of keys -func findFirstIn[K comparable, V any](set map[K]V, keys []K) (K, V, bool) { +func findFirstIn[K comparable, V any](set map[K]V, keys []K) (V, bool) { for _, k := range keys { if v, ok := set[k]; ok { - return k, v, true + return v, true } } - var zeroK K var zeroV V - return zeroK, zeroV, false + return zeroV, false } From f46c6819eda889dcef1ec4aa7748481028109484 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 30 Oct 2024 10:08:49 -0400 Subject: [PATCH 126/174] Update transaction_sender.go --- pkg/solana/client/multinode/transaction_sender.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go index 36219af2b..61976b76b 100644 --- a/pkg/solana/client/multinode/transaction_sender.go +++ b/pkg/solana/client/multinode/transaction_sender.go @@ -101,7 +101,7 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct } healthyNodesNum := 0 - err := txSender.multiNode.DoAll(ctx, func(ctx context.Context, rpc RPC, isSendOnly bool) { + err := txSender.multiNode.DoAll(context.WithoutCancel(ctx), func(ctx context.Context, rpc RPC, isSendOnly bool) { if isSendOnly { txSender.wg.Add(1) go func() { @@ -211,8 +211,6 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) collectTxResults(c if healthyNodesNum == 0 { return txSender.newResult(ErroringNodeError) } - ctx, cancel := txSender.chStop.Ctx(ctx) - defer cancel() requiredResults := int(math.Ceil(float64(healthyNodesNum) * sendTxQuorum)) errorsByCode := sendTxResults[RESULT]{} var softTimeoutChan <-chan time.Time From a647746d3ecf9db92ba5ba95f0b0d9cc6fd2c1be Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 30 Oct 2024 10:54:33 -0400 Subject: [PATCH 127/174] Update transaction_sender.go --- pkg/solana/client/multinode/transaction_sender.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go index 61976b76b..3cf6e4e45 100644 --- a/pkg/solana/client/multinode/transaction_sender.go +++ b/pkg/solana/client/multinode/transaction_sender.go @@ -100,8 +100,11 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct return txSender.newResult(errors.New("TransactionSender not started")) } + txSenderCtx, cancel := txSender.chStop.NewCtx() + defer cancel() + healthyNodesNum := 0 - err := txSender.multiNode.DoAll(context.WithoutCancel(ctx), func(ctx context.Context, rpc RPC, isSendOnly bool) { + err := txSender.multiNode.DoAll(txSenderCtx, func(ctx context.Context, rpc RPC, isSendOnly bool) { if isSendOnly { txSender.wg.Add(1) go func() { @@ -153,7 +156,7 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct } func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) broadcastTxAsync(ctx context.Context, rpc RPC, tx TX) RESULT { - result := rpc.SendTransaction(context.WithoutCancel(ctx), tx) // let rpc handle tx timeouts + result := rpc.SendTransaction(ctx, tx) txSender.lggr.Debugw("Node sent transaction", "tx", tx, "err", result.TxError()) if !slices.Contains(sendTxSuccessfulCodes, result.Code()) { txSender.lggr.Warnw("RPC returned error", "tx", tx, "err", result.TxError()) From 4428c1219ad97de3b6332c06121231612e945f6f Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 30 Oct 2024 10:59:09 -0400 Subject: [PATCH 128/174] Increase tx timeout --- integration-tests/testconfig/testconfig.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/integration-tests/testconfig/testconfig.go b/integration-tests/testconfig/testconfig.go index ea92ebdf5..09d6e5f63 100644 --- a/integration-tests/testconfig/testconfig.go +++ b/integration-tests/testconfig/testconfig.go @@ -280,11 +280,17 @@ func (c *TestConfig) GetNodeConfigTOML() (string, error) { }) } + chainCfg := solcfg.Chain{ + TxTimeout: config.MustNewDuration(2 * time.Minute), + } + chainCfg.SetDefaults() + solConfig := solcfg.TOMLConfig{ Enabled: ptr.Ptr(true), ChainID: ptr.Ptr(chainID), Nodes: nodes, MultiNode: mnConfig, + Chain: chainCfg, } baseConfig := node.NewBaseConfig() baseConfig.Solana = solcfg.TOMLConfigs{ From 8c624cef21bd0b6911af3c383f20322eb932133b Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 30 Oct 2024 11:42:37 -0400 Subject: [PATCH 129/174] Update transaction_sender.go --- pkg/solana/client/multinode/transaction_sender.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go index 3cf6e4e45..bef74193e 100644 --- a/pkg/solana/client/multinode/transaction_sender.go +++ b/pkg/solana/client/multinode/transaction_sender.go @@ -156,10 +156,11 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct } func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) broadcastTxAsync(ctx context.Context, rpc RPC, tx TX) RESULT { + elapsedTime := time.Now() result := rpc.SendTransaction(ctx, tx) - txSender.lggr.Debugw("Node sent transaction", "tx", tx, "err", result.TxError()) + txSender.lggr.Debugw("Node sent transaction", "tx", tx, "err", result.TxError(), "elapsedTime", time.Since(elapsedTime)) if !slices.Contains(sendTxSuccessfulCodes, result.Code()) { - txSender.lggr.Warnw("RPC returned error", "tx", tx, "err", result.TxError()) + txSender.lggr.Warnw("RPC returned error", "tx", tx, "err", result.TxError(), "elapsedTime", time.Since(elapsedTime)) } return result } From 329be9af7a2532a3100ca7003a40618d1d3f3b10 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 30 Oct 2024 11:59:20 -0400 Subject: [PATCH 130/174] Update ctx --- pkg/solana/chain.go | 2 ++ pkg/solana/client/multinode/transaction_sender.go | 9 +++------ 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/pkg/solana/chain.go b/pkg/solana/chain.go index c47e1cf1b..18c95a8ea 100644 --- a/pkg/solana/chain.go +++ b/pkg/solana/chain.go @@ -292,6 +292,8 @@ func newChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr logger.L // Send tx using MultiNode transaction sender sendTx = func(ctx context.Context, tx *solanago.Transaction) (solanago.Signature, error) { + // TODO: For testing + ch.lggr.Debug("TxTimeoutSeconds: ", cfg.TxTimeout().Seconds()) result := ch.txSender.SendTransaction(ctx, tx) if result == nil { return solanago.Signature{}, errors.New("tx sender returned nil result") diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go index bef74193e..71f58915d 100644 --- a/pkg/solana/client/multinode/transaction_sender.go +++ b/pkg/solana/client/multinode/transaction_sender.go @@ -100,11 +100,8 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct return txSender.newResult(errors.New("TransactionSender not started")) } - txSenderCtx, cancel := txSender.chStop.NewCtx() - defer cancel() - healthyNodesNum := 0 - err := txSender.multiNode.DoAll(txSenderCtx, func(ctx context.Context, rpc RPC, isSendOnly bool) { + err := txSender.multiNode.DoAll(context.WithoutCancel(ctx), func(ctx context.Context, rpc RPC, isSendOnly bool) { if isSendOnly { txSender.wg.Add(1) go func() { @@ -158,9 +155,9 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) broadcastTxAsync(ctx context.Context, rpc RPC, tx TX) RESULT { elapsedTime := time.Now() result := rpc.SendTransaction(ctx, tx) - txSender.lggr.Debugw("Node sent transaction", "tx", tx, "err", result.TxError(), "elapsedTime", time.Since(elapsedTime)) + txSender.lggr.Debugw("Node sent transaction", "tx", tx, "err", result.TxError(), "elapsedTimeSeconds", time.Since(elapsedTime).Seconds()) if !slices.Contains(sendTxSuccessfulCodes, result.Code()) { - txSender.lggr.Warnw("RPC returned error", "tx", tx, "err", result.TxError(), "elapsedTime", time.Since(elapsedTime)) + txSender.lggr.Warnw("RPC returned error", "tx", tx, "err", result.TxError(), "elapsedTimeSeconds", time.Since(elapsedTime).Seconds()) } return result } From 265fd67808d345a735ee53065c3c80ee67d8f72d Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 30 Oct 2024 12:20:12 -0400 Subject: [PATCH 131/174] Add timer --- pkg/solana/chain.go | 2 -- pkg/solana/client/multinode/transaction_sender.go | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/solana/chain.go b/pkg/solana/chain.go index 18c95a8ea..c47e1cf1b 100644 --- a/pkg/solana/chain.go +++ b/pkg/solana/chain.go @@ -292,8 +292,6 @@ func newChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr logger.L // Send tx using MultiNode transaction sender sendTx = func(ctx context.Context, tx *solanago.Transaction) (solanago.Signature, error) { - // TODO: For testing - ch.lggr.Debug("TxTimeoutSeconds: ", cfg.TxTimeout().Seconds()) result := ch.txSender.SendTransaction(ctx, tx) if result == nil { return solanago.Signature{}, errors.New("tx sender returned nil result") diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go index 71f58915d..1e0d98fef 100644 --- a/pkg/solana/client/multinode/transaction_sender.go +++ b/pkg/solana/client/multinode/transaction_sender.go @@ -92,6 +92,7 @@ type TransactionSender[TX any, RESULT SendTxResult, CHAIN_ID ID, RPC SendTxRPCCl // * If there is both success and terminal error - returns success and reports invariant violation // * Otherwise, returns any (effectively random) of the errors. func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ctx context.Context, tx TX) RESULT { + elapsed := time.Now() txResults := make(chan RESULT) txResultsToReport := make(chan RESULT) primaryNodeWg := sync.WaitGroup{} @@ -149,6 +150,7 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct txSender.wg.Add(1) go txSender.reportSendTxAnomalies(tx, txResultsToReport) + txSender.lggr.Debugw("Collecting Tx Results", "elapsedTimeSeconds", time.Since(elapsed).Seconds()) return txSender.collectTxResults(ctx, tx, healthyNodesNum, txResults) } From 359224361afa663595401322a5fe718b4b5d2595 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 30 Oct 2024 12:23:15 -0400 Subject: [PATCH 132/174] Update transaction_sender.go --- pkg/solana/client/multinode/transaction_sender.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go index 1e0d98fef..02d39eb46 100644 --- a/pkg/solana/client/multinode/transaction_sender.go +++ b/pkg/solana/client/multinode/transaction_sender.go @@ -151,6 +151,12 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct go txSender.reportSendTxAnomalies(tx, txResultsToReport) txSender.lggr.Debugw("Collecting Tx Results", "elapsedTimeSeconds", time.Since(elapsed).Seconds()) + select { + case <-ctx.Done(): + txSender.lggr.Errorw("ctx cancelled before collect results", "elapsedTimeSeconds", time.Since(elapsed).Seconds()) + default: + break + } return txSender.collectTxResults(ctx, tx, healthyNodesNum, txResults) } From abed47ccecd865c906471f43815c372f13a4e839 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 30 Oct 2024 12:57:27 -0400 Subject: [PATCH 133/174] Update transaction_sender.go --- pkg/solana/client/multinode/transaction_sender.go | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go index 02d39eb46..1d0b487c1 100644 --- a/pkg/solana/client/multinode/transaction_sender.go +++ b/pkg/solana/client/multinode/transaction_sender.go @@ -92,7 +92,6 @@ type TransactionSender[TX any, RESULT SendTxResult, CHAIN_ID ID, RPC SendTxRPCCl // * If there is both success and terminal error - returns success and reports invariant violation // * Otherwise, returns any (effectively random) of the errors. func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ctx context.Context, tx TX) RESULT { - elapsed := time.Now() txResults := make(chan RESULT) txResultsToReport := make(chan RESULT) primaryNodeWg := sync.WaitGroup{} @@ -150,14 +149,7 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct txSender.wg.Add(1) go txSender.reportSendTxAnomalies(tx, txResultsToReport) - txSender.lggr.Debugw("Collecting Tx Results", "elapsedTimeSeconds", time.Since(elapsed).Seconds()) - select { - case <-ctx.Done(): - txSender.lggr.Errorw("ctx cancelled before collect results", "elapsedTimeSeconds", time.Since(elapsed).Seconds()) - default: - break - } - return txSender.collectTxResults(ctx, tx, healthyNodesNum, txResults) + return txSender.collectTxResults(context.WithoutCancel(ctx), tx, healthyNodesNum, txResults) } func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) broadcastTxAsync(ctx context.Context, rpc RPC, tx TX) RESULT { @@ -224,11 +216,12 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) collectTxResults(c errorsByCode := sendTxResults[RESULT]{} var softTimeoutChan <-chan time.Time var resultsCount int + elapsedTime := time.Now() loop: for { select { case <-ctx.Done(): - txSender.lggr.Debugw("Failed to collect of the results before context was done", "tx", tx, "errorsByCode", errorsByCode) + txSender.lggr.Debugw("Failed to collect of the results before context was done", "tx", tx, "errorsByCode", errorsByCode, "elapsedTimeSeconds", time.Since(elapsedTime).Seconds()) return txSender.newResult(ctx.Err()) case r := <-txResults: errorsByCode[r.Code()] = append(errorsByCode[r.Code()], r) From b258370fb7c8813d6517ffd91f3fbbae9ac8f2bb Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 30 Oct 2024 12:59:52 -0400 Subject: [PATCH 134/174] Update testconfig.go --- integration-tests/testconfig/testconfig.go | 1 + 1 file changed, 1 insertion(+) diff --git a/integration-tests/testconfig/testconfig.go b/integration-tests/testconfig/testconfig.go index 09d6e5f63..9b74c7aff 100644 --- a/integration-tests/testconfig/testconfig.go +++ b/integration-tests/testconfig/testconfig.go @@ -281,6 +281,7 @@ func (c *TestConfig) GetNodeConfigTOML() (string, error) { } chainCfg := solcfg.Chain{ + // Increase timeout for TransactionSender TxTimeout: config.MustNewDuration(2 * time.Minute), } chainCfg.SetDefaults() From a75815faf7f4bac9d18ee92f38621fbf9948bcff Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 30 Oct 2024 15:59:34 -0400 Subject: [PATCH 135/174] Fix ctx --- pkg/solana/client/multinode/transaction_sender.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go index 1d0b487c1..fb206ea35 100644 --- a/pkg/solana/client/multinode/transaction_sender.go +++ b/pkg/solana/client/multinode/transaction_sender.go @@ -100,8 +100,11 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct return txSender.newResult(errors.New("TransactionSender not started")) } + txSenderCtx, cancel := txSender.chStop.NewCtx() + defer cancel() + healthyNodesNum := 0 - err := txSender.multiNode.DoAll(context.WithoutCancel(ctx), func(ctx context.Context, rpc RPC, isSendOnly bool) { + err := txSender.multiNode.DoAll(txSenderCtx, func(ctx context.Context, rpc RPC, isSendOnly bool) { if isSendOnly { txSender.wg.Add(1) go func() { @@ -149,7 +152,7 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct txSender.wg.Add(1) go txSender.reportSendTxAnomalies(tx, txResultsToReport) - return txSender.collectTxResults(context.WithoutCancel(ctx), tx, healthyNodesNum, txResults) + return txSender.collectTxResults(ctx, tx, healthyNodesNum, txResults) } func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) broadcastTxAsync(ctx context.Context, rpc RPC, tx TX) RESULT { From 1914f47a6aefadd5404ff98dfda10f87df3ed363 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 30 Oct 2024 16:33:08 -0400 Subject: [PATCH 136/174] Remove debug logging --- pkg/solana/client/multinode/transaction_sender.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go index fb206ea35..031755882 100644 --- a/pkg/solana/client/multinode/transaction_sender.go +++ b/pkg/solana/client/multinode/transaction_sender.go @@ -156,11 +156,10 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct } func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) broadcastTxAsync(ctx context.Context, rpc RPC, tx TX) RESULT { - elapsedTime := time.Now() result := rpc.SendTransaction(ctx, tx) - txSender.lggr.Debugw("Node sent transaction", "tx", tx, "err", result.TxError(), "elapsedTimeSeconds", time.Since(elapsedTime).Seconds()) + txSender.lggr.Debugw("Node sent transaction", "tx", tx, "err", result.TxError(), "elapsedTimeSeconds") if !slices.Contains(sendTxSuccessfulCodes, result.Code()) { - txSender.lggr.Warnw("RPC returned error", "tx", tx, "err", result.TxError(), "elapsedTimeSeconds", time.Since(elapsedTime).Seconds()) + txSender.lggr.Warnw("RPC returned error", "tx", tx, "err", result.TxError(), "elapsedTimeSeconds") } return result } @@ -219,12 +218,11 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) collectTxResults(c errorsByCode := sendTxResults[RESULT]{} var softTimeoutChan <-chan time.Time var resultsCount int - elapsedTime := time.Now() loop: for { select { case <-ctx.Done(): - txSender.lggr.Debugw("Failed to collect of the results before context was done", "tx", tx, "errorsByCode", errorsByCode, "elapsedTimeSeconds", time.Since(elapsedTime).Seconds()) + txSender.lggr.Debugw("Failed to collect of the results before context was done", "tx", tx, "errorsByCode", errorsByCode) return txSender.newResult(ctx.Err()) case r := <-txResults: errorsByCode[r.Code()] = append(errorsByCode[r.Code()], r) From 1643efc9e4f4d7d7a7e1dc403c893b729a35041b Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 30 Oct 2024 16:54:15 -0400 Subject: [PATCH 137/174] Update run_soak_test.sh --- integration-tests/{smoke => scripts}/run_soak_test.sh | 2 ++ 1 file changed, 2 insertions(+) rename integration-tests/{smoke => scripts}/run_soak_test.sh (98%) diff --git a/integration-tests/smoke/run_soak_test.sh b/integration-tests/scripts/run_soak_test.sh similarity index 98% rename from integration-tests/smoke/run_soak_test.sh rename to integration-tests/scripts/run_soak_test.sh index 9485fdf17..32caa3e93 100755 --- a/integration-tests/smoke/run_soak_test.sh +++ b/integration-tests/scripts/run_soak_test.sh @@ -2,6 +2,8 @@ NODE_VERSION=18 +cd ../smoke || exit + echo "Switching to required Node.js version $NODE_VERSION..." export NVM_DIR="$HOME/.nvm" [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" From e82103d29a71b2543469e45b358d7b6706e3671a Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 30 Oct 2024 16:58:11 -0400 Subject: [PATCH 138/174] lint --- pkg/solana/client/multinode/transaction_sender.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go index 031755882..3cf6e4e45 100644 --- a/pkg/solana/client/multinode/transaction_sender.go +++ b/pkg/solana/client/multinode/transaction_sender.go @@ -157,9 +157,9 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) broadcastTxAsync(ctx context.Context, rpc RPC, tx TX) RESULT { result := rpc.SendTransaction(ctx, tx) - txSender.lggr.Debugw("Node sent transaction", "tx", tx, "err", result.TxError(), "elapsedTimeSeconds") + txSender.lggr.Debugw("Node sent transaction", "tx", tx, "err", result.TxError()) if !slices.Contains(sendTxSuccessfulCodes, result.Code()) { - txSender.lggr.Warnw("RPC returned error", "tx", tx, "err", result.TxError(), "elapsedTimeSeconds") + txSender.lggr.Warnw("RPC returned error", "tx", tx, "err", result.TxError()) } return result } From 6392027693ddf31eae4f035934fd22ee40af52c4 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 31 Oct 2024 10:19:09 -0400 Subject: [PATCH 139/174] Add debugging logs --- integration-tests/testconfig/testconfig.go | 4 +++- pkg/solana/client/client.go | 5 +++++ pkg/solana/client/multinode/multi_node.go | 6 +++--- pkg/solana/client/multinode/transaction_sender.go | 8 +++++--- pkg/solana/client/multinode_client.go | 8 ++++++++ 5 files changed, 24 insertions(+), 7 deletions(-) diff --git a/integration-tests/testconfig/testconfig.go b/integration-tests/testconfig/testconfig.go index 9b74c7aff..91749d002 100644 --- a/integration-tests/testconfig/testconfig.go +++ b/integration-tests/testconfig/testconfig.go @@ -267,7 +267,8 @@ func (c *TestConfig) GetNodeConfigTOML() (string, error) { mnConfig := solcfg.MultiNodeConfig{ MultiNode: solcfg.MultiNode{ - Enabled: ptr.Ptr(true), + Enabled: ptr.Ptr(true), + SyncThreshold: ptr.Ptr(uint32(170)), }, } mnConfig.SetDefaults() @@ -283,6 +284,7 @@ func (c *TestConfig) GetNodeConfigTOML() (string, error) { chainCfg := solcfg.Chain{ // Increase timeout for TransactionSender TxTimeout: config.MustNewDuration(2 * time.Minute), + // TODO: Increase TxRetryTimeout? } chainCfg.SetDefaults() diff --git a/pkg/solana/client/client.go b/pkg/solana/client/client.go index 9b55cf595..589d1363a 100644 --- a/pkg/solana/client/client.go +++ b/pkg/solana/client/client.go @@ -242,6 +242,11 @@ func (c *Client) SendTx(ctx context.Context, tx *solana.Transaction) (solana.Sig done := c.latency("send_tx") defer done() + elapsedTime := time.Now() + defer func() { + c.log.Debugw("Client SendTx() elapsed time", "time", time.Since(elapsedTime).Seconds()) + }() + c.log.Debugw("Client SendTx()", "timeout", c.txTimeout) ctx, cancel := context.WithTimeout(ctx, c.txTimeout) defer cancel() diff --git a/pkg/solana/client/multinode/multi_node.go b/pkg/solana/client/multinode/multi_node.go index 596130f93..7a7b09231 100644 --- a/pkg/solana/client/multinode/multi_node.go +++ b/pkg/solana/client/multinode/multi_node.go @@ -90,15 +90,14 @@ func (c *MultiNode[CHAIN_ID, RPC]) ChainID() CHAIN_ID { return c.chainID } -func (c *MultiNode[CHAIN_ID, RPC]) DoAll(baseCtx context.Context, do func(ctx context.Context, rpc RPC, isSendOnly bool)) error { +func (c *MultiNode[CHAIN_ID, RPC]) DoAll(ctx context.Context, do func(ctx context.Context, rpc RPC, isSendOnly bool)) error { var err error ok := c.IfNotStopped(func() { - ctx, _ := c.chStop.Ctx(baseCtx) - callsCompleted := 0 for _, n := range c.primaryNodes { select { case <-ctx.Done(): + c.lggr.Errorw("DoAll: Context done on primary node", "err", ctx.Err()) err = ctx.Err() return default: @@ -118,6 +117,7 @@ func (c *MultiNode[CHAIN_ID, RPC]) DoAll(baseCtx context.Context, do func(ctx co for _, n := range c.sendOnlyNodes { select { case <-ctx.Done(): + c.lggr.Errorw("DoAll: Context done on send only node", "err", ctx.Err()) err = ctx.Err() return default: diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go index 3cf6e4e45..324060757 100644 --- a/pkg/solana/client/multinode/transaction_sender.go +++ b/pkg/solana/client/multinode/transaction_sender.go @@ -92,6 +92,7 @@ type TransactionSender[TX any, RESULT SendTxResult, CHAIN_ID ID, RPC SendTxRPCCl // * If there is both success and terminal error - returns success and reports invariant violation // * Otherwise, returns any (effectively random) of the errors. func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ctx context.Context, tx TX) RESULT { + startTime := time.Now() txResults := make(chan RESULT) txResultsToReport := make(chan RESULT) primaryNodeWg := sync.WaitGroup{} @@ -146,13 +147,14 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct }() if err != nil { + txSender.lggr.Errorw("Failed to broadcast transaction", "tx", tx, "err", err) return txSender.newResult(err) } txSender.wg.Add(1) go txSender.reportSendTxAnomalies(tx, txResultsToReport) - return txSender.collectTxResults(ctx, tx, healthyNodesNum, txResults) + return txSender.collectTxResults(ctx, tx, healthyNodesNum, txResults, startTime) } func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) broadcastTxAsync(ctx context.Context, rpc RPC, tx TX) RESULT { @@ -210,7 +212,7 @@ func aggregateTxResults[RESULT any](resultsByCode sendTxResults[RESULT]) (result return result, criticalErr } -func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) collectTxResults(ctx context.Context, tx TX, healthyNodesNum int, txResults <-chan RESULT) RESULT { +func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) collectTxResults(ctx context.Context, tx TX, healthyNodesNum int, txResults <-chan RESULT, startTime time.Time) RESULT { if healthyNodesNum == 0 { return txSender.newResult(ErroringNodeError) } @@ -222,7 +224,7 @@ loop: for { select { case <-ctx.Done(): - txSender.lggr.Debugw("Failed to collect of the results before context was done", "tx", tx, "errorsByCode", errorsByCode) + txSender.lggr.Debugw("Failed to collect of the results before context was done", "tx", tx, "errorsByCode", errorsByCode, "elapsedTime", time.Since(startTime).Seconds()) return txSender.newResult(ctx.Err()) case r := <-txResults: errorsByCode[r.Code()] = append(errorsByCode[r.Code()], r) diff --git a/pkg/solana/client/multinode_client.go b/pkg/solana/client/multinode_client.go index 6280cd65c..842dc7080 100644 --- a/pkg/solana/client/multinode_client.go +++ b/pkg/solana/client/multinode_client.go @@ -354,6 +354,14 @@ func (r *SendTxResult) Signature() solana.Signature { } func (m *MultiNodeClient) SendTransaction(ctx context.Context, tx *solana.Transaction) *SendTxResult { + // TODO: Debugging check if ctx is already cancelled + select { + case <-ctx.Done(): + // TODO: This context would be from the TxSenderCtx and not sendTx() + m.log.Errorf("SendTransaction: context already cancelled") + default: + break + } var sendTxResult = &SendTxResult{} sendTxResult.sig, sendTxResult.txErr = m.SendTx(ctx, tx) sendTxResult.code = ClassifySendError(tx, sendTxResult.txErr) From fe861683259e603a599d601a5eb6b1f1749283ea Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 31 Oct 2024 11:05:48 -0400 Subject: [PATCH 140/174] Fix ctx cancel --- pkg/solana/client/multinode/transaction_sender.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go index 324060757..259a78a9c 100644 --- a/pkg/solana/client/multinode/transaction_sender.go +++ b/pkg/solana/client/multinode/transaction_sender.go @@ -102,7 +102,13 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct } txSenderCtx, cancel := txSender.chStop.NewCtx() - defer cancel() + reportWg := sync.WaitGroup{} + defer func() { + go func() { + reportWg.Wait() + cancel() + }() + }() healthyNodesNum := 0 err := txSender.multiNode.DoAll(txSenderCtx, func(ctx context.Context, rpc RPC, isSendOnly bool) { @@ -152,7 +158,11 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct } txSender.wg.Add(1) - go txSender.reportSendTxAnomalies(tx, txResultsToReport) + reportWg.Add(1) + go func() { + txSender.reportSendTxAnomalies(tx, txResultsToReport) + reportWg.Done() + }() return txSender.collectTxResults(ctx, tx, healthyNodesNum, txResults, startTime) } From e5f03a03f84aebd66c3e6598cac3efb3bec9a1f3 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 31 Oct 2024 11:08:59 -0400 Subject: [PATCH 141/174] Fix ctx cancel --- pkg/solana/client/multinode/transaction_sender.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go index 3cf6e4e45..b548d8e42 100644 --- a/pkg/solana/client/multinode/transaction_sender.go +++ b/pkg/solana/client/multinode/transaction_sender.go @@ -101,7 +101,13 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct } txSenderCtx, cancel := txSender.chStop.NewCtx() - defer cancel() + reportWg := sync.WaitGroup{} + defer func() { + go func() { + reportWg.Wait() + cancel() + }() + }() healthyNodesNum := 0 err := txSender.multiNode.DoAll(txSenderCtx, func(ctx context.Context, rpc RPC, isSendOnly bool) { @@ -150,7 +156,11 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct } txSender.wg.Add(1) - go txSender.reportSendTxAnomalies(tx, txResultsToReport) + reportWg.Add(1) + go func() { + txSender.reportSendTxAnomalies(tx, txResultsToReport) + reportWg.Done() + }() return txSender.collectTxResults(ctx, tx, healthyNodesNum, txResults) } From 85165b6a218d14835215447f09d8e85c199210bd Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 31 Oct 2024 11:26:52 -0400 Subject: [PATCH 142/174] Fix DoAll ctx --- pkg/solana/client/multinode/multi_node.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/solana/client/multinode/multi_node.go b/pkg/solana/client/multinode/multi_node.go index 8b7efc46b..bd97ebc7b 100644 --- a/pkg/solana/client/multinode/multi_node.go +++ b/pkg/solana/client/multinode/multi_node.go @@ -90,11 +90,9 @@ func (c *MultiNode[CHAIN_ID, RPC]) ChainID() CHAIN_ID { return c.chainID } -func (c *MultiNode[CHAIN_ID, RPC]) DoAll(baseCtx context.Context, do func(ctx context.Context, rpc RPC, isSendOnly bool)) error { +func (c *MultiNode[CHAIN_ID, RPC]) DoAll(ctx context.Context, do func(ctx context.Context, rpc RPC, isSendOnly bool)) error { var err error ok := c.IfNotStopped(func() { - ctx, _ := c.chStop.Ctx(baseCtx) - callsCompleted := 0 for _, n := range c.primaryNodes { select { From 4f792d3b4e51e477cffacd942af22c4217142d80 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 31 Oct 2024 12:15:16 -0400 Subject: [PATCH 143/174] Remove debugging logs --- integration-tests/testconfig/testconfig.go | 1 - pkg/solana/client/client.go | 5 ----- pkg/solana/client/multinode/multi_node.go | 4 ---- pkg/solana/client/multinode/poller.go | 2 +- pkg/solana/client/multinode/transaction_sender.go | 8 ++++---- pkg/solana/client/multinode_client.go | 8 -------- pkg/solana/config/multinode.go | 10 +++++----- 7 files changed, 10 insertions(+), 28 deletions(-) diff --git a/integration-tests/testconfig/testconfig.go b/integration-tests/testconfig/testconfig.go index 91749d002..2f7fb9a01 100644 --- a/integration-tests/testconfig/testconfig.go +++ b/integration-tests/testconfig/testconfig.go @@ -284,7 +284,6 @@ func (c *TestConfig) GetNodeConfigTOML() (string, error) { chainCfg := solcfg.Chain{ // Increase timeout for TransactionSender TxTimeout: config.MustNewDuration(2 * time.Minute), - // TODO: Increase TxRetryTimeout? } chainCfg.SetDefaults() diff --git a/pkg/solana/client/client.go b/pkg/solana/client/client.go index 589d1363a..9b55cf595 100644 --- a/pkg/solana/client/client.go +++ b/pkg/solana/client/client.go @@ -242,11 +242,6 @@ func (c *Client) SendTx(ctx context.Context, tx *solana.Transaction) (solana.Sig done := c.latency("send_tx") defer done() - elapsedTime := time.Now() - defer func() { - c.log.Debugw("Client SendTx() elapsed time", "time", time.Since(elapsedTime).Seconds()) - }() - c.log.Debugw("Client SendTx()", "timeout", c.txTimeout) ctx, cancel := context.WithTimeout(ctx, c.txTimeout) defer cancel() diff --git a/pkg/solana/client/multinode/multi_node.go b/pkg/solana/client/multinode/multi_node.go index 7a7b09231..bd97ebc7b 100644 --- a/pkg/solana/client/multinode/multi_node.go +++ b/pkg/solana/client/multinode/multi_node.go @@ -97,15 +97,12 @@ func (c *MultiNode[CHAIN_ID, RPC]) DoAll(ctx context.Context, do func(ctx contex for _, n := range c.primaryNodes { select { case <-ctx.Done(): - c.lggr.Errorw("DoAll: Context done on primary node", "err", ctx.Err()) err = ctx.Err() return default: if n.State() != NodeStateAlive { - c.lggr.Warnw("DoAll: Node is not alive", "node", n.Name()) continue } - c.lggr.Debugw("DoAll: Calling do on primary node", "node", n.Name()) do(ctx, n.RPC(), false) callsCompleted++ } @@ -117,7 +114,6 @@ func (c *MultiNode[CHAIN_ID, RPC]) DoAll(ctx context.Context, do func(ctx contex for _, n := range c.sendOnlyNodes { select { case <-ctx.Done(): - c.lggr.Errorw("DoAll: Context done on send only node", "err", ctx.Err()) err = ctx.Err() return default: diff --git a/pkg/solana/client/multinode/poller.go b/pkg/solana/client/multinode/poller.go index f17d81458..4f426ec02 100644 --- a/pkg/solana/client/multinode/poller.go +++ b/pkg/solana/client/multinode/poller.go @@ -75,7 +75,7 @@ func (p *Poller[T]) pollingLoop(ctx context.Context) { return case <-ticker.C: // Set polling timeout - pollingCtx, cancelPolling := context.WithTimeout(context.Background(), p.pollingTimeout) + pollingCtx, cancelPolling := context.WithTimeout(ctx, p.pollingTimeout) // Execute polling function result, err := p.pollingFunc(pollingCtx) cancelPolling() diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go index 259a78a9c..7659cb4ff 100644 --- a/pkg/solana/client/multinode/transaction_sender.go +++ b/pkg/solana/client/multinode/transaction_sender.go @@ -92,7 +92,6 @@ type TransactionSender[TX any, RESULT SendTxResult, CHAIN_ID ID, RPC SendTxRPCCl // * If there is both success and terminal error - returns success and reports invariant violation // * Otherwise, returns any (effectively random) of the errors. func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ctx context.Context, tx TX) RESULT { - startTime := time.Now() txResults := make(chan RESULT) txResultsToReport := make(chan RESULT) primaryNodeWg := sync.WaitGroup{} @@ -101,6 +100,7 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct return txSender.newResult(errors.New("TransactionSender not started")) } + // Must wait for reportSendTxAnomalies and collectTxResults to complete before cancelling the context txSenderCtx, cancel := txSender.chStop.NewCtx() reportWg := sync.WaitGroup{} defer func() { @@ -164,7 +164,7 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct reportWg.Done() }() - return txSender.collectTxResults(ctx, tx, healthyNodesNum, txResults, startTime) + return txSender.collectTxResults(ctx, tx, healthyNodesNum, txResults) } func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) broadcastTxAsync(ctx context.Context, rpc RPC, tx TX) RESULT { @@ -222,7 +222,7 @@ func aggregateTxResults[RESULT any](resultsByCode sendTxResults[RESULT]) (result return result, criticalErr } -func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) collectTxResults(ctx context.Context, tx TX, healthyNodesNum int, txResults <-chan RESULT, startTime time.Time) RESULT { +func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) collectTxResults(ctx context.Context, tx TX, healthyNodesNum int, txResults <-chan RESULT) RESULT { if healthyNodesNum == 0 { return txSender.newResult(ErroringNodeError) } @@ -234,7 +234,7 @@ loop: for { select { case <-ctx.Done(): - txSender.lggr.Debugw("Failed to collect of the results before context was done", "tx", tx, "errorsByCode", errorsByCode, "elapsedTime", time.Since(startTime).Seconds()) + txSender.lggr.Debugw("Failed to collect of the results before context was done", "tx", tx, "errorsByCode", errorsByCode) return txSender.newResult(ctx.Err()) case r := <-txResults: errorsByCode[r.Code()] = append(errorsByCode[r.Code()], r) diff --git a/pkg/solana/client/multinode_client.go b/pkg/solana/client/multinode_client.go index 842dc7080..6280cd65c 100644 --- a/pkg/solana/client/multinode_client.go +++ b/pkg/solana/client/multinode_client.go @@ -354,14 +354,6 @@ func (r *SendTxResult) Signature() solana.Signature { } func (m *MultiNodeClient) SendTransaction(ctx context.Context, tx *solana.Transaction) *SendTxResult { - // TODO: Debugging check if ctx is already cancelled - select { - case <-ctx.Done(): - // TODO: This context would be from the TxSenderCtx and not sendTx() - m.log.Errorf("SendTransaction: context already cancelled") - default: - break - } var sendTxResult = &SendTxResult{} sendTxResult.sig, sendTxResult.txErr = m.SendTx(ctx, tx) sendTxResult.code = ClassifySendError(tx, sendTxResult.txErr) diff --git a/pkg/solana/config/multinode.go b/pkg/solana/config/multinode.go index 00b82f9cd..194e9bf55 100644 --- a/pkg/solana/config/multinode.go +++ b/pkg/solana/config/multinode.go @@ -95,7 +95,7 @@ func (c *MultiNodeConfig) SetDefaults() { /* Node Configs */ // Failure threshold for polling set to 5 to tolerate some polling failures before taking action. if c.MultiNode.PollFailureThreshold == nil { - c.MultiNode.PollFailureThreshold = ptr(uint32(8)) + c.MultiNode.PollFailureThreshold = ptr(uint32(5)) } // Poll interval is set to 10 seconds to ensure timely updates while minimizing resource usage. if c.MultiNode.PollInterval == nil { @@ -107,7 +107,7 @@ func (c *MultiNodeConfig) SetDefaults() { } // The sync threshold is set to 5 to allow for some flexibility in node synchronization before considering it out of sync. if c.MultiNode.SyncThreshold == nil { - c.MultiNode.SyncThreshold = ptr(uint32(160)) // TODO: Increased to 160 for slow test environment RPCs that are always behind + c.MultiNode.SyncThreshold = ptr(uint32(10)) } // Lease duration is set to 1 minute by default to allow node locks for a reasonable amount of time. if c.MultiNode.LeaseDuration == nil { @@ -127,17 +127,17 @@ func (c *MultiNodeConfig) SetDefaults() { } // The delay before declaring a node dead is set to 20 seconds to give nodes time to recover from temporary issues. if c.MultiNode.DeathDeclarationDelay == nil { - c.MultiNode.DeathDeclarationDelay = config.MustNewDuration(45 * time.Second) + c.MultiNode.DeathDeclarationDelay = config.MustNewDuration(20 * time.Second) } /* Chain Configs */ // Threshold for no new heads is set to 20 seconds, assuming that heads should update at a reasonable pace. if c.MultiNode.NodeNoNewHeadsThreshold == nil { - c.MultiNode.NodeNoNewHeadsThreshold = config.MustNewDuration(45 * time.Second) + c.MultiNode.NodeNoNewHeadsThreshold = config.MustNewDuration(20 * time.Second) } // Similar to heads, finalized heads should be updated within 20 seconds. if c.MultiNode.NoNewFinalizedHeadsThreshold == nil { - c.MultiNode.NoNewFinalizedHeadsThreshold = config.MustNewDuration(45 * time.Second) + c.MultiNode.NoNewFinalizedHeadsThreshold = config.MustNewDuration(20 * time.Second) } // Finality tags are used in Solana and enabled by default. if c.MultiNode.FinalityTagEnabled == nil { From 5891c3c7ad5e4ca6ef5d353a603a1a0d5ee55858 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 31 Oct 2024 12:53:18 -0400 Subject: [PATCH 144/174] Remove logs --- pkg/solana/client/multinode/node_fsm.go | 1 - pkg/solana/client/multinode/transaction_sender.go | 1 - 2 files changed, 2 deletions(-) diff --git a/pkg/solana/client/multinode/node_fsm.go b/pkg/solana/client/multinode/node_fsm.go index e44cdbb19..136910868 100644 --- a/pkg/solana/client/multinode/node_fsm.go +++ b/pkg/solana/client/multinode/node_fsm.go @@ -153,7 +153,6 @@ func (n *node[CHAIN_ID, HEAD, RPC]) isFinalizedBlockOutOfSync() bool { return latest.FinalizedBlockNumber < highestObservedByCaller.FinalizedBlockNumber-int64(n.chainCfg.FinalizedBlockOffset()) } - // TODO: Should this be using finality depth instead of non finalized blocks? return latest.BlockNumber < highestObservedByCaller.BlockNumber-int64(n.chainCfg.FinalizedBlockOffset()) } diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go index 7659cb4ff..147145b91 100644 --- a/pkg/solana/client/multinode/transaction_sender.go +++ b/pkg/solana/client/multinode/transaction_sender.go @@ -153,7 +153,6 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct }() if err != nil { - txSender.lggr.Errorw("Failed to broadcast transaction", "tx", tx, "err", err) return txSender.newResult(err) } From e1e458b9fd0e154f2a543bca5d2bb44e9d497953 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 5 Nov 2024 09:32:31 -0500 Subject: [PATCH 145/174] defer reportWg --- pkg/solana/client/multinode/transaction_sender.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go index b548d8e42..bd11a71a5 100644 --- a/pkg/solana/client/multinode/transaction_sender.go +++ b/pkg/solana/client/multinode/transaction_sender.go @@ -158,8 +158,8 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct txSender.wg.Add(1) reportWg.Add(1) go func() { + defer reportWg.Done() txSender.reportSendTxAnomalies(tx, txResultsToReport) - reportWg.Done() }() return txSender.collectTxResults(ctx, tx, healthyNodesNum, txResults) From e30f10ea1355f2c4cb2390f1db23bdcce9c7e567 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 5 Nov 2024 11:39:25 -0500 Subject: [PATCH 146/174] Add result ctx logging --- pkg/solana/client/multinode/transaction_sender.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go index 54c38298c..f910976ad 100644 --- a/pkg/solana/client/multinode/transaction_sender.go +++ b/pkg/solana/client/multinode/transaction_sender.go @@ -131,12 +131,14 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct r := txSender.broadcastTxAsync(ctx, rpc, tx) select { case <-ctx.Done(): + txSender.lggr.Warnw("Failed to send tx results", "err", ctx.Err()) return case txResults <- r: } select { case <-ctx.Done(): + txSender.lggr.Warnw("Failed to send tx results to report", "err", ctx.Err()) return case txResultsToReport <- r: } From b972c1b08758a33fdb5ad4ca43e41e4015be2d42 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 5 Nov 2024 12:10:50 -0500 Subject: [PATCH 147/174] log on close --- pkg/solana/client/multinode/transaction_sender.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go index fdfe61a1b..5ae11e97f 100644 --- a/pkg/solana/client/multinode/transaction_sender.go +++ b/pkg/solana/client/multinode/transaction_sender.go @@ -269,6 +269,7 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) Start(ctx context. func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) Close() error { return txSender.StopOnce("TransactionSender", func() error { + txSender.lggr.Debug("Closing TransactionSender") close(txSender.chStop) txSender.wg.Wait() return nil From 13d7072a726c59d133bf94308395df30af26b558 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 5 Nov 2024 13:16:32 -0500 Subject: [PATCH 148/174] Update transaction_sender.go --- .../client/multinode/transaction_sender.go | 119 +++++++++--------- 1 file changed, 57 insertions(+), 62 deletions(-) diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go index 5ae11e97f..f9fa57539 100644 --- a/pkg/solana/client/multinode/transaction_sender.go +++ b/pkg/solana/client/multinode/transaction_sender.go @@ -92,79 +92,73 @@ type TransactionSender[TX any, RESULT SendTxResult, CHAIN_ID ID, RPC SendTxRPCCl // * If there is both success and terminal error - returns success and reports invariant violation // * Otherwise, returns any (effectively random) of the errors. func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ctx context.Context, tx TX) RESULT { - txResults := make(chan RESULT) - txResultsToReport := make(chan RESULT) - primaryNodeWg := sync.WaitGroup{} + var result RESULT + if !txSender.IfStarted(func() { + txResults := make(chan RESULT) + txResultsToReport := make(chan RESULT) + primaryNodeWg := sync.WaitGroup{} + + ctx, cancel := txSender.chStop.Ctx(ctx) + + healthyNodesNum := 0 + err := txSender.multiNode.DoAll(ctx, func(ctx context.Context, rpc RPC, isSendOnly bool) { + if isSendOnly { + txSender.wg.Add(1) + go func() { + defer txSender.wg.Done() + // Send-only nodes' results are ignored as they tend to return false-positive responses. + // Broadcast to them is necessary to speed up the propagation of TX in the network. + _ = txSender.broadcastTxAsync(ctx, rpc, tx) + }() + return + } - if txSender.State() != "Started" { - return txSender.newResult(errors.New("TransactionSender not started")) - } + // Primary Nodes + healthyNodesNum++ + primaryNodeWg.Add(1) + go func() { + defer primaryNodeWg.Done() + r := txSender.broadcastTxAsync(ctx, rpc, tx) + select { + case <-ctx.Done(): + txSender.lggr.Warnw("Failed to send tx results", "err", ctx.Err()) + return + case txResults <- r: + } + + select { + case <-ctx.Done(): + txSender.lggr.Warnw("Failed to send tx results to report", "err", ctx.Err()) + return + case txResultsToReport <- r: + } + }() + }) - txSenderCtx, cancel := txSender.chStop.NewCtx() - reportWg := sync.WaitGroup{} - defer func() { + // This needs to be done in parallel so the reporting knows when it's done (when the channel is closed) + txSender.wg.Add(1) go func() { - reportWg.Wait() - cancel() + defer txSender.wg.Done() + primaryNodeWg.Wait() + close(txResultsToReport) + close(txResults) + cancel() // TODO: Will this guarantee that collectTxResults will read the last result before ctx.Done()? }() - }() - healthyNodesNum := 0 - err := txSender.multiNode.DoAll(txSenderCtx, func(ctx context.Context, rpc RPC, isSendOnly bool) { - if isSendOnly { - txSender.wg.Add(1) - go func() { - defer txSender.wg.Done() - // Send-only nodes' results are ignored as they tend to return false-positive responses. - // Broadcast to them is necessary to speed up the propagation of TX in the network. - _ = txSender.broadcastTxAsync(ctx, rpc, tx) - }() + if err != nil { + result = txSender.newResult(err) return } - // Primary Nodes - healthyNodesNum++ - primaryNodeWg.Add(1) - go func() { - defer primaryNodeWg.Done() - r := txSender.broadcastTxAsync(ctx, rpc, tx) - select { - case <-ctx.Done(): - txSender.lggr.Warnw("Failed to send tx results", "err", ctx.Err()) - return - case txResults <- r: - } - - select { - case <-ctx.Done(): - txSender.lggr.Warnw("Failed to send tx results to report", "err", ctx.Err()) - return - case txResultsToReport <- r: - } - }() - }) + txSender.wg.Add(1) + go txSender.reportSendTxAnomalies(tx, txResultsToReport) - // This needs to be done in parallel so the reporting knows when it's done (when the channel is closed) - txSender.wg.Add(1) - go func() { - defer txSender.wg.Done() - primaryNodeWg.Wait() - close(txResultsToReport) - close(txResults) - }() - - if err != nil { - return txSender.newResult(err) + result = txSender.collectTxResults(ctx, tx, healthyNodesNum, txResults) + }) { + result = txSender.newResult(errors.New("TransactionSender not started")) } - txSender.wg.Add(1) - reportWg.Add(1) - go func() { - defer reportWg.Done() - txSender.reportSendTxAnomalies(tx, txResultsToReport) - }() - - return txSender.collectTxResults(ctx, tx, healthyNodesNum, txResults) + return result } func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) broadcastTxAsync(ctx context.Context, rpc RPC, tx TX) RESULT { @@ -258,6 +252,7 @@ loop: // ignore critical error as it's reported in reportSendTxAnomalies result, _ := aggregateTxResults(errorsByCode) + txSender.lggr.Debugw("Collected results", "errorsByCode", errorsByCode, "result", result) return result } From 1306020fd8bf2ca667bb50157353c298ef95dadf Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 5 Nov 2024 13:21:51 -0500 Subject: [PATCH 149/174] add cancel func --- pkg/solana/client/multinode/transaction_sender.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go index f9fa57539..ea84dcbc0 100644 --- a/pkg/solana/client/multinode/transaction_sender.go +++ b/pkg/solana/client/multinode/transaction_sender.go @@ -98,7 +98,8 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct txResultsToReport := make(chan RESULT) primaryNodeWg := sync.WaitGroup{} - ctx, cancel := txSender.chStop.Ctx(ctx) + var cancel context.CancelFunc + ctx, cancel = txSender.chStop.Ctx(ctx) healthyNodesNum := 0 err := txSender.multiNode.DoAll(ctx, func(ctx context.Context, rpc RPC, isSendOnly bool) { From 71ed9fc5e77a935739f6ceca8f625be5faefd91d Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 5 Nov 2024 13:33:36 -0500 Subject: [PATCH 150/174] Update transaction_sender.go --- pkg/solana/client/multinode/transaction_sender.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go index ea84dcbc0..6b41275d7 100644 --- a/pkg/solana/client/multinode/transaction_sender.go +++ b/pkg/solana/client/multinode/transaction_sender.go @@ -98,8 +98,7 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct txResultsToReport := make(chan RESULT) primaryNodeWg := sync.WaitGroup{} - var cancel context.CancelFunc - ctx, cancel = txSender.chStop.Ctx(ctx) + _, cancel := txSender.chStop.Ctx(ctx) healthyNodesNum := 0 err := txSender.multiNode.DoAll(ctx, func(ctx context.Context, rpc RPC, isSendOnly bool) { From 50872b61f41393f85b3c58df493e19dbd1290461 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 5 Nov 2024 14:20:45 -0500 Subject: [PATCH 151/174] Update transaction_sender.go --- pkg/solana/client/multinode/transaction_sender.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go index 6b41275d7..80a39f0d4 100644 --- a/pkg/solana/client/multinode/transaction_sender.go +++ b/pkg/solana/client/multinode/transaction_sender.go @@ -98,10 +98,10 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct txResultsToReport := make(chan RESULT) primaryNodeWg := sync.WaitGroup{} - _, cancel := txSender.chStop.Ctx(ctx) + txSenderCtx, cancel := txSender.chStop.Ctx(ctx) healthyNodesNum := 0 - err := txSender.multiNode.DoAll(ctx, func(ctx context.Context, rpc RPC, isSendOnly bool) { + err := txSender.multiNode.DoAll(txSenderCtx, func(ctx context.Context, rpc RPC, isSendOnly bool) { if isSendOnly { txSender.wg.Add(1) go func() { @@ -142,7 +142,7 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct primaryNodeWg.Wait() close(txResultsToReport) close(txResults) - cancel() // TODO: Will this guarantee that collectTxResults will read the last result before ctx.Done()? + cancel() }() if err != nil { From b1c33a266b749acf18c2e0733a6e8db6966314e7 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 5 Nov 2024 14:25:38 -0500 Subject: [PATCH 152/174] Add ctx to reportSendTxAnomalies --- pkg/solana/client/multinode/transaction_sender.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go index 80a39f0d4..190c5dcb4 100644 --- a/pkg/solana/client/multinode/transaction_sender.go +++ b/pkg/solana/client/multinode/transaction_sender.go @@ -151,7 +151,7 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct } txSender.wg.Add(1) - go txSender.reportSendTxAnomalies(tx, txResultsToReport) + go txSender.reportSendTxAnomalies(ctx, tx, txResultsToReport) result = txSender.collectTxResults(ctx, tx, healthyNodesNum, txResults) }) { @@ -170,7 +170,7 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) broadcastTxAsync(c return result } -func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) reportSendTxAnomalies(tx TX, txResults <-chan RESULT) { +func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) reportSendTxAnomalies(ctx context.Context, tx TX, txResults <-chan RESULT) { defer txSender.wg.Done() resultsByCode := sendTxResults[RESULT]{} // txResults eventually will be closed @@ -179,7 +179,7 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) reportSendTxAnomal } _, criticalErr := aggregateTxResults[RESULT](resultsByCode) - if criticalErr != nil { + if criticalErr != nil && ctx.Err() == nil { txSender.lggr.Criticalw("observed invariant violation on SendTransaction", "tx", tx, "resultsByCode", resultsByCode, "err", criticalErr) PromMultiNodeInvariantViolations.WithLabelValues(txSender.chainFamily, txSender.chainID.String(), criticalErr.Error()).Inc() } From cd834722e8742afe043c61b7611f55a799d2efeb Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 6 Nov 2024 10:36:40 -0500 Subject: [PATCH 153/174] Update comments --- pkg/solana/client/multinode_client.go | 3 ++- pkg/solana/config/multinode.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/solana/client/multinode_client.go b/pkg/solana/client/multinode_client.go index 6280cd65c..5aa7d4820 100644 --- a/pkg/solana/client/multinode_client.go +++ b/pkg/solana/client/multinode_client.go @@ -102,7 +102,8 @@ func (m *MultiNodeClient) SubscribeToHeads(ctx context.Context) (<-chan *Head, m ctx, cancel, chStopInFlight, _ := m.acquireQueryCtx(ctx, m.cfg.TxTimeout()) defer cancel() - pollInterval := m.cfg.MultiNode.FinalizedBlockPollInterval() // TODO: Should have HeadPollInterval separate from Version Poll interval + // TODO: BCFR-1070 - Add BlockPollInterval + pollInterval := m.cfg.MultiNode.FinalizedBlockPollInterval() // Use same interval as finalized polling if pollInterval == 0 { return nil, nil, errors.New("PollInterval is 0") } diff --git a/pkg/solana/config/multinode.go b/pkg/solana/config/multinode.go index 194e9bf55..726e22166 100644 --- a/pkg/solana/config/multinode.go +++ b/pkg/solana/config/multinode.go @@ -149,7 +149,7 @@ func (c *MultiNodeConfig) SetDefaults() { } // Finalized block offset allows for RPCs to be slightly behind the finalized block. if c.MultiNode.FinalizedBlockOffset == nil { - c.MultiNode.FinalizedBlockOffset = ptr(uint32(50)) // TODO: Set to 50 for slow test environment + c.MultiNode.FinalizedBlockOffset = ptr(uint32(50)) } } From fce0ca919d406307a47535f9c18b63e91d4650df Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 6 Nov 2024 11:18:19 -0500 Subject: [PATCH 154/174] Fix comments --- pkg/solana/config/multinode.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/solana/config/multinode.go b/pkg/solana/config/multinode.go index 726e22166..d002d489e 100644 --- a/pkg/solana/config/multinode.go +++ b/pkg/solana/config/multinode.go @@ -97,7 +97,7 @@ func (c *MultiNodeConfig) SetDefaults() { if c.MultiNode.PollFailureThreshold == nil { c.MultiNode.PollFailureThreshold = ptr(uint32(5)) } - // Poll interval is set to 10 seconds to ensure timely updates while minimizing resource usage. + // Poll interval is set to 15 seconds to ensure timely updates while minimizing resource usage. if c.MultiNode.PollInterval == nil { c.MultiNode.PollInterval = config.MustNewDuration(15 * time.Second) } @@ -105,7 +105,7 @@ func (c *MultiNodeConfig) SetDefaults() { if c.MultiNode.SelectionMode == nil { c.MultiNode.SelectionMode = ptr(mn.NodeSelectionModePriorityLevel) } - // The sync threshold is set to 5 to allow for some flexibility in node synchronization before considering it out of sync. + // The sync threshold is set to 10 to allow for some flexibility in node synchronization before considering it out of sync. if c.MultiNode.SyncThreshold == nil { c.MultiNode.SyncThreshold = ptr(uint32(10)) } From d0adc9e7964e3c642d11d7611dfb860b9fd504f1 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 7 Nov 2024 10:17:04 -0500 Subject: [PATCH 155/174] Address comments --- pkg/solana/client/multinode/node_lifecycle.go | 24 +++++++++---------- .../client/multinode/transaction_sender.go | 9 +++---- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/pkg/solana/client/multinode/node_lifecycle.go b/pkg/solana/client/multinode/node_lifecycle.go index 2ab3cc8b1..10efc796e 100644 --- a/pkg/solana/client/multinode/node_lifecycle.go +++ b/pkg/solana/client/multinode/node_lifecycle.go @@ -167,9 +167,6 @@ func (n *node[CHAIN_ID, HEAD, RPC]) aliveLoop() { } if outOfSync, liveNodes := n.isOutOfSyncWithPool(); outOfSync { // note: there must be another live node for us to be out of sync - _, highest := n.poolInfoProvider.LatestChainInfo() - _, latestChainInfo := n.StateAndLatest() - lggr.Errorw("RPC endpoint has fallen behind", "blockNumber", latestChainInfo.BlockNumber, "bestLatestBlockNumber", highest.BlockNumber, "totalDifficulty", latestChainInfo.TotalDifficulty, "nodeState", n.getCachedState()) if liveNodes < 2 { lggr.Criticalf("RPC endpoint has fallen behind; %s %s", msgCannotDisable, msgDegradedState) continue @@ -194,8 +191,7 @@ func (n *node[CHAIN_ID, HEAD, RPC]) aliveLoop() { case <-headsSub.NoNewHeads: // We haven't received a head on the channel for at least the // threshold amount of time, mark it broken - _, latestChainInfo := n.StateAndLatest() - lggr.Errorw(fmt.Sprintf("RPC endpoint detected out of sync; no new heads received for %s (last head received was %v)", noNewHeadsTimeoutThreshold, latestChainInfo.BlockNumber), "nodeState", n.getCachedState(), "latestReceivedBlockNumber", latestChainInfo.BlockNumber, "noNewHeadsTimeoutThreshold", noNewHeadsTimeoutThreshold) + lggr.Errorw(fmt.Sprintf("RPC endpoint detected out of sync; no new heads received for %s (last head received was %v)", noNewHeadsTimeoutThreshold, localHighestChainInfo.BlockNumber), "nodeState", n.getCachedState(), "latestReceivedBlockNumber", localHighestChainInfo.BlockNumber, "noNewHeadsTimeoutThreshold", noNewHeadsTimeoutThreshold) if n.poolInfoProvider != nil { if l, _ := n.poolInfoProvider.LatestChainInfo(); l < 2 { lggr.Criticalf("RPC endpoint detected out of sync; %s %s", msgCannotDisable, msgDegradedState) @@ -368,17 +364,22 @@ func (n *node[CHAIN_ID, HEAD, RPC]) isOutOfSyncWithPool() (outOfSync bool, liveN } // Check against best node ln, ci := n.poolInfoProvider.LatestChainInfo() - _, localChainInfo := n.StateAndLatest() + localChainInfo, _ := n.rpc.GetInterceptedChainInfo() mode := n.nodePoolCfg.SelectionMode() switch mode { case NodeSelectionModeHighestHead, NodeSelectionModeRoundRobin, NodeSelectionModePriorityLevel: - return localChainInfo.BlockNumber < ci.BlockNumber-int64(threshold), ln + outOfSync = localChainInfo.BlockNumber < ci.BlockNumber-int64(threshold) case NodeSelectionModeTotalDifficulty: bigThreshold := big.NewInt(int64(threshold)) - return localChainInfo.TotalDifficulty.Cmp(bigmath.Sub(ci.TotalDifficulty, bigThreshold)) < 0, ln + outOfSync = localChainInfo.TotalDifficulty.Cmp(bigmath.Sub(ci.TotalDifficulty, bigThreshold)) < 0 default: panic("unrecognized NodeSelectionMode: " + mode) } + + if outOfSync && n.getCachedState() == NodeStateAlive { + n.lfcLog.Errorw("RPC endpoint has fallen behind", "blockNumber", localChainInfo.BlockNumber, "bestLatestBlockNumber", ci.BlockNumber, "totalDifficulty", localChainInfo.TotalDifficulty) + } + return outOfSync, ln } // outOfSyncLoop takes an OutOfSync node and waits until isOutOfSync returns false to go back to live status @@ -504,8 +505,7 @@ func (n *node[CHAIN_ID, HEAD, RPC]) outOfSyncLoop(syncIssues syncStatus) { continue } - _, latestChainInfo := n.StateAndLatest() - receivedNewHead := n.onNewFinalizedHead(lggr, &latestChainInfo, latestFinalized) + receivedNewHead := n.onNewFinalizedHead(lggr, &localHighestChainInfo, latestFinalized) if !receivedNewHead { continue } @@ -516,9 +516,9 @@ func (n *node[CHAIN_ID, HEAD, RPC]) outOfSyncLoop(syncIssues syncStatus) { finalizedHeadsSub.ResetTimer(noNewFinalizedBlocksTimeoutThreshold) } - _, highestSeen := n.poolInfoProvider.LatestChainInfo() + highestSeen := n.poolInfoProvider.HighestUserObservations() - lggr.Debugw(msgReceivedFinalizedBlock, "blockNumber", latestFinalized.BlockNumber(), "highestBlockNumber", highestSeen.FinalizedBlockNumber, "syncIssues", syncIssues) + lggr.Debugw(msgReceivedFinalizedBlock, "blockNumber", latestFinalized.BlockNumber(), "poolHighestBlockNumber", highestSeen.FinalizedBlockNumber, "syncIssues", syncIssues) case err := <-finalizedHeadsSub.Errors: lggr.Errorw("Finalized head subscription was terminated", "err", err) n.declareUnreachable() diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go index 190c5dcb4..bbc927410 100644 --- a/pkg/solana/client/multinode/transaction_sender.go +++ b/pkg/solana/client/multinode/transaction_sender.go @@ -98,13 +98,13 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct txResultsToReport := make(chan RESULT) primaryNodeWg := sync.WaitGroup{} - txSenderCtx, cancel := txSender.chStop.Ctx(ctx) - healthyNodesNum := 0 - err := txSender.multiNode.DoAll(txSenderCtx, func(ctx context.Context, rpc RPC, isSendOnly bool) { + err := txSender.multiNode.DoAll(ctx, func(ctx context.Context, rpc RPC, isSendOnly bool) { if isSendOnly { txSender.wg.Add(1) go func() { + ctx, cancel := txSender.chStop.Ctx(context.WithoutCancel(ctx)) + defer cancel() defer txSender.wg.Done() // Send-only nodes' results are ignored as they tend to return false-positive responses. // Broadcast to them is necessary to speed up the propagation of TX in the network. @@ -117,6 +117,8 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct healthyNodesNum++ primaryNodeWg.Add(1) go func() { + ctx, cancel := txSender.chStop.Ctx(context.WithoutCancel(ctx)) + defer cancel() defer primaryNodeWg.Done() r := txSender.broadcastTxAsync(ctx, rpc, tx) select { @@ -142,7 +144,6 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct primaryNodeWg.Wait() close(txResultsToReport) close(txResults) - cancel() }() if err != nil { From 66b1757df3018b653172513283162f8b21975caf Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 7 Nov 2024 10:22:59 -0500 Subject: [PATCH 156/174] lint --- pkg/solana/client/multinode/transaction_sender.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go index bbc927410..933a92dee 100644 --- a/pkg/solana/client/multinode/transaction_sender.go +++ b/pkg/solana/client/multinode/transaction_sender.go @@ -103,7 +103,8 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct if isSendOnly { txSender.wg.Add(1) go func() { - ctx, cancel := txSender.chStop.Ctx(context.WithoutCancel(ctx)) + var cancel context.CancelFunc + ctx, cancel = txSender.chStop.Ctx(context.WithoutCancel(ctx)) defer cancel() defer txSender.wg.Done() // Send-only nodes' results are ignored as they tend to return false-positive responses. @@ -117,7 +118,8 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct healthyNodesNum++ primaryNodeWg.Add(1) go func() { - ctx, cancel := txSender.chStop.Ctx(context.WithoutCancel(ctx)) + var cancel context.CancelFunc + ctx, cancel = txSender.chStop.Ctx(context.WithoutCancel(ctx)) defer cancel() defer primaryNodeWg.Done() r := txSender.broadcastTxAsync(ctx, rpc, tx) From 40448bf6c6f680ae1a606868420494ef26a2ba6e Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 7 Nov 2024 10:39:26 -0500 Subject: [PATCH 157/174] lint --- pkg/solana/client/multinode/transaction_sender.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go index 933a92dee..12bf73b55 100644 --- a/pkg/solana/client/multinode/transaction_sender.go +++ b/pkg/solana/client/multinode/transaction_sender.go @@ -103,8 +103,8 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct if isSendOnly { txSender.wg.Add(1) go func() { - var cancel context.CancelFunc - ctx, cancel = txSender.chStop.Ctx(context.WithoutCancel(ctx)) + //nolint:shadow + ctx, cancel := txSender.chStop.Ctx(context.WithoutCancel(ctx)) defer cancel() defer txSender.wg.Done() // Send-only nodes' results are ignored as they tend to return false-positive responses. @@ -118,21 +118,21 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct healthyNodesNum++ primaryNodeWg.Add(1) go func() { - var cancel context.CancelFunc - ctx, cancel = txSender.chStop.Ctx(context.WithoutCancel(ctx)) + //nolint:shadow + ctx, cancel := txSender.chStop.Ctx(context.WithoutCancel(ctx)) defer cancel() defer primaryNodeWg.Done() r := txSender.broadcastTxAsync(ctx, rpc, tx) select { case <-ctx.Done(): - txSender.lggr.Warnw("Failed to send tx results", "err", ctx.Err()) + txSender.lggr.Debugw("Failed to send tx results", "err", ctx.Err()) return case txResults <- r: } select { case <-ctx.Done(): - txSender.lggr.Warnw("Failed to send tx results to report", "err", ctx.Err()) + txSender.lggr.Debugw("Failed to send tx results to report", "err", ctx.Err()) return case txResultsToReport <- r: } @@ -167,7 +167,7 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) broadcastTxAsync(ctx context.Context, rpc RPC, tx TX) RESULT { result := rpc.SendTransaction(ctx, tx) txSender.lggr.Debugw("Node sent transaction", "tx", tx, "err", result.TxError()) - if !slices.Contains(sendTxSuccessfulCodes, result.Code()) { + if !slices.Contains(sendTxSuccessfulCodes, result.Code()) && ctx.Err() == nil { txSender.lggr.Warnw("RPC returned error", "tx", tx, "err", result.TxError()) } return result From 1cc97ed1a105c332ad19074df201ccf38c8a7d81 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 7 Nov 2024 12:27:04 -0500 Subject: [PATCH 158/174] Pass context --- pkg/solana/client/multinode/multi_node.go | 2 +- pkg/solana/client/multinode/transaction_sender.go | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/pkg/solana/client/multinode/multi_node.go b/pkg/solana/client/multinode/multi_node.go index bd97ebc7b..92a65912b 100644 --- a/pkg/solana/client/multinode/multi_node.go +++ b/pkg/solana/client/multinode/multi_node.go @@ -372,6 +372,6 @@ func (c *MultiNode[CHAIN_ID, RPC]) report(nodesStateInfo []nodeWithState) { c.lggr.Criticalw(rerr.Error(), "nodeStates", nodesStateInfo) c.SvcErrBuffer.Append(rerr) } else if dead > 0 { - c.lggr.Errorw(fmt.Sprintf("At least one primary node is dead: %d/%d nodes are alive", live, total), "nodeStates", nodesStateInfo) + c.lggr.Warnw(fmt.Sprintf("At least one primary node is dead: %d/%d nodes are alive", live, total), "nodeStates", nodesStateInfo) } } diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go index 12bf73b55..ad3575dbc 100644 --- a/pkg/solana/client/multinode/transaction_sender.go +++ b/pkg/solana/client/multinode/transaction_sender.go @@ -102,23 +102,21 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct err := txSender.multiNode.DoAll(ctx, func(ctx context.Context, rpc RPC, isSendOnly bool) { if isSendOnly { txSender.wg.Add(1) - go func() { - //nolint:shadow + go func(ctx context.Context) { ctx, cancel := txSender.chStop.Ctx(context.WithoutCancel(ctx)) defer cancel() defer txSender.wg.Done() // Send-only nodes' results are ignored as they tend to return false-positive responses. // Broadcast to them is necessary to speed up the propagation of TX in the network. _ = txSender.broadcastTxAsync(ctx, rpc, tx) - }() + }(ctx) return } // Primary Nodes healthyNodesNum++ primaryNodeWg.Add(1) - go func() { - //nolint:shadow + go func(ctx context.Context) { ctx, cancel := txSender.chStop.Ctx(context.WithoutCancel(ctx)) defer cancel() defer primaryNodeWg.Done() @@ -136,7 +134,7 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct return case txResultsToReport <- r: } - }() + }(ctx) }) // This needs to be done in parallel so the reporting knows when it's done (when the channel is closed) From 5c9fd29157c58eaf020f655a3a0ce70c45c23947 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 7 Nov 2024 12:46:46 -0500 Subject: [PATCH 159/174] Update node_lifecycle.go --- pkg/solana/client/multinode/node_lifecycle.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/solana/client/multinode/node_lifecycle.go b/pkg/solana/client/multinode/node_lifecycle.go index 10efc796e..bca637a22 100644 --- a/pkg/solana/client/multinode/node_lifecycle.go +++ b/pkg/solana/client/multinode/node_lifecycle.go @@ -516,7 +516,10 @@ func (n *node[CHAIN_ID, HEAD, RPC]) outOfSyncLoop(syncIssues syncStatus) { finalizedHeadsSub.ResetTimer(noNewFinalizedBlocksTimeoutThreshold) } - highestSeen := n.poolInfoProvider.HighestUserObservations() + var highestSeen ChainInfo + if n.poolInfoProvider != nil { + highestSeen = n.poolInfoProvider.HighestUserObservations() + } lggr.Debugw(msgReceivedFinalizedBlock, "blockNumber", latestFinalized.BlockNumber(), "poolHighestBlockNumber", highestSeen.FinalizedBlockNumber, "syncIssues", syncIssues) case err := <-finalizedHeadsSub.Errors: From fdbf11983a5aadf3a32a192e8686ac8974f604e7 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 8 Nov 2024 12:26:59 -0500 Subject: [PATCH 160/174] Use get reader function --- pkg/solana/cache_test.go | 14 +++++++++----- pkg/solana/config_tracker.go | 11 ++++++----- pkg/solana/config_tracker_test.go | 4 +++- pkg/solana/relay.go | 17 +++++++---------- pkg/solana/state_cache.go | 11 ++++++++--- pkg/solana/transmissions_cache.go | 14 ++++++++++---- pkg/solana/transmitter.go | 11 +++++++---- pkg/solana/transmitter_test.go | 3 ++- 8 files changed, 52 insertions(+), 33 deletions(-) diff --git a/pkg/solana/cache_test.go b/pkg/solana/cache_test.go index e39bb52ad..1db616783 100644 --- a/pkg/solana/cache_test.go +++ b/pkg/solana/cache_test.go @@ -106,8 +106,10 @@ func TestGetState(t *testing.T) { })) defer mockServer.Close() + reader := func() (client.Reader, error) { return testSetupReader(t, mockServer.URL), nil } + // happy path does not error (actual state decoding handled in types_test) - _, _, err := GetState(context.TODO(), testSetupReader(t, mockServer.URL), solana.PublicKey{}, "") + _, _, err := GetState(context.TODO(), reader, solana.PublicKey{}, "") require.NoError(t, err) } @@ -132,7 +134,7 @@ func TestGetLatestTransmission(t *testing.T) { })) defer mockServer.Close() - reader := testSetupReader(t, mockServer.URL) + reader := func() (client.Reader, error) { return testSetupReader(t, mockServer.URL), nil } a, _, err := GetLatestTransmission(context.TODO(), reader, solana.PublicKey{}, "") assert.NoError(t, err) assert.Equal(t, expectedTime, a.Timestamp) @@ -166,12 +168,14 @@ func TestCache(t *testing.T) { w.Write(testTransmissionsResponse(t, body, 0)) //nolint:errcheck })) + reader := func() (client.Reader, error) { return testSetupReader(t, mockServer.URL), nil } + lggr := logger.Test(t) stateCache := NewStateCache( solana.MustPublicKeyFromBase58("11111111111111111111111111111111"), "test-chain-id", config.NewDefault(), - testSetupReader(t, mockServer.URL), + reader, lggr, ) require.NoError(t, stateCache.Start(ctx)) @@ -186,7 +190,7 @@ func TestCache(t *testing.T) { solana.MustPublicKeyFromBase58("11111111111111111111111111111112"), "test-chain-id", config.NewDefault(), - testSetupReader(t, mockServer.URL), + reader, lggr, ) require.NoError(t, transmissionsCache.Start(ctx)) @@ -220,7 +224,7 @@ func TestNilPointerHandling(t *testing.T) { defer mockServer.Close() errString := "nil pointer returned in " - reader := testSetupReader(t, mockServer.URL) + reader := func() (client.Reader, error) { return testSetupReader(t, mockServer.URL), nil } // fail on get state query _, _, err := GetState(context.TODO(), reader, solana.PublicKey{}, "") diff --git a/pkg/solana/config_tracker.go b/pkg/solana/config_tracker.go index 998790b45..511771396 100644 --- a/pkg/solana/config_tracker.go +++ b/pkg/solana/config_tracker.go @@ -2,16 +2,13 @@ package solana import ( "context" - "github.com/smartcontractkit/libocr/offchainreporting2/reportingplugin/median" "github.com/smartcontractkit/libocr/offchainreporting2/types" - - "github.com/smartcontractkit/chainlink-solana/pkg/solana/client" ) type ConfigTracker struct { stateCache *StateCache - reader client.Reader + reader GetReader } func (c *ConfigTracker) Notify() <-chan struct{} { @@ -75,5 +72,9 @@ func (c *ConfigTracker) LatestConfig(ctx context.Context, changedInBlock uint64) // LatestBlockHeight returns the height of the most recent block in the chain. func (c *ConfigTracker) LatestBlockHeight(ctx context.Context) (blockHeight uint64, err error) { - return c.reader.SlotHeight(ctx) // this returns the latest slot height through CommitmentProcessed + reader, err := c.reader() + if err != nil { + return 0, err + } + return reader.SlotHeight(ctx) // this returns the latest slot height through CommitmentProcessed } diff --git a/pkg/solana/config_tracker_test.go b/pkg/solana/config_tracker_test.go index 1e88d4ecd..88ba0442e 100644 --- a/pkg/solana/config_tracker_test.go +++ b/pkg/solana/config_tracker_test.go @@ -8,6 +8,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/smartcontractkit/chainlink-solana/pkg/solana/client" ) func TestLatestBlockHeight(t *testing.T) { @@ -19,7 +21,7 @@ func TestLatestBlockHeight(t *testing.T) { ctx := context.Background() c := &ConfigTracker{ - reader: testSetupReader(t, mockServer.URL), + reader: func() (client.Reader, error) { return testSetupReader(t, mockServer.URL), nil }, } h, err := c.LatestBlockHeight(ctx) diff --git a/pkg/solana/relay.go b/pkg/solana/relay.go index 6edd11b4f..e2b5615cb 100644 --- a/pkg/solana/relay.go +++ b/pkg/solana/relay.go @@ -154,7 +154,7 @@ func (r *Relayer) NewMedianProvider(ctx context.Context, rargs relaytypes.RelayA } cfg := configWatcher.chain.Config() - transmissionsCache := NewTransmissionsCache(transmissionsID, relayConfig.ChainID, cfg, configWatcher.reader, r.lggr) + transmissionsCache := NewTransmissionsCache(transmissionsID, relayConfig.ChainID, cfg, configWatcher.chain.Reader, r.lggr) return &medianProvider{ configProvider: configWatcher, transmissionsCache: transmissionsCache, @@ -169,7 +169,7 @@ func (r *Relayer) NewMedianProvider(ctx context.Context, rargs relaytypes.RelayA storeProgramID: configWatcher.storeProgramID, transmissionsID: transmissionsID, transmissionSigner: transmitterAccount, - reader: configWatcher.reader, + reader: configWatcher.chain.Reader, stateCache: configWatcher.stateCache, lggr: r.lggr, txManager: configWatcher.chain.TxManager(), @@ -187,6 +187,8 @@ func (r *Relayer) NewAutomationProvider(ctx context.Context, rargs relaytypes.Re var _ relaytypes.ConfigProvider = &configProvider{} +type GetReader func() (client.Reader, error) + type configProvider struct { services.StateMachine chainID string @@ -195,7 +197,7 @@ type configProvider struct { offchainConfigDigester types.OffchainConfigDigester configTracker types.ContractConfigTracker chain Chain - reader client.Reader + reader GetReader } func newConfigProvider(_ context.Context, lggr logger.Logger, chain Chain, args relaytypes.RelayArgs) (*configProvider, error) { @@ -222,11 +224,7 @@ func newConfigProvider(_ context.Context, lggr logger.Logger, chain Chain, args StateID: stateID, } - reader, err := chain.Reader() - if err != nil { - return nil, fmt.Errorf("error in NewMedianProvider.chain.Reader: %w", err) - } - stateCache := NewStateCache(stateID, relayConfig.ChainID, chain.Config(), reader, lggr) + stateCache := NewStateCache(stateID, relayConfig.ChainID, chain.Config(), chain.Reader, lggr) return &configProvider{ chainID: relayConfig.ChainID, stateID: stateID, @@ -234,9 +232,8 @@ func newConfigProvider(_ context.Context, lggr logger.Logger, chain Chain, args storeProgramID: storeProgramID, stateCache: stateCache, offchainConfigDigester: offchainConfigDigester, - configTracker: &ConfigTracker{stateCache: stateCache, reader: reader}, + configTracker: &ConfigTracker{stateCache: stateCache, reader: chain.Reader}, chain: chain, - reader: reader, }, nil } diff --git a/pkg/solana/state_cache.go b/pkg/solana/state_cache.go index 9faa766d0..7d9794a10 100644 --- a/pkg/solana/state_cache.go +++ b/pkg/solana/state_cache.go @@ -23,7 +23,7 @@ type StateCache struct { *client.Cache[State] } -func NewStateCache(stateID solana.PublicKey, chainID string, cfg config.Config, reader client.Reader, lggr logger.Logger) *StateCache { +func NewStateCache(stateID solana.PublicKey, chainID string, cfg config.Config, reader GetReader, lggr logger.Logger) *StateCache { name := "ocr2_median_state" getter := func(ctx context.Context) (State, uint64, error) { return GetState(ctx, reader, stateID, cfg.Commitment()) @@ -31,8 +31,13 @@ func NewStateCache(stateID solana.PublicKey, chainID string, cfg config.Config, return &StateCache{client.NewCache(name, stateID, chainID, cfg, getter, logger.With(lggr, "cache", name))} } -func GetState(ctx context.Context, reader client.AccountReader, account solana.PublicKey, commitment rpc.CommitmentType) (State, uint64, error) { - res, err := reader.GetAccountInfoWithOpts(ctx, account, &rpc.GetAccountInfoOpts{ +func GetState(ctx context.Context, reader GetReader, account solana.PublicKey, commitment rpc.CommitmentType) (State, uint64, error) { + r, err := reader() + if err != nil { + return State{}, 0, fmt.Errorf("failed to get reader: %w", err) + } + + res, err := r.GetAccountInfoWithOpts(ctx, account, &rpc.GetAccountInfoOpts{ Commitment: commitment, Encoding: "base64", }) diff --git a/pkg/solana/transmissions_cache.go b/pkg/solana/transmissions_cache.go index 75ad30a6b..b572541e2 100644 --- a/pkg/solana/transmissions_cache.go +++ b/pkg/solana/transmissions_cache.go @@ -19,7 +19,7 @@ type TransmissionsCache struct { *client.Cache[Answer] } -func NewTransmissionsCache(transmissionsID solana.PublicKey, chainID string, cfg config.Config, reader client.Reader, lggr logger.Logger) *TransmissionsCache { +func NewTransmissionsCache(transmissionsID solana.PublicKey, chainID string, cfg config.Config, reader GetReader, lggr logger.Logger) *TransmissionsCache { name := "ocr2_median_transmissions" getter := func(ctx context.Context) (Answer, uint64, error) { return GetLatestTransmission(ctx, reader, transmissionsID, cfg.Commitment()) @@ -27,11 +27,17 @@ func NewTransmissionsCache(transmissionsID solana.PublicKey, chainID string, cfg return &TransmissionsCache{client.NewCache(name, transmissionsID, chainID, cfg, getter, logger.With(lggr, "cache", name))} } -func GetLatestTransmission(ctx context.Context, reader client.AccountReader, account solana.PublicKey, commitment rpc.CommitmentType) (Answer, uint64, error) { +func GetLatestTransmission(ctx context.Context, reader GetReader, account solana.PublicKey, commitment rpc.CommitmentType) (Answer, uint64, error) { // query for transmission header headerStart := AccountDiscriminatorLen // skip account discriminator headerLen := TransmissionsHeaderLen - res, err := reader.GetAccountInfoWithOpts(ctx, account, &rpc.GetAccountInfoOpts{ + + r, err := reader() + if err != nil { + return Answer{}, 0, fmt.Errorf("failed to get reader: %w", err) + } + + res, err := r.GetAccountInfoWithOpts(ctx, account, &rpc.GetAccountInfoOpts{ Encoding: "base64", Commitment: commitment, DataSlice: &rpc.DataSlice{ @@ -71,7 +77,7 @@ func GetLatestTransmission(ctx context.Context, reader client.AccountReader, acc transmissionOffset := AccountDiscriminatorLen + TransmissionsHeaderMaxSize + (uint64(cursor) * transmissionLen) - res, err = reader.GetAccountInfoWithOpts(ctx, account, &rpc.GetAccountInfoOpts{ + res, err = r.GetAccountInfoWithOpts(ctx, account, &rpc.GetAccountInfoOpts{ Encoding: "base64", Commitment: commitment, DataSlice: &rpc.DataSlice{ diff --git a/pkg/solana/transmitter.go b/pkg/solana/transmitter.go index 4a3731921..db4e4ac03 100644 --- a/pkg/solana/transmitter.go +++ b/pkg/solana/transmitter.go @@ -11,15 +11,13 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/utils" - - "github.com/smartcontractkit/chainlink-solana/pkg/solana/client" ) var _ types.ContractTransmitter = (*Transmitter)(nil) type Transmitter struct { stateID, programID, storeProgramID, transmissionsID, transmissionSigner solana.PublicKey - reader client.Reader + reader GetReader stateCache *StateCache lggr logger.Logger txManager TxManager @@ -32,7 +30,12 @@ func (c *Transmitter) Transmit( report types.Report, sigs []types.AttributedOnchainSignature, ) error { - blockhash, err := c.reader.LatestBlockhash(ctx) + reader, err := c.reader() + if err != nil { + return fmt.Errorf("error on Transmit.Reader: %w", err) + } + + blockhash, err := reader.LatestBlockhash(ctx) if err != nil { return fmt.Errorf("error on Transmit.GetRecentBlockhash: %w", err) } diff --git a/pkg/solana/transmitter_test.go b/pkg/solana/transmitter_test.go index 66dd8658c..718808679 100644 --- a/pkg/solana/transmitter_test.go +++ b/pkg/solana/transmitter_test.go @@ -14,6 +14,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" + "github.com/smartcontractkit/chainlink-solana/pkg/solana/client" clientmocks "github.com/smartcontractkit/chainlink-solana/pkg/solana/client/mocks" "github.com/smartcontractkit/chainlink-solana/pkg/solana/fees" "github.com/smartcontractkit/chainlink-solana/pkg/solana/txm" @@ -68,7 +69,7 @@ func TestTransmitter_TxSize(t *testing.T) { storeProgramID: mustNewRandomPublicKey(), transmissionsID: mustNewRandomPublicKey(), transmissionSigner: signer.PublicKey(), - reader: rw, + reader: func() (client.Reader, error) { return rw, nil }, stateCache: &StateCache{}, lggr: logger.Test(t), txManager: mockTxm, From e1eab95da92cb38572d72489267def62d96c1cbe Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 8 Nov 2024 12:52:46 -0500 Subject: [PATCH 161/174] Make rpcurls plural --- integration-tests/common/common.go | 14 +++++++------- integration-tests/common/test_common.go | 16 ++++++++-------- integration-tests/config/config.go | 12 ++++++------ integration-tests/testconfig/testconfig.go | 16 ++++++++-------- 4 files changed, 29 insertions(+), 29 deletions(-) diff --git a/integration-tests/common/common.go b/integration-tests/common/common.go index 57fb96452..05ccabbad 100644 --- a/integration-tests/common/common.go +++ b/integration-tests/common/common.go @@ -51,7 +51,7 @@ type TestEnvDetails struct { type ChainDetails struct { ChainName string ChainID string - RPCUrl []string + RPCUrls []string RPCURLExternal string WSURLExternal string ProgramAddresses *chainConfig.ProgramAddresses @@ -116,9 +116,9 @@ func New(testConfig *tc.TestConfig) *Common { config = chainConfig.DevnetConfig() privateKeyString = *testConfig.Common.PrivateKey - if len(*testConfig.Common.RPCURL) > 0 { - config.RPCUrl = *testConfig.Common.RPCURL - config.WSUrl = *testConfig.Common.WsURL + if len(*testConfig.Common.RPCURLs) > 0 { + config.RPCUrls = *testConfig.Common.RPCURLs + config.WSUrls = *testConfig.Common.WsURLs config.ProgramAddresses = &chainConfig.ProgramAddresses{ OCR2: *testConfig.SolanaConfig.OCR2ProgramID, AccessController: *testConfig.SolanaConfig.AccessControllerProgramID, @@ -130,7 +130,7 @@ func New(testConfig *tc.TestConfig) *Common { c = &Common{ ChainDetails: &ChainDetails{ ChainID: config.ChainID, - RPCUrl: config.RPCUrl, + RPCUrls: config.RPCUrls, ChainName: config.ChainName, ProgramAddresses: config.ProgramAddresses, }, @@ -146,7 +146,7 @@ func New(testConfig *tc.TestConfig) *Common { } // provide getters for TestConfig (pointers to chain details) c.TestConfig.GetChainID = func() string { return c.ChainDetails.ChainID } - c.TestConfig.GetURL = func() []string { return c.ChainDetails.RPCUrl } + c.TestConfig.GetURL = func() []string { return c.ChainDetails.RPCUrls } return c } @@ -298,7 +298,7 @@ func (c *Common) CreateJobsForContract(contractNodeInfo *ContractNodeInfo) error bootstrapNodeInternalIP = contractNodeInfo.BootstrapNode.InternalIP() } relayConfig := job.JSONConfig{ - "nodeEndpointHTTP": c.ChainDetails.RPCUrl, + "nodeEndpointHTTP": c.ChainDetails.RPCUrls, "ocr2ProgramID": contractNodeInfo.OCR2.ProgramAddress(), "transmissionsID": contractNodeInfo.Store.TransmissionsAddress(), "storeProgramID": contractNodeInfo.Store.ProgramAddress(), diff --git a/integration-tests/common/test_common.go b/integration-tests/common/test_common.go index c738f632c..b351ee73d 100644 --- a/integration-tests/common/test_common.go +++ b/integration-tests/common/test_common.go @@ -118,9 +118,9 @@ func (m *OCRv2TestState) DeployCluster(contractsDir string) { m.Common.ChainDetails.WSURLExternal = m.Common.Env.URLs["sol"][1] if *m.Config.TestConfig.Common.Network == "devnet" { - m.Common.ChainDetails.RPCUrl = *m.Config.TestConfig.Common.RPCURL - m.Common.ChainDetails.RPCURLExternal = (*m.Config.TestConfig.Common.RPCURL)[0] - m.Common.ChainDetails.WSURLExternal = (*m.Config.TestConfig.Common.WsURL)[0] + m.Common.ChainDetails.RPCUrls = *m.Config.TestConfig.Common.RPCURLs + m.Common.ChainDetails.RPCURLExternal = (*m.Config.TestConfig.Common.RPCURLs)[0] + m.Common.ChainDetails.WSURLExternal = (*m.Config.TestConfig.Common.WsURLs)[0] } m.Common.ChainDetails.MockserverURLInternal = m.Common.Env.URLs["qa_mock_adapter_internal"][0] @@ -133,14 +133,14 @@ func (m *OCRv2TestState) DeployCluster(contractsDir string) { require.NoError(m.Config.T, err) // Setting the External RPC url for Gauntlet - m.Common.ChainDetails.RPCUrl = []string{sol.InternalHTTPURL} + m.Common.ChainDetails.RPCUrls = []string{sol.InternalHTTPURL} m.Common.ChainDetails.RPCURLExternal = sol.ExternalHTTPURL m.Common.ChainDetails.WSURLExternal = sol.ExternalWsURL if *m.Config.TestConfig.Common.Network == "devnet" { - m.Common.ChainDetails.RPCUrl = *m.Config.TestConfig.Common.RPCURL - m.Common.ChainDetails.RPCURLExternal = (*m.Config.TestConfig.Common.RPCURL)[0] - m.Common.ChainDetails.WSURLExternal = (*m.Config.TestConfig.Common.WsURL)[0] + m.Common.ChainDetails.RPCUrls = *m.Config.TestConfig.Common.RPCURLs + m.Common.ChainDetails.RPCURLExternal = (*m.Config.TestConfig.Common.RPCURLs)[0] + m.Common.ChainDetails.WSURLExternal = (*m.Config.TestConfig.Common.WsURLs)[0] } b, err := test_env.NewCLTestEnvBuilder(). @@ -273,7 +273,7 @@ func (m *OCRv2TestState) CreateJobs() { require.NoError(m.Config.T, err, "Error connecting to websocket client") relayConfig := job.JSONConfig{ - "nodeEndpointHTTP": m.Common.ChainDetails.RPCUrl, + "nodeEndpointHTTP": m.Common.ChainDetails.RPCUrls, "ocr2ProgramID": m.Common.ChainDetails.ProgramAddresses.OCR2, "transmissionsID": m.Gauntlet.FeedAddress, "storeProgramID": m.Common.ChainDetails.ProgramAddresses.Store, diff --git a/integration-tests/config/config.go b/integration-tests/config/config.go index 303e1a5a0..1b96b1f77 100644 --- a/integration-tests/config/config.go +++ b/integration-tests/config/config.go @@ -3,8 +3,8 @@ package config type Config struct { ChainName string ChainID string - RPCUrl []string - WSUrl []string + RPCUrls []string + WSUrls []string ProgramAddresses *ProgramAddresses PrivateKey string } @@ -20,8 +20,8 @@ func DevnetConfig() *Config { ChainName: "solana", ChainID: "devnet", // Will be overridden if set in toml - RPCUrl: []string{"https://api.devnet.solana.com"}, - WSUrl: []string{"wss://api.devnet.solana.com/"}, + RPCUrls: []string{"https://api.devnet.solana.com"}, + WSUrls: []string{"wss://api.devnet.solana.com/"}, } } @@ -30,8 +30,8 @@ func LocalNetConfig() *Config { ChainName: "solana", ChainID: "localnet", // Will be overridden if set in toml - RPCUrl: []string{"http://sol:8899"}, - WSUrl: []string{"ws://sol:8900"}, + RPCUrls: []string{"http://sol:8899"}, + WSUrls: []string{"ws://sol:8900"}, ProgramAddresses: &ProgramAddresses{ OCR2: "E3j24rx12SyVsG6quKuZPbQqZPkhAUCh8Uek4XrKYD2x", AccessController: "2ckhep7Mvy1dExenBqpcdevhRu7CLuuctMcx7G9mWEvo", diff --git a/integration-tests/testconfig/testconfig.go b/integration-tests/testconfig/testconfig.go index 2f7fb9a01..1f482b7f5 100644 --- a/integration-tests/testconfig/testconfig.go +++ b/integration-tests/testconfig/testconfig.go @@ -193,8 +193,8 @@ func (c *TestConfig) ReadFromEnvVar() error { if c.Common == nil { c.Common = &Common{} } - logger.Info().Msgf("Using %s env var to override Common.RPCURL", E2E_TEST_COMMON_RPC_URL_ENV) - c.Common.RPCURL = &commonRPCURL + logger.Info().Msgf("Using %s env var to override Common.RPCURLs", E2E_TEST_COMMON_RPC_URL_ENV) + c.Common.RPCURLs = &commonRPCURL } commonWSURL := ctf_config.MustReadEnvVar_Strings(E2E_TEST_COMMON_WS_URL_ENV, ",") @@ -202,8 +202,8 @@ func (c *TestConfig) ReadFromEnvVar() error { if c.Common == nil { c.Common = &Common{} } - logger.Info().Msgf("Using %s env var to override Common.WsURL", E2E_TEST_COMMON_WS_URL_ENV) - c.Common.WsURL = &commonWSURL + logger.Info().Msgf("Using %s env var to override Common.WsURLs", E2E_TEST_COMMON_WS_URL_ENV) + c.Common.WsURLs = &commonWSURL } commonPrivateKey := ctf_config.MustReadEnvVar_String(E2E_TEST_COMMON_PRIVATE_KEY_ENV) @@ -377,8 +377,8 @@ type Common struct { InsideK8s *bool `toml:"inside_k8"` User *string `toml:"user"` // if rpc requires api key to be passed as an HTTP header - RPCURL *[]string `toml:"-"` - WsURL *[]string `toml:"-"` + RPCURLs *[]string `toml:"-"` + WsURLs *[]string `toml:"-"` PrivateKey *string `toml:"-"` Stateful *bool `toml:"stateful_db"` InternalDockerRepo *string `toml:"internal_docker_repo"` @@ -430,10 +430,10 @@ func (c *Common) Validate() error { if c.PrivateKey == nil { return fmt.Errorf("private_key must be set") } - if c.RPCURL == nil { + if c.RPCURLs == nil { return fmt.Errorf("rpc_url must be set") } - if c.WsURL == nil { + if c.WsURLs == nil { return fmt.Errorf("rpc_url must be set") } From 5c0079af14f6a4a49be6e0b12a4cd51905acd50e Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 8 Nov 2024 13:14:28 -0500 Subject: [PATCH 162/174] Fix reader getters --- integration-tests/solclient/store.go | 4 +++- pkg/monitoring/chain_reader.go | 8 +++++--- pkg/solana/cache_test.go | 7 +++---- pkg/solana/relay.go | 1 - pkg/solana/state_cache.go | 7 +++++-- pkg/solana/transmissions_cache.go | 5 +++-- 6 files changed, 19 insertions(+), 13 deletions(-) diff --git a/integration-tests/solclient/store.go b/integration-tests/solclient/store.go index 238d5cc31..3bc48bec9 100644 --- a/integration-tests/solclient/store.go +++ b/integration-tests/solclient/store.go @@ -8,6 +8,7 @@ import ( "github.com/smartcontractkit/chainlink-solana/contracts/generated/store" relaySol "github.com/smartcontractkit/chainlink-solana/pkg/solana" + "github.com/smartcontractkit/chainlink-solana/pkg/solana/client" ) type Store struct { @@ -19,7 +20,8 @@ type Store struct { } func (m *Store) GetLatestRoundData() (uint64, uint64, uint64, error) { - a, _, err := relaySol.GetLatestTransmission(context.Background(), m.Client.RPC, m.Feed.PublicKey(), rpc.CommitmentConfirmed) + getReader := func() (client.AccountReader, error) { return m.Client.RPC, nil } + a, _, err := relaySol.GetLatestTransmission(context.Background(), getReader, m.Feed.PublicKey(), rpc.CommitmentConfirmed) if err != nil { return 0, 0, 0, err } diff --git a/pkg/monitoring/chain_reader.go b/pkg/monitoring/chain_reader.go index eb4d4b8e5..dee5ddeca 100644 --- a/pkg/monitoring/chain_reader.go +++ b/pkg/monitoring/chain_reader.go @@ -2,9 +2,9 @@ package monitoring import ( "context" - "github.com/gagliardetto/solana-go" "github.com/gagliardetto/solana-go/rpc" + "github.com/smartcontractkit/chainlink-solana/pkg/solana/client" pkgSolana "github.com/smartcontractkit/chainlink-solana/pkg/solana" ) @@ -31,11 +31,13 @@ type chainReader struct { } func (c *chainReader) GetState(ctx context.Context, account solana.PublicKey, commitment rpc.CommitmentType) (state pkgSolana.State, blockHeight uint64, err error) { - return pkgSolana.GetState(ctx, c.client, account, commitment) + getReader := func() (client.AccountReader, error) { return c.client, nil } + return pkgSolana.GetState(ctx, getReader, account, commitment) } func (c *chainReader) GetLatestTransmission(ctx context.Context, account solana.PublicKey, commitment rpc.CommitmentType) (answer pkgSolana.Answer, blockHeight uint64, err error) { - return pkgSolana.GetLatestTransmission(ctx, c.client, account, commitment) + getReader := func() (client.AccountReader, error) { return c.client, nil } + return pkgSolana.GetLatestTransmission(ctx, getReader, account, commitment) } func (c *chainReader) GetTokenAccountBalance(ctx context.Context, account solana.PublicKey, commitment rpc.CommitmentType) (out *rpc.GetTokenAccountBalanceResult, err error) { diff --git a/pkg/solana/cache_test.go b/pkg/solana/cache_test.go index 1db616783..ef1fb7cdb 100644 --- a/pkg/solana/cache_test.go +++ b/pkg/solana/cache_test.go @@ -106,8 +106,7 @@ func TestGetState(t *testing.T) { })) defer mockServer.Close() - reader := func() (client.Reader, error) { return testSetupReader(t, mockServer.URL), nil } - + reader := func() (client.AccountReader, error) { return testSetupReader(t, mockServer.URL), nil } // happy path does not error (actual state decoding handled in types_test) _, _, err := GetState(context.TODO(), reader, solana.PublicKey{}, "") require.NoError(t, err) @@ -134,7 +133,7 @@ func TestGetLatestTransmission(t *testing.T) { })) defer mockServer.Close() - reader := func() (client.Reader, error) { return testSetupReader(t, mockServer.URL), nil } + reader := func() (client.AccountReader, error) { return testSetupReader(t, mockServer.URL), nil } a, _, err := GetLatestTransmission(context.TODO(), reader, solana.PublicKey{}, "") assert.NoError(t, err) assert.Equal(t, expectedTime, a.Timestamp) @@ -224,7 +223,7 @@ func TestNilPointerHandling(t *testing.T) { defer mockServer.Close() errString := "nil pointer returned in " - reader := func() (client.Reader, error) { return testSetupReader(t, mockServer.URL), nil } + reader := func() (client.AccountReader, error) { return testSetupReader(t, mockServer.URL), nil } // fail on get state query _, _, err := GetState(context.TODO(), reader, solana.PublicKey{}, "") diff --git a/pkg/solana/relay.go b/pkg/solana/relay.go index e2b5615cb..d67373b31 100644 --- a/pkg/solana/relay.go +++ b/pkg/solana/relay.go @@ -197,7 +197,6 @@ type configProvider struct { offchainConfigDigester types.OffchainConfigDigester configTracker types.ContractConfigTracker chain Chain - reader GetReader } func newConfigProvider(_ context.Context, lggr logger.Logger, chain Chain, args relaytypes.RelayArgs) (*configProvider, error) { diff --git a/pkg/solana/state_cache.go b/pkg/solana/state_cache.go index 7d9794a10..614ad0d81 100644 --- a/pkg/solana/state_cache.go +++ b/pkg/solana/state_cache.go @@ -23,15 +23,18 @@ type StateCache struct { *client.Cache[State] } +type GetAccountReader func() (client.AccountReader, error) + func NewStateCache(stateID solana.PublicKey, chainID string, cfg config.Config, reader GetReader, lggr logger.Logger) *StateCache { name := "ocr2_median_state" getter := func(ctx context.Context) (State, uint64, error) { - return GetState(ctx, reader, stateID, cfg.Commitment()) + getAccountReader := func() (client.AccountReader, error) { return reader() } + return GetState(ctx, getAccountReader, stateID, cfg.Commitment()) } return &StateCache{client.NewCache(name, stateID, chainID, cfg, getter, logger.With(lggr, "cache", name))} } -func GetState(ctx context.Context, reader GetReader, account solana.PublicKey, commitment rpc.CommitmentType) (State, uint64, error) { +func GetState(ctx context.Context, reader GetAccountReader, account solana.PublicKey, commitment rpc.CommitmentType) (State, uint64, error) { r, err := reader() if err != nil { return State{}, 0, fmt.Errorf("failed to get reader: %w", err) diff --git a/pkg/solana/transmissions_cache.go b/pkg/solana/transmissions_cache.go index b572541e2..25c6125ea 100644 --- a/pkg/solana/transmissions_cache.go +++ b/pkg/solana/transmissions_cache.go @@ -22,12 +22,13 @@ type TransmissionsCache struct { func NewTransmissionsCache(transmissionsID solana.PublicKey, chainID string, cfg config.Config, reader GetReader, lggr logger.Logger) *TransmissionsCache { name := "ocr2_median_transmissions" getter := func(ctx context.Context) (Answer, uint64, error) { - return GetLatestTransmission(ctx, reader, transmissionsID, cfg.Commitment()) + getAccountReader := func() (client.AccountReader, error) { return reader() } + return GetLatestTransmission(ctx, getAccountReader, transmissionsID, cfg.Commitment()) } return &TransmissionsCache{client.NewCache(name, transmissionsID, chainID, cfg, getter, logger.With(lggr, "cache", name))} } -func GetLatestTransmission(ctx context.Context, reader GetReader, account solana.PublicKey, commitment rpc.CommitmentType) (Answer, uint64, error) { +func GetLatestTransmission(ctx context.Context, reader GetAccountReader, account solana.PublicKey, commitment rpc.CommitmentType) (Answer, uint64, error) { // query for transmission header headerStart := AccountDiscriminatorLen // skip account discriminator headerLen := TransmissionsHeaderLen From 9838345eaee618e688f4e63b65d2450e52ba30e1 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 8 Nov 2024 13:19:59 -0500 Subject: [PATCH 163/174] lint --- pkg/monitoring/chain_reader.go | 2 +- pkg/solana/config_tracker.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/monitoring/chain_reader.go b/pkg/monitoring/chain_reader.go index dee5ddeca..196f224cf 100644 --- a/pkg/monitoring/chain_reader.go +++ b/pkg/monitoring/chain_reader.go @@ -4,9 +4,9 @@ import ( "context" "github.com/gagliardetto/solana-go" "github.com/gagliardetto/solana-go/rpc" - "github.com/smartcontractkit/chainlink-solana/pkg/solana/client" pkgSolana "github.com/smartcontractkit/chainlink-solana/pkg/solana" + "github.com/smartcontractkit/chainlink-solana/pkg/solana/client" ) //go:generate mockery --name ChainReader --output ./mocks/ diff --git a/pkg/solana/config_tracker.go b/pkg/solana/config_tracker.go index 511771396..9a1fa8e28 100644 --- a/pkg/solana/config_tracker.go +++ b/pkg/solana/config_tracker.go @@ -2,6 +2,7 @@ package solana import ( "context" + "github.com/smartcontractkit/libocr/offchainreporting2/reportingplugin/median" "github.com/smartcontractkit/libocr/offchainreporting2/types" ) From c4cef31791fa17f0fa5d5acc7d78f2b5b883c4d0 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 8 Nov 2024 13:27:54 -0500 Subject: [PATCH 164/174] fix imports --- pkg/monitoring/chain_reader.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/monitoring/chain_reader.go b/pkg/monitoring/chain_reader.go index 196f224cf..9b8c8ebff 100644 --- a/pkg/monitoring/chain_reader.go +++ b/pkg/monitoring/chain_reader.go @@ -2,6 +2,7 @@ package monitoring import ( "context" + "github.com/gagliardetto/solana-go" "github.com/gagliardetto/solana-go/rpc" From bbb714cbaec76a3cdb83fa4a4054d1d8fac73fe5 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Mon, 11 Nov 2024 16:38:43 -0500 Subject: [PATCH 165/174] Update transaction_sender.go --- pkg/solana/client/multinode/transaction_sender.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go index ad3575dbc..082f56c05 100644 --- a/pkg/solana/client/multinode/transaction_sender.go +++ b/pkg/solana/client/multinode/transaction_sender.go @@ -123,14 +123,12 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct r := txSender.broadcastTxAsync(ctx, rpc, tx) select { case <-ctx.Done(): - txSender.lggr.Debugw("Failed to send tx results", "err", ctx.Err()) return case txResults <- r: } select { case <-ctx.Done(): - txSender.lggr.Debugw("Failed to send tx results to report", "err", ctx.Err()) return case txResultsToReport <- r: } From 91c64b38747bec073bb7ae412aa0eb7bf0f7c240 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 13 Nov 2024 11:18:09 -0500 Subject: [PATCH 166/174] Remove TxError --- pkg/solana/chain.go | 5 +---- .../client/multinode/transaction_sender.go | 5 ++--- pkg/solana/client/multinode_client.go | 18 ++++++------------ 3 files changed, 9 insertions(+), 19 deletions(-) diff --git a/pkg/solana/chain.go b/pkg/solana/chain.go index c47e1cf1b..5148b11d4 100644 --- a/pkg/solana/chain.go +++ b/pkg/solana/chain.go @@ -296,10 +296,7 @@ func newChain(id string, cfg *config.TOMLConfig, ks loop.Keystore, lggr logger.L if result == nil { return solanago.Signature{}, errors.New("tx sender returned nil result") } - if result.Error() != nil { - return solanago.Signature{}, result.Error() - } - return result.Signature(), result.TxError() + return result.Signature(), result.Error() } tc = internal.NewLoader[client.ReaderWriter](func() (client.ReaderWriter, error) { return ch.multiNode.SelectRPC() }) diff --git a/pkg/solana/client/multinode/transaction_sender.go b/pkg/solana/client/multinode/transaction_sender.go index 082f56c05..06b2e18be 100644 --- a/pkg/solana/client/multinode/transaction_sender.go +++ b/pkg/solana/client/multinode/transaction_sender.go @@ -26,7 +26,6 @@ var ( type SendTxResult interface { Code() SendTxReturnCode - TxError() error Error() error } @@ -162,9 +161,9 @@ func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) SendTransaction(ct func (txSender *TransactionSender[TX, RESULT, CHAIN_ID, RPC]) broadcastTxAsync(ctx context.Context, rpc RPC, tx TX) RESULT { result := rpc.SendTransaction(ctx, tx) - txSender.lggr.Debugw("Node sent transaction", "tx", tx, "err", result.TxError()) + txSender.lggr.Debugw("Node sent transaction", "tx", tx, "err", result.Error()) if !slices.Contains(sendTxSuccessfulCodes, result.Code()) && ctx.Err() == nil { - txSender.lggr.Warnw("RPC returned error", "tx", tx, "err", result.TxError()) + txSender.lggr.Warnw("RPC returned error", "tx", tx, "err", result.Error()) } return result } diff --git a/pkg/solana/client/multinode_client.go b/pkg/solana/client/multinode_client.go index 5aa7d4820..e6a70de9c 100644 --- a/pkg/solana/client/multinode_client.go +++ b/pkg/solana/client/multinode_client.go @@ -321,18 +321,16 @@ func (m *MultiNodeClient) GetInterceptedChainInfo() (latest, highestUserObservat } type SendTxResult struct { - err error - txErr error - code mn.SendTxReturnCode - sig solana.Signature + err error + code mn.SendTxReturnCode + sig solana.Signature } var _ mn.SendTxResult = (*SendTxResult)(nil) func NewSendTxResult(err error) *SendTxResult { result := &SendTxResult{ - err: err, - txErr: err, + err: err, } result.code = ClassifySendError(nil, err) return result @@ -342,10 +340,6 @@ func (r *SendTxResult) Error() error { return r.err } -func (r *SendTxResult) TxError() error { - return r.txErr -} - func (r *SendTxResult) Code() mn.SendTxReturnCode { return r.code } @@ -356,7 +350,7 @@ func (r *SendTxResult) Signature() solana.Signature { func (m *MultiNodeClient) SendTransaction(ctx context.Context, tx *solana.Transaction) *SendTxResult { var sendTxResult = &SendTxResult{} - sendTxResult.sig, sendTxResult.txErr = m.SendTx(ctx, tx) - sendTxResult.code = ClassifySendError(tx, sendTxResult.txErr) + sendTxResult.sig, sendTxResult.err = m.SendTx(ctx, tx) + sendTxResult.code = ClassifySendError(tx, sendTxResult.err) return sendTxResult } From 83cf57ba3f8ef86413146cf5d5bdbdeb769c76d9 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 13 Nov 2024 11:31:46 -0500 Subject: [PATCH 167/174] Rename getReader --- pkg/solana/transmitter.go | 4 ++-- pkg/solana/transmitter_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/solana/transmitter.go b/pkg/solana/transmitter.go index db4e4ac03..63ab09840 100644 --- a/pkg/solana/transmitter.go +++ b/pkg/solana/transmitter.go @@ -17,7 +17,7 @@ var _ types.ContractTransmitter = (*Transmitter)(nil) type Transmitter struct { stateID, programID, storeProgramID, transmissionsID, transmissionSigner solana.PublicKey - reader GetReader + getReader GetReader stateCache *StateCache lggr logger.Logger txManager TxManager @@ -30,7 +30,7 @@ func (c *Transmitter) Transmit( report types.Report, sigs []types.AttributedOnchainSignature, ) error { - reader, err := c.reader() + reader, err := c.getReader() if err != nil { return fmt.Errorf("error on Transmit.Reader: %w", err) } diff --git a/pkg/solana/transmitter_test.go b/pkg/solana/transmitter_test.go index 718808679..ba036bece 100644 --- a/pkg/solana/transmitter_test.go +++ b/pkg/solana/transmitter_test.go @@ -69,7 +69,7 @@ func TestTransmitter_TxSize(t *testing.T) { storeProgramID: mustNewRandomPublicKey(), transmissionsID: mustNewRandomPublicKey(), transmissionSigner: signer.PublicKey(), - reader: func() (client.Reader, error) { return rw, nil }, + getReader: func() (client.Reader, error) { return rw, nil }, stateCache: &StateCache{}, lggr: logger.Test(t), txManager: mockTxm, From 6408ea3073432fd609c9d7ebbe532a5f0aadb61f Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 13 Nov 2024 11:51:06 -0500 Subject: [PATCH 168/174] lint --- pkg/solana/relay.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/solana/relay.go b/pkg/solana/relay.go index d67373b31..ddac29730 100644 --- a/pkg/solana/relay.go +++ b/pkg/solana/relay.go @@ -169,7 +169,7 @@ func (r *Relayer) NewMedianProvider(ctx context.Context, rargs relaytypes.RelayA storeProgramID: configWatcher.storeProgramID, transmissionsID: transmissionsID, transmissionSigner: transmitterAccount, - reader: configWatcher.chain.Reader, + getReader: configWatcher.chain.Reader, stateCache: configWatcher.stateCache, lggr: r.lggr, txManager: configWatcher.chain.TxManager(), From 50e3a214203fd575316ab797de078cf75b9146c9 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 13 Nov 2024 12:09:48 -0500 Subject: [PATCH 169/174] Update chain_test.go --- pkg/solana/chain_test.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pkg/solana/chain_test.go b/pkg/solana/chain_test.go index b705860c9..0980799e1 100644 --- a/pkg/solana/chain_test.go +++ b/pkg/solana/chain_test.go @@ -472,8 +472,7 @@ func TestChain_MultiNode_TransactionSender(t *testing.T) { // Send tx using transaction sender result := c.txSender.SendTransaction(ctx, unsignedTx(receiver.PublicKey())) require.NotNil(t, result) - require.NoError(t, result.Error()) - require.Error(t, result.TxError()) + require.Error(t, result.Error()) require.Equal(t, mn.Fatal, result.Code()) require.Empty(t, result.Signature()) }) @@ -481,8 +480,7 @@ func TestChain_MultiNode_TransactionSender(t *testing.T) { t.Run("empty transaction", func(t *testing.T) { result := c.txSender.SendTransaction(ctx, &solana.Transaction{}) require.NotNil(t, result) - require.NoError(t, result.Error()) - require.Error(t, result.TxError()) + require.Error(t, result.Error()) require.Equal(t, mn.Fatal, result.Code()) require.Empty(t, result.Signature()) }) From 40e280ff5609d1ede45122a7d7da1b2b42e5faba Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 14 Nov 2024 11:18:48 -0500 Subject: [PATCH 170/174] Update transmissions_cache.go --- pkg/solana/transmissions_cache.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/solana/transmissions_cache.go b/pkg/solana/transmissions_cache.go index 25c6125ea..bdf258180 100644 --- a/pkg/solana/transmissions_cache.go +++ b/pkg/solana/transmissions_cache.go @@ -19,21 +19,21 @@ type TransmissionsCache struct { *client.Cache[Answer] } -func NewTransmissionsCache(transmissionsID solana.PublicKey, chainID string, cfg config.Config, reader GetReader, lggr logger.Logger) *TransmissionsCache { +func NewTransmissionsCache(transmissionsID solana.PublicKey, chainID string, cfg config.Config, getReader GetReader, lggr logger.Logger) *TransmissionsCache { name := "ocr2_median_transmissions" getter := func(ctx context.Context) (Answer, uint64, error) { - getAccountReader := func() (client.AccountReader, error) { return reader() } + getAccountReader := func() (client.AccountReader, error) { return getReader() } return GetLatestTransmission(ctx, getAccountReader, transmissionsID, cfg.Commitment()) } return &TransmissionsCache{client.NewCache(name, transmissionsID, chainID, cfg, getter, logger.With(lggr, "cache", name))} } -func GetLatestTransmission(ctx context.Context, reader GetAccountReader, account solana.PublicKey, commitment rpc.CommitmentType) (Answer, uint64, error) { +func GetLatestTransmission(ctx context.Context, getReader GetAccountReader, account solana.PublicKey, commitment rpc.CommitmentType) (Answer, uint64, error) { // query for transmission header headerStart := AccountDiscriminatorLen // skip account discriminator headerLen := TransmissionsHeaderLen - r, err := reader() + r, err := getReader() if err != nil { return Answer{}, 0, fmt.Errorf("failed to get reader: %w", err) } From 8e9ab203448027853983d6332d4480ea328d98e0 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 14 Nov 2024 13:02:22 -0500 Subject: [PATCH 171/174] Update run_soak_test.sh --- integration-tests/scripts/run_soak_test.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/integration-tests/scripts/run_soak_test.sh b/integration-tests/scripts/run_soak_test.sh index 32caa3e93..7e5490859 100755 --- a/integration-tests/scripts/run_soak_test.sh +++ b/integration-tests/scripts/run_soak_test.sh @@ -22,7 +22,6 @@ while IFS= read -r line; do fi done < <(sudo go test -timeout 24h -count=1 -run TestSolanaOCRV2Smoke/embedded -test.timeout 30m 2>&1) - # Capture the PID of the background process READER_PID=$! From ca7c98259f21439e004507edc03046347bcf9766 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 14 Nov 2024 13:59:03 -0500 Subject: [PATCH 172/174] Fix deprecated method --- integration-tests/solclient/solclient.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/solclient/solclient.go b/integration-tests/solclient/solclient.go index 7b3921c19..2d5f52ac7 100644 --- a/integration-tests/solclient/solclient.go +++ b/integration-tests/solclient/solclient.go @@ -481,7 +481,7 @@ func SendFunds(senderPrivateKey string, receiverPublicKey string, lamports uint6 accountTo := solana.MustPublicKeyFromBase58(receiverPublicKey) // Get recent blockhash - recent, err := rpcClient.GetRecentBlockhash(context.Background(), rpc.CommitmentFinalized) + recent, err := rpcClient.GetLatestBlockhash(context.Background(), rpc.CommitmentFinalized) if err != nil { return err } From e671b69c0a02c7a3fe13f97823a9693eeaee48b0 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 15 Nov 2024 11:43:24 -0500 Subject: [PATCH 173/174] Clean up getReader --- pkg/solana/cache_test.go | 32 ++++++++++++++++----------- pkg/solana/client/multinode/poller.go | 3 +-- pkg/solana/config_tracker.go | 4 ++-- pkg/solana/config_tracker_test.go | 2 +- pkg/solana/relay.go | 7 +++--- pkg/solana/state_cache.go | 9 ++++---- pkg/solana/transmissions_cache.go | 5 ++--- 7 files changed, 33 insertions(+), 29 deletions(-) diff --git a/pkg/solana/cache_test.go b/pkg/solana/cache_test.go index ef1fb7cdb..75649d35a 100644 --- a/pkg/solana/cache_test.go +++ b/pkg/solana/cache_test.go @@ -106,9 +106,10 @@ func TestGetState(t *testing.T) { })) defer mockServer.Close() - reader := func() (client.AccountReader, error) { return testSetupReader(t, mockServer.URL), nil } + reader := testSetupReader(t, mockServer.URL) + getReader := func() (client.AccountReader, error) { return reader, nil } // happy path does not error (actual state decoding handled in types_test) - _, _, err := GetState(context.TODO(), reader, solana.PublicKey{}, "") + _, _, err := GetState(context.TODO(), getReader, solana.PublicKey{}, "") require.NoError(t, err) } @@ -133,18 +134,19 @@ func TestGetLatestTransmission(t *testing.T) { })) defer mockServer.Close() - reader := func() (client.AccountReader, error) { return testSetupReader(t, mockServer.URL), nil } - a, _, err := GetLatestTransmission(context.TODO(), reader, solana.PublicKey{}, "") + reader := testSetupReader(t, mockServer.URL) + getReader := func() (client.AccountReader, error) { return reader, nil } + a, _, err := GetLatestTransmission(context.TODO(), getReader, solana.PublicKey{}, "") assert.NoError(t, err) assert.Equal(t, expectedTime, a.Timestamp) assert.Equal(t, expectedAns, a.Data.String()) // fail if returned transmission header is too short - _, _, err = GetLatestTransmission(context.TODO(), reader, solana.PublicKey{}, "") + _, _, err = GetLatestTransmission(context.TODO(), getReader, solana.PublicKey{}, "") assert.Error(t, err) // fail if returned transmission is too short - _, _, err = GetLatestTransmission(context.TODO(), reader, solana.PublicKey{}, "") + _, _, err = GetLatestTransmission(context.TODO(), getReader, solana.PublicKey{}, "") assert.Error(t, err) } @@ -167,14 +169,16 @@ func TestCache(t *testing.T) { w.Write(testTransmissionsResponse(t, body, 0)) //nolint:errcheck })) - reader := func() (client.Reader, error) { return testSetupReader(t, mockServer.URL), nil } + reader := testSetupReader(t, mockServer.URL) + getReader := func() (client.Reader, error) { return reader, nil } + getAccountReader := func() (client.AccountReader, error) { return reader, nil } lggr := logger.Test(t) stateCache := NewStateCache( solana.MustPublicKeyFromBase58("11111111111111111111111111111111"), "test-chain-id", config.NewDefault(), - reader, + getReader, lggr, ) require.NoError(t, stateCache.Start(ctx)) @@ -189,7 +193,7 @@ func TestCache(t *testing.T) { solana.MustPublicKeyFromBase58("11111111111111111111111111111112"), "test-chain-id", config.NewDefault(), - reader, + getAccountReader, lggr, ) require.NoError(t, transmissionsCache.Start(ctx)) @@ -223,17 +227,19 @@ func TestNilPointerHandling(t *testing.T) { defer mockServer.Close() errString := "nil pointer returned in " - reader := func() (client.AccountReader, error) { return testSetupReader(t, mockServer.URL), nil } + + reader := testSetupReader(t, mockServer.URL) + getReader := func() (client.AccountReader, error) { return reader, nil } // fail on get state query - _, _, err := GetState(context.TODO(), reader, solana.PublicKey{}, "") + _, _, err := GetState(context.TODO(), getReader, solana.PublicKey{}, "") assert.EqualError(t, err, errString+"GetState.GetAccountInfoWithOpts") // fail on transmissions header query - _, _, err = GetLatestTransmission(context.TODO(), reader, solana.PublicKey{}, "") + _, _, err = GetLatestTransmission(context.TODO(), getReader, solana.PublicKey{}, "") assert.EqualError(t, err, errString+"GetLatestTransmission.GetAccountInfoWithOpts.Header") passFirst = true // allow proper response for header query, fail on transmission - _, _, err = GetLatestTransmission(context.TODO(), reader, solana.PublicKey{}, "") + _, _, err = GetLatestTransmission(context.TODO(), getReader, solana.PublicKey{}, "") assert.EqualError(t, err, errString+"GetLatestTransmission.GetAccountInfoWithOpts.Transmission") } diff --git a/pkg/solana/client/multinode/poller.go b/pkg/solana/client/multinode/poller.go index 4f426ec02..0ce87fade 100644 --- a/pkg/solana/client/multinode/poller.go +++ b/pkg/solana/client/multinode/poller.go @@ -65,8 +65,7 @@ func (p *Poller[T]) Err() <-chan error { } func (p *Poller[T]) pollingLoop(ctx context.Context) { - tickerCfg := services.TickerConfig{Initial: 0, JitterPct: services.DefaultJitter} - ticker := tickerCfg.NewTicker(p.pollingInterval) + ticker := services.NewTicker(p.pollingInterval) defer ticker.Stop() for { diff --git a/pkg/solana/config_tracker.go b/pkg/solana/config_tracker.go index 9a1fa8e28..3ddff2715 100644 --- a/pkg/solana/config_tracker.go +++ b/pkg/solana/config_tracker.go @@ -9,7 +9,7 @@ import ( type ConfigTracker struct { stateCache *StateCache - reader GetReader + getReader GetReader } func (c *ConfigTracker) Notify() <-chan struct{} { @@ -73,7 +73,7 @@ func (c *ConfigTracker) LatestConfig(ctx context.Context, changedInBlock uint64) // LatestBlockHeight returns the height of the most recent block in the chain. func (c *ConfigTracker) LatestBlockHeight(ctx context.Context) (blockHeight uint64, err error) { - reader, err := c.reader() + reader, err := c.getReader() if err != nil { return 0, err } diff --git a/pkg/solana/config_tracker_test.go b/pkg/solana/config_tracker_test.go index 88ba0442e..d0e2d8625 100644 --- a/pkg/solana/config_tracker_test.go +++ b/pkg/solana/config_tracker_test.go @@ -21,7 +21,7 @@ func TestLatestBlockHeight(t *testing.T) { ctx := context.Background() c := &ConfigTracker{ - reader: func() (client.Reader, error) { return testSetupReader(t, mockServer.URL), nil }, + getReader: func() (client.Reader, error) { return testSetupReader(t, mockServer.URL), nil }, } h, err := c.LatestBlockHeight(ctx) diff --git a/pkg/solana/relay.go b/pkg/solana/relay.go index bbae6b961..4e6009027 100644 --- a/pkg/solana/relay.go +++ b/pkg/solana/relay.go @@ -154,7 +154,8 @@ func (r *Relayer) NewMedianProvider(ctx context.Context, rargs relaytypes.RelayA } cfg := configWatcher.chain.Config() - transmissionsCache := NewTransmissionsCache(transmissionsID, relayConfig.ChainID, cfg, configWatcher.chain.Reader, r.lggr) + getReader := func() (client.AccountReader, error) { return configWatcher.chain.Reader() } + transmissionsCache := NewTransmissionsCache(transmissionsID, relayConfig.ChainID, cfg, getReader, r.lggr) return &medianProvider{ configProvider: configWatcher, transmissionsCache: transmissionsCache, @@ -187,8 +188,6 @@ func (r *Relayer) NewAutomationProvider(ctx context.Context, rargs relaytypes.Re var _ relaytypes.ConfigProvider = &configProvider{} -type GetReader func() (client.Reader, error) - type configProvider struct { services.StateMachine chainID string @@ -231,7 +230,7 @@ func newConfigProvider(_ context.Context, lggr logger.Logger, chain Chain, args storeProgramID: storeProgramID, stateCache: stateCache, offchainConfigDigester: offchainConfigDigester, - configTracker: &ConfigTracker{stateCache: stateCache, reader: chain.Reader}, + configTracker: &ConfigTracker{stateCache: stateCache, getReader: chain.Reader}, chain: chain, }, nil } diff --git a/pkg/solana/state_cache.go b/pkg/solana/state_cache.go index 614ad0d81..06fc4c62b 100644 --- a/pkg/solana/state_cache.go +++ b/pkg/solana/state_cache.go @@ -23,19 +23,20 @@ type StateCache struct { *client.Cache[State] } +type GetReader func() (client.Reader, error) type GetAccountReader func() (client.AccountReader, error) -func NewStateCache(stateID solana.PublicKey, chainID string, cfg config.Config, reader GetReader, lggr logger.Logger) *StateCache { +func NewStateCache(stateID solana.PublicKey, chainID string, cfg config.Config, getReader GetReader, lggr logger.Logger) *StateCache { name := "ocr2_median_state" getter := func(ctx context.Context) (State, uint64, error) { - getAccountReader := func() (client.AccountReader, error) { return reader() } + getAccountReader := func() (client.AccountReader, error) { return getReader() } return GetState(ctx, getAccountReader, stateID, cfg.Commitment()) } return &StateCache{client.NewCache(name, stateID, chainID, cfg, getter, logger.With(lggr, "cache", name))} } -func GetState(ctx context.Context, reader GetAccountReader, account solana.PublicKey, commitment rpc.CommitmentType) (State, uint64, error) { - r, err := reader() +func GetState(ctx context.Context, getReader GetAccountReader, account solana.PublicKey, commitment rpc.CommitmentType) (State, uint64, error) { + r, err := getReader() if err != nil { return State{}, 0, fmt.Errorf("failed to get reader: %w", err) } diff --git a/pkg/solana/transmissions_cache.go b/pkg/solana/transmissions_cache.go index bdf258180..acc530cbb 100644 --- a/pkg/solana/transmissions_cache.go +++ b/pkg/solana/transmissions_cache.go @@ -19,11 +19,10 @@ type TransmissionsCache struct { *client.Cache[Answer] } -func NewTransmissionsCache(transmissionsID solana.PublicKey, chainID string, cfg config.Config, getReader GetReader, lggr logger.Logger) *TransmissionsCache { +func NewTransmissionsCache(transmissionsID solana.PublicKey, chainID string, cfg config.Config, getReader GetAccountReader, lggr logger.Logger) *TransmissionsCache { name := "ocr2_median_transmissions" getter := func(ctx context.Context) (Answer, uint64, error) { - getAccountReader := func() (client.AccountReader, error) { return getReader() } - return GetLatestTransmission(ctx, getAccountReader, transmissionsID, cfg.Commitment()) + return GetLatestTransmission(ctx, getReader, transmissionsID, cfg.Commitment()) } return &TransmissionsCache{client.NewCache(name, transmissionsID, chainID, cfg, getter, logger.With(lggr, "cache", name))} } From 7aa905dee210989ef9e046a1ae8269faf6520261 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 15 Nov 2024 13:23:17 -0500 Subject: [PATCH 174/174] Use AccountReader --- pkg/solana/cache_test.go | 3 +-- pkg/solana/relay.go | 3 ++- pkg/solana/state_cache.go | 5 ++--- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/pkg/solana/cache_test.go b/pkg/solana/cache_test.go index 75649d35a..a9126d0ac 100644 --- a/pkg/solana/cache_test.go +++ b/pkg/solana/cache_test.go @@ -170,7 +170,6 @@ func TestCache(t *testing.T) { })) reader := testSetupReader(t, mockServer.URL) - getReader := func() (client.Reader, error) { return reader, nil } getAccountReader := func() (client.AccountReader, error) { return reader, nil } lggr := logger.Test(t) @@ -178,7 +177,7 @@ func TestCache(t *testing.T) { solana.MustPublicKeyFromBase58("11111111111111111111111111111111"), "test-chain-id", config.NewDefault(), - getReader, + getAccountReader, lggr, ) require.NoError(t, stateCache.Start(ctx)) diff --git a/pkg/solana/relay.go b/pkg/solana/relay.go index 4e6009027..d98ab0442 100644 --- a/pkg/solana/relay.go +++ b/pkg/solana/relay.go @@ -222,7 +222,8 @@ func newConfigProvider(_ context.Context, lggr logger.Logger, chain Chain, args StateID: stateID, } - stateCache := NewStateCache(stateID, relayConfig.ChainID, chain.Config(), chain.Reader, lggr) + getAccountReader := func() (client.AccountReader, error) { return chain.Reader() } + stateCache := NewStateCache(stateID, relayConfig.ChainID, chain.Config(), getAccountReader, lggr) return &configProvider{ chainID: relayConfig.ChainID, stateID: stateID, diff --git a/pkg/solana/state_cache.go b/pkg/solana/state_cache.go index 06fc4c62b..4f6f2b084 100644 --- a/pkg/solana/state_cache.go +++ b/pkg/solana/state_cache.go @@ -26,11 +26,10 @@ type StateCache struct { type GetReader func() (client.Reader, error) type GetAccountReader func() (client.AccountReader, error) -func NewStateCache(stateID solana.PublicKey, chainID string, cfg config.Config, getReader GetReader, lggr logger.Logger) *StateCache { +func NewStateCache(stateID solana.PublicKey, chainID string, cfg config.Config, getReader GetAccountReader, lggr logger.Logger) *StateCache { name := "ocr2_median_state" getter := func(ctx context.Context) (State, uint64, error) { - getAccountReader := func() (client.AccountReader, error) { return getReader() } - return GetState(ctx, getAccountReader, stateID, cfg.Commitment()) + return GetState(ctx, getReader, stateID, cfg.Commitment()) } return &StateCache{client.NewCache(name, stateID, chainID, cfg, getter, logger.With(lggr, "cache", name))} }