From 3af440736a7fd02cd51d914dd77fdde3965f462c Mon Sep 17 00:00:00 2001 From: Dan Kanefsky Date: Tue, 12 Dec 2023 15:46:42 -0800 Subject: [PATCH 01/13] batch tx's --- cmd/ethereum/broadcast.go | 164 ++++++++------- cmd/ethereum/listener.go | 23 ++- cmd/noble/broadcast.go | 2 +- cmd/process.go | 141 ++++++++----- cmd/process_test.go | 102 ++++++++-- cmd/root.go | 29 +-- ...eth_burn_to_noble_mint_and_forward_test.go | 4 +- integration/eth_burn_to_noble_mint_test.go | 4 +- integration/eth_multi_send_test.go | 4 +- integration/noble_burn_to_eth_mint_test.go | 4 +- integration/noble_multi_send_test.go | 4 +- types/message_state.go | 27 ++- types/message_state.go.bak | 192 ++++++++++++++++++ types/message_state_test.go | 85 +++++--- types/state.go | 10 +- types/state_test.go | 31 +-- 16 files changed, 601 insertions(+), 225 deletions(-) create mode 100644 types/message_state.go.bak diff --git a/cmd/ethereum/broadcast.go b/cmd/ethereum/broadcast.go index 2fe1a95..7ffe0c3 100644 --- a/cmd/ethereum/broadcast.go +++ b/cmd/ethereum/broadcast.go @@ -25,9 +25,9 @@ func Broadcast( ctx context.Context, cfg config.Config, logger log.Logger, - msg *types.MessageState, + msgs []*types.MessageState, sequenceMap *types.SequenceMap, -) (*ethtypes.Transaction, error) { +) ([]*ethtypes.Transaction, error) { // set up eth client client, err := ethclient.Dial(cfg.Networks.Destination.Ethereum.RPC) @@ -53,99 +53,105 @@ func Broadcast( return nil, fmt.Errorf("unable to create message transmitter: %w", err) } - attestationBytes, err := hex.DecodeString(msg.Attestation[2:]) - if err != nil { - return nil, errors.New("unable to decode message attestation") - } - - for attempt := 0; attempt <= cfg.Networks.Destination.Ethereum.BroadcastRetries; attempt++ { - logger.Info(fmt.Sprintf( - "Broadcasting %s message from %d to %d: with source tx hash %s", - msg.Type, - msg.SourceDomain, - msg.DestDomain, - msg.SourceTxHash)) - - nonce := sequenceMap.Next(cfg.Networks.Destination.Ethereum.DomainId) - auth.Nonce = big.NewInt(nonce) + var broadcastErrors error + var txs []*ethtypes.Transaction + for _, msg := range msgs { - // TODO remove - nextNonce, err := GetEthereumAccountNonce(cfg.Networks.Destination.Ethereum.RPC, ethereumAddress) + attestationBytes, err := hex.DecodeString(msg.Attestation[2:]) if err != nil { - logger.Error("unable to retrieve account number") - } else { - auth.Nonce = big.NewInt(nextNonce) + return nil, errors.New("unable to decode message attestation") } - // TODO end remove - // check if nonce already used - co := &bind.CallOpts{ - Pending: true, - Context: ctx, - } + for attempt := 0; attempt <= cfg.Networks.Destination.Ethereum.BroadcastRetries; attempt++ { + logger.Info(fmt.Sprintf( + "Broadcasting message from %d to %d: with source tx hash %s", + msg.SourceDomain, + msg.DestDomain, + msg.SourceTxHash)) + + nonce := sequenceMap.Next(cfg.Networks.Destination.Ethereum.DomainId) + auth.Nonce = big.NewInt(nonce) + + // TODO remove + nextNonce, err := GetEthereumAccountNonce(cfg.Networks.Destination.Ethereum.RPC, ethereumAddress) + if err != nil { + logger.Error("unable to retrieve account number") + } else { + auth.Nonce = big.NewInt(nextNonce) + } + // TODO end remove - logger.Debug("Checking if nonce was used for broadcast to Ethereum", "source_domain", msg.SourceDomain, "nonce", msg.Nonce) - - key := append( - common.LeftPadBytes((big.NewInt(int64(msg.SourceDomain))).Bytes(), 4), - common.LeftPadBytes((big.NewInt(int64(msg.Nonce))).Bytes(), 8)..., - ) - - response, nonceErr := messageTransmitter.UsedNonces(co, [32]byte(crypto.Keccak256(key))) - if nonceErr != nil { - logger.Debug("Error querying whether nonce was used. Continuing...") - } else { - fmt.Printf("received used nonce response: %d\n", response) - if response.Uint64() == uint64(1) { - // nonce has already been used, mark as complete - logger.Debug(fmt.Sprintf("This source domain/nonce has already been used: %d %d", - msg.SourceDomain, msg.Nonce)) - msg.Status = types.Complete - return nil, errors.New("receive message was already broadcasted") + // check if nonce already used + co := &bind.CallOpts{ + Pending: true, + Context: ctx, } - } - // broadcast txn - tx, err := messageTransmitter.ReceiveMessage( - auth, - msg.MsgSentBytes, - attestationBytes, - ) - if err == nil { - msg.Status = types.Complete - return tx, nil - } else { - logger.Error(fmt.Sprintf("error during broadcast: %s", err.Error())) - if parsedErr, ok := err.(JsonError); ok { - if parsedErr.ErrorCode() == 3 && parsedErr.Error() == "execution reverted: Nonce already used" { + logger.Debug("Checking if nonce was used for broadcast to Ethereum", "source_domain", msg.SourceDomain, "nonce", msg.Nonce) + + key := append( + common.LeftPadBytes((big.NewInt(int64(msg.SourceDomain))).Bytes(), 4), + common.LeftPadBytes((big.NewInt(int64(msg.Nonce))).Bytes(), 8)..., + ) + + response, nonceErr := messageTransmitter.UsedNonces(co, [32]byte(crypto.Keccak256(key))) + if nonceErr != nil { + logger.Debug("Error querying whether nonce was used. Continuing...") + } else { + fmt.Printf("received used nonce response: %d\n", response) + if response.Uint64() == uint64(1) { + // nonce has already been used, mark as complete + logger.Debug(fmt.Sprintf("This source domain/nonce has already been used: %d %d", + msg.SourceDomain, msg.Nonce)) msg.Status = types.Complete - return nil, parsedErr + return nil, errors.New("receive message was already broadcasted") } + } + + // broadcast txn + tx, err := messageTransmitter.ReceiveMessage( + auth, + msg.MsgSentBytes, + attestationBytes, + ) + if err == nil { + msg.Status = types.Complete + txs = append(txs, tx) + continue + } else { + logger.Error(fmt.Sprintf("error during broadcast: %s", err.Error())) + if parsedErr, ok := err.(JsonError); ok { + if parsedErr.ErrorCode() == 3 && parsedErr.Error() == "execution reverted: Nonce already used" { + msg.Status = types.Complete + return nil, parsedErr + } - match, _ := regexp.MatchString("nonce too low: next nonce [0-9]+, tx nonce [0-9]+", parsedErr.Error()) - if match { - numberRegex := regexp.MustCompile("[0-9]+") - nextNonce, err := strconv.ParseInt(numberRegex.FindAllString(parsedErr.Error(), 1)[0], 10, 0) - if err != nil { - nextNonce, err = GetEthereumAccountNonce(cfg.Networks.Destination.Ethereum.RPC, ethereumAddress) + match, _ := regexp.MatchString("nonce too low: next nonce [0-9]+, tx nonce [0-9]+", parsedErr.Error()) + if match { + numberRegex := regexp.MustCompile("[0-9]+") + nextNonce, err := strconv.ParseInt(numberRegex.FindAllString(parsedErr.Error(), 1)[0], 10, 0) if err != nil { - logger.Error("unable to retrieve account number") + nextNonce, err = GetEthereumAccountNonce(cfg.Networks.Destination.Ethereum.RPC, ethereumAddress) + if err != nil { + logger.Error("unable to retrieve account number") + } } + sequenceMap.Put(cfg.Networks.Destination.Ethereum.DomainId, nextNonce) } - sequenceMap.Put(cfg.Networks.Destination.Ethereum.DomainId, nextNonce) } - } - // if it's not the last attempt, retry - // TODO increase the destination.ethereum.broadcast retries (3-5) and retry interval (15s). By checking for used nonces, there is no gas cost for failed mints. - if attempt != cfg.Networks.Destination.Ethereum.BroadcastRetries { - logger.Info(fmt.Sprintf("Retrying in %d seconds", cfg.Networks.Destination.Ethereum.BroadcastRetryInterval)) - time.Sleep(time.Duration(cfg.Networks.Destination.Ethereum.BroadcastRetryInterval) * time.Second) + // if it's not the last attempt, retry + // TODO increase the destination.ethereum.broadcast retries (3-5) and retry interval (15s). By checking for used nonces, there is no gas cost for failed mints. + if attempt != cfg.Networks.Destination.Ethereum.BroadcastRetries { + logger.Info(fmt.Sprintf("Retrying in %d seconds", cfg.Networks.Destination.Ethereum.BroadcastRetryInterval)) + time.Sleep(time.Duration(cfg.Networks.Destination.Ethereum.BroadcastRetryInterval) * time.Second) + } + continue } - continue } - } - msg.Status = types.Failed + msg.Status = types.Failed - return nil, errors.New("reached max number of broadcast attempts") + broadcastErrors = errors.Join(broadcastErrors, errors.New("reached max number of broadcast attempts")) + } + return nil, broadcastErrors } diff --git a/cmd/ethereum/listener.go b/cmd/ethereum/listener.go index d1ca85b..1e7088b 100644 --- a/cmd/ethereum/listener.go +++ b/cmd/ethereum/listener.go @@ -21,7 +21,7 @@ import ( //go:embed abi/MessageTransmitter.json var content embed.FS -func StartListener(cfg config.Config, logger log.Logger, processingQueue chan *types.MessageState) { +func StartListener(cfg config.Config, logger log.Logger, processingQueue chan *types.TxState) { // set up client messageTransmitter, err := content.ReadFile("abi/MessageTransmitter.json") if err != nil { @@ -72,7 +72,7 @@ func StartListener(cfg config.Config, logger log.Logger, processingQueue chan *t } logger.Info(fmt.Sprintf("New historical msg from source domain %d with tx hash %s", parsedMsg.SourceDomain, parsedMsg.SourceTxHash)) - processingQueue <- parsedMsg + processingQueue <- &types.TxState{TxHash: parsedMsg.SourceTxHash, Msgs: []*types.MessageState{parsedMsg}} // It might help to wait a small amount of time between sending messages into the processing queue // so that account sequences / nonces are set correctly @@ -81,6 +81,7 @@ func StartListener(cfg config.Config, logger log.Logger, processingQueue chan *t // consume stream go func() { + var txState *types.TxState for { select { case err := <-sub.Err(): @@ -93,12 +94,20 @@ func StartListener(cfg config.Config, logger log.Logger, processingQueue chan *t continue } logger.Info(fmt.Sprintf("New stream msg from %d with tx hash %s", parsedMsg.SourceDomain, parsedMsg.SourceTxHash)) + if txState == nil { + txState = &types.TxState{TxHash: parsedMsg.SourceTxHash, Msgs: []*types.MessageState{parsedMsg}} + } else if parsedMsg.SourceTxHash != txState.TxHash { + processingQueue <- txState + txState = &types.TxState{TxHash: parsedMsg.SourceTxHash, Msgs: []*types.MessageState{parsedMsg}} + } else { + txState.Msgs = append(txState.Msgs, parsedMsg) - processingQueue <- parsedMsg - - // It might help to wait a small amount of time between sending messages into the processing queue - // so that account sequences / nonces are set correctly - // time.Sleep(10 * time.Millisecond) + } + default: + if txState != nil { + processingQueue <- txState + txState = nil + } } } }() diff --git a/cmd/noble/broadcast.go b/cmd/noble/broadcast.go index 4848489..c8c7f50 100644 --- a/cmd/noble/broadcast.go +++ b/cmd/noble/broadcast.go @@ -246,7 +246,7 @@ func GetNobleAccountNumberSequence(urlBase string, address string) (int64, int64 var resp types.AccountResp err = json.Unmarshal(body, &resp) if err != nil { - return 0, 0, errors.New("unable to parse account number, sequence") + return 0, 0, fmt.Errorf("unable to parse account number, sequence. Raw HHTP Get response: %s", string(body)) } accountNumber, _ := strconv.ParseInt(resp.AccountNumber, 10, 0) accountSequence, _ := strconv.ParseInt(resp.Sequence, 10, 0) diff --git a/cmd/process.go b/cmd/process.go index ad7adc5..6c2996b 100644 --- a/cmd/process.go +++ b/cmd/process.go @@ -19,6 +19,14 @@ import ( "github.com/strangelove-ventures/noble-cctp-relayer/types" ) +type Processor struct { + Mu sync.RWMutex +} + +func NewProcessor() *Processor { + return &Processor{} +} + var startCmd = &cobra.Command{ Use: "start", Short: "Start relaying CCTP transactions from Ethereum to Noble", @@ -35,6 +43,8 @@ var sequenceMap = types.NewSequenceMap() func Start(cmd *cobra.Command, args []string) { + p := NewProcessor() + var wg sync.WaitGroup wg.Add(1) @@ -71,7 +81,7 @@ func Start(cmd *cobra.Command, args []string) { // spin up Processor worker pool for i := 0; i < int(Cfg.ProcessorWorkerCount); i++ { - go StartProcessor(cmd.Context(), Cfg, Logger, processingQueue, sequenceMap) + go p.StartProcessor(cmd.Context(), Cfg, Logger, processingQueue, sequenceMap) } // listeners listen for events, parse them, and enqueue them to processingQueue @@ -87,92 +97,111 @@ func Start(cmd *cobra.Command, args []string) { } // StartProcessor is the main processing pipeline. -func StartProcessor(ctx context.Context, cfg config.Config, logger log.Logger, processingQueue chan *types.MessageState, sequenceMap *types.SequenceMap) { +func (p *Processor) StartProcessor(ctx context.Context, cfg config.Config, logger log.Logger, processingQueue chan *types.TxState, sequenceMap *types.SequenceMap) { for { - dequeuedMsg := <-processingQueue + dequeuedTx := <-processingQueue + // if this is the first time seeing this message, add it to the State - msg, ok := State.Load(LookupKey(dequeuedMsg.SourceTxHash, dequeuedMsg.Type)) + tx, ok := State.Load(LookupKey(dequeuedTx.TxHash)) if !ok { - State.Store(LookupKey(dequeuedMsg.SourceTxHash, dequeuedMsg.Type), dequeuedMsg) - msg, _ = State.Load(LookupKey(dequeuedMsg.SourceTxHash, dequeuedMsg.Type)) - msg.Status = types.Created + State.Store(LookupKey(dequeuedTx.TxHash), dequeuedTx) + tx, _ = State.Load(LookupKey(dequeuedTx.TxHash)) + for _, msg := range tx.Msgs { + msg.Status = types.Created + } } - // if a filter's condition is met, mark as filtered - if filterDisabledCCTPRoutes(cfg, logger, msg) || - filterInvalidDestinationCallers(cfg, logger, msg) || - filterNonWhitelistedChannels(cfg, logger, msg) || - filterMessages(cfg, logger, msg) { - msg.Status = types.Filtered - } + var broadcastMsgs = make(map[uint32][]*types.MessageState) + var requeue bool + for _, msg := range tx.Msgs { - // if the message is burned or pending, check for an attestation - if msg.Status == types.Created || msg.Status == types.Pending { - response := circle.CheckAttestation(cfg, logger, msg.IrisLookupId, msg.SourceTxHash, msg.SourceDomain, msg.DestDomain) - if response != nil { - if msg.Status == types.Created && response.Status == "pending_confirmations" { - logger.Debug("Attestation is created but still pending confirmations for 0x" + msg.IrisLookupId + ". Retrying...") - msg.Status = types.Pending - msg.Updated = time.Now() - time.Sleep(10 * time.Second) - processingQueue <- msg - continue - } else if response.Status == "pending_confirmations" { - logger.Debug("Attestation is still pending for 0x" + msg.IrisLookupId + ". Retrying...") + // if a filter's condition is met, mark as filtered + if filterDisabledCCTPRoutes(cfg, logger, msg) || + filterInvalidDestinationCallers(cfg, logger, msg) || + filterNonWhitelistedChannels(cfg, logger, msg) || + filterMessages(cfg, logger, msg) { + msg.Status = types.Filtered + } + + // if the message is burned or pending, check for an attestation + if msg.Status == types.Created || msg.Status == types.Pending { + response := circle.CheckAttestation(cfg, logger, msg.IrisLookupId, msg.SourceTxHash, msg.SourceDomain, msg.DestDomain) + if response != nil { + if msg.Status == types.Created && response.Status == "pending_confirmations" { + logger.Debug("Attestation is created but still pending confirmations for 0x" + msg.IrisLookupId + ". Retrying...") + msg.Status = types.Pending + msg.Updated = time.Now() + time.Sleep(10 * time.Second) + requeue = true + continue + } else if response.Status == "pending_confirmations" { + logger.Debug("Attestation is still pending for 0x" + msg.IrisLookupId + ". Retrying...") + time.Sleep(10 * time.Second) + requeue = true + continue + } else if response.Status == "complete" { + logger.Debug("Attestation is complete for 0x" + msg.IrisLookupId + ". Retrying...") + msg.Status = types.Attested + msg.Attestation = response.Attestation + msg.Updated = time.Now() + broadcastMsgs[msg.DestDomain] = append(broadcastMsgs[msg.DestDomain], msg) + } + } else { + // add attestation retry intervals per domain here + logger.Debug("Attestation is still processing for 0x" + msg.IrisLookupId + ". Retrying...") time.Sleep(10 * time.Second) - processingQueue <- msg + // retry + requeue = true continue - } else if response.Status == "complete" { - logger.Debug("Attestation is complete for 0x" + msg.IrisLookupId + ". Retrying...") - msg.Status = types.Attested - msg.Attestation = response.Attestation - msg.Updated = time.Now() } - } else { - // add attestation retry intervals per domain here - logger.Debug("Attestation is still processing for 0x" + msg.IrisLookupId + ". Retrying...") - time.Sleep(10 * time.Second) - // retry - processingQueue <- msg - continue } } // if the message is attested to, try to broadcast - if msg.Status == types.Attested { - switch msg.DestDomain { + for domain, msgs := range broadcastMsgs { + switch domain { case 0: // ethereum - response, err := ethereum.Broadcast(ctx, cfg, logger, msg, sequenceMap) + response, err := ethereum.Broadcast(ctx, cfg, logger, msgs, sequenceMap) if err != nil { logger.Error("unable to mint on Ethereum", "err", err) - processingQueue <- msg + requeue = true continue } fullLog, err := response.MarshalJSON() if err != nil { logger.Error("error on marshall", err) } - msg.DestTxHash = response.Hash().Hex() - logger.Info(fmt.Sprintf("Successfully broadcast %s to Ethereum. Tx hash: %s, FULL LOG: %s", msg.SourceTxHash, msg.DestTxHash, string(fullLog))) + for _, msg := range msgs { + msg.DestTxHash = response.Hash().Hex() + } + logger.Info(fmt.Sprintf("Successfully broadcast %s to Ethereum. Tx hash: %s, FULL LOG: %s", msgs[0].SourceTxHash, msgs[0].DestTxHash, string(fullLog))) case 4: // noble - response, err := noble.Broadcast(ctx, cfg, logger, msg, sequenceMap) + response, err := noble.Broadcast(ctx, cfg, logger, msgs, sequenceMap) if err != nil { logger.Error("unable to mint on Noble", "err", err) - processingQueue <- msg + requeue = true continue } if response.Code != 0 { logger.Error("nonzero response code received", "err", err) - processingQueue <- msg + requeue = true continue } // success! - msg.DestTxHash = response.Hash.String() - logger.Info(fmt.Sprintf("Successfully broadcast %s to Noble. Tx hash: %s", msg.SourceTxHash, msg.DestTxHash)) + for _, msg := range msgs { + msg.DestTxHash = response.Hash.String() + } + logger.Info(fmt.Sprintf("Successfully broadcast %s to Noble. Tx hash: %s", msgs[0].SourceTxHash, msgs[0].DestTxHash)) } // ...add minters for different domains here - msg.Status = types.Complete - msg.Updated = time.Now() + for _, msg := range msgs { + msg.Status = types.Complete + msg.Updated = time.Now() + } + + } + if requeue { + processingQueue <- tx } } } @@ -250,8 +279,10 @@ func filterMessages(_ config.Config, logger log.Logger, msg *types.MessageState) return false } -func LookupKey(sourceTxHash string, messageType string) string { - return fmt.Sprintf("%s-%s", sourceTxHash, messageType) +func LookupKey(sourceTxHash string) string { + // return fmt.Sprintf("%s-%s", sourceTxHash, messageType) + return sourceTxHash + } func init() { diff --git a/cmd/process_test.go b/cmd/process_test.go index de02250..88e0b08 100644 --- a/cmd/process_test.go +++ b/cmd/process_test.go @@ -30,7 +30,7 @@ func setupTest() { cfg.Networks.Minters[4].MinterAddress) if err != nil { - logger.Error("Error retrieving account sequence") + logger.Error("Error retrieving account sequence", "err: ", err) os.Exit(1) } sequenceMap = types.NewSequenceMap() @@ -42,7 +42,9 @@ func setupTest() { func TestProcessNewLog(t *testing.T) { setupTest() - go cmd.StartProcessor(context.TODO(), cfg, logger, processingQueue, sequenceMap) + p := cmd.Processor{} + + go p.StartProcessor(context.TODO(), cfg, logger, processingQueue, sequenceMap) emptyBz := make([]byte, 32) expectedState := &types.MessageState{ @@ -55,11 +57,13 @@ func TestProcessNewLog(t *testing.T) { processingQueue <- expectedState - time.Sleep(2 * time.Second) + time.Sleep(5 * time.Second) - actualState, _ := cmd.State.Load(cmd.LookupKey(expectedState.SourceTxHash, expectedState.Type)) + actualState, _ := cmd.State.Load(expectedState.SourceTxHash) - require.Equal(t, types.Created, actualState.Status) + p.Mu.RLock() + require.Equal(t, types.Created, actualState[0].Status) + p.Mu.RUnlock() } @@ -68,7 +72,9 @@ func TestProcessCreatedLog(t *testing.T) { setupTest() cfg.Networks.EnabledRoutes[0] = 5 // skip mint - go cmd.StartProcessor(context.TODO(), cfg, logger, processingQueue, sequenceMap) + p := cmd.NewProcessor() + + go p.StartProcessor(context.TODO(), cfg, logger, processingQueue, sequenceMap) emptyBz := make([]byte, 32) expectedState := &types.MessageState{ @@ -85,10 +91,11 @@ func TestProcessCreatedLog(t *testing.T) { time.Sleep(2 * time.Second) - actualState, ok := cmd.State.Load(cmd.LookupKey(expectedState.SourceTxHash, expectedState.Type)) + actualState, ok := cmd.State.Load(expectedState.SourceTxHash) require.True(t, ok) - require.Equal(t, types.Complete, actualState.Status) - + p.Mu.RLock() + require.Equal(t, types.Complete, actualState[0].Status) + p.Mu.RUnlock() } // created message -> disabled cctp route -> filtered @@ -97,7 +104,9 @@ func TestProcessDisabledCctpRoute(t *testing.T) { delete(cfg.Networks.EnabledRoutes, 0) - go cmd.StartProcessor(context.TODO(), cfg, logger, processingQueue, sequenceMap) + p := cmd.NewProcessor() + + go p.StartProcessor(context.TODO(), cfg, logger, processingQueue, sequenceMap) emptyBz := make([]byte, 32) expectedState := &types.MessageState{ @@ -113,17 +122,20 @@ func TestProcessDisabledCctpRoute(t *testing.T) { time.Sleep(2 * time.Second) - actualState, ok := cmd.State.Load(cmd.LookupKey(expectedState.SourceTxHash, expectedState.Type)) + actualState, ok := cmd.State.Load(expectedState.SourceTxHash) require.True(t, ok) - require.Equal(t, types.Filtered, actualState.Status) - + p.Mu.RLock() + require.Equal(t, types.Filtered, actualState[0].Status) + p.Mu.RUnlock() } // created message -> different destination caller -> filtered func TestProcessInvalidDestinationCaller(t *testing.T) { setupTest() - go cmd.StartProcessor(context.TODO(), cfg, logger, processingQueue, sequenceMap) + p := cmd.NewProcessor() + + go p.StartProcessor(context.TODO(), cfg, logger, processingQueue, sequenceMap) nonEmptyBytes := make([]byte, 31) nonEmptyBytes = append(nonEmptyBytes, 0x1) @@ -141,10 +153,11 @@ func TestProcessInvalidDestinationCaller(t *testing.T) { time.Sleep(2 * time.Second) - actualState, ok := cmd.State.Load(cmd.LookupKey(expectedState.SourceTxHash, expectedState.Type)) + actualState, ok := cmd.State.Load(expectedState.SourceTxHash) require.True(t, ok) - require.Equal(t, types.Filtered, actualState.Status) - + p.Mu.RLock() + require.Equal(t, types.Filtered, actualState[0].Status) + p.Mu.RUnlock() } // created message -> nonwhitelisted channel -> filtered @@ -152,7 +165,9 @@ func TestProcessNonWhitelistedChannel(t *testing.T) { setupTest() cfg.Networks.Destination.Noble.FilterForwardsByIbcChannel = true - go cmd.StartProcessor(context.TODO(), cfg, logger, processingQueue, sequenceMap) + p := cmd.NewProcessor() + + go p.StartProcessor(context.TODO(), cfg, logger, processingQueue, sequenceMap) emptyBz := make([]byte, 32) expectedState := &types.MessageState{ @@ -168,17 +183,60 @@ func TestProcessNonWhitelistedChannel(t *testing.T) { time.Sleep(2 * time.Second) - actualState, ok := cmd.State.Load(cmd.LookupKey(expectedState.SourceTxHash, expectedState.Type)) + actualState, ok := cmd.State.Load(expectedState.SourceTxHash) require.True(t, ok) - require.Equal(t, types.Filtered, actualState.Status) + p.Mu.RLock() + require.Equal(t, types.Filtered, actualState[0].Status) + p.Mu.RUnlock() +} + +// created message -> nonwhitelisted channel -> filtered +func TestBatchTx(t *testing.T) { + setupTest() + + p := cmd.NewProcessor() + + go p.StartProcessor(context.TODO(), cfg, logger, processingQueue, sequenceMap) + emptyBz := make([]byte, 32) + expectedState := &types.MessageState{ + SourceTxHash: "123", + IrisLookupId: "a404f4155166a1fc7ffee145b5cac6d0f798333745289ab1db171344e226ef0c", + Status: types.Created, + SourceDomain: 0, + DestDomain: 4, + DestinationCaller: emptyBz, + MsgSentBytes: []byte("mock bytes 1"), + } + processingQueue <- expectedState + + expectedState2 := &types.MessageState{ + SourceTxHash: "123", + IrisLookupId: "a404f4155166a1fc7ffee145b5cac6d0f798333745289ab1db171344e226ef0c", + Status: types.Created, + SourceDomain: 0, + DestDomain: 4, + DestinationCaller: emptyBz, + MsgSentBytes: []byte("mock bytes 2"), + } + + processingQueue <- expectedState2 + time.Sleep(6 * time.Second) + + actualState, ok := cmd.State.Load(expectedState.SourceTxHash) + require.True(t, ok) + p.Mu.RLock() + require.Equal(t, 2, len(actualState)) + p.Mu.RUnlock() } // created message -> not \ -> filtered func TestProcessNonBurnMessageWhenDisabled(t *testing.T) { setupTest() - go cmd.StartProcessor(context.TODO(), cfg, logger, processingQueue, sequenceMap) + p := cmd.NewProcessor() + + go p.StartProcessor(context.TODO(), cfg, logger, processingQueue, sequenceMap) emptyBz := make([]byte, 32) expectedState := &types.MessageState{ @@ -197,6 +255,6 @@ func TestProcessNonBurnMessageWhenDisabled(t *testing.T) { actualState, ok := cmd.State.Load(cmd.LookupKey(expectedState.SourceTxHash, expectedState.Type)) require.True(t, ok) - require.Equal(t, types.Filtered, actualState.Status) + require.Equal(t, types.Filtered, actualState[0].Status) } diff --git a/cmd/root.go b/cmd/root.go index 0d18aa6..8354762 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -5,16 +5,17 @@ import ( "encoding/hex" "encoding/json" "fmt" + "io" + "net/http" + "os" + "strconv" + "github.com/cosmos/cosmos-sdk/crypto/keys/secp256k1" "github.com/cosmos/cosmos-sdk/types/bech32" "github.com/ethereum/go-ethereum/ethclient" "github.com/gin-gonic/gin" "github.com/strangelove-ventures/noble-cctp-relayer/cmd/ethereum" "github.com/strangelove-ventures/noble-cctp-relayer/types" - "io" - "net/http" - "os" - "strconv" "cosmossdk.io/log" "github.com/rs/zerolog" @@ -139,18 +140,22 @@ func getTxByHash(c *gin.Context) { var result []types.MessageState msgType := c.Query("type") // mint or forward if msgType == types.Mint || msgType == "" { - if message, ok := State.Load(LookupKey(txHash, types.Mint)); ok { - if domain == "" || (domain != "" && message.SourceDomain == uint32(domainInt)) { - result = append(result, *message) - found = true + if message, ok := State.Load(txHash); ok { + for _, msg := range message { + if msg.Type == types.Mint && domain == "" || (domain != "" && msg.SourceDomain == uint32(domainInt)) { + result = append(result, *msg) + found = true + } } } } if msgType == types.Forward || msgType == "" { - if message, ok := State.Load(LookupKey(txHash, types.Forward)); ok { - if domain == "" || (domain != "" && message.SourceDomain == uint32(domainInt)) { - result = append(result, *message) - found = true + if message, ok := State.Load(txHash); ok { + for _, msg := range message { + if msg.Type == types.Forward && domain == "" || (domain != "" && msg.SourceDomain == uint32(domainInt)) { + result = append(result, *msg) + found = true + } } } } diff --git a/integration/eth_burn_to_noble_mint_and_forward_test.go b/integration/eth_burn_to_noble_mint_and_forward_test.go index 9c1d1cb..3e8b332 100644 --- a/integration/eth_burn_to_noble_mint_and_forward_test.go +++ b/integration/eth_burn_to_noble_mint_and_forward_test.go @@ -23,6 +23,8 @@ import ( func TestEthBurnToNobleMintAndForward(t *testing.T) { setupTest() + p := cmd.NewProcessor() + // start up relayer cfg.Networks.Source.Ethereum.StartBlock = getEthereumLatestBlockHeight(t) cfg.Networks.Source.Ethereum.LookbackPeriod = 0 @@ -30,7 +32,7 @@ func TestEthBurnToNobleMintAndForward(t *testing.T) { fmt.Println("Starting relayer...") processingQueue := make(chan *types.MessageState, 10) go eth.StartListener(cfg, logger, processingQueue) - go cmd.StartProcessor(context.TODO(), cfg, logger, processingQueue, sequenceMap) + go p.StartProcessor(context.TODO(), cfg, logger, processingQueue, sequenceMap) fmt.Println("Building Ethereum depositForBurnWithMetadata txn...") _, _, cosmosAddress := testdata.KeyTestPubAddr() diff --git a/integration/eth_burn_to_noble_mint_test.go b/integration/eth_burn_to_noble_mint_test.go index 9be6c05..5b8d777 100644 --- a/integration/eth_burn_to_noble_mint_test.go +++ b/integration/eth_burn_to_noble_mint_test.go @@ -23,6 +23,8 @@ import ( func TestEthBurnToNobleMint(t *testing.T) { setupTest() + p := cmd.NewProcessor() + // start up relayer cfg.Networks.Source.Ethereum.StartBlock = getEthereumLatestBlockHeight(t) cfg.Networks.Source.Ethereum.LookbackPeriod = 0 @@ -30,7 +32,7 @@ func TestEthBurnToNobleMint(t *testing.T) { fmt.Println("Starting relayer...") processingQueue := make(chan *types.MessageState, 10) go eth.StartListener(cfg, logger, processingQueue) - go cmd.StartProcessor(context.TODO(), cfg, logger, processingQueue, sequenceMap) + go p.StartProcessor(context.TODO(), cfg, logger, processingQueue, sequenceMap) fmt.Println("Building Ethereum depositForBurnWithMetadata txn...") _, _, cosmosAddress := testdata.KeyTestPubAddr() diff --git a/integration/eth_multi_send_test.go b/integration/eth_multi_send_test.go index 224cf50..9e36746 100644 --- a/integration/eth_multi_send_test.go +++ b/integration/eth_multi_send_test.go @@ -176,8 +176,10 @@ func TestEthereumMultiSend(t *testing.T) { processingQueue := make(chan *types.MessageState, 100) + p := cmd.NewProcessor() + go noble.StartListener(cfg, logger, processingQueue) - go cmd.StartProcessor(context.TODO(), cfg, logger, processingQueue, sequenceMap) + go p.StartProcessor(context.TODO(), cfg, logger, processingQueue, sequenceMap) fmt.Println("Checking eth wallet...") for i := 0; i < 60; i++ { diff --git a/integration/noble_burn_to_eth_mint_test.go b/integration/noble_burn_to_eth_mint_test.go index 1aa6c5b..cfbe32c 100644 --- a/integration/noble_burn_to_eth_mint_test.go +++ b/integration/noble_burn_to_eth_mint_test.go @@ -42,6 +42,8 @@ import ( // and broadcasts on Ethereum Goerli func TestNobleBurnToEthMint(t *testing.T) { setupTest() + p := cmd.NewProcessor() + cfg.Networks.Source.Ethereum.Enabled = false // start up relayer @@ -50,7 +52,7 @@ func TestNobleBurnToEthMint(t *testing.T) { fmt.Println("Starting relayer...") processingQueue := make(chan *types.MessageState, 10) go noble.StartListener(cfg, logger, processingQueue) - go cmd.StartProcessor(context.TODO(), cfg, logger, processingQueue, sequenceMap) + go p.StartProcessor(context.TODO(), cfg, logger, processingQueue, sequenceMap) fmt.Println("Building Noble depositForBurn txn...") ethDestinationAddress := "0x971c54a6Eb782fAccD00bc3Ed5E934Cc5bD8e3Ef" diff --git a/integration/noble_multi_send_test.go b/integration/noble_multi_send_test.go index c8a4293..a965650 100644 --- a/integration/noble_multi_send_test.go +++ b/integration/noble_multi_send_test.go @@ -126,8 +126,10 @@ func TestNobleMultiSend(t *testing.T) { fmt.Println("Starting relayer...") processingQueue := make(chan *types.MessageState, 100) + p := cmd.NewProcessor() + go eth.StartListener(cfg, logger, processingQueue) - go cmd.StartProcessor(context.TODO(), cfg, logger, processingQueue, sequenceMap) + go p.StartProcessor(context.TODO(), cfg, logger, processingQueue, sequenceMap) fmt.Println("Checking noble wallet...") for i := 0; i < 250; i++ { diff --git a/types/message_state.go b/types/message_state.go index d28c0d3..f7d106c 100644 --- a/types/message_state.go +++ b/types/message_state.go @@ -1,6 +1,7 @@ package types import ( + "bytes" "encoding/base64" "encoding/hex" "encoding/json" @@ -28,9 +29,14 @@ const ( Forward string = "forward" ) +type TxState struct { + TxHash string + Msgs []*MessageState +} + type MessageState struct { - IrisLookupId string // hex encoded MessageSent bytes - Type string // 'mint' or 'forward' + IrisLookupId string // hex encoded MessageSent bytes + // Type string // 'mint' or 'forward' Status string // created, pending, attested, complete, failed, filtered Attestation string // hex encoded attestation SourceDomain uint32 // source domain id @@ -167,3 +173,20 @@ func DecodeDestinationCaller(input []byte) (string, error) { } return output, nil } + +// Equal checks if two MessageState instances are equal +func (m *MessageState) Equal(other *MessageState) bool { + return (m.IrisLookupId == other.IrisLookupId && + m.Type == other.Type && + m.Status == other.Status && + m.Attestation == other.Attestation && + m.SourceDomain == other.SourceDomain && + m.DestDomain == other.DestDomain && + m.SourceTxHash == other.SourceTxHash && + m.DestTxHash == other.DestTxHash && + bytes.Equal(m.MsgSentBytes, other.MsgSentBytes) && + bytes.Equal(m.DestinationCaller, other.DestinationCaller) && + m.Channel == other.Channel && + m.Created == other.Created && + m.Updated == other.Updated) +} diff --git a/types/message_state.go.bak b/types/message_state.go.bak new file mode 100644 index 0000000..f7d106c --- /dev/null +++ b/types/message_state.go.bak @@ -0,0 +1,192 @@ +package types + +import ( + "bytes" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "strconv" + "time" + + "github.com/circlefin/noble-cctp/x/cctp/types" + "github.com/cosmos/cosmos-sdk/types/bech32" + "github.com/ethereum/go-ethereum/accounts/abi" + ethtypes "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" +) + +const ( + Created string = "created" + Pending string = "pending" + Attested string = "attested" + Complete string = "complete" + Failed string = "failed" + Filtered string = "filtered" + + Mint string = "mint" + Forward string = "forward" +) + +type TxState struct { + TxHash string + Msgs []*MessageState +} + +type MessageState struct { + IrisLookupId string // hex encoded MessageSent bytes + // Type string // 'mint' or 'forward' + Status string // created, pending, attested, complete, failed, filtered + Attestation string // hex encoded attestation + SourceDomain uint32 // source domain id + DestDomain uint32 // destination domain id + SourceTxHash string + DestTxHash string + MsgSentBytes []byte // bytes of the MessageSent message transmitter event + DestinationCaller []byte // address authorized to call transaction + Channel string // "channel-%d" if a forward, empty if not a forward + Created time.Time + Updated time.Time + Nonce uint64 +} + +// EvmLogToMessageState transforms an evm log into a messageState given an ABI +func EvmLogToMessageState(abi abi.ABI, messageSent abi.Event, log *ethtypes.Log) (messageState *MessageState, err error) { + event := make(map[string]interface{}) + _ = abi.UnpackIntoMap(event, messageSent.Name, log.Data) + + rawMessageSentBytes := event["message"].([]byte) + message, _ := new(types.Message).Parse(rawMessageSentBytes) + + hashed := crypto.Keccak256(rawMessageSentBytes) + hashedHexStr := hex.EncodeToString(hashed) + + messageState = &MessageState{ + IrisLookupId: hashedHexStr, + Status: Created, + SourceDomain: message.SourceDomain, + DestDomain: message.DestinationDomain, + SourceTxHash: log.TxHash.Hex(), + MsgSentBytes: rawMessageSentBytes, + DestinationCaller: message.DestinationCaller, + Nonce: message.Nonce, + Created: time.Now(), + Updated: time.Now(), + } + + if _, err := new(BurnMessage).Parse(message.MessageBody); err == nil { + messageState.Type = Mint + return messageState, nil + } + + if forward, err := new(MetadataMessage).Parse(message.MessageBody); err == nil { + messageState.Type = Forward + // add forward channel to object so we can filter later + messageState.Channel = "channel-" + strconv.Itoa(int(forward.Channel)) + return messageState, nil + } + + return nil, fmt.Errorf("unable to parse tx into message, tx hash %s", log.TxHash.Hex()) +} + +// NobleLogToMessageState transforms a Noble log into a messageState +func NobleLogToMessageState(tx Tx) ([]*MessageState, error) { + var eventsList []struct { + Events []Event `json:"events"` + } + if tx.TxResult.Code != 0 { + return nil, nil + } + if err := json.Unmarshal([]byte(tx.TxResult.Log), &eventsList); err != nil { + return nil, fmt.Errorf("unable to parse log events: %s", tx.TxResult.Log) + } + + var messageStates []*MessageState + + for i, log := range eventsList { + for j, event := range log.Events { + if event.Type == "circle.cctp.v1.MessageSent" { + fmt.Printf("Saw cctp message %s - %d:%d\n", tx.Hash, i, j) + var parsed bool + var parseErrs error + for _, attr := range event.Attributes { + if attr.Key == "message" { + fmt.Printf("Saw message attribute %s - %d:%d\n", tx.Hash, i, j) + encoded := attr.Value[1 : len(attr.Value)-1] + rawMessageSentBytes, err := base64.StdEncoding.DecodeString(encoded) + if err != nil { + parseErrs = errors.Join(parseErrs, fmt.Errorf("failed to decode message: %w", err)) + continue + } + + hashed := crypto.Keccak256(rawMessageSentBytes) + hashedHexStr := hex.EncodeToString(hashed) + + msg, err := new(types.Message).Parse(rawMessageSentBytes) + if err != nil { + parseErrs = errors.Join(parseErrs, fmt.Errorf("failed to parse message: %w", err)) + continue + } + + parsed = true + + messageState := &MessageState{ + IrisLookupId: hashedHexStr, + Type: Mint, + Status: Created, + SourceDomain: msg.SourceDomain, + DestDomain: msg.DestinationDomain, + Nonce: msg.Nonce, + SourceTxHash: tx.Hash, + MsgSentBytes: rawMessageSentBytes, + DestinationCaller: msg.DestinationCaller, + Created: time.Now(), + Updated: time.Now(), + } + + messageStates = append(messageStates, messageState) + + fmt.Printf("Appended transfer from 4 to %d\n", msg.DestinationDomain) + } + } + if !parsed { + return nil, fmt.Errorf("unable to parse cctp message. tx hash %s: %w", tx.Hash, parseErrs) + } + } + } + } + + return messageStates, nil + +} + +// DecodeDestinationCaller transforms an encoded Noble cctp address into a noble bech32 address +// left padded input -> bech32 output +func DecodeDestinationCaller(input []byte) (string, error) { + if len(input) <= 12 { + return "", errors.New("destinationCaller is too short") + } + output, err := bech32.ConvertAndEncode("noble", input[12:]) + if err != nil { + return "", errors.New("unable to encode destination caller") + } + return output, nil +} + +// Equal checks if two MessageState instances are equal +func (m *MessageState) Equal(other *MessageState) bool { + return (m.IrisLookupId == other.IrisLookupId && + m.Type == other.Type && + m.Status == other.Status && + m.Attestation == other.Attestation && + m.SourceDomain == other.SourceDomain && + m.DestDomain == other.DestDomain && + m.SourceTxHash == other.SourceTxHash && + m.DestTxHash == other.DestTxHash && + bytes.Equal(m.MsgSentBytes, other.MsgSentBytes) && + bytes.Equal(m.DestinationCaller, other.DestinationCaller) && + m.Channel == other.Channel && + m.Created == other.Created && + m.Updated == other.Updated) +} diff --git a/types/message_state_test.go b/types/message_state_test.go index 1917948..971e4f1 100644 --- a/types/message_state_test.go +++ b/types/message_state_test.go @@ -3,6 +3,10 @@ package types_test import ( "context" "fmt" + "math/big" + "os" + "testing" + "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" @@ -10,11 +14,7 @@ import ( "github.com/pascaldekloe/etherstream" "github.com/strangelove-ventures/noble-cctp-relayer/config" "github.com/strangelove-ventures/noble-cctp-relayer/types" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "math/big" - "os" - "testing" ) var cfg config.Config @@ -36,13 +36,15 @@ func TestToMessageStateSuccess(t *testing.T) { ethClient, err := ethclient.DialContext(context.Background(), cfg.Networks.Source.Ethereum.RPC) require.Nil(t, err) - messageTransmitterAddress := common.HexToAddress("0x26413e8157CD32011E726065a5462e97dD4d03D9") + // changed to mainnet address + messageTransmitterAddress := common.HexToAddress("0x0a992d191deec32afe36203ad87d7d289a738f81") query := ethereum.FilterQuery{ Addresses: []common.Address{messageTransmitterAddress}, Topics: [][]common.Hash{{messageSent.ID}}, - FromBlock: big.NewInt(9573853), - ToBlock: big.NewInt(9573853), + // Changed + FromBlock: big.NewInt(18685801), + ToBlock: big.NewInt(18685801), } etherReader := etherstream.Reader{Backend: ethClient} @@ -52,23 +54,56 @@ func TestToMessageStateSuccess(t *testing.T) { messageState, err := types.EvmLogToMessageState(messageTransmitterABI, messageSent, &history[0]) - event := make(map[string]interface{}) - _ = messageTransmitterABI.UnpackIntoMap(event, messageSent.Name, history[0].Data) - - rawMessageSentBytes := event["message"].([]byte) - - destCaller := make([]byte, 32) - assert.Equal(t, "e40ed0e983675678715972bd50d6abc417735051b0255f3c0916911957eda603", messageState.IrisLookupId) - assert.Equal(t, "mint", messageState.Type) - assert.Equal(t, "created", messageState.Status) - assert.Equal(t, "", messageState.Attestation) - assert.Equal(t, uint32(0), messageState.SourceDomain) - assert.Equal(t, uint32(4), messageState.DestDomain) - assert.Equal(t, "0xed567f5a62166d0a5df6cdcec710640b1c8079758cd1e1ac95085742f06afb04", messageState.SourceTxHash) - assert.Equal(t, "", messageState.DestTxHash) - assert.Equal(t, rawMessageSentBytes, messageState.MsgSentBytes) - assert.Equal(t, destCaller, messageState.DestinationCaller) - assert.Equal(t, "", messageState.Channel) fmt.Println(messageState) - require.Nil(t, err) + } + +// func TestToMessageStateSuccess(t *testing.T) { + +// messageTransmitter, err := os.Open("../cmd/ethereum/abi/MessageTransmitter.json") +// require.Nil(t, err) + +// messageTransmitterABI, err := abi.JSON(messageTransmitter) +// require.Nil(t, err) + +// messageSent := messageTransmitterABI.Events["MessageSent"] + +// ethClient, err := ethclient.DialContext(context.Background(), cfg.Networks.Source.Ethereum.RPC) +// require.Nil(t, err) + +// messageTransmitterAddress := common.HexToAddress("0x26413e8157CD32011E726065a5462e97dD4d03D9") + +// query := ethereum.FilterQuery{ +// Addresses: []common.Address{messageTransmitterAddress}, +// Topics: [][]common.Hash{{messageSent.ID}}, +// FromBlock: big.NewInt(9573853), +// ToBlock: big.NewInt(9573853), +// } + +// etherReader := etherstream.Reader{Backend: ethClient} + +// _, _, history, err := etherReader.QueryWithHistory(context.Background(), &query) +// require.Nil(t, err) + +// messageState, err := types.EvmLogToMessageState(messageTransmitterABI, messageSent, &history[0]) + +// event := make(map[string]interface{}) +// _ = messageTransmitterABI.UnpackIntoMap(event, messageSent.Name, history[0].Data) + +// rawMessageSentBytes := event["message"].([]byte) + +// destCaller := make([]byte, 32) +// assert.Equal(t, "e40ed0e983675678715972bd50d6abc417735051b0255f3c0916911957eda603", messageState.IrisLookupId) +// assert.Equal(t, "mint", messageState.Type) +// assert.Equal(t, "created", messageState.Status) +// assert.Equal(t, "", messageState.Attestation) +// assert.Equal(t, uint32(0), messageState.SourceDomain) +// assert.Equal(t, uint32(4), messageState.DestDomain) +// assert.Equal(t, "0xed567f5a62166d0a5df6cdcec710640b1c8079758cd1e1ac95085742f06afb04", messageState.SourceTxHash) +// assert.Equal(t, "", messageState.DestTxHash) +// assert.Equal(t, rawMessageSentBytes, messageState.MsgSentBytes) +// assert.Equal(t, destCaller, messageState.DestinationCaller) +// assert.Equal(t, "", messageState.Channel) +// fmt.Println(messageState) +// require.Nil(t, err) +// } diff --git a/types/state.go b/types/state.go index a957b0f..c687241 100644 --- a/types/state.go +++ b/types/state.go @@ -5,7 +5,7 @@ import ( ) // StateMap wraps sync.Map with type safety -// maps source tx hash -> MessageState +// maps source tx hash -> TxState type StateMap struct { internal sync.Map } @@ -16,18 +16,20 @@ func NewStateMap() *StateMap { } } -func (sm *StateMap) Load(key string) (value *MessageState, ok bool) { +// load loads the message states tied to a specific transaction hash +func (sm *StateMap) Load(key string) (value *TxState, ok bool) { internalResult, ok := sm.internal.Load(key) if !ok { return nil, ok } - return internalResult.(*MessageState), ok + return internalResult.(*TxState), ok } func (sm *StateMap) Delete(key string) { sm.internal.Delete(key) } -func (sm *StateMap) Store(key string, value *MessageState) { +// store stores the message states tied to a specific transaction hash +func (sm *StateMap) Store(key string, value *TxState) { sm.internal.Store(key, value) } diff --git a/types/state_test.go b/types/state_test.go index 40971e5..985deb2 100644 --- a/types/state_test.go +++ b/types/state_test.go @@ -1,28 +1,33 @@ package types import ( - "fmt" "testing" -) -var stateMap StateMap + "github.com/stretchr/testify/require" +) func TestX(t *testing.T) { stateMap := NewStateMap() - msg := MessageState{IrisLookupId: "123", Status: Filtered} - stateMap.Store("123", &msg) - - lMsg, _ := stateMap.Load("123") - fmt.Println(lMsg) - msg.Status = Complete + txHash := "123456789" + msg := MessageState{SourceTxHash: txHash, IrisLookupId: "123", Status: Filtered, MsgSentBytes: []byte("i like turtles")} + stateMap.Store(txHash, []*MessageState{&msg}) - f, _ := stateMap.Load("123") + loadedMsg, _ := stateMap.Load(txHash) + require.True(t, msg.Equal(loadedMsg[0])) - lMsg.Status = Created + loadedMsg[0].Status = Complete - f, _ = stateMap.Load("123") + // Becasue it is a pointer, no need to re-store to state + // message status should be updated with out re-storing. + loadedMsg2, _ := stateMap.Load(txHash) + require.Equal(t, Complete, loadedMsg2[0].Status) - fmt.Println(f) + // even though loadedMsg is a pointer, if we add to the array, we need to re-store in cache. + msg2 := MessageState{SourceTxHash: txHash, IrisLookupId: "123", Status: Filtered, MsgSentBytes: []byte("mock bytes 2")} + loadedMsg = append(loadedMsg, &msg2) + stateMap.Store(txHash, loadedMsg) + loadedMsg3, _ := stateMap.Load(txHash) + require.Equal(t, 2, len(loadedMsg3)) } From 9817ceceb3e4eca9a754b2914aa44a7481a32e05 Mon Sep 17 00:00:00 2001 From: Dan Kanefsky Date: Tue, 12 Dec 2023 15:53:30 -0800 Subject: [PATCH 02/13] update test --- cmd/process_test.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/cmd/process_test.go b/cmd/process_test.go index 88e0b08..eae7888 100644 --- a/cmd/process_test.go +++ b/cmd/process_test.go @@ -190,7 +190,8 @@ func TestProcessNonWhitelistedChannel(t *testing.T) { p.Mu.RUnlock() } -// created message -> nonwhitelisted channel -> filtered +// test batch transactions where multiple messages can be sent with the same tx hash +// MsgSentBytes defer between messages func TestBatchTx(t *testing.T) { setupTest() @@ -200,24 +201,24 @@ func TestBatchTx(t *testing.T) { emptyBz := make([]byte, 32) expectedState := &types.MessageState{ - SourceTxHash: "123", + SourceTxHash: "123", // same source tx hash IrisLookupId: "a404f4155166a1fc7ffee145b5cac6d0f798333745289ab1db171344e226ef0c", Status: types.Created, SourceDomain: 0, DestDomain: 4, DestinationCaller: emptyBz, - MsgSentBytes: []byte("mock bytes 1"), + MsgSentBytes: []byte("mock bytes 1"), // different message sent bytes } processingQueue <- expectedState expectedState2 := &types.MessageState{ - SourceTxHash: "123", + SourceTxHash: "123", // same source tx hash IrisLookupId: "a404f4155166a1fc7ffee145b5cac6d0f798333745289ab1db171344e226ef0c", Status: types.Created, SourceDomain: 0, DestDomain: 4, DestinationCaller: emptyBz, - MsgSentBytes: []byte("mock bytes 2"), + MsgSentBytes: []byte("mock bytes 2"), // different message sent bytes } processingQueue <- expectedState2 From 040e22370ed28ba58ef08598b50f0488c5b8a927 Mon Sep 17 00:00:00 2001 From: Dan Kanefsky Date: Tue, 12 Dec 2023 16:20:39 -0800 Subject: [PATCH 03/13] fix test --- cmd/process_test.go | 58 +++++++++++++++++++++++---------------------- 1 file changed, 30 insertions(+), 28 deletions(-) diff --git a/cmd/process_test.go b/cmd/process_test.go index eae7888..e86d3dd 100644 --- a/cmd/process_test.go +++ b/cmd/process_test.go @@ -190,49 +190,40 @@ func TestProcessNonWhitelistedChannel(t *testing.T) { p.Mu.RUnlock() } -// test batch transactions where multiple messages can be sent with the same tx hash -// MsgSentBytes defer between messages -func TestBatchTx(t *testing.T) { +// created message -> not \ -> filtered +func TestProcessNonBurnMessageWhenDisabled(t *testing.T) { setupTest() p := cmd.NewProcessor() - go p.StartProcessor(context.TODO(), cfg, logger, processingQueue, sequenceMap) + go p.StartProcessor(cfg, logger, processingQueue, sequenceMap) emptyBz := make([]byte, 32) expectedState := &types.MessageState{ - SourceTxHash: "123", // same source tx hash + SourceTxHash: "123", + Type: "", IrisLookupId: "a404f4155166a1fc7ffee145b5cac6d0f798333745289ab1db171344e226ef0c", Status: types.Created, SourceDomain: 0, DestDomain: 4, DestinationCaller: emptyBz, - MsgSentBytes: []byte("mock bytes 1"), // different message sent bytes } - processingQueue <- expectedState - expectedState2 := &types.MessageState{ - SourceTxHash: "123", // same source tx hash - IrisLookupId: "a404f4155166a1fc7ffee145b5cac6d0f798333745289ab1db171344e226ef0c", - Status: types.Created, - SourceDomain: 0, - DestDomain: 4, - DestinationCaller: emptyBz, - MsgSentBytes: []byte("mock bytes 2"), // different message sent bytes - } + processingQueue <- expectedState - processingQueue <- expectedState2 - time.Sleep(6 * time.Second) + time.Sleep(2 * time.Second) actualState, ok := cmd.State.Load(expectedState.SourceTxHash) require.True(t, ok) p.Mu.RLock() - require.Equal(t, 2, len(actualState)) + require.Equal(t, types.Filtered, actualState[0].Status) p.Mu.RUnlock() + } -// created message -> not \ -> filtered -func TestProcessNonBurnMessageWhenDisabled(t *testing.T) { +// test batch transactions where multiple messages can be sent with the same tx hash +// MsgSentBytes defer between messages +func TestBatchTx(t *testing.T) { setupTest() p := cmd.NewProcessor() @@ -241,21 +232,32 @@ func TestProcessNonBurnMessageWhenDisabled(t *testing.T) { emptyBz := make([]byte, 32) expectedState := &types.MessageState{ - SourceTxHash: "123", - Type: "", + SourceTxHash: "123", // same source tx hash IrisLookupId: "a404f4155166a1fc7ffee145b5cac6d0f798333745289ab1db171344e226ef0c", Status: types.Created, SourceDomain: 0, DestDomain: 4, DestinationCaller: emptyBz, + MsgSentBytes: []byte("mock bytes 1"), // different message sent bytes } - processingQueue <- expectedState - time.Sleep(2 * time.Second) + expectedState2 := &types.MessageState{ + SourceTxHash: "123", // same source tx hash + IrisLookupId: "a404f4155166a1fc7ffee145b5cac6d0f798333745289ab1db171344e226ef0c", + Status: types.Created, + SourceDomain: 0, + DestDomain: 4, + DestinationCaller: emptyBz, + MsgSentBytes: []byte("mock bytes 2"), // different message sent bytes + } - actualState, ok := cmd.State.Load(cmd.LookupKey(expectedState.SourceTxHash, expectedState.Type)) - require.True(t, ok) - require.Equal(t, types.Filtered, actualState[0].Status) + processingQueue <- expectedState2 + time.Sleep(6 * time.Second) + actualState, ok := cmd.State.Load(expectedState.SourceTxHash) + require.True(t, ok) + p.Mu.RLock() + require.Equal(t, 2, len(actualState)) + p.Mu.RUnlock() } From 9eb062af60fe169f13bc73a07677a7924f113c15 Mon Sep 17 00:00:00 2001 From: Dan Kanefsky Date: Wed, 13 Dec 2023 09:50:46 -0800 Subject: [PATCH 04/13] fix nits --- types/state_test.go | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/types/state_test.go b/types/state_test.go index 985deb2..1d87cde 100644 --- a/types/state_test.go +++ b/types/state_test.go @@ -6,11 +6,17 @@ import ( "github.com/stretchr/testify/require" ) -func TestX(t *testing.T) { +func TestStateHandling(t *testing.T) { stateMap := NewStateMap() txHash := "123456789" - msg := MessageState{SourceTxHash: txHash, IrisLookupId: "123", Status: Filtered, MsgSentBytes: []byte("i like turtles")} + msg := MessageState{ + SourceTxHash: txHash, + IrisLookupId: "123", + Status: Filtered, + MsgSentBytes: []byte("i like turtles"), + } + stateMap.Store(txHash, []*MessageState{&msg}) loadedMsg, _ := stateMap.Load(txHash) @@ -24,7 +30,13 @@ func TestX(t *testing.T) { require.Equal(t, Complete, loadedMsg2[0].Status) // even though loadedMsg is a pointer, if we add to the array, we need to re-store in cache. - msg2 := MessageState{SourceTxHash: txHash, IrisLookupId: "123", Status: Filtered, MsgSentBytes: []byte("mock bytes 2")} + msg2 := MessageState{ + SourceTxHash: txHash, + IrisLookupId: "123", + Status: Filtered, + MsgSentBytes: []byte("mock bytes 2"), + } + loadedMsg = append(loadedMsg, &msg2) stateMap.Store(txHash, loadedMsg) From 366a992d5163cd35bf26cb80c2975972a9efbed2 Mon Sep 17 00:00:00 2001 From: Dan Kanefsky Date: Thu, 18 Jan 2024 14:47:47 -0800 Subject: [PATCH 05/13] finish batch tx's --- cmd/circle/attestation.go | 2 +- cmd/ethereum/broadcast.go | 36 +++--- cmd/ethereum/listener_test.go | 25 ++--- cmd/ethereum/util_test.go | 7 +- cmd/noble/broadcast.go | 105 ++++++++++-------- cmd/noble/cosmos/query.go | 8 +- cmd/noble/listener.go | 6 +- cmd/noble/listener_test.go | 25 ++--- cmd/process.go | 74 +++---------- cmd/process_test.go | 200 +++++++++++++++++----------------- cmd/root.go | 31 +----- config/config.go | 61 ++++++----- types/message_state.go | 25 ++--- types/sequence_map.go | 8 +- types/state_test.go | 17 ++- 15 files changed, 289 insertions(+), 341 deletions(-) diff --git a/cmd/circle/attestation.go b/cmd/circle/attestation.go index e083ce3..0fac048 100644 --- a/cmd/circle/attestation.go +++ b/cmd/circle/attestation.go @@ -13,7 +13,7 @@ import ( ) // CheckAttestation checks the iris api for attestation status and returns true if attestation is complete -func CheckAttestation(cfg config.Config, logger log.Logger, irisLookupId string, txHash string, sourceDomain, destDomain uint32) *types.AttestationResponse { +func CheckAttestation(cfg config.Config, logger log.Logger, irisLookupId string, txHash string, sourceDomain, destDomain types.Domain) *types.AttestationResponse { logger.Debug(fmt.Sprintf("Checking attestation for %s%s%s for source tx %s from %d to %d", cfg.Circle.AttestationBaseUrl, "0x", irisLookupId, txHash, sourceDomain, destDomain)) client := http.Client{Timeout: 2 * time.Second} diff --git a/cmd/ethereum/broadcast.go b/cmd/ethereum/broadcast.go index 7ffe0c3..31cef84 100644 --- a/cmd/ethereum/broadcast.go +++ b/cmd/ethereum/broadcast.go @@ -13,7 +13,6 @@ import ( "cosmossdk.io/log" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" - ethtypes "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethclient" "github.com/strangelove-ventures/noble-cctp-relayer/config" @@ -27,12 +26,12 @@ func Broadcast( logger log.Logger, msgs []*types.MessageState, sequenceMap *types.SequenceMap, -) ([]*ethtypes.Transaction, error) { +) error { // set up eth client client, err := ethclient.Dial(cfg.Networks.Destination.Ethereum.RPC) if err != nil { - return nil, fmt.Errorf("unable to dial ethereum client: %w", err) + return fmt.Errorf("unable to dial ethereum client: %w", err) } defer client.Close() @@ -40,26 +39,29 @@ func Broadcast( privEcdsaKey, ethereumAddress, err := GetEcdsaKeyAddress(cfg.Networks.Minters[0].MinterPrivateKey) if err != nil { - return nil, err + return err } auth, err := bind.NewKeyedTransactorWithChainID(privEcdsaKey, big.NewInt(cfg.Networks.Destination.Ethereum.ChainId)) if err != nil { - return nil, fmt.Errorf("unable to create auth: %w", err) + return fmt.Errorf("unable to create auth: %w", err) } messageTransmitter, err := NewMessageTransmitter(common.HexToAddress(cfg.Networks.Source.Ethereum.MessageTransmitter), backend) if err != nil { - return nil, fmt.Errorf("unable to create message transmitter: %w", err) + return fmt.Errorf("unable to create message transmitter: %w", err) } var broadcastErrors error - var txs []*ethtypes.Transaction for _, msg := range msgs { + if msg.Status == types.Complete { + continue + } + attestationBytes, err := hex.DecodeString(msg.Attestation[2:]) if err != nil { - return nil, errors.New("unable to decode message attestation") + return errors.New("unable to decode message attestation") } for attempt := 0; attempt <= cfg.Networks.Destination.Ethereum.BroadcastRetries; attempt++ { @@ -104,7 +106,7 @@ func Broadcast( logger.Debug(fmt.Sprintf("This source domain/nonce has already been used: %d %d", msg.SourceDomain, msg.Nonce)) msg.Status = types.Complete - return nil, errors.New("receive message was already broadcasted") + return errors.New("receive message was already broadcasted") } } @@ -116,14 +118,22 @@ func Broadcast( ) if err == nil { msg.Status = types.Complete - txs = append(txs, tx) + + fullLog, err := tx.MarshalJSON() + if err != nil { + logger.Error("error marshalling eth tx log", err) + } + + msg.DestTxHash = tx.Hash().Hex() + + logger.Info(fmt.Sprintf("Successfully broadcast %s to Ethereum. Tx hash: %s, FULL LOG: %s", msg.SourceTxHash, msg.DestTxHash, string(fullLog))) continue } else { logger.Error(fmt.Sprintf("error during broadcast: %s", err.Error())) if parsedErr, ok := err.(JsonError); ok { if parsedErr.ErrorCode() == 3 && parsedErr.Error() == "execution reverted: Nonce already used" { msg.Status = types.Complete - return nil, parsedErr + return parsedErr } match, _ := regexp.MatchString("nonce too low: next nonce [0-9]+, tx nonce [0-9]+", parsedErr.Error()) @@ -149,9 +159,9 @@ func Broadcast( continue } } + // retried max times with failure msg.Status = types.Failed - broadcastErrors = errors.Join(broadcastErrors, errors.New("reached max number of broadcast attempts")) } - return nil, broadcastErrors + return broadcastErrors } diff --git a/cmd/ethereum/listener_test.go b/cmd/ethereum/listener_test.go index 24367db..97577d8 100644 --- a/cmd/ethereum/listener_test.go +++ b/cmd/ethereum/listener_test.go @@ -1,26 +1,27 @@ package ethereum_test import ( + "os" + "testing" + "time" + "cosmossdk.io/log" "github.com/rs/zerolog" eth "github.com/strangelove-ventures/noble-cctp-relayer/cmd/ethereum" "github.com/strangelove-ventures/noble-cctp-relayer/config" "github.com/strangelove-ventures/noble-cctp-relayer/types" "github.com/stretchr/testify/require" - "os" - "testing" - "time" ) var cfg config.Config var logger log.Logger -var processingQueue chan *types.MessageState +var processingQueue chan *types.TxState func init() { cfg = config.Parse("../../.ignore/unit_tests.yaml") logger = log.NewLogger(os.Stdout, log.LevelOption(zerolog.ErrorLevel)) - processingQueue = make(chan *types.MessageState, 10000) + processingQueue = make(chan *types.TxState, 10000) } // tests for a historical log @@ -32,21 +33,19 @@ func TestStartListener(t *testing.T) { time.Sleep(5 * time.Second) - msg := <-processingQueue + tx := <-processingQueue expectedMsg := &types.MessageState{ IrisLookupId: "a404f4155166a1fc7ffee145b5cac6d0f798333745289ab1db171344e226ef0c", - Type: "mint", Status: "created", SourceDomain: 0, DestDomain: 4, SourceTxHash: "0xe1d7729de300274ee3a2fd20ba179b14a8e3ffcd9d847c506b06760f0dad7802", } - require.Equal(t, expectedMsg.IrisLookupId, msg.IrisLookupId) - require.Equal(t, expectedMsg.Type, msg.Type) - require.Equal(t, expectedMsg.Status, msg.Status) - require.Equal(t, expectedMsg.SourceDomain, msg.SourceDomain) - require.Equal(t, expectedMsg.DestDomain, msg.DestDomain) - require.Equal(t, expectedMsg.SourceTxHash, msg.SourceTxHash) + require.Equal(t, expectedMsg.IrisLookupId, tx.Msgs[0].IrisLookupId) + require.Equal(t, expectedMsg.Status, tx.Msgs[0].Status) + require.Equal(t, expectedMsg.SourceDomain, tx.Msgs[0].SourceDomain) + require.Equal(t, expectedMsg.DestDomain, tx.Msgs[0].DestDomain) + require.Equal(t, expectedMsg.SourceTxHash, tx.Msgs[0].SourceTxHash) } diff --git a/cmd/ethereum/util_test.go b/cmd/ethereum/util_test.go index b39cb2d..0856a5a 100644 --- a/cmd/ethereum/util_test.go +++ b/cmd/ethereum/util_test.go @@ -1,21 +1,22 @@ package ethereum_test import ( + "os" + "testing" + "cosmossdk.io/log" "github.com/rs/zerolog" "github.com/strangelove-ventures/noble-cctp-relayer/cmd/ethereum" "github.com/strangelove-ventures/noble-cctp-relayer/config" "github.com/strangelove-ventures/noble-cctp-relayer/types" "github.com/stretchr/testify/require" - "os" - "testing" ) func init() { cfg = config.Parse("../../.ignore/unit_tests.yaml") logger = log.NewLogger(os.Stdout, log.LevelOption(zerolog.ErrorLevel)) - processingQueue = make(chan *types.MessageState, 10000) + processingQueue = make(chan *types.TxState, 10000) } func TestGetEthereumAccountNonce(t *testing.T) { diff --git a/cmd/noble/broadcast.go b/cmd/noble/broadcast.go index c8c7f50..8cd77b7 100644 --- a/cmd/noble/broadcast.go +++ b/cmd/noble/broadcast.go @@ -23,6 +23,7 @@ import ( "github.com/cosmos/cosmos-sdk/codec" codectypes "github.com/cosmos/cosmos-sdk/codec/types" "github.com/cosmos/cosmos-sdk/crypto/keys/secp256k1" + sdk "github.com/cosmos/cosmos-sdk/types" "github.com/cosmos/cosmos-sdk/types/tx/signing" xauthsigning "github.com/cosmos/cosmos-sdk/x/auth/signing" xauthtx "github.com/cosmos/cosmos-sdk/x/auth/tx" @@ -36,9 +37,9 @@ func Broadcast( ctx context.Context, cfg config.Config, logger log.Logger, - msg *types.MessageState, + msgs []*types.MessageState, sequenceMap *types.SequenceMap, -) (*ctypes.ResultBroadcastTx, error) { +) error { // set up sdk context interfaceRegistry := codectypes.NewInterfaceRegistry() nobletypes.RegisterInterfaces(interfaceRegistry) @@ -49,66 +50,75 @@ func Broadcast( // build txn txBuilder := sdkContext.TxConfig.NewTxBuilder() - attestationBytes, err := hex.DecodeString(msg.Attestation[2:]) - if err != nil { - return nil, fmt.Errorf("unable to decode message attestation") - } // get priv key nobleAddress := cfg.Networks.Minters[4].MinterAddress keyBz, err := hex.DecodeString(cfg.Networks.Minters[4].MinterPrivateKey) if err != nil { - return nil, fmt.Errorf("unable to parse Noble private key") + return fmt.Errorf("unable to parse Noble private key") } privKey := secp256k1.PrivKey{Key: keyBz} - receiveMsg := nobletypes.NewMsgReceiveMessage( - nobleAddress, - msg.MsgSentBytes, - attestationBytes, - ) - err = txBuilder.SetMsgs(receiveMsg) - if err != nil { - return nil, err - } - - txBuilder.SetGasLimit(cfg.Networks.Destination.Noble.GasLimit) - txBuilder.SetMemo("Thank you for relaying with Strangelove") - - // sign and broadcast txn rpcClient, err := NewRPCClient(cfg.Networks.Destination.Noble.RPC, 10*time.Second) if err != nil { - return nil, errors.New("failed to set up rpc client") + return errors.New("failed to set up rpc client") } cc, err := cosmos.NewProvider(cfg.Networks.Source.Noble.RPC) if err != nil { - return nil, fmt.Errorf("unable to build cosmos provider for noble: %w", err) + return fmt.Errorf("unable to build cosmos provider for noble: %w", err) } + // sign and broadcast txn for attempt := 0; attempt <= cfg.Networks.Destination.Noble.BroadcastRetries; attempt++ { - used, err := cc.QueryUsedNonce(ctx, msg.SourceDomain, msg.Nonce) - if err != nil { - return nil, fmt.Errorf("unable to query used nonce: %w", err) + + var receiveMsgs []sdk.Msg + for _, msg := range msgs { + + used, err := cc.QueryUsedNonce(ctx, types.Domain(msg.SourceDomain), msg.Nonce) + if err != nil { + return fmt.Errorf("unable to query used nonce: %w", err) + } + + if used { + msg.Status = types.Complete + logger.Info(fmt.Sprintf("Noble cctp minter nonce %d already used", msg.Nonce)) + continue + } + + attestationBytes, err := hex.DecodeString(msg.Attestation[2:]) + if err != nil { + return fmt.Errorf("unable to decode message attestation") + } + + receiveMsgs = append(receiveMsgs, nobletypes.NewMsgReceiveMessage( + nobleAddress, + msg.MsgSentBytes, + attestationBytes, + )) + + logger.Info(fmt.Sprintf( + "Broadcasting message from %d to %d: with source tx hash %s", + msg.SourceDomain, + msg.DestDomain, + msg.SourceTxHash)) } - if used { - msg.Status = types.Complete - return nil, fmt.Errorf("noble cctp minter nonce %d already used", msg.Nonce) + err = txBuilder.SetMsgs(receiveMsgs...) + if err != nil { + return fmt.Errorf("failed to set messages on tx: %w", err) } - logger.Info(fmt.Sprintf( - "Broadcasting %s message from %d to %d: with source tx hash %s", - msg.Type, - msg.SourceDomain, - msg.DestDomain, - msg.SourceTxHash)) + txBuilder.SetGasLimit(cfg.Networks.Destination.Noble.GasLimit) + // TODO: make configurable + txBuilder.SetMemo("Thank you for relaying with Strangelove") accountSequence := sequenceMap.Next(cfg.Networks.Destination.Noble.DomainId) + //TODO: don't need to fetch this everytime accountNumber, _, err := GetNobleAccountNumberSequence(cfg.Networks.Destination.Noble.API, nobleAddress) if err != nil { - return nil, fmt.Errorf("failed to retrieve account number and sequence: %w", err) + return fmt.Errorf("failed to retrieve account number and sequence: %w", err) } sigV2 := signing.SignatureV2{ @@ -137,17 +147,17 @@ func Broadcast( uint64(accountSequence), ) if err != nil { - return nil, fmt.Errorf("failed to sign tx: %w", err) + return fmt.Errorf("failed to sign tx: %w", err) } if err := txBuilder.SetSignatures(sigV2); err != nil { - return nil, fmt.Errorf("failed to set signatures: %w", err) + return fmt.Errorf("failed to set signatures: %w", err) } // Generated Protobuf-encoded bytes. txBytes, err := sdkContext.TxConfig.TxEncoder()(txBuilder.GetTx()) if err != nil { - return nil, fmt.Errorf("failed to proto encode tx: %w", err) + return fmt.Errorf("failed to proto encode tx: %w", err) } rpcResponse, err := rpcClient.BroadcastTxSync(context.Background(), txBytes) @@ -183,13 +193,22 @@ func Broadcast( } // Tx was successfully broadcast - msg.Status = types.Complete - return rpcResponse, nil + for _, msg := range msgs { + msg.DestTxHash = rpcResponse.Hash.String() + msg.Status = types.Complete + } + logger.Info(fmt.Sprintf("Successfully broadcast %s to Noble. Tx hash: %s", msgs[0].SourceTxHash, msgs[0].DestTxHash)) + + return nil } - msg.Status = types.Failed + for _, msg := range msgs { + if msg.Status != types.Complete { + msg.Status = types.Failed + } + } - return nil, errors.New("reached max number of broadcast attempts") + return errors.New("reached max number of broadcast attempts") } // getErrorString returns the appropriate value to log when tx broadcast errors are encountered. diff --git a/cmd/noble/cosmos/query.go b/cmd/noble/cosmos/query.go index 804c806..35073f8 100644 --- a/cmd/noble/cosmos/query.go +++ b/cmd/noble/cosmos/query.go @@ -4,11 +4,11 @@ import ( "context" "fmt" - "github.com/circlefin/noble-cctp/x/cctp/types" cctptypes "github.com/circlefin/noble-cctp/x/cctp/types" abci "github.com/cometbft/cometbft/abci/types" rpcclient "github.com/cometbft/cometbft/rpc/client" coretypes "github.com/cometbft/cometbft/rpc/core/types" + "github.com/strangelove-ventures/noble-cctp-relayer/types" ) // func defaultPageRequest() *querytypes.PageRequest { @@ -38,11 +38,11 @@ func (cc *CosmosProvider) QueryABCI(ctx context.Context, req abci.RequestQuery) return result.Response, nil } -func (cc *CosmosProvider) QueryUsedNonce(ctx context.Context, sourceDomain uint32, nonce uint64) (bool, error) { +func (cc *CosmosProvider) QueryUsedNonce(ctx context.Context, sourceDomain types.Domain, nonce uint64) (bool, error) { qc := cctptypes.NewQueryClient(cc) - params := &types.QueryGetUsedNonceRequest{ - SourceDomain: sourceDomain, + params := &cctptypes.QueryGetUsedNonceRequest{ + SourceDomain: uint32(sourceDomain), Nonce: nonce, } diff --git a/cmd/noble/listener.go b/cmd/noble/listener.go index b52e824..c978f07 100644 --- a/cmd/noble/listener.go +++ b/cmd/noble/listener.go @@ -14,7 +14,7 @@ import ( "github.com/strangelove-ventures/noble-cctp-relayer/types" ) -func StartListener(cfg config.Config, logger log.Logger, processingQueue chan *types.MessageState) { +func StartListener(cfg config.Config, logger log.Logger, processingQueue chan *types.TxState) { // set up client logger.Info(fmt.Sprintf("Starting Noble listener at block %d looking back %d blocks", @@ -88,9 +88,9 @@ func StartListener(cfg config.Config, logger log.Logger, processingQueue chan *t continue } for _, parsedMsg := range parsedMsgs { - logger.Info(fmt.Sprintf("New stream msg from %d with tx hash %s", parsedMsg.SourceDomain, parsedMsg.SourceTxHash)) - processingQueue <- parsedMsg + logger.Info(fmt.Sprintf("New stream msg with nonce %d from %d with tx hash %s", parsedMsg.Nonce, parsedMsg.SourceDomain, parsedMsg.SourceTxHash)) } + processingQueue <- &types.TxState{TxHash: tx.Hash, Msgs: parsedMsgs} } } }() diff --git a/cmd/noble/listener_test.go b/cmd/noble/listener_test.go index f92616c..a1d668a 100644 --- a/cmd/noble/listener_test.go +++ b/cmd/noble/listener_test.go @@ -1,26 +1,27 @@ package noble_test import ( + "os" + "testing" + "time" + "cosmossdk.io/log" "github.com/rs/zerolog" "github.com/strangelove-ventures/noble-cctp-relayer/cmd/noble" "github.com/strangelove-ventures/noble-cctp-relayer/config" "github.com/strangelove-ventures/noble-cctp-relayer/types" "github.com/stretchr/testify/require" - "os" - "testing" - "time" ) var cfg config.Config var logger log.Logger -var processingQueue chan *types.MessageState +var processingQueue chan *types.TxState func init() { cfg = config.Parse("../../.ignore/unit_tests.yaml") logger = log.NewLogger(os.Stdout, log.LevelOption(zerolog.DebugLevel)) - processingQueue = make(chan *types.MessageState, 10000) + processingQueue = make(chan *types.TxState, 10000) cfg.Networks.Source.Noble.Workers = 1 } @@ -30,21 +31,19 @@ func TestStartListener(t *testing.T) { time.Sleep(20 * time.Second) - msg := <-processingQueue + tx := <-processingQueue expectedMsg := &types.MessageState{ IrisLookupId: "efe7cea3fd4785c3beab7f37876bdd48c5d4689c84d85a250813a2a7f01fe765", - Type: "mint", Status: "created", SourceDomain: 4, DestDomain: 0, SourceTxHash: "5002A249B1353FA59C1660EBAE5FA7FC652AC1E77F69CEF3A4533B0DF2864012", } - require.Equal(t, expectedMsg.IrisLookupId, msg.IrisLookupId) - require.Equal(t, expectedMsg.Type, msg.Type) - require.Equal(t, expectedMsg.Status, msg.Status) - require.Equal(t, expectedMsg.SourceDomain, msg.SourceDomain) - require.Equal(t, expectedMsg.DestDomain, msg.DestDomain) - require.Equal(t, expectedMsg.SourceTxHash, msg.SourceTxHash) + require.Equal(t, expectedMsg.IrisLookupId, tx.Msgs[0].IrisLookupId) + require.Equal(t, expectedMsg.Status, tx.Msgs[0].Status) + require.Equal(t, expectedMsg.SourceDomain, tx.Msgs[0].SourceDomain) + require.Equal(t, expectedMsg.DestDomain, tx.Msgs[0].DestDomain) + require.Equal(t, expectedMsg.SourceTxHash, tx.Msgs[0].SourceTxHash) } diff --git a/cmd/process.go b/cmd/process.go index 6c2996b..9c6cbde 100644 --- a/cmd/process.go +++ b/cmd/process.go @@ -77,7 +77,7 @@ func Start(cmd *cobra.Command, args []string) { } // messageState processing queue - var processingQueue = make(chan *types.MessageState, 10000) + var processingQueue = make(chan *types.TxState, 10000) // spin up Processor worker pool for i := 0; i < int(Cfg.ProcessorWorkerCount); i++ { @@ -111,15 +111,13 @@ func (p *Processor) StartProcessor(ctx context.Context, cfg config.Config, logge } } - var broadcastMsgs = make(map[uint32][]*types.MessageState) + var broadcastMsgs = make(map[types.Domain][]*types.MessageState) var requeue bool for _, msg := range tx.Msgs { // if a filter's condition is met, mark as filtered if filterDisabledCCTPRoutes(cfg, logger, msg) || - filterInvalidDestinationCallers(cfg, logger, msg) || - filterNonWhitelistedChannels(cfg, logger, msg) || - filterMessages(cfg, logger, msg) { + filterInvalidDestinationCallers(cfg, logger, msg) { msg.Status = types.Filtered } @@ -158,39 +156,19 @@ func (p *Processor) StartProcessor(ctx context.Context, cfg config.Config, logge } // if the message is attested to, try to broadcast for domain, msgs := range broadcastMsgs { + var err error switch domain { case 0: // ethereum - response, err := ethereum.Broadcast(ctx, cfg, logger, msgs, sequenceMap) - if err != nil { - logger.Error("unable to mint on Ethereum", "err", err) - requeue = true - continue - } - fullLog, err := response.MarshalJSON() - if err != nil { - logger.Error("error on marshall", err) - } - for _, msg := range msgs { - msg.DestTxHash = response.Hash().Hex() - } - logger.Info(fmt.Sprintf("Successfully broadcast %s to Ethereum. Tx hash: %s, FULL LOG: %s", msgs[0].SourceTxHash, msgs[0].DestTxHash, string(fullLog))) + err = ethereum.Broadcast(ctx, cfg, logger, msgs, sequenceMap) case 4: // noble - response, err := noble.Broadcast(ctx, cfg, logger, msgs, sequenceMap) - if err != nil { - logger.Error("unable to mint on Noble", "err", err) - requeue = true - continue - } - if response.Code != 0 { - logger.Error("nonzero response code received", "err", err) - requeue = true - continue - } - // success! - for _, msg := range msgs { - msg.DestTxHash = response.Hash.String() - } - logger.Info(fmt.Sprintf("Successfully broadcast %s to Noble. Tx hash: %s", msgs[0].SourceTxHash, msgs[0].DestTxHash)) + err = noble.Broadcast(ctx, cfg, logger, msgs, sequenceMap) + } + + if err != nil { + // TODO: add dest domain to error log + logger.Error("unable to mint one or more transfers", "error(s)", err, "total_transfers", len(msgs)) + requeue = true + continue } // ...add minters for different domains here @@ -254,35 +232,9 @@ func filterInvalidDestinationCallers(cfg config.Config, logger log.Logger, msg * return result } -// filterNonWhitelistedChannels is a Noble specific filter that returns true -// if the channel is not in the forwarding_channel_whitelist -func filterNonWhitelistedChannels(cfg config.Config, logger log.Logger, msg *types.MessageState) bool { - if !cfg.Networks.Destination.Noble.FilterForwardsByIbcChannel { - return false - } - for _, channel := range cfg.Networks.Destination.Noble.ForwardingChannelWhitelist { - if msg.Channel == channel { - return false - } - } - logger.Info(fmt.Sprintf("Filtered tx %s because channel whitelisting is enabled and the tx's channel is not in the whitelist: %s", - msg.SourceTxHash, msg.Channel)) - return true -} - -// filterMessages filters out non-burn messages. It returns true if the message is not a burn. -func filterMessages(_ config.Config, logger log.Logger, msg *types.MessageState) bool { - if msg.Type != types.Mint { - logger.Info(fmt.Sprintf("Filtered tx %s because it's a not a burn", msg.SourceTxHash)) - return true - } - return false -} - func LookupKey(sourceTxHash string) string { // return fmt.Sprintf("%s-%s", sourceTxHash, messageType) return sourceTxHash - } func init() { diff --git a/cmd/process_test.go b/cmd/process_test.go index e86d3dd..43fb3cb 100644 --- a/cmd/process_test.go +++ b/cmd/process_test.go @@ -17,13 +17,13 @@ import ( var cfg config.Config var logger log.Logger -var processingQueue chan *types.MessageState +var processingQueue chan *types.TxState var sequenceMap *types.SequenceMap func setupTest() { cfg = config.Parse("../.ignore/unit_tests.yaml") logger = log.NewLogger(os.Stdout, log.LevelOption(zerolog.DebugLevel)) - processingQueue = make(chan *types.MessageState, 10000) + processingQueue = make(chan *types.TxState, 10000) _, nextMinterSequence, err := noble.GetNobleAccountNumberSequence( cfg.Networks.Destination.Noble.API, @@ -34,7 +34,7 @@ func setupTest() { os.Exit(1) } sequenceMap = types.NewSequenceMap() - sequenceMap.Put(uint32(4), nextMinterSequence) + sequenceMap.Put(types.Domain(4), nextMinterSequence) } @@ -47,22 +47,26 @@ func TestProcessNewLog(t *testing.T) { go p.StartProcessor(context.TODO(), cfg, logger, processingQueue, sequenceMap) emptyBz := make([]byte, 32) - expectedState := &types.MessageState{ - SourceTxHash: "1", - Type: types.Mint, - SourceDomain: 0, - DestDomain: 4, - DestinationCaller: emptyBz, + expectedState := &types.TxState{ + TxHash: "1", + Msgs: []*types.MessageState{ + &types.MessageState{ + SourceTxHash: "1", + SourceDomain: 0, + DestDomain: 4, + DestinationCaller: emptyBz, + }, + }, } processingQueue <- expectedState time.Sleep(5 * time.Second) - actualState, _ := cmd.State.Load(expectedState.SourceTxHash) + actualState, _ := cmd.State.Load(expectedState.TxHash) p.Mu.RLock() - require.Equal(t, types.Created, actualState[0].Status) + require.Equal(t, types.Created, actualState.Msgs[0].Status) p.Mu.RUnlock() } @@ -77,24 +81,29 @@ func TestProcessCreatedLog(t *testing.T) { go p.StartProcessor(context.TODO(), cfg, logger, processingQueue, sequenceMap) emptyBz := make([]byte, 32) - expectedState := &types.MessageState{ - SourceTxHash: "123", - Type: types.Mint, - IrisLookupId: "a404f4155166a1fc7ffee145b5cac6d0f798333745289ab1db171344e226ef0c", - Status: types.Created, - SourceDomain: 0, - DestDomain: 5, - DestinationCaller: emptyBz, + + expectedState := &types.TxState{ + TxHash: "123", + Msgs: []*types.MessageState{ + &types.MessageState{ + SourceTxHash: "123", + IrisLookupId: "a404f4155166a1fc7ffee145b5cac6d0f798333745289ab1db171344e226ef0c", + Status: types.Created, + SourceDomain: 0, + DestDomain: 5, + DestinationCaller: emptyBz, + }, + }, } processingQueue <- expectedState time.Sleep(2 * time.Second) - actualState, ok := cmd.State.Load(expectedState.SourceTxHash) + actualState, ok := cmd.State.Load(expectedState.TxHash) require.True(t, ok) p.Mu.RLock() - require.Equal(t, types.Complete, actualState[0].Status) + require.Equal(t, types.Complete, actualState.Msgs[0].Status) p.Mu.RUnlock() } @@ -109,23 +118,28 @@ func TestProcessDisabledCctpRoute(t *testing.T) { go p.StartProcessor(context.TODO(), cfg, logger, processingQueue, sequenceMap) emptyBz := make([]byte, 32) - expectedState := &types.MessageState{ - SourceTxHash: "123", - IrisLookupId: "a404f4155166a1fc7ffee145b5cac6d0f798333745289ab1db171344e226ef0c", - Status: types.Created, - SourceDomain: 0, - DestDomain: 5, - DestinationCaller: emptyBz, + expectedState := &types.TxState{ + TxHash: "123", + Msgs: []*types.MessageState{ + &types.MessageState{ + SourceTxHash: "123", + IrisLookupId: "a404f4155166a1fc7ffee145b5cac6d0f798333745289ab1db171344e226ef0c", + Status: types.Created, + SourceDomain: 0, + DestDomain: 5, + DestinationCaller: emptyBz, + }, + }, } processingQueue <- expectedState time.Sleep(2 * time.Second) - actualState, ok := cmd.State.Load(expectedState.SourceTxHash) + actualState, ok := cmd.State.Load(expectedState.TxHash) require.True(t, ok) p.Mu.RLock() - require.Equal(t, types.Filtered, actualState[0].Status) + require.Equal(t, types.Filtered, actualState.Msgs[0].Status) p.Mu.RUnlock() } @@ -140,53 +154,28 @@ func TestProcessInvalidDestinationCaller(t *testing.T) { nonEmptyBytes := make([]byte, 31) nonEmptyBytes = append(nonEmptyBytes, 0x1) - expectedState := &types.MessageState{ - SourceTxHash: "123", - IrisLookupId: "a404f4155166a1fc7ffee145b5cac6d0f798333745289ab1db171344e226ef0c", - Status: types.Created, - SourceDomain: 0, - DestDomain: 4, - DestinationCaller: nonEmptyBytes, - } - - processingQueue <- expectedState - - time.Sleep(2 * time.Second) - - actualState, ok := cmd.State.Load(expectedState.SourceTxHash) - require.True(t, ok) - p.Mu.RLock() - require.Equal(t, types.Filtered, actualState[0].Status) - p.Mu.RUnlock() -} - -// created message -> nonwhitelisted channel -> filtered -func TestProcessNonWhitelistedChannel(t *testing.T) { - setupTest() - cfg.Networks.Destination.Noble.FilterForwardsByIbcChannel = true - - p := cmd.NewProcessor() - - go p.StartProcessor(context.TODO(), cfg, logger, processingQueue, sequenceMap) - - emptyBz := make([]byte, 32) - expectedState := &types.MessageState{ - SourceTxHash: "123", - IrisLookupId: "a404f4155166a1fc7ffee145b5cac6d0f798333745289ab1db171344e226ef0c", - Status: types.Created, - SourceDomain: 0, - DestDomain: 4, - DestinationCaller: emptyBz, + expectedState := &types.TxState{ + TxHash: "123", + Msgs: []*types.MessageState{ + &types.MessageState{ + SourceTxHash: "123", + IrisLookupId: "a404f4155166a1fc7ffee145b5cac6d0f798333745289ab1db171344e226ef0c", + Status: types.Created, + SourceDomain: 0, + DestDomain: 4, + DestinationCaller: nonEmptyBytes, + }, + }, } processingQueue <- expectedState time.Sleep(2 * time.Second) - actualState, ok := cmd.State.Load(expectedState.SourceTxHash) + actualState, ok := cmd.State.Load(expectedState.TxHash) require.True(t, ok) p.Mu.RLock() - require.Equal(t, types.Filtered, actualState[0].Status) + require.Equal(t, types.Filtered, actualState.Msgs[0].Status) p.Mu.RUnlock() } @@ -196,27 +185,31 @@ func TestProcessNonBurnMessageWhenDisabled(t *testing.T) { p := cmd.NewProcessor() - go p.StartProcessor(cfg, logger, processingQueue, sequenceMap) + go p.StartProcessor(context.TODO(), cfg, logger, processingQueue, sequenceMap) emptyBz := make([]byte, 32) - expectedState := &types.MessageState{ - SourceTxHash: "123", - Type: "", - IrisLookupId: "a404f4155166a1fc7ffee145b5cac6d0f798333745289ab1db171344e226ef0c", - Status: types.Created, - SourceDomain: 0, - DestDomain: 4, - DestinationCaller: emptyBz, + expectedState := &types.TxState{ + TxHash: "123", + Msgs: []*types.MessageState{ + &types.MessageState{ + SourceTxHash: "123", + IrisLookupId: "a404f4155166a1fc7ffee145b5cac6d0f798333745289ab1db171344e226ef0c", + Status: types.Created, + SourceDomain: 0, + DestDomain: 4, + DestinationCaller: emptyBz, + }, + }, } processingQueue <- expectedState time.Sleep(2 * time.Second) - actualState, ok := cmd.State.Load(expectedState.SourceTxHash) + actualState, ok := cmd.State.Load(expectedState.TxHash) require.True(t, ok) p.Mu.RLock() - require.Equal(t, types.Filtered, actualState[0].Status) + require.Equal(t, types.Filtered, actualState.Msgs[0].Status) p.Mu.RUnlock() } @@ -231,33 +224,34 @@ func TestBatchTx(t *testing.T) { go p.StartProcessor(context.TODO(), cfg, logger, processingQueue, sequenceMap) emptyBz := make([]byte, 32) - expectedState := &types.MessageState{ - SourceTxHash: "123", // same source tx hash - IrisLookupId: "a404f4155166a1fc7ffee145b5cac6d0f798333745289ab1db171344e226ef0c", - Status: types.Created, - SourceDomain: 0, - DestDomain: 4, - DestinationCaller: emptyBz, - MsgSentBytes: []byte("mock bytes 1"), // different message sent bytes - } - processingQueue <- expectedState - - expectedState2 := &types.MessageState{ - SourceTxHash: "123", // same source tx hash - IrisLookupId: "a404f4155166a1fc7ffee145b5cac6d0f798333745289ab1db171344e226ef0c", - Status: types.Created, - SourceDomain: 0, - DestDomain: 4, - DestinationCaller: emptyBz, - MsgSentBytes: []byte("mock bytes 2"), // different message sent bytes + expectedState := &types.TxState{ + TxHash: "123", + Msgs: []*types.MessageState{ + &types.MessageState{ + SourceTxHash: "123", + IrisLookupId: "a404f4155166a1fc7ffee145b5cac6d0f798333745289ab1db171344e226ef0c", + Status: types.Created, + SourceDomain: 0, + DestDomain: 4, + MsgSentBytes: []byte("mock bytes 1"), // different message sent bytes + }, + &types.MessageState{ + SourceTxHash: "123", // same source tx hash + IrisLookupId: "a404f4155166a1fc7ffee145b5cac6d0f798333745289ab1db171344e226ef0c", + Status: types.Created, + SourceDomain: 0, + DestDomain: 4, + DestinationCaller: emptyBz, + MsgSentBytes: []byte("mock bytes 2"), // different message sent bytes + }, + }, } - processingQueue <- expectedState2 - time.Sleep(6 * time.Second) + processingQueue <- expectedState - actualState, ok := cmd.State.Load(expectedState.SourceTxHash) + actualState, ok := cmd.State.Load(expectedState.TxHash) require.True(t, ok) p.Mu.RLock() - require.Equal(t, 2, len(actualState)) + require.Equal(t, 2, len(actualState.Msgs)) p.Mu.RUnlock() } diff --git a/cmd/root.go b/cmd/root.go index 8354762..4d7c173 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -136,33 +136,10 @@ func getTxByHash(c *gin.Context) { c.JSON(http.StatusBadRequest, gin.H{"message": "unable to parse domain"}) } - found := false - var result []types.MessageState - msgType := c.Query("type") // mint or forward - if msgType == types.Mint || msgType == "" { - if message, ok := State.Load(txHash); ok { - for _, msg := range message { - if msg.Type == types.Mint && domain == "" || (domain != "" && msg.SourceDomain == uint32(domainInt)) { - result = append(result, *msg) - found = true - } - } - } - } - if msgType == types.Forward || msgType == "" { - if message, ok := State.Load(txHash); ok { - for _, msg := range message { - if msg.Type == types.Forward && domain == "" || (domain != "" && msg.SourceDomain == uint32(domainInt)) { - result = append(result, *msg) - found = true - } - } - } + if tx, ok := State.Load(txHash); ok && domain == "" || (domain != "" && tx.Msgs[0].SourceDomain == types.Domain(domainInt)) { + c.JSON(http.StatusOK, tx.Msgs) + return } - if found { - c.JSON(http.StatusOK, result) - } else { - c.JSON(http.StatusNotFound, gin.H{"message": "message not found"}) - } + c.JSON(http.StatusNotFound, gin.H{"message": "message not found"}) } diff --git a/config/config.go b/config/config.go index b0c29b6..2e85000 100644 --- a/config/config.go +++ b/config/config.go @@ -3,6 +3,7 @@ package config import ( "os" + "github.com/strangelove-ventures/noble-cctp-relayer/types" "gopkg.in/yaml.v3" ) @@ -10,46 +11,46 @@ type Config struct { Networks struct { Source struct { Ethereum struct { - DomainId uint32 `yaml:"domain-id"` - RPC string `yaml:"rpc"` - MessageTransmitter string `yaml:"message-transmitter"` - RequestQueueSize uint32 `yaml:"request-queue-size"` - StartBlock uint64 `yaml:"start-block"` - LookbackPeriod uint64 `yaml:"lookback-period"` - Enabled bool `yaml:"enabled"` + DomainId types.Domain `yaml:"domain-id"` + RPC string `yaml:"rpc"` + MessageTransmitter string `yaml:"message-transmitter"` + RequestQueueSize uint32 `yaml:"request-queue-size"` + StartBlock uint64 `yaml:"start-block"` + LookbackPeriod uint64 `yaml:"lookback-period"` + Enabled bool `yaml:"enabled"` } `yaml:"ethereum"` Noble struct { - DomainId uint32 `yaml:"domain-id"` - RPC string `yaml:"rpc"` - RequestQueueSize uint32 `yaml:"request-queue-size"` - StartBlock uint64 `yaml:"start-block"` - LookbackPeriod uint64 `yaml:"lookback-period"` - Workers uint32 `yaml:"workers"` - Enabled bool `yaml:"enabled"` + DomainId types.Domain `yaml:"domain-id"` + RPC string `yaml:"rpc"` + RequestQueueSize uint32 `yaml:"request-queue-size"` + StartBlock uint64 `yaml:"start-block"` + LookbackPeriod uint64 `yaml:"lookback-period"` + Workers uint32 `yaml:"workers"` + Enabled bool `yaml:"enabled"` } `yaml:"noble"` } `yaml:"source"` Destination struct { Ethereum struct { - DomainId uint32 `yaml:"domain-id"` - ChainId int64 `yaml:"chain-id"` - RPC string `yaml:"rpc"` - BroadcastRetries int `yaml:"broadcast-retries"` - BroadcastRetryInterval int `yaml:"broadcast-retry-interval"` + DomainId types.Domain `yaml:"domain-id"` + ChainId int64 `yaml:"chain-id"` + RPC string `yaml:"rpc"` + BroadcastRetries int `yaml:"broadcast-retries"` + BroadcastRetryInterval int `yaml:"broadcast-retry-interval"` } `yaml:"ethereum"` Noble struct { - DomainId uint32 `yaml:"domain-id"` - RPC string `yaml:"rpc"` - API string `yaml:"api"` - ChainId string `yaml:"chain-id"` - GasLimit uint64 `yaml:"gas-limit"` - BroadcastRetries int `yaml:"broadcast-retries"` - BroadcastRetryInterval int `yaml:"broadcast-retry-interval"` - FilterForwardsByIbcChannel bool `yaml:"filter-forwards-by-ibc-channel"` - ForwardingChannelWhitelist []string `yaml:"forwarding-channel-whitelist"` + DomainId types.Domain `yaml:"domain-id"` + RPC string `yaml:"rpc"` + API string `yaml:"api"` + ChainId string `yaml:"chain-id"` + GasLimit uint64 `yaml:"gas-limit"` + BroadcastRetries int `yaml:"broadcast-retries"` + BroadcastRetryInterval int `yaml:"broadcast-retry-interval"` + FilterForwardsByIbcChannel bool `yaml:"filter-forwards-by-ibc-channel"` + ForwardingChannelWhitelist []string `yaml:"forwarding-channel-whitelist"` } `yaml:"noble"` } `yaml:"destination"` - EnabledRoutes map[uint32]uint32 `yaml:"enabled-routes"` - Minters map[uint32]struct { + EnabledRoutes map[types.Domain]types.Domain `yaml:"enabled-routes"` + Minters map[types.Domain]struct { MinterAddress string `yaml:"minter-address"` MinterPrivateKey string `yaml:"minter-private-key"` } `yaml:"minters"` diff --git a/types/message_state.go b/types/message_state.go index f7d106c..4d0630d 100644 --- a/types/message_state.go +++ b/types/message_state.go @@ -7,7 +7,6 @@ import ( "encoding/json" "errors" "fmt" - "strconv" "time" "github.com/circlefin/noble-cctp/x/cctp/types" @@ -29,6 +28,8 @@ const ( Forward string = "forward" ) +type Domain uint32 + type TxState struct { TxHash string Msgs []*MessageState @@ -39,8 +40,8 @@ type MessageState struct { // Type string // 'mint' or 'forward' Status string // created, pending, attested, complete, failed, filtered Attestation string // hex encoded attestation - SourceDomain uint32 // source domain id - DestDomain uint32 // destination domain id + SourceDomain Domain // uint32 source domain id + DestDomain Domain // uint32 destination domain id SourceTxHash string DestTxHash string MsgSentBytes []byte // bytes of the MessageSent message transmitter event @@ -65,8 +66,8 @@ func EvmLogToMessageState(abi abi.ABI, messageSent abi.Event, log *ethtypes.Log) messageState = &MessageState{ IrisLookupId: hashedHexStr, Status: Created, - SourceDomain: message.SourceDomain, - DestDomain: message.DestinationDomain, + SourceDomain: Domain(message.SourceDomain), + DestDomain: Domain(message.DestinationDomain), SourceTxHash: log.TxHash.Hex(), MsgSentBytes: rawMessageSentBytes, DestinationCaller: message.DestinationCaller, @@ -76,14 +77,6 @@ func EvmLogToMessageState(abi abi.ABI, messageSent abi.Event, log *ethtypes.Log) } if _, err := new(BurnMessage).Parse(message.MessageBody); err == nil { - messageState.Type = Mint - return messageState, nil - } - - if forward, err := new(MetadataMessage).Parse(message.MessageBody); err == nil { - messageState.Type = Forward - // add forward channel to object so we can filter later - messageState.Channel = "channel-" + strconv.Itoa(int(forward.Channel)) return messageState, nil } @@ -133,10 +126,9 @@ func NobleLogToMessageState(tx Tx) ([]*MessageState, error) { messageState := &MessageState{ IrisLookupId: hashedHexStr, - Type: Mint, Status: Created, - SourceDomain: msg.SourceDomain, - DestDomain: msg.DestinationDomain, + SourceDomain: Domain(msg.SourceDomain), + DestDomain: Domain(msg.DestinationDomain), Nonce: msg.Nonce, SourceTxHash: tx.Hash, MsgSentBytes: rawMessageSentBytes, @@ -177,7 +169,6 @@ func DecodeDestinationCaller(input []byte) (string, error) { // Equal checks if two MessageState instances are equal func (m *MessageState) Equal(other *MessageState) bool { return (m.IrisLookupId == other.IrisLookupId && - m.Type == other.Type && m.Status == other.Status && m.Attestation == other.Attestation && m.SourceDomain == other.SourceDomain && diff --git a/types/sequence_map.go b/types/sequence_map.go index dab63f1..917dd5e 100644 --- a/types/sequence_map.go +++ b/types/sequence_map.go @@ -8,22 +8,22 @@ import ( type SequenceMap struct { mu sync.Mutex // map destination domain -> minter account sequence - sequenceMap map[uint32]int64 + sequenceMap map[Domain]int64 } func NewSequenceMap() *SequenceMap { return &SequenceMap{ - sequenceMap: map[uint32]int64{}, + sequenceMap: map[Domain]int64{}, } } -func (m *SequenceMap) Put(destDomain uint32, val int64) { +func (m *SequenceMap) Put(destDomain Domain, val int64) { m.mu.Lock() defer m.mu.Unlock() m.sequenceMap[destDomain] = val } -func (m *SequenceMap) Next(destDomain uint32) int64 { +func (m *SequenceMap) Next(destDomain Domain) int64 { m.mu.Lock() defer m.mu.Unlock() result := m.sequenceMap[destDomain] diff --git a/types/state_test.go b/types/state_test.go index 1d87cde..7bb5e1e 100644 --- a/types/state_test.go +++ b/types/state_test.go @@ -17,17 +17,22 @@ func TestStateHandling(t *testing.T) { MsgSentBytes: []byte("i like turtles"), } - stateMap.Store(txHash, []*MessageState{&msg}) + stateMap.Store(txHash, &TxState{ + TxHash: txHash, + Msgs: []*MessageState{ + &msg, + }, + }) loadedMsg, _ := stateMap.Load(txHash) - require.True(t, msg.Equal(loadedMsg[0])) + require.True(t, msg.Equal(loadedMsg.Msgs[0])) - loadedMsg[0].Status = Complete + loadedMsg.Msgs[0].Status = Complete // Becasue it is a pointer, no need to re-store to state // message status should be updated with out re-storing. loadedMsg2, _ := stateMap.Load(txHash) - require.Equal(t, Complete, loadedMsg2[0].Status) + require.Equal(t, Complete, loadedMsg2.Msgs[0].Status) // even though loadedMsg is a pointer, if we add to the array, we need to re-store in cache. msg2 := MessageState{ @@ -37,9 +42,9 @@ func TestStateHandling(t *testing.T) { MsgSentBytes: []byte("mock bytes 2"), } - loadedMsg = append(loadedMsg, &msg2) + loadedMsg.Msgs = append(loadedMsg.Msgs, &msg2) stateMap.Store(txHash, loadedMsg) loadedMsg3, _ := stateMap.Load(txHash) - require.Equal(t, 2, len(loadedMsg3)) + require.Equal(t, 2, len(loadedMsg3.Msgs)) } From 6d7f649c0081515c8565d93a5488f1cde237cd87 Mon Sep 17 00:00:00 2001 From: Andrew Gouin Date: Thu, 18 Jan 2024 19:20:06 -0700 Subject: [PATCH 06/13] Refactorrrrrrrrrr --- README.md | 8 +- {cmd/circle => circle}/attestation.go | 9 +- {cmd/circle => circle}/attestation_test.go | 10 +- cmd/ethereum/broadcast.go | 167 ------- cmd/ethereum/listener.go | 114 ----- cmd/noble/broadcast.go | 274 ----------- cmd/noble/listener.go | 113 ----- cmd/process.go | 154 ++---- cmd/process_test.go | 88 ++-- cmd/root.go | 68 +-- config/config.go | 73 --- {cmd/noble/cosmos => cosmos}/codec.go | 0 .../noble/cosmos => cosmos}/cosmosprovider.go | 0 {cmd/noble/cosmos => cosmos}/grpc_shim.go | 0 {cmd/noble/cosmos => cosmos}/query.go | 0 {cmd/noble/cosmos => cosmos}/query_test.go | 2 +- {cmd/ethereum => ethereum}/abi/ERC20.json | 0 .../abi/MessageTransmitter.json | 0 .../abi/TokenMessenger.json | 0 .../abi/TokenMessengerWithMetadata.json | 0 {cmd/ethereum => ethereum}/broadcast_test.go | 4 +- ethereum/chain.go | 380 +++++++++++++++ ethereum/config.go | 38 ++ .../contract_backend_wrapper.go | 0 .../contracts}/MessageTransmitter.go | 2 +- {cmd => ethereum/contracts}/TokenMessenger.go | 2 +- .../contracts}/TokenMessengerWithMetadata.go | 2 +- {cmd/ethereum => ethereum}/listener_test.go | 22 +- {cmd/ethereum => ethereum}/util.go | 0 {cmd/ethereum => ethereum}/util_test.go | 13 +- noble/chain.go | 459 ++++++++++++++++++ noble/config.go | 37 ++ {cmd/noble => noble}/listener_test.go | 21 +- noble/message_state.go | 77 +++ types/chain.go | 43 ++ types/config.go | 34 ++ types/message_state.go | 87 ---- types/message_state.go.bak | 192 -------- types/message_state_test.go | 13 +- types/sequence_map.go | 8 +- 40 files changed, 1231 insertions(+), 1283 deletions(-) rename {cmd/circle => circle}/attestation.go (62%) rename {cmd/circle => circle}/attestation_test.go (56%) delete mode 100644 cmd/ethereum/broadcast.go delete mode 100644 cmd/ethereum/listener.go delete mode 100644 cmd/noble/broadcast.go delete mode 100644 cmd/noble/listener.go delete mode 100644 config/config.go rename {cmd/noble/cosmos => cosmos}/codec.go (100%) rename {cmd/noble/cosmos => cosmos}/cosmosprovider.go (100%) rename {cmd/noble/cosmos => cosmos}/grpc_shim.go (100%) rename {cmd/noble/cosmos => cosmos}/query.go (100%) rename {cmd/noble/cosmos => cosmos}/query_test.go (86%) rename {cmd/ethereum => ethereum}/abi/ERC20.json (100%) rename {cmd/ethereum => ethereum}/abi/MessageTransmitter.json (100%) rename {cmd/ethereum => ethereum}/abi/TokenMessenger.json (100%) rename {cmd/ethereum => ethereum}/abi/TokenMessengerWithMetadata.json (100%) rename {cmd/ethereum => ethereum}/broadcast_test.go (82%) create mode 100644 ethereum/chain.go create mode 100644 ethereum/config.go rename {cmd/ethereum => ethereum}/contract_backend_wrapper.go (100%) rename {cmd/ethereum => ethereum/contracts}/MessageTransmitter.go (99%) rename {cmd => ethereum/contracts}/TokenMessenger.go (99%) rename {cmd => ethereum/contracts}/TokenMessengerWithMetadata.go (99%) rename {cmd/ethereum => ethereum}/listener_test.go (73%) rename {cmd/ethereum => ethereum}/util.go (100%) rename {cmd/ethereum => ethereum}/util_test.go (58%) create mode 100644 noble/chain.go create mode 100644 noble/config.go rename {cmd/noble => noble}/listener_test.go (70%) create mode 100644 noble/message_state.go create mode 100644 types/chain.go create mode 100644 types/config.go delete mode 100644 types/message_state.go.bak diff --git a/README.md b/README.md index 3012fd8..103698e 100644 --- a/README.md +++ b/README.md @@ -42,10 +42,10 @@ localhost:8000/tx/?type=forward ### Generating Go ABI bindings ```shell -abigen --abi cmd/ethereum/abi/TokenMessenger.json --pkg cmd --type TokenMessenger --out cmd/TokenMessenger.go -abigen --abi cmd/ethereum/abi/TokenMessengerWithMetadata.json --pkg cmd --type TokenMessengerWithMetadata --out cmd/TokenMessengerWithMetadata.go -abigen --abi cmd/ethereum/abi/ERC20.json --pkg integration_testing --type ERC20 --out integration/ERC20.go -abigen --abi cmd/ethereum/abi/MessageTransmitter.json --pkg cmd --type MessageTransmitter --out cmd/MessageTransmitter.go +abigen --abi ethereum/abi/TokenMessenger.json --pkg contracts --type TokenMessenger --out ethereum/contracts/TokenMessenger.go +abigen --abi ethereum/abi/TokenMessengerWithMetadata.json --pkg contracts --type TokenMessengerWithMetadata --out ethereum/contracts/TokenMessengerWithMetadata.go +abigen --abi ethereum/abi/ERC20.json --pkg integration_testing --type ERC20 --out integration/ERC20.go +abigen --abi ethereum/abi/MessageTransmitter.json --pkg contracts- --type MessageTransmitter --out ethereum/contracts/MessageTransmitter.go ``` ### Useful links diff --git a/cmd/circle/attestation.go b/circle/attestation.go similarity index 62% rename from cmd/circle/attestation.go rename to circle/attestation.go index 0fac048..c51da8a 100644 --- a/cmd/circle/attestation.go +++ b/circle/attestation.go @@ -8,17 +8,16 @@ import ( "time" "cosmossdk.io/log" - "github.com/strangelove-ventures/noble-cctp-relayer/config" "github.com/strangelove-ventures/noble-cctp-relayer/types" ) // CheckAttestation checks the iris api for attestation status and returns true if attestation is complete -func CheckAttestation(cfg config.Config, logger log.Logger, irisLookupId string, txHash string, sourceDomain, destDomain types.Domain) *types.AttestationResponse { - logger.Debug(fmt.Sprintf("Checking attestation for %s%s%s for source tx %s from %d to %d", cfg.Circle.AttestationBaseUrl, "0x", irisLookupId, txHash, sourceDomain, destDomain)) +func CheckAttestation(attestationURL string, logger log.Logger, irisLookupId string, txHash string, sourceDomain, destDomain types.Domain) *types.AttestationResponse { + logger.Debug(fmt.Sprintf("Checking attestation for %s%s%s for source tx %s from %d to %d", attestationURL, "0x", irisLookupId, txHash, sourceDomain, destDomain)) client := http.Client{Timeout: 2 * time.Second} - rawResponse, err := client.Get(cfg.Circle.AttestationBaseUrl + "0x" + irisLookupId) + rawResponse, err := client.Get(attestationURL + "0x" + irisLookupId) if err != nil { logger.Debug("error during request: " + err.Error()) return nil @@ -39,7 +38,7 @@ func CheckAttestation(cfg config.Config, logger log.Logger, irisLookupId string, logger.Debug("unable to unmarshal response") return nil } - logger.Info(fmt.Sprintf("Attestation found for %s%s%s", cfg.Circle.AttestationBaseUrl, "0x", irisLookupId)) + logger.Info(fmt.Sprintf("Attestation found for %s%s%s", attestationURL, "0x", irisLookupId)) return &response } diff --git a/cmd/circle/attestation_test.go b/circle/attestation_test.go similarity index 56% rename from cmd/circle/attestation_test.go rename to circle/attestation_test.go index 1e68cc5..a2fa252 100644 --- a/cmd/circle/attestation_test.go +++ b/circle/attestation_test.go @@ -6,12 +6,12 @@ import ( "cosmossdk.io/log" "github.com/rs/zerolog" - "github.com/strangelove-ventures/noble-cctp-relayer/cmd/circle" - "github.com/strangelove-ventures/noble-cctp-relayer/config" + "github.com/strangelove-ventures/noble-cctp-relayer/circle" + "github.com/strangelove-ventures/noble-cctp-relayer/types" "github.com/stretchr/testify/require" ) -var cfg config.Config +var cfg types.Config var logger log.Logger func init() { @@ -20,12 +20,12 @@ func init() { } func TestAttestationIsReady(t *testing.T) { - resp := circle.CheckAttestation(cfg, logger, "85bbf7e65a5992e6317a61f005e06d9972a033d71b514be183b179e1b47723fe", "", 0, 4) + resp := circle.CheckAttestation(cfg.Circle.AttestationBaseUrl, logger, "85bbf7e65a5992e6317a61f005e06d9972a033d71b514be183b179e1b47723fe", "", 0, 4) require.NotNil(t, resp) require.Equal(t, "complete", resp.Status) } func TestAttestationNotFound(t *testing.T) { - resp := circle.CheckAttestation(cfg, logger, "not an attestation", "", 0, 4) + resp := circle.CheckAttestation(cfg.Circle.AttestationBaseUrl, logger, "not an attestation", "", 0, 4) require.Nil(t, resp) } diff --git a/cmd/ethereum/broadcast.go b/cmd/ethereum/broadcast.go deleted file mode 100644 index 31cef84..0000000 --- a/cmd/ethereum/broadcast.go +++ /dev/null @@ -1,167 +0,0 @@ -package ethereum - -import ( - "context" - "encoding/hex" - "errors" - "fmt" - "math/big" - "regexp" - "strconv" - "time" - - "cosmossdk.io/log" - "github.com/ethereum/go-ethereum/accounts/abi/bind" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethclient" - "github.com/strangelove-ventures/noble-cctp-relayer/config" - "github.com/strangelove-ventures/noble-cctp-relayer/types" -) - -// Broadcast broadcasts a message to Ethereum -func Broadcast( - ctx context.Context, - cfg config.Config, - logger log.Logger, - msgs []*types.MessageState, - sequenceMap *types.SequenceMap, -) error { - - // set up eth client - client, err := ethclient.Dial(cfg.Networks.Destination.Ethereum.RPC) - if err != nil { - return fmt.Errorf("unable to dial ethereum client: %w", err) - } - defer client.Close() - - backend := NewContractBackendWrapper(client) - - privEcdsaKey, ethereumAddress, err := GetEcdsaKeyAddress(cfg.Networks.Minters[0].MinterPrivateKey) - if err != nil { - return err - } - - auth, err := bind.NewKeyedTransactorWithChainID(privEcdsaKey, big.NewInt(cfg.Networks.Destination.Ethereum.ChainId)) - if err != nil { - return fmt.Errorf("unable to create auth: %w", err) - } - - messageTransmitter, err := NewMessageTransmitter(common.HexToAddress(cfg.Networks.Source.Ethereum.MessageTransmitter), backend) - if err != nil { - return fmt.Errorf("unable to create message transmitter: %w", err) - } - - var broadcastErrors error - for _, msg := range msgs { - - if msg.Status == types.Complete { - continue - } - - attestationBytes, err := hex.DecodeString(msg.Attestation[2:]) - if err != nil { - return errors.New("unable to decode message attestation") - } - - for attempt := 0; attempt <= cfg.Networks.Destination.Ethereum.BroadcastRetries; attempt++ { - logger.Info(fmt.Sprintf( - "Broadcasting message from %d to %d: with source tx hash %s", - msg.SourceDomain, - msg.DestDomain, - msg.SourceTxHash)) - - nonce := sequenceMap.Next(cfg.Networks.Destination.Ethereum.DomainId) - auth.Nonce = big.NewInt(nonce) - - // TODO remove - nextNonce, err := GetEthereumAccountNonce(cfg.Networks.Destination.Ethereum.RPC, ethereumAddress) - if err != nil { - logger.Error("unable to retrieve account number") - } else { - auth.Nonce = big.NewInt(nextNonce) - } - // TODO end remove - - // check if nonce already used - co := &bind.CallOpts{ - Pending: true, - Context: ctx, - } - - logger.Debug("Checking if nonce was used for broadcast to Ethereum", "source_domain", msg.SourceDomain, "nonce", msg.Nonce) - - key := append( - common.LeftPadBytes((big.NewInt(int64(msg.SourceDomain))).Bytes(), 4), - common.LeftPadBytes((big.NewInt(int64(msg.Nonce))).Bytes(), 8)..., - ) - - response, nonceErr := messageTransmitter.UsedNonces(co, [32]byte(crypto.Keccak256(key))) - if nonceErr != nil { - logger.Debug("Error querying whether nonce was used. Continuing...") - } else { - fmt.Printf("received used nonce response: %d\n", response) - if response.Uint64() == uint64(1) { - // nonce has already been used, mark as complete - logger.Debug(fmt.Sprintf("This source domain/nonce has already been used: %d %d", - msg.SourceDomain, msg.Nonce)) - msg.Status = types.Complete - return errors.New("receive message was already broadcasted") - } - } - - // broadcast txn - tx, err := messageTransmitter.ReceiveMessage( - auth, - msg.MsgSentBytes, - attestationBytes, - ) - if err == nil { - msg.Status = types.Complete - - fullLog, err := tx.MarshalJSON() - if err != nil { - logger.Error("error marshalling eth tx log", err) - } - - msg.DestTxHash = tx.Hash().Hex() - - logger.Info(fmt.Sprintf("Successfully broadcast %s to Ethereum. Tx hash: %s, FULL LOG: %s", msg.SourceTxHash, msg.DestTxHash, string(fullLog))) - continue - } else { - logger.Error(fmt.Sprintf("error during broadcast: %s", err.Error())) - if parsedErr, ok := err.(JsonError); ok { - if parsedErr.ErrorCode() == 3 && parsedErr.Error() == "execution reverted: Nonce already used" { - msg.Status = types.Complete - return parsedErr - } - - match, _ := regexp.MatchString("nonce too low: next nonce [0-9]+, tx nonce [0-9]+", parsedErr.Error()) - if match { - numberRegex := regexp.MustCompile("[0-9]+") - nextNonce, err := strconv.ParseInt(numberRegex.FindAllString(parsedErr.Error(), 1)[0], 10, 0) - if err != nil { - nextNonce, err = GetEthereumAccountNonce(cfg.Networks.Destination.Ethereum.RPC, ethereumAddress) - if err != nil { - logger.Error("unable to retrieve account number") - } - } - sequenceMap.Put(cfg.Networks.Destination.Ethereum.DomainId, nextNonce) - } - } - - // if it's not the last attempt, retry - // TODO increase the destination.ethereum.broadcast retries (3-5) and retry interval (15s). By checking for used nonces, there is no gas cost for failed mints. - if attempt != cfg.Networks.Destination.Ethereum.BroadcastRetries { - logger.Info(fmt.Sprintf("Retrying in %d seconds", cfg.Networks.Destination.Ethereum.BroadcastRetryInterval)) - time.Sleep(time.Duration(cfg.Networks.Destination.Ethereum.BroadcastRetryInterval) * time.Second) - } - continue - } - } - // retried max times with failure - msg.Status = types.Failed - broadcastErrors = errors.Join(broadcastErrors, errors.New("reached max number of broadcast attempts")) - } - return broadcastErrors -} diff --git a/cmd/ethereum/listener.go b/cmd/ethereum/listener.go deleted file mode 100644 index 1e7088b..0000000 --- a/cmd/ethereum/listener.go +++ /dev/null @@ -1,114 +0,0 @@ -package ethereum - -import ( - "bytes" - "context" - "embed" - "fmt" - "math/big" - "os" - - "cosmossdk.io/log" - "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/accounts/abi" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/ethclient" - "github.com/pascaldekloe/etherstream" - "github.com/strangelove-ventures/noble-cctp-relayer/config" - "github.com/strangelove-ventures/noble-cctp-relayer/types" -) - -//go:embed abi/MessageTransmitter.json -var content embed.FS - -func StartListener(cfg config.Config, logger log.Logger, processingQueue chan *types.TxState) { - // set up client - messageTransmitter, err := content.ReadFile("abi/MessageTransmitter.json") - if err != nil { - logger.Error("unable to read MessageTransmitter abi", "err", err) - os.Exit(1) - } - messageTransmitterABI, err := abi.JSON(bytes.NewReader(messageTransmitter)) - if err != nil { - logger.Error("unable to parse MessageTransmitter abi", "err", err) - } - - messageSent := messageTransmitterABI.Events["MessageSent"] - - ethClient, err := ethclient.DialContext(context.Background(), cfg.Networks.Source.Ethereum.RPC) - if err != nil { - logger.Error("unable to initialize ethereum client", "err", err) - os.Exit(1) - } - - messageTransmitterAddress := common.HexToAddress(cfg.Networks.Source.Ethereum.MessageTransmitter) - etherReader := etherstream.Reader{Backend: ethClient} - - query := ethereum.FilterQuery{ - Addresses: []common.Address{messageTransmitterAddress}, - Topics: [][]common.Hash{{messageSent.ID}}, - FromBlock: big.NewInt(int64(cfg.Networks.Source.Ethereum.StartBlock - cfg.Networks.Source.Ethereum.LookbackPeriod)), - } - - logger.Info(fmt.Sprintf( - "Starting Ethereum listener at block %d looking back %d blocks", - cfg.Networks.Source.Ethereum.StartBlock, - cfg.Networks.Source.Ethereum.LookbackPeriod)) - - // websockets do not query history - // https://github.com/ethereum/go-ethereum/issues/15063 - stream, sub, history, err := etherReader.QueryWithHistory(context.Background(), &query) - if err != nil { - logger.Error("unable to subscribe to logs", "err", err) - os.Exit(1) - } - - // process history - for _, historicalLog := range history { - parsedMsg, err := types.EvmLogToMessageState(messageTransmitterABI, messageSent, &historicalLog) - if err != nil { - logger.Error("Unable to parse history log into MessageState, skipping", "err", err) - continue - } - logger.Info(fmt.Sprintf("New historical msg from source domain %d with tx hash %s", parsedMsg.SourceDomain, parsedMsg.SourceTxHash)) - - processingQueue <- &types.TxState{TxHash: parsedMsg.SourceTxHash, Msgs: []*types.MessageState{parsedMsg}} - - // It might help to wait a small amount of time between sending messages into the processing queue - // so that account sequences / nonces are set correctly - // time.Sleep(10 * time.Millisecond) - } - - // consume stream - go func() { - var txState *types.TxState - for { - select { - case err := <-sub.Err(): - logger.Error("connection closed", "err", err) - os.Exit(1) - case streamLog := <-stream: - parsedMsg, err := types.EvmLogToMessageState(messageTransmitterABI, messageSent, &streamLog) - if err != nil { - logger.Error("Unable to parse ws log into MessageState, skipping") - continue - } - logger.Info(fmt.Sprintf("New stream msg from %d with tx hash %s", parsedMsg.SourceDomain, parsedMsg.SourceTxHash)) - if txState == nil { - txState = &types.TxState{TxHash: parsedMsg.SourceTxHash, Msgs: []*types.MessageState{parsedMsg}} - } else if parsedMsg.SourceTxHash != txState.TxHash { - processingQueue <- txState - txState = &types.TxState{TxHash: parsedMsg.SourceTxHash, Msgs: []*types.MessageState{parsedMsg}} - } else { - txState.Msgs = append(txState.Msgs, parsedMsg) - - } - default: - if txState != nil { - processingQueue <- txState - txState = nil - } - } - } - }() -} diff --git a/cmd/noble/broadcast.go b/cmd/noble/broadcast.go deleted file mode 100644 index 8cd77b7..0000000 --- a/cmd/noble/broadcast.go +++ /dev/null @@ -1,274 +0,0 @@ -package noble - -import ( - "context" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "io" - "math/rand" - "net/http" - "regexp" - "strconv" - "time" - - "cosmossdk.io/log" - nobletypes "github.com/circlefin/noble-cctp/x/cctp/types" - rpchttp "github.com/cometbft/cometbft/rpc/client/http" - ctypes "github.com/cometbft/cometbft/rpc/core/types" - libclient "github.com/cometbft/cometbft/rpc/jsonrpc/client" - sdkClient "github.com/cosmos/cosmos-sdk/client" - clientTx "github.com/cosmos/cosmos-sdk/client/tx" - "github.com/cosmos/cosmos-sdk/codec" - codectypes "github.com/cosmos/cosmos-sdk/codec/types" - "github.com/cosmos/cosmos-sdk/crypto/keys/secp256k1" - sdk "github.com/cosmos/cosmos-sdk/types" - "github.com/cosmos/cosmos-sdk/types/tx/signing" - xauthsigning "github.com/cosmos/cosmos-sdk/x/auth/signing" - xauthtx "github.com/cosmos/cosmos-sdk/x/auth/tx" - "github.com/strangelove-ventures/noble-cctp-relayer/cmd/noble/cosmos" - "github.com/strangelove-ventures/noble-cctp-relayer/config" - "github.com/strangelove-ventures/noble-cctp-relayer/types" -) - -// Broadcast broadcasts a message to Noble -func Broadcast( - ctx context.Context, - cfg config.Config, - logger log.Logger, - msgs []*types.MessageState, - sequenceMap *types.SequenceMap, -) error { - // set up sdk context - interfaceRegistry := codectypes.NewInterfaceRegistry() - nobletypes.RegisterInterfaces(interfaceRegistry) - cdc := codec.NewProtoCodec(interfaceRegistry) - sdkContext := sdkClient.Context{ - TxConfig: xauthtx.NewTxConfig(cdc, xauthtx.DefaultSignModes), - } - - // build txn - txBuilder := sdkContext.TxConfig.NewTxBuilder() - - // get priv key - nobleAddress := cfg.Networks.Minters[4].MinterAddress - keyBz, err := hex.DecodeString(cfg.Networks.Minters[4].MinterPrivateKey) - if err != nil { - return fmt.Errorf("unable to parse Noble private key") - } - privKey := secp256k1.PrivKey{Key: keyBz} - - rpcClient, err := NewRPCClient(cfg.Networks.Destination.Noble.RPC, 10*time.Second) - if err != nil { - return errors.New("failed to set up rpc client") - } - - cc, err := cosmos.NewProvider(cfg.Networks.Source.Noble.RPC) - if err != nil { - return fmt.Errorf("unable to build cosmos provider for noble: %w", err) - } - - // sign and broadcast txn - for attempt := 0; attempt <= cfg.Networks.Destination.Noble.BroadcastRetries; attempt++ { - - var receiveMsgs []sdk.Msg - for _, msg := range msgs { - - used, err := cc.QueryUsedNonce(ctx, types.Domain(msg.SourceDomain), msg.Nonce) - if err != nil { - return fmt.Errorf("unable to query used nonce: %w", err) - } - - if used { - msg.Status = types.Complete - logger.Info(fmt.Sprintf("Noble cctp minter nonce %d already used", msg.Nonce)) - continue - } - - attestationBytes, err := hex.DecodeString(msg.Attestation[2:]) - if err != nil { - return fmt.Errorf("unable to decode message attestation") - } - - receiveMsgs = append(receiveMsgs, nobletypes.NewMsgReceiveMessage( - nobleAddress, - msg.MsgSentBytes, - attestationBytes, - )) - - logger.Info(fmt.Sprintf( - "Broadcasting message from %d to %d: with source tx hash %s", - msg.SourceDomain, - msg.DestDomain, - msg.SourceTxHash)) - } - - err = txBuilder.SetMsgs(receiveMsgs...) - if err != nil { - return fmt.Errorf("failed to set messages on tx: %w", err) - } - - txBuilder.SetGasLimit(cfg.Networks.Destination.Noble.GasLimit) - // TODO: make configurable - txBuilder.SetMemo("Thank you for relaying with Strangelove") - - accountSequence := sequenceMap.Next(cfg.Networks.Destination.Noble.DomainId) - //TODO: don't need to fetch this everytime - accountNumber, _, err := GetNobleAccountNumberSequence(cfg.Networks.Destination.Noble.API, nobleAddress) - - if err != nil { - return fmt.Errorf("failed to retrieve account number and sequence: %w", err) - } - - sigV2 := signing.SignatureV2{ - PubKey: privKey.PubKey(), - Data: &signing.SingleSignatureData{ - SignMode: sdkContext.TxConfig.SignModeHandler().DefaultMode(), - Signature: nil, - }, - Sequence: uint64(accountSequence), - } - - signerData := xauthsigning.SignerData{ - ChainID: cfg.Networks.Destination.Noble.ChainId, - AccountNumber: uint64(accountNumber), - Sequence: uint64(accountSequence), - } - - txBuilder.SetSignatures(sigV2) - - sigV2, err = clientTx.SignWithPrivKey( - sdkContext.TxConfig.SignModeHandler().DefaultMode(), - signerData, - txBuilder, - &privKey, - sdkContext.TxConfig, - uint64(accountSequence), - ) - if err != nil { - return fmt.Errorf("failed to sign tx: %w", err) - } - - if err := txBuilder.SetSignatures(sigV2); err != nil { - return fmt.Errorf("failed to set signatures: %w", err) - } - - // Generated Protobuf-encoded bytes. - txBytes, err := sdkContext.TxConfig.TxEncoder()(txBuilder.GetTx()) - if err != nil { - return fmt.Errorf("failed to proto encode tx: %w", err) - } - - rpcResponse, err := rpcClient.BroadcastTxSync(context.Background(), txBytes) - if err != nil || (rpcResponse != nil && rpcResponse.Code != 0) { - // Log the error - logger.Error(fmt.Sprintf("error during broadcast: %s", getErrorString(err, rpcResponse))) - - if err != nil || rpcResponse == nil { - // Log retry information - logger.Info(fmt.Sprintf("Retrying in %d seconds", cfg.Networks.Destination.Noble.BroadcastRetryInterval)) - time.Sleep(time.Duration(cfg.Networks.Destination.Noble.BroadcastRetryInterval) * time.Second) - // wait a random amount of time to lower probability of concurrent message nonce collision - time.Sleep(time.Duration(rand.Intn(5)) * time.Second) - continue - } - - // Log details for non-zero response code - logger.Error(fmt.Sprintf("received non-zero: %d - %s", rpcResponse.Code, rpcResponse.Log)) - - // Handle specific error code (32) - if rpcResponse.Code == 32 { - newAccountSequence := extractAccountSequence(logger, rpcResponse.Log, nobleAddress, cfg.Networks.Destination.Noble.API) - logger.Debug(fmt.Sprintf("retrying with new account sequence: %d", newAccountSequence)) - sequenceMap.Put(cfg.Networks.Destination.Noble.DomainId, newAccountSequence) - } - - // Log retry information - logger.Info(fmt.Sprintf("Retrying in %d seconds", cfg.Networks.Destination.Noble.BroadcastRetryInterval)) - time.Sleep(time.Duration(cfg.Networks.Destination.Noble.BroadcastRetryInterval) * time.Second) - // wait a random amount of time to lower probability of concurrent message nonce collision - time.Sleep(time.Duration(rand.Intn(5)) * time.Second) - continue - } - - // Tx was successfully broadcast - for _, msg := range msgs { - msg.DestTxHash = rpcResponse.Hash.String() - msg.Status = types.Complete - } - logger.Info(fmt.Sprintf("Successfully broadcast %s to Noble. Tx hash: %s", msgs[0].SourceTxHash, msgs[0].DestTxHash)) - - return nil - } - - for _, msg := range msgs { - if msg.Status != types.Complete { - msg.Status = types.Failed - } - } - - return errors.New("reached max number of broadcast attempts") -} - -// getErrorString returns the appropriate value to log when tx broadcast errors are encountered. -func getErrorString(err error, rpcResponse *ctypes.ResultBroadcastTx) string { - if rpcResponse != nil { - return rpcResponse.Log - } - return err.Error() -} - -// extractAccountSequence attempts to extract the account sequence number from the RPC response logs when -// account sequence mismatch errors are encountered. If the account sequence number cannot be extracted from the logs, -// it is retrieved by making a request to the API endpoint. -func extractAccountSequence(logger log.Logger, rpcResponseLog, nobleAddress, nobleAPI string) int64 { - pattern := `expected (\d+), got (\d+)` - re := regexp.MustCompile(pattern) - match := re.FindStringSubmatch(rpcResponseLog) - - if len(match) == 3 { - // Extract the numbers from the match. - newAccountSequence, _ := strconv.ParseInt(match[1], 10, 0) - return newAccountSequence - } - - // Otherwise, just request the account sequence - _, newAccountSequence, err := GetNobleAccountNumberSequence(nobleAPI, nobleAddress) - if err != nil { - logger.Error("unable to retrieve account number") - } - - return newAccountSequence -} - -// NewRPCClient initializes a new tendermint RPC client connected to the specified address. -func NewRPCClient(addr string, timeout time.Duration) (*rpchttp.HTTP, error) { - httpClient, err := libclient.DefaultHTTPClient(addr) - if err != nil { - return nil, err - } - httpClient.Timeout = timeout - rpcClient, err := rpchttp.NewWithClient(addr, "/websocket", httpClient) - if err != nil { - return nil, err - } - return rpcClient, nil -} - -func GetNobleAccountNumberSequence(urlBase string, address string) (int64, int64, error) { - rawResp, err := http.Get(fmt.Sprintf("%s/cosmos/auth/v1beta1/accounts/%s", urlBase, address)) - if err != nil { - return 0, 0, errors.New("unable to fetch account number, sequence") - } - body, _ := io.ReadAll(rawResp.Body) - var resp types.AccountResp - err = json.Unmarshal(body, &resp) - if err != nil { - return 0, 0, fmt.Errorf("unable to parse account number, sequence. Raw HHTP Get response: %s", string(body)) - } - accountNumber, _ := strconv.ParseInt(resp.AccountNumber, 10, 0) - accountSequence, _ := strconv.ParseInt(resp.Sequence, 10, 0) - - return accountNumber, accountSequence, nil -} diff --git a/cmd/noble/listener.go b/cmd/noble/listener.go deleted file mode 100644 index c978f07..0000000 --- a/cmd/noble/listener.go +++ /dev/null @@ -1,113 +0,0 @@ -package noble - -import ( - "encoding/json" - "fmt" - "io" - "net/http" - "strconv" - "sync" - "time" - - "cosmossdk.io/log" - "github.com/strangelove-ventures/noble-cctp-relayer/config" - "github.com/strangelove-ventures/noble-cctp-relayer/types" -) - -func StartListener(cfg config.Config, logger log.Logger, processingQueue chan *types.TxState) { - // set up client - - logger.Info(fmt.Sprintf("Starting Noble listener at block %d looking back %d blocks", - cfg.Networks.Source.Noble.StartBlock, - cfg.Networks.Source.Noble.LookbackPeriod)) - - var wg sync.WaitGroup - wg.Add(1) - - // enqueue block heights - currentBlock := cfg.Networks.Source.Noble.StartBlock - lookback := cfg.Networks.Source.Noble.LookbackPeriod - chainTip := GetNobleChainTip(cfg) - blockQueue := make(chan uint64, 1000000) - - // history - currentBlock = currentBlock - lookback - for currentBlock <= chainTip { - blockQueue <- currentBlock - currentBlock++ - } - - // listen for new blocks - go func() { - for { - chainTip = GetNobleChainTip(cfg) - if chainTip >= currentBlock { - for i := currentBlock; i <= chainTip; i++ { - blockQueue <- i - } - currentBlock = chainTip + 1 - } - time.Sleep(6 * time.Second) - } - }() - - // constantly query for blocks - for i := 0; i < int(cfg.Networks.Source.Noble.Workers); i++ { - go func() { - for { - block := <-blockQueue - rawResponse, err := http.Get(fmt.Sprintf("%s/tx_search?query=\"tx.height=%d\"", cfg.Networks.Source.Noble.RPC, block)) - if err != nil { - logger.Debug(fmt.Sprintf("unable to query Noble block %d", block)) - continue - } - if rawResponse.StatusCode != http.StatusOK { - logger.Debug(fmt.Sprintf("non 200 response received for Noble block %d", block)) - time.Sleep(5 * time.Second) - blockQueue <- block - continue - } - - body, err := io.ReadAll(rawResponse.Body) - if err != nil { - logger.Debug(fmt.Sprintf("unable to parse Noble block %d", block)) - continue - } - - response := types.BlockResultsResponse{} - err = json.Unmarshal(body, &response) - if err != nil { - logger.Debug(fmt.Sprintf("unable to unmarshal Noble block %d", block)) - continue - } - - for _, tx := range response.Result.Txs { - parsedMsgs, err := types.NobleLogToMessageState(tx) - if err != nil { - logger.Error("unable to parse Noble log to message state", "err", err.Error()) - continue - } - for _, parsedMsg := range parsedMsgs { - logger.Info(fmt.Sprintf("New stream msg with nonce %d from %d with tx hash %s", parsedMsg.Nonce, parsedMsg.SourceDomain, parsedMsg.SourceTxHash)) - } - processingQueue <- &types.TxState{TxHash: tx.Hash, Msgs: parsedMsgs} - } - } - }() - } - - wg.Wait() -} - -func GetNobleChainTip(cfg config.Config) uint64 { - rawResponse, _ := http.Get(cfg.Networks.Source.Noble.RPC + "/block") - body, _ := io.ReadAll(rawResponse.Body) - - response := types.BlockResponse{} - err := json.Unmarshal(body, &response) - if err != nil { - fmt.Println(err.Error()) - } - res, _ := strconv.ParseInt(response.Result.Block.Header.Height, 10, 0) - return uint64(res) -} diff --git a/cmd/process.go b/cmd/process.go index 9c6cbde..7c4f91d 100644 --- a/cmd/process.go +++ b/cmd/process.go @@ -1,32 +1,19 @@ package cmd import ( - "bytes" "context" - "encoding/hex" "fmt" "os" - "strings" - "sync" + "os/signal" + "syscall" "time" "cosmossdk.io/log" "github.com/spf13/cobra" - "github.com/strangelove-ventures/noble-cctp-relayer/cmd/circle" - "github.com/strangelove-ventures/noble-cctp-relayer/cmd/ethereum" - "github.com/strangelove-ventures/noble-cctp-relayer/cmd/noble" - "github.com/strangelove-ventures/noble-cctp-relayer/config" + "github.com/strangelove-ventures/noble-cctp-relayer/circle" "github.com/strangelove-ventures/noble-cctp-relayer/types" ) -type Processor struct { - Mu sync.RWMutex -} - -func NewProcessor() *Processor { - return &Processor{} -} - var startCmd = &cobra.Command{ Use: "start", Short: "Start relaying CCTP transactions from Ethereum to Noble", @@ -42,62 +29,53 @@ var State = types.NewStateMap() var sequenceMap = types.NewSequenceMap() func Start(cmd *cobra.Command, args []string) { + // messageState processing queue + var processingQueue = make(chan *types.TxState, 10000) - p := NewProcessor() + sigTerm := make(chan os.Signal, 1) - var wg sync.WaitGroup - wg.Add(1) + registeredDomains := make(map[types.Domain]types.Chain) - // initialize minter account sequences - for key := range Cfg.Networks.Minters { - switch key { - case 0: - ethNonce, err := ethereum.GetEthereumAccountNonce( - Cfg.Networks.Destination.Ethereum.RPC, - Cfg.Networks.Minters[0].MinterAddress) + for name, cfg := range Cfg.Chains { + c, err := cfg.Chain(name) + if err != nil { + Logger.Error("Error creating chain", "err: ", err) + os.Exit(1) + } - if err != nil { - Logger.Error("Error retrieving Ethereum account nonce") - os.Exit(1) - } - sequenceMap.Put(key, ethNonce) - case 4: - _, nextMinterSequence, err := noble.GetNobleAccountNumberSequence( - Cfg.Networks.Destination.Noble.API, - Cfg.Networks.Minters[4].MinterAddress) - - if err != nil { - Logger.Error("Error retrieving Noble account sequence") - os.Exit(1) - } - sequenceMap.Put(key, nextMinterSequence) + if err := c.InitializeBroadcaster(cmd.Context(), Logger, sequenceMap); err != nil { + Logger.Error("Error initializing broadcaster", "err: ", err) + os.Exit(1) } - // ...initialize more here - } + go c.StartListener(cmd.Context(), Logger, processingQueue, sigTerm) - // messageState processing queue - var processingQueue = make(chan *types.TxState, 10000) + if _, ok := registeredDomains[c.Domain()]; ok { + Logger.Error("Duplicate domain found", "domain", c.Domain()) + os.Exit(1) + } - // spin up Processor worker pool - for i := 0; i < int(Cfg.ProcessorWorkerCount); i++ { - go p.StartProcessor(cmd.Context(), Cfg, Logger, processingQueue, sequenceMap) + registeredDomains[c.Domain()] = c } - // listeners listen for events, parse them, and enqueue them to processingQueue - if Cfg.Networks.Source.Ethereum.Enabled { - ethereum.StartListener(Cfg, Logger, processingQueue) - } - if Cfg.Networks.Source.Noble.Enabled { - noble.StartListener(Cfg, Logger, processingQueue) + // spin up Processor worker pool + for i := 0; i < int(Cfg.ProcessorWorkerCount); i++ { + go StartProcessor(cmd.Context(), Cfg, Logger, registeredDomains, processingQueue, sequenceMap) } - // ...register more chain listeners here - wg.Wait() + signal.Notify(sigTerm, os.Interrupt, syscall.SIGTERM) + <-sigTerm } // StartProcessor is the main processing pipeline. -func (p *Processor) StartProcessor(ctx context.Context, cfg config.Config, logger log.Logger, processingQueue chan *types.TxState, sequenceMap *types.SequenceMap) { +func StartProcessor( + ctx context.Context, + cfg types.Config, + logger log.Logger, + registeredDomains map[types.Domain]types.Chain, + processingQueue chan *types.TxState, + sequenceMap *types.SequenceMap, +) { for { dequeuedTx := <-processingQueue @@ -117,13 +95,13 @@ func (p *Processor) StartProcessor(ctx context.Context, cfg config.Config, logge // if a filter's condition is met, mark as filtered if filterDisabledCCTPRoutes(cfg, logger, msg) || - filterInvalidDestinationCallers(cfg, logger, msg) { + filterInvalidDestinationCallers(registeredDomains, logger, msg) { msg.Status = types.Filtered } // if the message is burned or pending, check for an attestation if msg.Status == types.Created || msg.Status == types.Pending { - response := circle.CheckAttestation(cfg, logger, msg.IrisLookupId, msg.SourceTxHash, msg.SourceDomain, msg.DestDomain) + response := circle.CheckAttestation(cfg.Circle.AttestationBaseUrl, logger, msg.IrisLookupId, msg.SourceTxHash, msg.SourceDomain, msg.DestDomain) if response != nil { if msg.Status == types.Created && response.Status == "pending_confirmations" { logger.Debug("Attestation is created but still pending confirmations for 0x" + msg.IrisLookupId + ". Retrying...") @@ -156,21 +134,17 @@ func (p *Processor) StartProcessor(ctx context.Context, cfg config.Config, logge } // if the message is attested to, try to broadcast for domain, msgs := range broadcastMsgs { - var err error - switch domain { - case 0: // ethereum - err = ethereum.Broadcast(ctx, cfg, logger, msgs, sequenceMap) - case 4: // noble - err = noble.Broadcast(ctx, cfg, logger, msgs, sequenceMap) + chain, ok := registeredDomains[domain] + if !ok { + logger.Error("No chain registered for domain", "domain", domain) + continue } - if err != nil { - // TODO: add dest domain to error log - logger.Error("unable to mint one or more transfers", "error(s)", err, "total_transfers", len(msgs)) + if err := chain.Broadcast(ctx, logger, msgs, sequenceMap); err != nil { + logger.Error("unable to mint one or more transfers", "error(s)", err, "total_transfers", len(msgs), "name", chain.Name(), "domain", domain) requeue = true continue } - // ...add minters for different domains here for _, msg := range msgs { msg.Status = types.Complete @@ -185,8 +159,8 @@ func (p *Processor) StartProcessor(ctx context.Context, cfg config.Config, logge } // filterDisabledCCTPRoutes returns true if we haven't enabled relaying from a source domain to a destination domain -func filterDisabledCCTPRoutes(cfg config.Config, logger log.Logger, msg *types.MessageState) bool { - val, ok := cfg.Networks.EnabledRoutes[msg.SourceDomain] +func filterDisabledCCTPRoutes(cfg types.Config, logger log.Logger, msg *types.MessageState) bool { + val, ok := cfg.EnabledRoutes[msg.SourceDomain] result := !(ok && val == msg.DestDomain) if result { logger.Info(fmt.Sprintf("Filtered tx %s because relaying from %d to %d is not enabled", @@ -196,40 +170,14 @@ func filterDisabledCCTPRoutes(cfg config.Config, logger log.Logger, msg *types.M } // filterInvalidDestinationCallers returns true if the minter is not the destination caller for the specified domain -func filterInvalidDestinationCallers(cfg config.Config, logger log.Logger, msg *types.MessageState) bool { - zeroByteArr := make([]byte, 32) - result := false - - switch msg.DestDomain { - case 4: - bech32DestinationCaller, err := types.DecodeDestinationCaller(msg.DestinationCaller) - if err != nil { - result = true - } - if !bytes.Equal(msg.DestinationCaller, zeroByteArr) && - bech32DestinationCaller != cfg.Networks.Minters[msg.DestDomain].MinterAddress { - result = true - } - if result { - logger.Info(fmt.Sprintf("Filtered tx %s because the destination caller %s is specified and it's not the minter %s", - msg.SourceTxHash, msg.DestinationCaller, cfg.Networks.Minters[msg.DestDomain].MinterAddress)) - } - - default: // minting to evm - decodedMinter, err := hex.DecodeString(strings.ReplaceAll(cfg.Networks.Minters[0].MinterAddress, "0x", "")) - if err != nil { - return !bytes.Equal(msg.DestinationCaller, zeroByteArr) - } - - decodedMinterPadded := make([]byte, 32) - copy(decodedMinterPadded[12:], decodedMinter) - - if !bytes.Equal(msg.DestinationCaller, zeroByteArr) && !bytes.Equal(msg.DestinationCaller, decodedMinterPadded) { - result = true - } +func filterInvalidDestinationCallers(registeredDomains map[types.Domain]types.Chain, logger log.Logger, msg *types.MessageState) bool { + chain, ok := registeredDomains[msg.DestDomain] + if !ok { + logger.Error("No chain registered for domain", "domain", msg.DestDomain) + return true } - return result + return !chain.IsDestinationCaller(msg.DestinationCaller) } func LookupKey(sourceTxHash string) string { diff --git a/cmd/process_test.go b/cmd/process_test.go index 43fb3cb..3f56198 100644 --- a/cmd/process_test.go +++ b/cmd/process_test.go @@ -9,42 +9,50 @@ import ( "cosmossdk.io/log" "github.com/rs/zerolog" "github.com/strangelove-ventures/noble-cctp-relayer/cmd" - "github.com/strangelove-ventures/noble-cctp-relayer/cmd/noble" - "github.com/strangelove-ventures/noble-cctp-relayer/config" + "github.com/strangelove-ventures/noble-cctp-relayer/noble" "github.com/strangelove-ventures/noble-cctp-relayer/types" "github.com/stretchr/testify/require" ) -var cfg config.Config +var cfg types.Config var logger log.Logger var processingQueue chan *types.TxState var sequenceMap *types.SequenceMap -func setupTest() { - cfg = config.Parse("../.ignore/unit_tests.yaml") +func setupTest(t *testing.T) map[types.Domain]types.Chain { + var err error + cfg, err = types.Parse("../.ignore/unit_tests.yaml") + require.NoError(t, err, "Error parsing config") + logger = log.NewLogger(os.Stdout, log.LevelOption(zerolog.DebugLevel)) processingQueue = make(chan *types.TxState, 10000) - _, nextMinterSequence, err := noble.GetNobleAccountNumberSequence( - cfg.Networks.Destination.Noble.API, - cfg.Networks.Minters[4].MinterAddress) + n, err := cfg.Chains["noble"].(*noble.ChainConfig).Chain("noble") + require.NoError(t, err, "Error creating noble chain") + + _, nextMinterSequence, err := n.(*noble.Noble).AccountInfo(context.TODO()) + require.NoError(t, err, "Error retrieving account sequence") - if err != nil { - logger.Error("Error retrieving account sequence", "err: ", err) - os.Exit(1) - } sequenceMap = types.NewSequenceMap() sequenceMap.Put(types.Domain(4), nextMinterSequence) + registeredDomains := make(map[types.Domain]types.Chain) + for name, cfgg := range cfg.Chains { + c, err := cfgg.Chain(name) + require.NoError(t, err, "Error creating chain") + + registeredDomains[c.Domain()] = c + } + + return registeredDomains + } // new log -> create state entry func TestProcessNewLog(t *testing.T) { - setupTest() - - p := cmd.Processor{} + registeredDomains := setupTest(t) - go p.StartProcessor(context.TODO(), cfg, logger, processingQueue, sequenceMap) + go cmd.StartProcessor(context.TODO(), cfg, logger, registeredDomains, processingQueue, sequenceMap) emptyBz := make([]byte, 32) expectedState := &types.TxState{ @@ -65,20 +73,15 @@ func TestProcessNewLog(t *testing.T) { actualState, _ := cmd.State.Load(expectedState.TxHash) - p.Mu.RLock() require.Equal(t, types.Created, actualState.Msgs[0].Status) - p.Mu.RUnlock() - } // created message -> check attestation -> mark as attested -> mark as complete -> remove from state func TestProcessCreatedLog(t *testing.T) { - setupTest() - cfg.Networks.EnabledRoutes[0] = 5 // skip mint - - p := cmd.NewProcessor() + registeredDomains := setupTest(t) + cfg.EnabledRoutes[0] = 5 // skip mint - go p.StartProcessor(context.TODO(), cfg, logger, processingQueue, sequenceMap) + go cmd.StartProcessor(context.TODO(), cfg, logger, registeredDomains, processingQueue, sequenceMap) emptyBz := make([]byte, 32) @@ -102,20 +105,16 @@ func TestProcessCreatedLog(t *testing.T) { actualState, ok := cmd.State.Load(expectedState.TxHash) require.True(t, ok) - p.Mu.RLock() require.Equal(t, types.Complete, actualState.Msgs[0].Status) - p.Mu.RUnlock() } // created message -> disabled cctp route -> filtered func TestProcessDisabledCctpRoute(t *testing.T) { - setupTest() + registeredDomains := setupTest(t) - delete(cfg.Networks.EnabledRoutes, 0) + delete(cfg.EnabledRoutes, 0) - p := cmd.NewProcessor() - - go p.StartProcessor(context.TODO(), cfg, logger, processingQueue, sequenceMap) + go cmd.StartProcessor(context.TODO(), cfg, logger, registeredDomains, processingQueue, sequenceMap) emptyBz := make([]byte, 32) expectedState := &types.TxState{ @@ -138,18 +137,14 @@ func TestProcessDisabledCctpRoute(t *testing.T) { actualState, ok := cmd.State.Load(expectedState.TxHash) require.True(t, ok) - p.Mu.RLock() require.Equal(t, types.Filtered, actualState.Msgs[0].Status) - p.Mu.RUnlock() } // created message -> different destination caller -> filtered func TestProcessInvalidDestinationCaller(t *testing.T) { - setupTest() - - p := cmd.NewProcessor() + registeredDomains := setupTest(t) - go p.StartProcessor(context.TODO(), cfg, logger, processingQueue, sequenceMap) + go cmd.StartProcessor(context.TODO(), cfg, logger, registeredDomains, processingQueue, sequenceMap) nonEmptyBytes := make([]byte, 31) nonEmptyBytes = append(nonEmptyBytes, 0x1) @@ -174,18 +169,14 @@ func TestProcessInvalidDestinationCaller(t *testing.T) { actualState, ok := cmd.State.Load(expectedState.TxHash) require.True(t, ok) - p.Mu.RLock() require.Equal(t, types.Filtered, actualState.Msgs[0].Status) - p.Mu.RUnlock() } // created message -> not \ -> filtered func TestProcessNonBurnMessageWhenDisabled(t *testing.T) { - setupTest() + registeredDomains := setupTest(t) - p := cmd.NewProcessor() - - go p.StartProcessor(context.TODO(), cfg, logger, processingQueue, sequenceMap) + go cmd.StartProcessor(context.TODO(), cfg, logger, registeredDomains, processingQueue, sequenceMap) emptyBz := make([]byte, 32) expectedState := &types.TxState{ @@ -208,20 +199,15 @@ func TestProcessNonBurnMessageWhenDisabled(t *testing.T) { actualState, ok := cmd.State.Load(expectedState.TxHash) require.True(t, ok) - p.Mu.RLock() require.Equal(t, types.Filtered, actualState.Msgs[0].Status) - p.Mu.RUnlock() - } // test batch transactions where multiple messages can be sent with the same tx hash // MsgSentBytes defer between messages func TestBatchTx(t *testing.T) { - setupTest() - - p := cmd.NewProcessor() + registeredDomains := setupTest(t) - go p.StartProcessor(context.TODO(), cfg, logger, processingQueue, sequenceMap) + go cmd.StartProcessor(context.TODO(), cfg, logger, registeredDomains, processingQueue, sequenceMap) emptyBz := make([]byte, 32) expectedState := &types.TxState{ @@ -251,7 +237,5 @@ func TestBatchTx(t *testing.T) { actualState, ok := cmd.State.Load(expectedState.TxHash) require.True(t, ok) - p.Mu.RLock() require.Equal(t, 2, len(actualState.Msgs)) - p.Mu.RUnlock() } diff --git a/cmd/root.go b/cmd/root.go index 4d7c173..95385a4 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -1,30 +1,20 @@ package cmd import ( - "context" - "encoding/hex" - "encoding/json" - "fmt" - "io" "net/http" "os" "strconv" - "github.com/cosmos/cosmos-sdk/crypto/keys/secp256k1" - "github.com/cosmos/cosmos-sdk/types/bech32" - "github.com/ethereum/go-ethereum/ethclient" "github.com/gin-gonic/gin" - "github.com/strangelove-ventures/noble-cctp-relayer/cmd/ethereum" "github.com/strangelove-ventures/noble-cctp-relayer/types" "cosmossdk.io/log" "github.com/rs/zerolog" "github.com/spf13/cobra" - "github.com/strangelove-ventures/noble-cctp-relayer/config" ) var ( - Cfg config.Config + Cfg types.Config cfgFile string verbose bool @@ -56,57 +46,13 @@ func init() { Logger = log.NewLogger(os.Stdout, log.LevelOption(zerolog.InfoLevel)) } - Cfg = config.Parse(cfgFile) - Logger.Info("successfully parsed config file", "location", cfgFile) - - Logger.Info(Cfg.Networks.Source.Ethereum.RPC) - // Set minter addresses from priv keys - for i, minter := range Cfg.Networks.Minters { - switch i { - case 0: - _, address, err := ethereum.GetEcdsaKeyAddress(minter.MinterPrivateKey) - if err != nil { - Logger.Error(fmt.Sprintf("Unable to parse ecdsa key from source %d", i)) - os.Exit(1) - } - minter.MinterAddress = address - Cfg.Networks.Minters[0] = minter - case 4: - keyBz, err := hex.DecodeString(minter.MinterPrivateKey) - if err != nil { - Logger.Error(fmt.Sprintf("Unable to parse key from source %d", i)) - os.Exit(1) - } - privKey := secp256k1.PrivKey{Key: keyBz} - address, err := bech32.ConvertAndEncode("noble", privKey.PubKey().Address()) - if err != nil { - Logger.Error(fmt.Sprintf("Unable to parse ecdsa key from source %d", i)) - os.Exit(1) - } - minter.MinterAddress = address - Cfg.Networks.Minters[4] = minter - } - } - - // Set default listener blocks - - // if Ethereum start block not set, default to latest - if Cfg.Networks.Source.Ethereum.Enabled && Cfg.Networks.Source.Ethereum.StartBlock == 0 { - client, _ := ethclient.Dial(Cfg.Networks.Source.Ethereum.RPC) - defer client.Close() - header, _ := client.HeaderByNumber(context.Background(), nil) - Cfg.Networks.Source.Ethereum.StartBlock = header.Number.Uint64() - } - - // if Noble start block not set, default to latest - if Cfg.Networks.Source.Noble.Enabled && Cfg.Networks.Source.Noble.StartBlock == 0 { - rawResponse, _ := http.Get(Cfg.Networks.Source.Noble.RPC + "/block") - body, _ := io.ReadAll(rawResponse.Body) - response := types.BlockResponse{} - _ = json.Unmarshal(body, &response) - height, _ := strconv.ParseInt(response.Result.Block.Header.Height, 10, 0) - Cfg.Networks.Source.Noble.StartBlock = uint64(height) + var err error + Cfg, err = types.Parse(cfgFile) + if err != nil { + Logger.Error("unable to parse config file", "location", cfgFile, "err", err) + os.Exit(1) } + Logger.Info("successfully parsed config file", "location", cfgFile) // start api server go startApi() diff --git a/config/config.go b/config/config.go deleted file mode 100644 index 2e85000..0000000 --- a/config/config.go +++ /dev/null @@ -1,73 +0,0 @@ -package config - -import ( - "os" - - "github.com/strangelove-ventures/noble-cctp-relayer/types" - "gopkg.in/yaml.v3" -) - -type Config struct { - Networks struct { - Source struct { - Ethereum struct { - DomainId types.Domain `yaml:"domain-id"` - RPC string `yaml:"rpc"` - MessageTransmitter string `yaml:"message-transmitter"` - RequestQueueSize uint32 `yaml:"request-queue-size"` - StartBlock uint64 `yaml:"start-block"` - LookbackPeriod uint64 `yaml:"lookback-period"` - Enabled bool `yaml:"enabled"` - } `yaml:"ethereum"` - Noble struct { - DomainId types.Domain `yaml:"domain-id"` - RPC string `yaml:"rpc"` - RequestQueueSize uint32 `yaml:"request-queue-size"` - StartBlock uint64 `yaml:"start-block"` - LookbackPeriod uint64 `yaml:"lookback-period"` - Workers uint32 `yaml:"workers"` - Enabled bool `yaml:"enabled"` - } `yaml:"noble"` - } `yaml:"source"` - Destination struct { - Ethereum struct { - DomainId types.Domain `yaml:"domain-id"` - ChainId int64 `yaml:"chain-id"` - RPC string `yaml:"rpc"` - BroadcastRetries int `yaml:"broadcast-retries"` - BroadcastRetryInterval int `yaml:"broadcast-retry-interval"` - } `yaml:"ethereum"` - Noble struct { - DomainId types.Domain `yaml:"domain-id"` - RPC string `yaml:"rpc"` - API string `yaml:"api"` - ChainId string `yaml:"chain-id"` - GasLimit uint64 `yaml:"gas-limit"` - BroadcastRetries int `yaml:"broadcast-retries"` - BroadcastRetryInterval int `yaml:"broadcast-retry-interval"` - FilterForwardsByIbcChannel bool `yaml:"filter-forwards-by-ibc-channel"` - ForwardingChannelWhitelist []string `yaml:"forwarding-channel-whitelist"` - } `yaml:"noble"` - } `yaml:"destination"` - EnabledRoutes map[types.Domain]types.Domain `yaml:"enabled-routes"` - Minters map[types.Domain]struct { - MinterAddress string `yaml:"minter-address"` - MinterPrivateKey string `yaml:"minter-private-key"` - } `yaml:"minters"` - } `yaml:"networks"` - Circle struct { - AttestationBaseUrl string `yaml:"attestation-base-url"` - FetchRetries int `yaml:"fetch-retries"` - FetchRetryInterval int `yaml:"fetch-retry-interval"` - } `yaml:"circle"` - ProcessorWorkerCount uint32 `yaml:"processor-worker-count"` - Api struct { - TrustedProxies []string `yaml:"trusted-proxies"` - } `yaml:"api"` -} - -func Parse(file string) (cfg Config) { - data, _ := os.ReadFile(file) - _ = yaml.Unmarshal(data, &cfg) - return -} diff --git a/cmd/noble/cosmos/codec.go b/cosmos/codec.go similarity index 100% rename from cmd/noble/cosmos/codec.go rename to cosmos/codec.go diff --git a/cmd/noble/cosmos/cosmosprovider.go b/cosmos/cosmosprovider.go similarity index 100% rename from cmd/noble/cosmos/cosmosprovider.go rename to cosmos/cosmosprovider.go diff --git a/cmd/noble/cosmos/grpc_shim.go b/cosmos/grpc_shim.go similarity index 100% rename from cmd/noble/cosmos/grpc_shim.go rename to cosmos/grpc_shim.go diff --git a/cmd/noble/cosmos/query.go b/cosmos/query.go similarity index 100% rename from cmd/noble/cosmos/query.go rename to cosmos/query.go diff --git a/cmd/noble/cosmos/query_test.go b/cosmos/query_test.go similarity index 86% rename from cmd/noble/cosmos/query_test.go rename to cosmos/query_test.go index d0156c5..0ecd20f 100644 --- a/cmd/noble/cosmos/query_test.go +++ b/cosmos/query_test.go @@ -4,7 +4,7 @@ import ( "context" "testing" - "github.com/strangelove-ventures/noble-cctp-relayer/cmd/noble/cosmos" + "github.com/strangelove-ventures/noble-cctp-relayer/cosmos" "github.com/stretchr/testify/require" ) diff --git a/cmd/ethereum/abi/ERC20.json b/ethereum/abi/ERC20.json similarity index 100% rename from cmd/ethereum/abi/ERC20.json rename to ethereum/abi/ERC20.json diff --git a/cmd/ethereum/abi/MessageTransmitter.json b/ethereum/abi/MessageTransmitter.json similarity index 100% rename from cmd/ethereum/abi/MessageTransmitter.json rename to ethereum/abi/MessageTransmitter.json diff --git a/cmd/ethereum/abi/TokenMessenger.json b/ethereum/abi/TokenMessenger.json similarity index 100% rename from cmd/ethereum/abi/TokenMessenger.json rename to ethereum/abi/TokenMessenger.json diff --git a/cmd/ethereum/abi/TokenMessengerWithMetadata.json b/ethereum/abi/TokenMessengerWithMetadata.json similarity index 100% rename from cmd/ethereum/abi/TokenMessengerWithMetadata.json rename to ethereum/abi/TokenMessengerWithMetadata.json diff --git a/cmd/ethereum/broadcast_test.go b/ethereum/broadcast_test.go similarity index 82% rename from cmd/ethereum/broadcast_test.go rename to ethereum/broadcast_test.go index 76ad9f6..40e1fd7 100644 --- a/cmd/ethereum/broadcast_test.go +++ b/ethereum/broadcast_test.go @@ -9,7 +9,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethclient" - "github.com/strangelove-ventures/noble-cctp-relayer/cmd/ethereum" + "github.com/strangelove-ventures/noble-cctp-relayer/ethereum/contracts" "github.com/stretchr/testify/require" ) @@ -28,7 +28,7 @@ func TestEthUsedNonce(t *testing.T) { require.NoError(t, err) defer client.Close() - messageTransmitter, err := ethereum.NewMessageTransmitter(common.HexToAddress("0x0a992d191deec32afe36203ad87d7d289a738f81"), client) + messageTransmitter, err := contracts.NewMessageTransmitter(common.HexToAddress("0x0a992d191deec32afe36203ad87d7d289a738f81"), client) require.NoError(t, err) co := &bind.CallOpts{ diff --git a/ethereum/chain.go b/ethereum/chain.go new file mode 100644 index 0000000..8722217 --- /dev/null +++ b/ethereum/chain.go @@ -0,0 +1,380 @@ +package ethereum + +import ( + "bytes" + "context" + "crypto/ecdsa" + "embed" + "encoding/hex" + "errors" + "fmt" + "math/big" + "os" + "regexp" + "strconv" + "strings" + "sync" + "time" + + "cosmossdk.io/log" + ethereum "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/accounts/abi" + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethclient" + "github.com/pascaldekloe/etherstream" + "github.com/strangelove-ventures/noble-cctp-relayer/ethereum/contracts" + "github.com/strangelove-ventures/noble-cctp-relayer/types" +) + +//go:embed abi/MessageTransmitter.json +var content embed.FS + +var _ types.Chain = (*Ethereum)(nil) + +type Ethereum struct { + name string + chainID int64 + domain types.Domain + rpcURL string + wsURL string + messageTransmitterAddress string + startBlock uint64 + lookbackPeriod uint64 + privateKey *ecdsa.PrivateKey + minterAddress string + maxRetries int + retryIntervalSeconds int + + mu sync.Mutex +} + +func NewChain( + name string, + chainID int64, + domain types.Domain, + rpcURL string, + wsURL string, + messageTransmitterAddress string, + startBlock uint64, + lookbackPeriod uint64, + privateKey string, + maxRetries int, + retryIntervalSeconds int, +) (*Ethereum, error) { + privEcdsaKey, ethereumAddress, err := GetEcdsaKeyAddress(privateKey) + if err != nil { + return nil, err + } + return &Ethereum{ + name: name, + chainID: chainID, + domain: domain, + rpcURL: rpcURL, + wsURL: wsURL, + messageTransmitterAddress: messageTransmitterAddress, + startBlock: startBlock, + lookbackPeriod: lookbackPeriod, + privateKey: privEcdsaKey, + minterAddress: ethereumAddress, + maxRetries: maxRetries, + retryIntervalSeconds: retryIntervalSeconds, + }, nil +} + +func (e *Ethereum) Name() string { + return e.name +} + +func (e *Ethereum) Domain() types.Domain { + return e.domain +} + +func (e *Ethereum) IsDestinationCaller(destinationCaller []byte) bool { + zeroByteArr := make([]byte, 32) + + decodedMinter, err := hex.DecodeString(strings.ReplaceAll(e.minterAddress, "0x", "")) + if err != nil && bytes.Equal(destinationCaller, zeroByteArr) { + return true + } + + decodedMinterPadded := make([]byte, 32) + copy(decodedMinterPadded[12:], decodedMinter) + + return bytes.Equal(destinationCaller, zeroByteArr) || bytes.Equal(destinationCaller, decodedMinterPadded) +} + +func (e *Ethereum) InitializeBroadcaster( + ctx context.Context, + logger log.Logger, + sequenceMap *types.SequenceMap, +) error { + nextNonce, err := GetEthereumAccountNonce(e.rpcURL, e.minterAddress) + if err != nil { + return fmt.Errorf("unable to retrieve evm account nonce: %w", err) + } + sequenceMap.Put(e.Domain(), uint64(nextNonce)) + + return nil +} + +func (e *Ethereum) StartListener( + ctx context.Context, + logger log.Logger, + processingQueue chan *types.TxState, + quit chan os.Signal, +) { + logger = logger.With("chain", e.name, "chain_id", e.chainID, "domain", e.domain) + + // set up client + messageTransmitter, err := content.ReadFile("abi/MessageTransmitter.json") + if err != nil { + logger.Error("unable to read MessageTransmitter abi", "err", err) + os.Exit(1) + } + messageTransmitterABI, err := abi.JSON(bytes.NewReader(messageTransmitter)) + if err != nil { + logger.Error("unable to parse MessageTransmitter abi", "err", err) + } + + messageSent := messageTransmitterABI.Events["MessageSent"] + + ethClient, err := ethclient.DialContext(context.Background(), e.wsURL) + if err != nil { + logger.Error("unable to initialize ethereum client", "err", err) + os.Exit(1) + } + + defer ethClient.Close() + + messageTransmitterAddress := common.HexToAddress(e.messageTransmitterAddress) + etherReader := etherstream.Reader{Backend: ethClient} + + if e.startBlock == 0 { + header, err := ethClient.HeaderByNumber(context.Background(), nil) + if err != nil { + logger.Error("unable to retrieve latest eth block header", "err", err) + os.Exit(1) + } + + e.startBlock = header.Number.Uint64() + } + + query := ethereum.FilterQuery{ + Addresses: []common.Address{messageTransmitterAddress}, + Topics: [][]common.Hash{{messageSent.ID}}, + FromBlock: big.NewInt(int64(e.startBlock - e.lookbackPeriod)), + } + + logger.Info(fmt.Sprintf( + "Starting Ethereum listener at block %d looking back %d blocks", + e.startBlock, + e.lookbackPeriod)) + + // websockets do not query history + // https://github.com/ethereum/go-ethereum/issues/15063 + stream, sub, history, err := etherReader.QueryWithHistory(context.Background(), &query) + if err != nil { + logger.Error("unable to subscribe to logs", "err", err) + os.Exit(1) + } + + // process history + for _, historicalLog := range history { + parsedMsg, err := types.EvmLogToMessageState(messageTransmitterABI, messageSent, &historicalLog) + if err != nil { + logger.Error("Unable to parse history log into MessageState, skipping", "err", err) + continue + } + logger.Info(fmt.Sprintf("New historical msg from source domain %d with tx hash %s", parsedMsg.SourceDomain, parsedMsg.SourceTxHash)) + + processingQueue <- &types.TxState{TxHash: parsedMsg.SourceTxHash, Msgs: []*types.MessageState{parsedMsg}} + + // It might help to wait a small amount of time between sending messages into the processing queue + // so that account sequences / nonces are set correctly + // time.Sleep(10 * time.Millisecond) + } + + // consume stream + go func() { + var txState *types.TxState + for { + select { + case <-quit: + return + case err := <-sub.Err(): + logger.Error("connection closed", "err", err) + os.Exit(1) + case streamLog := <-stream: + parsedMsg, err := types.EvmLogToMessageState(messageTransmitterABI, messageSent, &streamLog) + if err != nil { + logger.Error("Unable to parse ws log into MessageState, skipping") + continue + } + logger.Info(fmt.Sprintf("New stream msg from %d with tx hash %s", parsedMsg.SourceDomain, parsedMsg.SourceTxHash)) + if txState == nil { + txState = &types.TxState{TxHash: parsedMsg.SourceTxHash, Msgs: []*types.MessageState{parsedMsg}} + } else if parsedMsg.SourceTxHash != txState.TxHash { + processingQueue <- txState + txState = &types.TxState{TxHash: parsedMsg.SourceTxHash, Msgs: []*types.MessageState{parsedMsg}} + } else { + txState.Msgs = append(txState.Msgs, parsedMsg) + + } + default: + if txState != nil { + processingQueue <- txState + txState = nil + } + } + } + }() +} + +func (e *Ethereum) Broadcast( + ctx context.Context, + logger log.Logger, + msgs []*types.MessageState, + sequenceMap *types.SequenceMap, +) error { + + // set up eth client + client, err := ethclient.Dial(e.rpcURL) + if err != nil { + return fmt.Errorf("unable to dial ethereum client: %w", err) + } + defer client.Close() + + backend := NewContractBackendWrapper(client) + + auth, err := bind.NewKeyedTransactorWithChainID(e.privateKey, big.NewInt(e.chainID)) + if err != nil { + return fmt.Errorf("unable to create auth: %w", err) + } + + messageTransmitter, err := contracts.NewMessageTransmitter(common.HexToAddress(e.messageTransmitterAddress), backend) + if err != nil { + return fmt.Errorf("unable to create message transmitter: %w", err) + } + + var broadcastErrors error +MsgLoop: + for _, msg := range msgs { + + if msg.Status == types.Complete { + continue + } + + attestationBytes, err := hex.DecodeString(msg.Attestation[2:]) + if err != nil { + return errors.New("unable to decode message attestation") + } + + for attempt := 0; attempt <= e.maxRetries; attempt++ { + logger.Info(fmt.Sprintf( + "Broadcasting message from %d to %d: with source tx hash %s", + msg.SourceDomain, + msg.DestDomain, + msg.SourceTxHash)) + + nonce := sequenceMap.Next(e.domain) + auth.Nonce = big.NewInt(int64(nonce)) + + e.mu.Lock() + + // TODO remove + nextNonce, err := GetEthereumAccountNonce(e.rpcURL, e.minterAddress) + if err != nil { + logger.Error("unable to retrieve account number") + } else { + auth.Nonce = big.NewInt(nextNonce) + } + // TODO end remove + + // check if nonce already used + co := &bind.CallOpts{ + Pending: true, + Context: ctx, + } + + logger.Debug("Checking if nonce was used for broadcast to Ethereum", "source_domain", msg.SourceDomain, "nonce", msg.Nonce) + + key := append( + common.LeftPadBytes((big.NewInt(int64(msg.SourceDomain))).Bytes(), 4), + common.LeftPadBytes((big.NewInt(int64(msg.Nonce))).Bytes(), 8)..., + ) + + response, nonceErr := messageTransmitter.UsedNonces(co, [32]byte(crypto.Keccak256(key))) + if nonceErr != nil { + logger.Debug("Error querying whether nonce was used. Continuing...") + } else { + fmt.Printf("received used nonce response: %d\n", response) + if response.Uint64() == uint64(1) { + // nonce has already been used, mark as complete + logger.Debug(fmt.Sprintf("This source domain/nonce has already been used: %d %d", + msg.SourceDomain, msg.Nonce)) + msg.Status = types.Complete + e.mu.Unlock() + continue MsgLoop + } + } + + // broadcast txn + tx, err := messageTransmitter.ReceiveMessage( + auth, + msg.MsgSentBytes, + attestationBytes, + ) + if err == nil { + msg.Status = types.Complete + + fullLog, err := tx.MarshalJSON() + if err != nil { + logger.Error("error marshalling eth tx log", err) + } + + msg.DestTxHash = tx.Hash().Hex() + + logger.Info(fmt.Sprintf("Successfully broadcast %s to Ethereum. Tx hash: %s, FULL LOG: %s", msg.SourceTxHash, msg.DestTxHash, string(fullLog))) + e.mu.Unlock() + continue MsgLoop + } + + logger.Error(fmt.Sprintf("error during broadcast: %s", err.Error())) + if parsedErr, ok := err.(JsonError); ok { + if parsedErr.ErrorCode() == 3 && parsedErr.Error() == "execution reverted: Nonce already used" { + msg.Status = types.Complete + logger.Error(fmt.Sprintf("This account nonce has already been used: %d", nonce)) + e.mu.Unlock() + continue MsgLoop + } + + match, _ := regexp.MatchString("nonce too low: next nonce [0-9]+, tx nonce [0-9]+", parsedErr.Error()) + if match { + numberRegex := regexp.MustCompile("[0-9]+") + nextNonce, err := strconv.ParseInt(numberRegex.FindAllString(parsedErr.Error(), 1)[0], 10, 0) + if err != nil { + nextNonce, err = GetEthereumAccountNonce(e.rpcURL, e.minterAddress) + if err != nil { + logger.Error("unable to retrieve account number") + } + } + sequenceMap.Put(e.domain, uint64(nextNonce)) + } + } + e.mu.Unlock() + + // if it's not the last attempt, retry + // TODO increase the destination.ethereum.broadcast retries (3-5) and retry interval (15s). By checking for used nonces, there is no gas cost for failed mints. + if attempt != e.maxRetries { + logger.Info(fmt.Sprintf("Retrying in %d seconds", e.retryIntervalSeconds)) + time.Sleep(time.Duration(e.retryIntervalSeconds) * time.Second) + } + } + // retried max times with failure + msg.Status = types.Failed + broadcastErrors = errors.Join(broadcastErrors, errors.New("reached max number of broadcast attempts")) + } + return broadcastErrors +} diff --git a/ethereum/config.go b/ethereum/config.go new file mode 100644 index 0000000..2be09d1 --- /dev/null +++ b/ethereum/config.go @@ -0,0 +1,38 @@ +package ethereum + +import "github.com/strangelove-ventures/noble-cctp-relayer/types" + +var _ types.ChainConfig = (*ChainConfig)(nil) + +type ChainConfig struct { + DomainID types.Domain `yaml:"domain-id"` + ChainID int64 `yaml:"chain-id"` + RPC string `yaml:"rpc"` + WS string `yaml:"ws"` + MessageTransmitter string `yaml:"message-transmitter"` + + StartBlock uint64 `yaml:"start-block"` + LookbackPeriod uint64 `yaml:"lookback-period"` + + BroadcastRetries int `yaml:"broadcast-retries"` + BroadcastRetryInterval int `yaml:"broadcast-retry-interval"` + + // TODO move to keyring + MinterPrivateKey string `yaml:"minter-private-key"` +} + +func (c *ChainConfig) Chain(name string) (types.Chain, error) { + return NewChain( + name, + c.ChainID, + c.DomainID, + c.RPC, + c.WS, + c.MessageTransmitter, + c.StartBlock, + c.LookbackPeriod, + c.MinterPrivateKey, + c.BroadcastRetries, + c.BroadcastRetryInterval, + ) +} diff --git a/cmd/ethereum/contract_backend_wrapper.go b/ethereum/contract_backend_wrapper.go similarity index 100% rename from cmd/ethereum/contract_backend_wrapper.go rename to ethereum/contract_backend_wrapper.go diff --git a/cmd/ethereum/MessageTransmitter.go b/ethereum/contracts/MessageTransmitter.go similarity index 99% rename from cmd/ethereum/MessageTransmitter.go rename to ethereum/contracts/MessageTransmitter.go index 6f448db..7a49132 100644 --- a/cmd/ethereum/MessageTransmitter.go +++ b/ethereum/contracts/MessageTransmitter.go @@ -1,7 +1,7 @@ // Code generated - DO NOT EDIT. // This file is a generated binding and any manual changes will be lost. -package ethereum +package contracts import ( "errors" diff --git a/cmd/TokenMessenger.go b/ethereum/contracts/TokenMessenger.go similarity index 99% rename from cmd/TokenMessenger.go rename to ethereum/contracts/TokenMessenger.go index a168c20..92aede5 100644 --- a/cmd/TokenMessenger.go +++ b/ethereum/contracts/TokenMessenger.go @@ -1,7 +1,7 @@ // Code generated - DO NOT EDIT. // This file is a generated binding and any manual changes will be lost. -package cmd +package contracts import ( "errors" diff --git a/cmd/TokenMessengerWithMetadata.go b/ethereum/contracts/TokenMessengerWithMetadata.go similarity index 99% rename from cmd/TokenMessengerWithMetadata.go rename to ethereum/contracts/TokenMessengerWithMetadata.go index 86bfa73..aaa66ab 100644 --- a/cmd/TokenMessengerWithMetadata.go +++ b/ethereum/contracts/TokenMessengerWithMetadata.go @@ -1,7 +1,7 @@ // Code generated - DO NOT EDIT. // This file is a generated binding and any manual changes will be lost. -package cmd +package contracts import ( "errors" diff --git a/cmd/ethereum/listener_test.go b/ethereum/listener_test.go similarity index 73% rename from cmd/ethereum/listener_test.go rename to ethereum/listener_test.go index 97577d8..1418e06 100644 --- a/cmd/ethereum/listener_test.go +++ b/ethereum/listener_test.go @@ -1,24 +1,28 @@ package ethereum_test import ( + "context" "os" "testing" "time" "cosmossdk.io/log" "github.com/rs/zerolog" - eth "github.com/strangelove-ventures/noble-cctp-relayer/cmd/ethereum" - "github.com/strangelove-ventures/noble-cctp-relayer/config" + "github.com/strangelove-ventures/noble-cctp-relayer/ethereum" "github.com/strangelove-ventures/noble-cctp-relayer/types" "github.com/stretchr/testify/require" ) -var cfg config.Config +var cfg types.Config var logger log.Logger var processingQueue chan *types.TxState func init() { - cfg = config.Parse("../../.ignore/unit_tests.yaml") + var err error + cfg, err = types.Parse("../../.ignore/unit_tests.yaml") + if err != nil { + panic(err) + } logger = log.NewLogger(os.Stdout, log.LevelOption(zerolog.ErrorLevel)) processingQueue = make(chan *types.TxState, 10000) @@ -26,10 +30,14 @@ func init() { // tests for a historical log func TestStartListener(t *testing.T) { + ethCfg := ethereum.ChainConfig{ + StartBlock: 9702735, + LookbackPeriod: 0, + } + eth, err := ethCfg.Chain("ethereum") + require.NoError(t, err) - cfg.Networks.Source.Ethereum.StartBlock = 9702735 - cfg.Networks.Source.Ethereum.LookbackPeriod = 0 - go eth.StartListener(cfg, logger, processingQueue) + go eth.StartListener(context.TODO(), logger, processingQueue, nil) time.Sleep(5 * time.Second) diff --git a/cmd/ethereum/util.go b/ethereum/util.go similarity index 100% rename from cmd/ethereum/util.go rename to ethereum/util.go diff --git a/cmd/ethereum/util_test.go b/ethereum/util_test.go similarity index 58% rename from cmd/ethereum/util_test.go rename to ethereum/util_test.go index 0856a5a..4406b82 100644 --- a/cmd/ethereum/util_test.go +++ b/ethereum/util_test.go @@ -6,27 +6,30 @@ import ( "cosmossdk.io/log" "github.com/rs/zerolog" - "github.com/strangelove-ventures/noble-cctp-relayer/cmd/ethereum" - "github.com/strangelove-ventures/noble-cctp-relayer/config" + "github.com/strangelove-ventures/noble-cctp-relayer/ethereum" "github.com/strangelove-ventures/noble-cctp-relayer/types" "github.com/stretchr/testify/require" ) func init() { - cfg = config.Parse("../../.ignore/unit_tests.yaml") + var err error + cfg, err = types.Parse("../../.ignore/unit_tests.yaml") + if err != nil { + panic(err) + } logger = log.NewLogger(os.Stdout, log.LevelOption(zerolog.ErrorLevel)) processingQueue = make(chan *types.TxState, 10000) } func TestGetEthereumAccountNonce(t *testing.T) { - _, err := ethereum.GetEthereumAccountNonce(cfg.Networks.Destination.Ethereum.RPC, "0x4996f29b254c77972fff8f25e6f7797b3c9a0eb6") + _, err := ethereum.GetEthereumAccountNonce(cfg.Chains["ethereum"].(*ethereum.ChainConfig).RPC, "0x4996f29b254c77972fff8f25e6f7797b3c9a0eb6") require.Nil(t, err) } // Return public ecdsa key and address given the private key func TestGetEcdsaKeyAddress(t *testing.T) { - key, addr, err := ethereum.GetEcdsaKeyAddress(cfg.Networks.Minters[0].MinterPrivateKey) + key, addr, err := ethereum.GetEcdsaKeyAddress(cfg.Chains["ethereum"].(*ethereum.ChainConfig).MinterPrivateKey) require.NotNil(t, key) require.NotNil(t, addr) require.Nil(t, err) diff --git a/noble/chain.go b/noble/chain.go new file mode 100644 index 0000000..cebe3a2 --- /dev/null +++ b/noble/chain.go @@ -0,0 +1,459 @@ +package noble + +import ( + "bytes" + "context" + "encoding/hex" + "errors" + "fmt" + "math/rand" + "os" + "regexp" + "strconv" + "sync" + "time" + + "cosmossdk.io/log" + nobletypes "github.com/circlefin/noble-cctp/x/cctp/types" + ctypes "github.com/cometbft/cometbft/rpc/core/types" + sdkClient "github.com/cosmos/cosmos-sdk/client" + clientTx "github.com/cosmos/cosmos-sdk/client/tx" + "github.com/cosmos/cosmos-sdk/codec" + codectypes "github.com/cosmos/cosmos-sdk/codec/types" + "github.com/cosmos/cosmos-sdk/crypto/keys/secp256k1" + sdk "github.com/cosmos/cosmos-sdk/types" + "github.com/cosmos/cosmos-sdk/types/bech32" + "github.com/cosmos/cosmos-sdk/types/tx/signing" + xauthsigning "github.com/cosmos/cosmos-sdk/x/auth/signing" + xauthtx "github.com/cosmos/cosmos-sdk/x/auth/tx" + authtypes "github.com/cosmos/cosmos-sdk/x/auth/types" + "github.com/strangelove-ventures/noble-cctp-relayer/cosmos" + "github.com/strangelove-ventures/noble-cctp-relayer/types" +) + +var _ types.Chain = (*Noble)(nil) + +type Noble struct { + cc *cosmos.CosmosProvider + chainID string + + privateKey *secp256k1.PrivKey + minterAddress string + accountNumber uint64 + + startBlock uint64 + lookbackPeriod uint64 + workers uint32 + + gasLimit uint64 + txMemo string + maxRetries int + retryIntervalSeconds int + + mu sync.Mutex +} + +func NewChain( + rpcURL string, + chainID string, + privateKey string, + startBlock uint64, + lookbackPeriod uint64, + workers uint32, + gasLimit uint64, + txMemo string, + maxRetries int, + retryIntervalSeconds int, +) (*Noble, error) { + cc, err := cosmos.NewProvider(rpcURL) + if err != nil { + return nil, fmt.Errorf("unable to build cosmos provider for noble: %w", err) + } + + keyBz, err := hex.DecodeString(privateKey) + if err != nil { + return nil, fmt.Errorf("unable to parse noble private key: %w", err) + } + + privKey := secp256k1.PrivKey{Key: keyBz} + + address := privKey.PubKey().Address() + minterAddress := sdk.MustBech32ifyAddressBytes("noble", address) + + return &Noble{ + cc: cc, + chainID: chainID, + startBlock: startBlock, + lookbackPeriod: lookbackPeriod, + workers: workers, + privateKey: &privKey, + minterAddress: minterAddress, + gasLimit: gasLimit, + txMemo: txMemo, + maxRetries: maxRetries, + retryIntervalSeconds: retryIntervalSeconds, + }, nil +} + +func (n *Noble) AccountInfo(ctx context.Context) (uint64, uint64, error) { + res, err := authtypes.NewQueryClient(n.cc).Account(ctx, &authtypes.QueryAccountRequest{ + Address: n.minterAddress, + }) + if err != nil { + return 0, 0, fmt.Errorf("unable to query account for noble: %w", err) + } + var acc authtypes.AccountI + if err := n.cc.Cdc.InterfaceRegistry.UnpackAny(res.Account, &acc); err != nil { + return 0, 0, fmt.Errorf("unable to unpack account for noble: %w", err) + } + + return acc.GetAccountNumber(), acc.GetSequence(), nil +} + +func (n *Noble) Name() string { + return "Noble" +} + +func (n *Noble) Domain() types.Domain { + return 4 +} + +func (n *Noble) IsDestinationCaller(destinationCaller []byte) bool { + zeroByteArr := make([]byte, 32) + + if bytes.Equal(destinationCaller, zeroByteArr) { + return true + } + + bech32DestinationCaller, err := decodeDestinationCaller(destinationCaller) + if err != nil { + return false + } + + return bech32DestinationCaller == n.minterAddress +} + +// DecodeDestinationCaller transforms an encoded Noble cctp address into a noble bech32 address +// left padded input -> bech32 output +func decodeDestinationCaller(input []byte) (string, error) { + if len(input) <= 12 { + return "", errors.New("destinationCaller is too short") + } + output, err := bech32.ConvertAndEncode("noble", input[12:]) + if err != nil { + return "", errors.New("unable to encode destination caller") + } + return output, nil +} + +func (n *Noble) InitializeBroadcaster( + ctx context.Context, + logger log.Logger, + sequenceMap *types.SequenceMap, +) error { + accountNumber, accountSequence, err := n.AccountInfo(ctx) + if err != nil { + return fmt.Errorf("unable to get account info for noble: %w", err) + } + + n.accountNumber = accountNumber + sequenceMap.Put(n.Domain(), accountSequence) + + return nil +} + +func (n *Noble) StartListener( + ctx context.Context, + logger log.Logger, + processingQueue chan *types.TxState, + quit chan os.Signal, +) { + logger = logger.With("chain", n.Name(), "chain_id", n.chainID, "domain", n.Domain()) + + if n.startBlock == 0 { + // get the latest block + chainTip, err := n.chainTip(ctx) + if err != nil { + panic(fmt.Errorf("unable to get chain tip for noble: %w", err)) + } + n.startBlock = chainTip + } + + logger.Info(fmt.Sprintf("Starting Noble listener at block %d looking back %d blocks", + n.startBlock, + n.lookbackPeriod)) + + accountNumber, _, err := n.AccountInfo(ctx) + if err != nil { + panic(fmt.Errorf("unable to get account info for noble: %w", err)) + } + + n.accountNumber = accountNumber + + // enqueue block heights + currentBlock := n.startBlock + lookback := n.lookbackPeriod + chainTip, err := n.chainTip(ctx) + blockQueue := make(chan uint64, 1000000) + + // history + currentBlock = currentBlock - lookback + for currentBlock <= chainTip { + blockQueue <- currentBlock + currentBlock++ + } + + // listen for new blocks + go func() { + for { + select { + case <-quit: + return + default: + chainTip, err = n.chainTip(ctx) + if err == nil { + if chainTip >= currentBlock { + for i := currentBlock; i <= chainTip; i++ { + blockQueue <- i + } + currentBlock = chainTip + 1 + } + } + time.Sleep(6 * time.Second) + } + } + }() + + // constantly query for blocks + for i := 0; i < int(n.workers); i++ { + go func() { + for { + select { + case <-quit: + return + default: + block := <-blockQueue + res, err := n.cc.RPCClient.TxSearch(ctx, fmt.Sprintf("tx.height=%d", block), false, nil, nil, "") + if err != nil { + logger.Debug(fmt.Sprintf("unable to query Noble block %d", block)) + blockQueue <- block + } + + for _, tx := range res.Txs { + parsedMsgs, err := txToMessageState(tx) + if err != nil { + logger.Error("unable to parse Noble log to message state", "err", err.Error()) + continue + } + for _, parsedMsg := range parsedMsgs { + logger.Info(fmt.Sprintf("New stream msg with nonce %d from %d with tx hash %s", parsedMsg.Nonce, parsedMsg.SourceDomain, parsedMsg.SourceTxHash)) + } + processingQueue <- &types.TxState{TxHash: tx.Hash.String(), Msgs: parsedMsgs} + } + } + } + }() + } + + <-quit +} + +func (n *Noble) chainTip(ctx context.Context) (uint64, error) { + res, err := n.cc.RPCClient.Status(ctx) + if err != nil { + return 0, fmt.Errorf("unable to query status for noble: %w", err) + } + return uint64(res.SyncInfo.LatestBlockHeight), nil +} + +func (n *Noble) Broadcast( + ctx context.Context, + logger log.Logger, + msgs []*types.MessageState, + sequenceMap *types.SequenceMap, +) error { + // set up sdk context + interfaceRegistry := codectypes.NewInterfaceRegistry() + nobletypes.RegisterInterfaces(interfaceRegistry) + cdc := codec.NewProtoCodec(interfaceRegistry) + sdkContext := sdkClient.Context{ + TxConfig: xauthtx.NewTxConfig(cdc, xauthtx.DefaultSignModes), + } + + // build txn + txBuilder := sdkContext.TxConfig.NewTxBuilder() + + // sign and broadcast txn + for attempt := 0; attempt <= n.maxRetries; attempt++ { + + var receiveMsgs []sdk.Msg + for _, msg := range msgs { + + used, err := n.cc.QueryUsedNonce(ctx, types.Domain(msg.SourceDomain), msg.Nonce) + if err != nil { + return fmt.Errorf("unable to query used nonce: %w", err) + } + + if used { + msg.Status = types.Complete + logger.Info(fmt.Sprintf("Noble cctp minter nonce %d already used", msg.Nonce)) + continue + } + + attestationBytes, err := hex.DecodeString(msg.Attestation[2:]) + if err != nil { + return fmt.Errorf("unable to decode message attestation") + } + + receiveMsgs = append(receiveMsgs, nobletypes.NewMsgReceiveMessage( + n.minterAddress, + msg.MsgSentBytes, + attestationBytes, + )) + + logger.Info(fmt.Sprintf( + "Broadcasting message from %d to %d: with source tx hash %s", + msg.SourceDomain, + msg.DestDomain, + msg.SourceTxHash)) + } + + if err := txBuilder.SetMsgs(receiveMsgs...); err != nil { + return fmt.Errorf("failed to set messages on tx: %w", err) + } + + txBuilder.SetGasLimit(n.gasLimit) + + txBuilder.SetMemo(n.txMemo) + + n.mu.Lock() + + accountSequence := sequenceMap.Next(n.Domain()) + + sigV2 := signing.SignatureV2{ + PubKey: n.privateKey.PubKey(), + Data: &signing.SingleSignatureData{ + SignMode: sdkContext.TxConfig.SignModeHandler().DefaultMode(), + Signature: nil, + }, + Sequence: uint64(accountSequence), + } + + signerData := xauthsigning.SignerData{ + ChainID: n.chainID, + AccountNumber: uint64(n.accountNumber), + Sequence: uint64(accountSequence), + } + + txBuilder.SetSignatures(sigV2) + + sigV2, err := clientTx.SignWithPrivKey( + sdkContext.TxConfig.SignModeHandler().DefaultMode(), + signerData, + txBuilder, + n.privateKey, + sdkContext.TxConfig, + uint64(accountSequence), + ) + if err != nil { + n.mu.Unlock() + return fmt.Errorf("failed to sign tx: %w", err) + } + + if err := txBuilder.SetSignatures(sigV2); err != nil { + n.mu.Unlock() + return fmt.Errorf("failed to set signatures: %w", err) + } + + // Generated Protobuf-encoded bytes. + txBytes, err := sdkContext.TxConfig.TxEncoder()(txBuilder.GetTx()) + if err != nil { + n.mu.Unlock() + return fmt.Errorf("failed to proto encode tx: %w", err) + } + + rpcResponse, err := n.cc.RPCClient.BroadcastTxSync(context.Background(), txBytes) + if err != nil || (rpcResponse != nil && rpcResponse.Code != 0) { + // Log the error + logger.Error(fmt.Sprintf("error during broadcast: %s", getErrorString(err, rpcResponse))) + + if err != nil || rpcResponse == nil { + // Log retry information + logger.Info(fmt.Sprintf("Retrying in %d seconds", n.retryIntervalSeconds)) + time.Sleep(time.Duration(n.retryIntervalSeconds) * time.Second) + // wait a random amount of time to lower probability of concurrent message nonce collision + time.Sleep(time.Duration(rand.Intn(5)) * time.Second) + n.mu.Unlock() + continue + } + + // Log details for non-zero response code + logger.Error(fmt.Sprintf("received non-zero: %d - %s", rpcResponse.Code, rpcResponse.Log)) + + // Handle specific error code (32) + if rpcResponse.Code == 32 { + newAccountSequence := n.extractAccountSequence(ctx, logger, rpcResponse.Log) + logger.Debug(fmt.Sprintf("retrying with new account sequence: %d", newAccountSequence)) + sequenceMap.Put(n.Domain(), newAccountSequence) + } + + // Log retry information + logger.Info(fmt.Sprintf("Retrying in %d seconds", n.retryIntervalSeconds)) + time.Sleep(time.Duration(n.retryIntervalSeconds) * time.Second) + // wait a random amount of time to lower probability of concurrent message nonce collision + time.Sleep(time.Duration(rand.Intn(5)) * time.Second) + n.mu.Unlock() + continue + } + + n.mu.Unlock() + + // Tx was successfully broadcast + for _, msg := range msgs { + msg.DestTxHash = rpcResponse.Hash.String() + msg.Status = types.Complete + } + logger.Info(fmt.Sprintf("Successfully broadcast %s to Noble. Tx hash: %s", msgs[0].SourceTxHash, msgs[0].DestTxHash)) + + return nil + } + + for _, msg := range msgs { + if msg.Status != types.Complete { + msg.Status = types.Failed + } + } + + return errors.New("reached max number of broadcast attempts") +} + +// getErrorString returns the appropriate value to log when tx broadcast errors are encountered. +func getErrorString(err error, rpcResponse *ctypes.ResultBroadcastTx) string { + if rpcResponse != nil { + return rpcResponse.Log + } + return err.Error() +} + +// extractAccountSequence attempts to extract the account sequence number from the RPC response logs when +// account sequence mismatch errors are encountered. If the account sequence number cannot be extracted from the logs, +// it is retrieved by making a request to the API endpoint. +func (n *Noble) extractAccountSequence(ctx context.Context, logger log.Logger, rpcResponseLog string) uint64 { + pattern := `expected (\d+), got (\d+)` + re := regexp.MustCompile(pattern) + match := re.FindStringSubmatch(rpcResponseLog) + + if len(match) == 3 { + // Extract the numbers from the match. + newAccountSequence, _ := strconv.ParseUint(match[1], 10, 64) + return newAccountSequence + } + + // Otherwise, just request the account sequence + _, newAccountSequence, err := n.AccountInfo(ctx) + if err != nil { + logger.Error("unable to retrieve account sequence") + } + + return newAccountSequence +} diff --git a/noble/config.go b/noble/config.go new file mode 100644 index 0000000..a9085a1 --- /dev/null +++ b/noble/config.go @@ -0,0 +1,37 @@ +package noble + +import "github.com/strangelove-ventures/noble-cctp-relayer/types" + +var _ types.ChainConfig = (*ChainConfig)(nil) + +type ChainConfig struct { + RPC string `yaml:"rpc"` + ChainID string `yaml:"chain-id"` + + StartBlock uint64 `yaml:"start-block"` + LookbackPeriod uint64 `yaml:"lookback-period"` + Workers uint32 `yaml:"workers"` + + TxMemo string `yaml:"tx-memo"` + GasLimit uint64 `yaml:"gas-limit"` + BroadcastRetries int `yaml:"broadcast-retries"` + BroadcastRetryInterval int `yaml:"broadcast-retry-interval"` + + // TODO move to keyring + MinterPrivateKey string `yaml:"minter-private-key"` +} + +func (c *ChainConfig) Chain(name string) (types.Chain, error) { + return NewChain( + c.RPC, + c.ChainID, + c.MinterPrivateKey, + c.StartBlock, + c.LookbackPeriod, + c.Workers, + c.GasLimit, + c.TxMemo, + c.BroadcastRetries, + c.BroadcastRetryInterval, + ) +} diff --git a/cmd/noble/listener_test.go b/noble/listener_test.go similarity index 70% rename from cmd/noble/listener_test.go rename to noble/listener_test.go index a1d668a..108f982 100644 --- a/cmd/noble/listener_test.go +++ b/noble/listener_test.go @@ -1,33 +1,40 @@ package noble_test import ( + "context" "os" "testing" "time" "cosmossdk.io/log" "github.com/rs/zerolog" - "github.com/strangelove-ventures/noble-cctp-relayer/cmd/noble" - "github.com/strangelove-ventures/noble-cctp-relayer/config" + "github.com/strangelove-ventures/noble-cctp-relayer/noble" "github.com/strangelove-ventures/noble-cctp-relayer/types" "github.com/stretchr/testify/require" ) -var cfg config.Config +var cfg types.Config var logger log.Logger var processingQueue chan *types.TxState func init() { - cfg = config.Parse("../../.ignore/unit_tests.yaml") + var err error + cfg, err = types.Parse("../../.ignore/unit_tests.yaml") + if err != nil { + panic(err) + } logger = log.NewLogger(os.Stdout, log.LevelOption(zerolog.DebugLevel)) processingQueue = make(chan *types.TxState, 10000) - cfg.Networks.Source.Noble.Workers = 1 + cfg.Chains["noble"].(*noble.ChainConfig).Workers = 1 } func TestStartListener(t *testing.T) { - cfg.Networks.Source.Noble.StartBlock = 3273557 - go noble.StartListener(cfg, logger, processingQueue) + cfg.Chains["noble"].(*noble.ChainConfig).StartBlock = 3273557 + n, err := cfg.Chains["noble"].(*noble.ChainConfig).Chain("noble") + require.NoError(t, err) + + go n.StartListener(context.TODO(), logger, processingQueue, nil) time.Sleep(20 * time.Second) diff --git a/noble/message_state.go b/noble/message_state.go new file mode 100644 index 0000000..bf99555 --- /dev/null +++ b/noble/message_state.go @@ -0,0 +1,77 @@ +package noble + +import ( + "encoding/base64" + "encoding/hex" + "errors" + "fmt" + "time" + + ctypes "github.com/cometbft/cometbft/rpc/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/strangelove-ventures/noble-cctp-relayer/types" +) + +// NobleLogToMessageState transforms a Noble log into a messageState +func txToMessageState(tx *ctypes.ResultTx) ([]*types.MessageState, error) { + if tx.TxResult.Code != 0 { + return nil, nil + } + + var messageStates []*types.MessageState + + for i, event := range tx.TxResult.Events { + if event.Type == "circle.cctp.v1.MessageSent" { + //fmt.Printf("Saw cctp message %s - %d:%d\n", tx., i, j) + var parsed bool + var parseErrs error + for _, attr := range event.Attributes { + if attr.Key == "message" { + fmt.Printf("Saw message attribute %s - %d\n", tx.Hash, i) + encoded := attr.Value[1 : len(attr.Value)-1] + rawMessageSentBytes, err := base64.StdEncoding.DecodeString(encoded) + if err != nil { + parseErrs = errors.Join(parseErrs, fmt.Errorf("failed to decode message: %w", err)) + continue + } + + hashed := crypto.Keccak256(rawMessageSentBytes) + hashedHexStr := hex.EncodeToString(hashed) + + msg, err := new(types.Message).Parse(rawMessageSentBytes) + if err != nil { + parseErrs = errors.Join(parseErrs, fmt.Errorf("failed to parse message: %w", err)) + continue + } + + parsed = true + + now := time.Now() + + messageState := &types.MessageState{ + IrisLookupId: hashedHexStr, + Status: types.Created, + SourceDomain: types.Domain(msg.SourceDomain), + DestDomain: types.Domain(msg.DestinationDomain), + Nonce: msg.Nonce, + SourceTxHash: tx.Hash.String(), + MsgSentBytes: rawMessageSentBytes, + DestinationCaller: msg.DestinationCaller, + Created: now, + Updated: now, + } + + messageStates = append(messageStates, messageState) + + fmt.Printf("Appended transfer from 4 to %d\n", msg.DestinationDomain) + } + } + if !parsed { + return nil, fmt.Errorf("unable to parse cctp message. tx hash %s: %w", tx.Hash, parseErrs) + } + } + } + + return messageStates, nil + +} diff --git a/types/chain.go b/types/chain.go new file mode 100644 index 0000000..967b7b8 --- /dev/null +++ b/types/chain.go @@ -0,0 +1,43 @@ +package types + +import ( + "context" + "os" + + "cosmossdk.io/log" +) + +// Chain is an interface for common CCTP source and destination chain operations. +type Chain interface { + // Name returns the name of the chain. + Name() string + + // Domain returns the domain ID of the chain. + Domain() Domain + + // IsDestinationCaller returns true if the specified destination caller is the minter for the specified domain. + IsDestinationCaller(destinationCaller []byte) bool + + // InitializeBroadcaster initializes the minter account info for the chain. + InitializeBroadcaster( + ctx context.Context, + logger log.Logger, + sequenceMap *SequenceMap, + ) error + + // StartListener starts a listener for observing new CCTP burn messages. + StartListener( + ctx context.Context, + logger log.Logger, + processingQueue chan *TxState, + quit chan os.Signal, + ) + + // Broadcast broadcasts CCTP mint messages to the chain. + Broadcast( + ctx context.Context, + logger log.Logger, + msgs []*MessageState, + sequenceMap *SequenceMap, + ) error +} diff --git a/types/config.go b/types/config.go new file mode 100644 index 0000000..2da8dcb --- /dev/null +++ b/types/config.go @@ -0,0 +1,34 @@ +package types + +import ( + "os" + + "gopkg.in/yaml.v3" +) + +type Config struct { + Chains map[string]ChainConfig `yaml:"chains"` + EnabledRoutes map[Domain]Domain `yaml:"enabled-routes"` + Circle struct { + AttestationBaseUrl string `yaml:"attestation-base-url"` + FetchRetries int `yaml:"fetch-retries"` + FetchRetryInterval int `yaml:"fetch-retry-interval"` + } `yaml:"circle"` + ProcessorWorkerCount uint32 `yaml:"processor-worker-count"` + Api struct { + TrustedProxies []string `yaml:"trusted-proxies"` + } `yaml:"api"` +} + +func Parse(file string) (cfg Config, err error) { + data, err := os.ReadFile(file) + if err != nil { + return + } + err = yaml.Unmarshal(data, &cfg) + return cfg, err +} + +type ChainConfig interface { + Chain(name string) (Chain, error) +} diff --git a/types/message_state.go b/types/message_state.go index 4d0630d..3bfcd0d 100644 --- a/types/message_state.go +++ b/types/message_state.go @@ -2,15 +2,11 @@ package types import ( "bytes" - "encoding/base64" "encoding/hex" - "encoding/json" - "errors" "fmt" "time" "github.com/circlefin/noble-cctp/x/cctp/types" - "github.com/cosmos/cosmos-sdk/types/bech32" "github.com/ethereum/go-ethereum/accounts/abi" ethtypes "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" @@ -83,89 +79,6 @@ func EvmLogToMessageState(abi abi.ABI, messageSent abi.Event, log *ethtypes.Log) return nil, fmt.Errorf("unable to parse tx into message, tx hash %s", log.TxHash.Hex()) } -// NobleLogToMessageState transforms a Noble log into a messageState -func NobleLogToMessageState(tx Tx) ([]*MessageState, error) { - var eventsList []struct { - Events []Event `json:"events"` - } - if tx.TxResult.Code != 0 { - return nil, nil - } - if err := json.Unmarshal([]byte(tx.TxResult.Log), &eventsList); err != nil { - return nil, fmt.Errorf("unable to parse log events: %s", tx.TxResult.Log) - } - - var messageStates []*MessageState - - for i, log := range eventsList { - for j, event := range log.Events { - if event.Type == "circle.cctp.v1.MessageSent" { - fmt.Printf("Saw cctp message %s - %d:%d\n", tx.Hash, i, j) - var parsed bool - var parseErrs error - for _, attr := range event.Attributes { - if attr.Key == "message" { - fmt.Printf("Saw message attribute %s - %d:%d\n", tx.Hash, i, j) - encoded := attr.Value[1 : len(attr.Value)-1] - rawMessageSentBytes, err := base64.StdEncoding.DecodeString(encoded) - if err != nil { - parseErrs = errors.Join(parseErrs, fmt.Errorf("failed to decode message: %w", err)) - continue - } - - hashed := crypto.Keccak256(rawMessageSentBytes) - hashedHexStr := hex.EncodeToString(hashed) - - msg, err := new(types.Message).Parse(rawMessageSentBytes) - if err != nil { - parseErrs = errors.Join(parseErrs, fmt.Errorf("failed to parse message: %w", err)) - continue - } - - parsed = true - - messageState := &MessageState{ - IrisLookupId: hashedHexStr, - Status: Created, - SourceDomain: Domain(msg.SourceDomain), - DestDomain: Domain(msg.DestinationDomain), - Nonce: msg.Nonce, - SourceTxHash: tx.Hash, - MsgSentBytes: rawMessageSentBytes, - DestinationCaller: msg.DestinationCaller, - Created: time.Now(), - Updated: time.Now(), - } - - messageStates = append(messageStates, messageState) - - fmt.Printf("Appended transfer from 4 to %d\n", msg.DestinationDomain) - } - } - if !parsed { - return nil, fmt.Errorf("unable to parse cctp message. tx hash %s: %w", tx.Hash, parseErrs) - } - } - } - } - - return messageStates, nil - -} - -// DecodeDestinationCaller transforms an encoded Noble cctp address into a noble bech32 address -// left padded input -> bech32 output -func DecodeDestinationCaller(input []byte) (string, error) { - if len(input) <= 12 { - return "", errors.New("destinationCaller is too short") - } - output, err := bech32.ConvertAndEncode("noble", input[12:]) - if err != nil { - return "", errors.New("unable to encode destination caller") - } - return output, nil -} - // Equal checks if two MessageState instances are equal func (m *MessageState) Equal(other *MessageState) bool { return (m.IrisLookupId == other.IrisLookupId && diff --git a/types/message_state.go.bak b/types/message_state.go.bak deleted file mode 100644 index f7d106c..0000000 --- a/types/message_state.go.bak +++ /dev/null @@ -1,192 +0,0 @@ -package types - -import ( - "bytes" - "encoding/base64" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "strconv" - "time" - - "github.com/circlefin/noble-cctp/x/cctp/types" - "github.com/cosmos/cosmos-sdk/types/bech32" - "github.com/ethereum/go-ethereum/accounts/abi" - ethtypes "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto" -) - -const ( - Created string = "created" - Pending string = "pending" - Attested string = "attested" - Complete string = "complete" - Failed string = "failed" - Filtered string = "filtered" - - Mint string = "mint" - Forward string = "forward" -) - -type TxState struct { - TxHash string - Msgs []*MessageState -} - -type MessageState struct { - IrisLookupId string // hex encoded MessageSent bytes - // Type string // 'mint' or 'forward' - Status string // created, pending, attested, complete, failed, filtered - Attestation string // hex encoded attestation - SourceDomain uint32 // source domain id - DestDomain uint32 // destination domain id - SourceTxHash string - DestTxHash string - MsgSentBytes []byte // bytes of the MessageSent message transmitter event - DestinationCaller []byte // address authorized to call transaction - Channel string // "channel-%d" if a forward, empty if not a forward - Created time.Time - Updated time.Time - Nonce uint64 -} - -// EvmLogToMessageState transforms an evm log into a messageState given an ABI -func EvmLogToMessageState(abi abi.ABI, messageSent abi.Event, log *ethtypes.Log) (messageState *MessageState, err error) { - event := make(map[string]interface{}) - _ = abi.UnpackIntoMap(event, messageSent.Name, log.Data) - - rawMessageSentBytes := event["message"].([]byte) - message, _ := new(types.Message).Parse(rawMessageSentBytes) - - hashed := crypto.Keccak256(rawMessageSentBytes) - hashedHexStr := hex.EncodeToString(hashed) - - messageState = &MessageState{ - IrisLookupId: hashedHexStr, - Status: Created, - SourceDomain: message.SourceDomain, - DestDomain: message.DestinationDomain, - SourceTxHash: log.TxHash.Hex(), - MsgSentBytes: rawMessageSentBytes, - DestinationCaller: message.DestinationCaller, - Nonce: message.Nonce, - Created: time.Now(), - Updated: time.Now(), - } - - if _, err := new(BurnMessage).Parse(message.MessageBody); err == nil { - messageState.Type = Mint - return messageState, nil - } - - if forward, err := new(MetadataMessage).Parse(message.MessageBody); err == nil { - messageState.Type = Forward - // add forward channel to object so we can filter later - messageState.Channel = "channel-" + strconv.Itoa(int(forward.Channel)) - return messageState, nil - } - - return nil, fmt.Errorf("unable to parse tx into message, tx hash %s", log.TxHash.Hex()) -} - -// NobleLogToMessageState transforms a Noble log into a messageState -func NobleLogToMessageState(tx Tx) ([]*MessageState, error) { - var eventsList []struct { - Events []Event `json:"events"` - } - if tx.TxResult.Code != 0 { - return nil, nil - } - if err := json.Unmarshal([]byte(tx.TxResult.Log), &eventsList); err != nil { - return nil, fmt.Errorf("unable to parse log events: %s", tx.TxResult.Log) - } - - var messageStates []*MessageState - - for i, log := range eventsList { - for j, event := range log.Events { - if event.Type == "circle.cctp.v1.MessageSent" { - fmt.Printf("Saw cctp message %s - %d:%d\n", tx.Hash, i, j) - var parsed bool - var parseErrs error - for _, attr := range event.Attributes { - if attr.Key == "message" { - fmt.Printf("Saw message attribute %s - %d:%d\n", tx.Hash, i, j) - encoded := attr.Value[1 : len(attr.Value)-1] - rawMessageSentBytes, err := base64.StdEncoding.DecodeString(encoded) - if err != nil { - parseErrs = errors.Join(parseErrs, fmt.Errorf("failed to decode message: %w", err)) - continue - } - - hashed := crypto.Keccak256(rawMessageSentBytes) - hashedHexStr := hex.EncodeToString(hashed) - - msg, err := new(types.Message).Parse(rawMessageSentBytes) - if err != nil { - parseErrs = errors.Join(parseErrs, fmt.Errorf("failed to parse message: %w", err)) - continue - } - - parsed = true - - messageState := &MessageState{ - IrisLookupId: hashedHexStr, - Type: Mint, - Status: Created, - SourceDomain: msg.SourceDomain, - DestDomain: msg.DestinationDomain, - Nonce: msg.Nonce, - SourceTxHash: tx.Hash, - MsgSentBytes: rawMessageSentBytes, - DestinationCaller: msg.DestinationCaller, - Created: time.Now(), - Updated: time.Now(), - } - - messageStates = append(messageStates, messageState) - - fmt.Printf("Appended transfer from 4 to %d\n", msg.DestinationDomain) - } - } - if !parsed { - return nil, fmt.Errorf("unable to parse cctp message. tx hash %s: %w", tx.Hash, parseErrs) - } - } - } - } - - return messageStates, nil - -} - -// DecodeDestinationCaller transforms an encoded Noble cctp address into a noble bech32 address -// left padded input -> bech32 output -func DecodeDestinationCaller(input []byte) (string, error) { - if len(input) <= 12 { - return "", errors.New("destinationCaller is too short") - } - output, err := bech32.ConvertAndEncode("noble", input[12:]) - if err != nil { - return "", errors.New("unable to encode destination caller") - } - return output, nil -} - -// Equal checks if two MessageState instances are equal -func (m *MessageState) Equal(other *MessageState) bool { - return (m.IrisLookupId == other.IrisLookupId && - m.Type == other.Type && - m.Status == other.Status && - m.Attestation == other.Attestation && - m.SourceDomain == other.SourceDomain && - m.DestDomain == other.DestDomain && - m.SourceTxHash == other.SourceTxHash && - m.DestTxHash == other.DestTxHash && - bytes.Equal(m.MsgSentBytes, other.MsgSentBytes) && - bytes.Equal(m.DestinationCaller, other.DestinationCaller) && - m.Channel == other.Channel && - m.Created == other.Created && - m.Updated == other.Updated) -} diff --git a/types/message_state_test.go b/types/message_state_test.go index 971e4f1..2036c3b 100644 --- a/types/message_state_test.go +++ b/types/message_state_test.go @@ -12,15 +12,19 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethclient" "github.com/pascaldekloe/etherstream" - "github.com/strangelove-ventures/noble-cctp-relayer/config" + ethinternal "github.com/strangelove-ventures/noble-cctp-relayer/ethereum" "github.com/strangelove-ventures/noble-cctp-relayer/types" "github.com/stretchr/testify/require" ) -var cfg config.Config +var cfg types.Config func init() { - cfg = config.Parse("../.ignore/unit_tests.yaml") + var err error + cfg, err = types.Parse("../../.ignore/unit_tests.yaml") + if err != nil { + panic(err) + } } func TestToMessageStateSuccess(t *testing.T) { @@ -33,7 +37,7 @@ func TestToMessageStateSuccess(t *testing.T) { messageSent := messageTransmitterABI.Events["MessageSent"] - ethClient, err := ethclient.DialContext(context.Background(), cfg.Networks.Source.Ethereum.RPC) + ethClient, err := ethclient.DialContext(context.Background(), cfg.Chains["ethereum"].(*ethinternal.ChainConfig).RPC) require.Nil(t, err) // changed to mainnet address @@ -53,6 +57,7 @@ func TestToMessageStateSuccess(t *testing.T) { require.Nil(t, err) messageState, err := types.EvmLogToMessageState(messageTransmitterABI, messageSent, &history[0]) + require.NoError(t, err) fmt.Println(messageState) diff --git a/types/sequence_map.go b/types/sequence_map.go index 917dd5e..e997b4b 100644 --- a/types/sequence_map.go +++ b/types/sequence_map.go @@ -8,22 +8,22 @@ import ( type SequenceMap struct { mu sync.Mutex // map destination domain -> minter account sequence - sequenceMap map[Domain]int64 + sequenceMap map[Domain]uint64 } func NewSequenceMap() *SequenceMap { return &SequenceMap{ - sequenceMap: map[Domain]int64{}, + sequenceMap: map[Domain]uint64{}, } } -func (m *SequenceMap) Put(destDomain Domain, val int64) { +func (m *SequenceMap) Put(destDomain Domain, val uint64) { m.mu.Lock() defer m.mu.Unlock() m.sequenceMap[destDomain] = val } -func (m *SequenceMap) Next(destDomain Domain) int64 { +func (m *SequenceMap) Next(destDomain Domain) uint64 { m.mu.Lock() defer m.mu.Unlock() result := m.sequenceMap[destDomain] From d4d198a9cb11b33cd061f35ab85629909df252ed Mon Sep 17 00:00:00 2001 From: Dan Kanefsky Date: Fri, 19 Jan 2024 21:39:31 -0800 Subject: [PATCH 07/13] fix config parsing --- cmd/process.go | 4 +-- cmd/process_test.go | 4 +-- cmd/root.go | 51 +++++++++++++++++++++++++++++++++++-- cmd/root_test.go | 25 ++++++++++++++++++ config/sample.yaml | 43 +++++++++++++++++++++++++++++++ ethereum/chain.go | 2 -- ethereum/config.go | 14 +++++----- ethereum/listener_test.go | 5 ++-- ethereum/util_test.go | 3 ++- integration/config.go | 14 +++++----- noble/chain.go | 5 ++++ noble/listener_test.go | 5 ++-- types/config.go | 25 +++++++++--------- types/message_state_test.go | 7 ++--- 14 files changed, 165 insertions(+), 42 deletions(-) create mode 100644 cmd/root_test.go create mode 100644 config/sample.yaml diff --git a/cmd/process.go b/cmd/process.go index 7c4f91d..f9be5de 100644 --- a/cmd/process.go +++ b/cmd/process.go @@ -70,7 +70,7 @@ func Start(cmd *cobra.Command, args []string) { // StartProcessor is the main processing pipeline. func StartProcessor( ctx context.Context, - cfg types.Config, + cfg *types.Config, logger log.Logger, registeredDomains map[types.Domain]types.Chain, processingQueue chan *types.TxState, @@ -159,7 +159,7 @@ func StartProcessor( } // filterDisabledCCTPRoutes returns true if we haven't enabled relaying from a source domain to a destination domain -func filterDisabledCCTPRoutes(cfg types.Config, logger log.Logger, msg *types.MessageState) bool { +func filterDisabledCCTPRoutes(cfg *types.Config, logger log.Logger, msg *types.MessageState) bool { val, ok := cfg.EnabledRoutes[msg.SourceDomain] result := !(ok && val == msg.DestDomain) if result { diff --git a/cmd/process_test.go b/cmd/process_test.go index 3f56198..afb98a1 100644 --- a/cmd/process_test.go +++ b/cmd/process_test.go @@ -14,14 +14,14 @@ import ( "github.com/stretchr/testify/require" ) -var cfg types.Config +var cfg *types.Config var logger log.Logger var processingQueue chan *types.TxState var sequenceMap *types.SequenceMap func setupTest(t *testing.T) map[types.Domain]types.Chain { var err error - cfg, err = types.Parse("../.ignore/unit_tests.yaml") + cfg, err = cmd.Parse("../.ignore/unit_tests.yaml") require.NoError(t, err, "Error parsing config") logger = log.NewLogger(os.Stdout, log.LevelOption(zerolog.DebugLevel)) diff --git a/cmd/root.go b/cmd/root.go index 95385a4..f21a545 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -1,12 +1,16 @@ package cmd import ( + "fmt" "net/http" "os" "strconv" "github.com/gin-gonic/gin" + "github.com/strangelove-ventures/noble-cctp-relayer/ethereum" + "github.com/strangelove-ventures/noble-cctp-relayer/noble" "github.com/strangelove-ventures/noble-cctp-relayer/types" + "gopkg.in/yaml.v2" "cosmossdk.io/log" "github.com/rs/zerolog" @@ -14,7 +18,7 @@ import ( ) var ( - Cfg types.Config + Cfg *types.Config cfgFile string verbose bool @@ -47,7 +51,7 @@ func init() { } var err error - Cfg, err = types.Parse(cfgFile) + Cfg, err = Parse(cfgFile) if err != nil { Logger.Error("unable to parse config file", "location", cfgFile, "err", err) os.Exit(1) @@ -89,3 +93,46 @@ func getTxByHash(c *gin.Context) { c.JSON(http.StatusNotFound, gin.H{"message": "message not found"}) } + +func Parse(file string) (*types.Config, error) { + data, err := os.ReadFile(file) + if err != nil { + return nil, fmt.Errorf("failed to read file %w", err) + } + + var cfg types.ConfigWrapper + if err := yaml.Unmarshal(data, &cfg); err != nil { + return nil, fmt.Errorf("error unmarshalling config: %w", err) + } + + c := types.Config{ + EnabledRoutes: cfg.EnabledRoutes, + Circle: cfg.Circle, + ProcessorWorkerCount: cfg.ProcessorWorkerCount, + Api: cfg.Api, + Chains: make(map[string]types.ChainConfig), + } + + for name, chain := range cfg.Chains { + yamlbz, err := yaml.Marshal(chain) + if err != nil { + return nil, err + } + + switch name { + case "noble": + var cc noble.ChainConfig + if err := yaml.Unmarshal(yamlbz, &cc); err != nil { + return nil, err + } + c.Chains[name] = &cc + default: + var cc ethereum.ChainConfig + if err := yaml.Unmarshal(yamlbz, &cc); err != nil { + return nil, err + } + c.Chains[name] = &cc + } + } + return &c, err +} diff --git a/cmd/root_test.go b/cmd/root_test.go new file mode 100644 index 0000000..6e4388c --- /dev/null +++ b/cmd/root_test.go @@ -0,0 +1,25 @@ +package cmd_test + +import ( + "testing" + + "github.com/strangelove-ventures/noble-cctp-relayer/cmd" + "github.com/strangelove-ventures/noble-cctp-relayer/ethereum" + "github.com/strangelove-ventures/noble-cctp-relayer/noble" + "github.com/stretchr/testify/require" +) + +func TestConfig(t *testing.T) { + file, err := cmd.Parse("../config/sample.yaml") + require.NoError(t, err, "Error parsing config") + + // assert noble chainConfig correctly parsed + var nobleType interface{} = file.Chains["noble"] + _, ok := nobleType.(*noble.ChainConfig) + require.True(t, ok) + + // assert ethereum chainConfig correctly parsed + var ethType interface{} = file.Chains["ethereum"] + _, ok = ethType.(*ethereum.ChainConfig) + require.True(t, ok) +} diff --git a/config/sample.yaml b/config/sample.yaml new file mode 100644 index 0000000..1c8bd1d --- /dev/null +++ b/config/sample.yaml @@ -0,0 +1,43 @@ +chains: + ethereum: + chain-id: 5 + rpc: "https://goerli.infura.io/v3/apiKey" + ws: "wss://goerli.infura.io/ws/v3/apiKey" + message-transmitter: "0x26413e8157CD32011E726065a5462e97dD4d03D9" + + start-block: 9737196 + lookback-period: 5 # historical blocks to look back on launch + + broadcast-retries: 5 # number of times to attempt the broadcast + broadcast-retry-interval: 10 # time between retries in seconds + + minter-private-key: "privateKey" + + + noble: + rpc: "https://rpc.testnet.noble.strange.love:443" + chain-id: "grand-1" + + start-block: 0 # set to 0 to default to latest block + lookback-period: 5 # historical blocks to look back on launch + workers: 8 + + tx-memo: "Relayed by Strangelove" + gas-limit: 200000 + broadcast-retries: 5 # number of times to attempt the broadcast + broadcast-retry-interval: 5 # time between retries in seconds + + # hex encoded + minter-private-key: "hexEncodedPrivateKey" + +# source domain id -> destination domain id +enabled-routes: + 0: 4 # ethereum to noble + 4: 0 # noble to ethereum + +circle: + attestation-base-url: "https://iris-api-sandbox.circle.com/attestations/" + fetch-retries: 0 # additional times to fetch an attestation + fetch-retry-interval: 3 # time between retries in seconds + +processor-worker-count: 16 diff --git a/ethereum/chain.go b/ethereum/chain.go index 8722217..b228102 100644 --- a/ethereum/chain.go +++ b/ethereum/chain.go @@ -53,7 +53,6 @@ type Ethereum struct { func NewChain( name string, chainID int64, - domain types.Domain, rpcURL string, wsURL string, messageTransmitterAddress string, @@ -70,7 +69,6 @@ func NewChain( return &Ethereum{ name: name, chainID: chainID, - domain: domain, rpcURL: rpcURL, wsURL: wsURL, messageTransmitterAddress: messageTransmitterAddress, diff --git a/ethereum/config.go b/ethereum/config.go index 2be09d1..1dc2558 100644 --- a/ethereum/config.go +++ b/ethereum/config.go @@ -1,15 +1,16 @@ package ethereum -import "github.com/strangelove-ventures/noble-cctp-relayer/types" +import ( + "github.com/strangelove-ventures/noble-cctp-relayer/types" +) var _ types.ChainConfig = (*ChainConfig)(nil) type ChainConfig struct { - DomainID types.Domain `yaml:"domain-id"` - ChainID int64 `yaml:"chain-id"` - RPC string `yaml:"rpc"` - WS string `yaml:"ws"` - MessageTransmitter string `yaml:"message-transmitter"` + RPC string `yaml:"rpc"` + WS string `yaml:"ws"` + ChainID int64 `yaml:"chain-id"` + MessageTransmitter string `yaml:"message-transmitter"` StartBlock uint64 `yaml:"start-block"` LookbackPeriod uint64 `yaml:"lookback-period"` @@ -25,7 +26,6 @@ func (c *ChainConfig) Chain(name string) (types.Chain, error) { return NewChain( name, c.ChainID, - c.DomainID, c.RPC, c.WS, c.MessageTransmitter, diff --git a/ethereum/listener_test.go b/ethereum/listener_test.go index 1418e06..4548a8c 100644 --- a/ethereum/listener_test.go +++ b/ethereum/listener_test.go @@ -8,18 +8,19 @@ import ( "cosmossdk.io/log" "github.com/rs/zerolog" + "github.com/strangelove-ventures/noble-cctp-relayer/cmd" "github.com/strangelove-ventures/noble-cctp-relayer/ethereum" "github.com/strangelove-ventures/noble-cctp-relayer/types" "github.com/stretchr/testify/require" ) -var cfg types.Config +var cfg *types.Config var logger log.Logger var processingQueue chan *types.TxState func init() { var err error - cfg, err = types.Parse("../../.ignore/unit_tests.yaml") + cfg, err = cmd.Parse("../.ignore/unit_tests.yaml") if err != nil { panic(err) } diff --git a/ethereum/util_test.go b/ethereum/util_test.go index 4406b82..e4e8f83 100644 --- a/ethereum/util_test.go +++ b/ethereum/util_test.go @@ -6,6 +6,7 @@ import ( "cosmossdk.io/log" "github.com/rs/zerolog" + "github.com/strangelove-ventures/noble-cctp-relayer/cmd" "github.com/strangelove-ventures/noble-cctp-relayer/ethereum" "github.com/strangelove-ventures/noble-cctp-relayer/types" "github.com/stretchr/testify/require" @@ -13,7 +14,7 @@ import ( func init() { var err error - cfg, err = types.Parse("../../.ignore/unit_tests.yaml") + cfg, err = cmd.Parse("../.ignore/unit_tests.yaml") if err != nil { panic(err) } diff --git a/integration/config.go b/integration/config.go index 8ddec3d..cb6e600 100644 --- a/integration/config.go +++ b/integration/config.go @@ -1,19 +1,21 @@ package integration_testing import ( + "os" + "cosmossdk.io/log" "github.com/rs/zerolog" + "github.com/strangelove-ventures/noble-cctp-relayer/cmd" "github.com/strangelove-ventures/noble-cctp-relayer/cmd/noble" - "github.com/strangelove-ventures/noble-cctp-relayer/config" "github.com/strangelove-ventures/noble-cctp-relayer/types" - "os" "gopkg.in/yaml.v3" ) -var testCfg Config // for testing secrets -var cfg config.Config // app config +var testCfg *types.Config // for testing secrets +var cfg *types.Config // app config var logger log.Logger +var err error // goerli const TokenMessengerAddress = "0xd0c3da58f55358142b8d3e06c1c30c5c6114efe8" @@ -24,8 +26,8 @@ var sequenceMap *types.SequenceMap func setupTest() func() { // setup - testCfg = Parse("../.ignore/integration.yaml") - cfg = config.Parse("../.ignore/testnet.yaml") + testCfg, err = cmd.Parse("../.ignore/integration.yaml") + cfg, err = cmd.Parse("../.ignore/testnet.yaml") logger = log.NewLogger(os.Stdout, log.LevelOption(zerolog.DebugLevel)) _, nextMinterSequence, err := noble.GetNobleAccountNumberSequence( diff --git a/noble/chain.go b/noble/chain.go index cebe3a2..67ca19b 100644 --- a/noble/chain.go +++ b/noble/chain.go @@ -286,6 +286,9 @@ func (n *Noble) Broadcast( // sign and broadcast txn for attempt := 0; attempt <= n.maxRetries; attempt++ { + //TODO: MOVE EVERYTHING IN FOR LOOP TO FUNCTION. Same for ETH. + // see todo below. + var receiveMsgs []sdk.Msg for _, msg := range msgs { @@ -327,6 +330,8 @@ func (n *Noble) Broadcast( txBuilder.SetMemo(n.txMemo) n.mu.Lock() + // TODO: uncomment this & remove all remainin n.mu.Unlock() 's after moving loop body to its own function + // defer n.mu.Unlock() accountSequence := sequenceMap.Next(n.Domain()) diff --git a/noble/listener_test.go b/noble/listener_test.go index 108f982..9de5bd1 100644 --- a/noble/listener_test.go +++ b/noble/listener_test.go @@ -8,18 +8,19 @@ import ( "cosmossdk.io/log" "github.com/rs/zerolog" + "github.com/strangelove-ventures/noble-cctp-relayer/cmd" "github.com/strangelove-ventures/noble-cctp-relayer/noble" "github.com/strangelove-ventures/noble-cctp-relayer/types" "github.com/stretchr/testify/require" ) -var cfg types.Config +var cfg *types.Config var logger log.Logger var processingQueue chan *types.TxState func init() { var err error - cfg, err = types.Parse("../../.ignore/unit_tests.yaml") + cfg, err = cmd.Parse("../../.ignore/unit_tests.yaml") if err != nil { panic(err) } diff --git a/types/config.go b/types/config.go index 2da8dcb..92d2c09 100644 --- a/types/config.go +++ b/types/config.go @@ -1,11 +1,5 @@ package types -import ( - "os" - - "gopkg.in/yaml.v3" -) - type Config struct { Chains map[string]ChainConfig `yaml:"chains"` EnabledRoutes map[Domain]Domain `yaml:"enabled-routes"` @@ -20,13 +14,18 @@ type Config struct { } `yaml:"api"` } -func Parse(file string) (cfg Config, err error) { - data, err := os.ReadFile(file) - if err != nil { - return - } - err = yaml.Unmarshal(data, &cfg) - return cfg, err +type ConfigWrapper struct { + Chains map[string]map[string]any `yaml:"chains"` + EnabledRoutes map[Domain]Domain `yaml:"enabled-routes"` + Circle struct { + AttestationBaseUrl string `yaml:"attestation-base-url"` + FetchRetries int `yaml:"fetch-retries"` + FetchRetryInterval int `yaml:"fetch-retry-interval"` + } `yaml:"circle"` + ProcessorWorkerCount uint32 `yaml:"processor-worker-count"` + Api struct { + TrustedProxies []string `yaml:"trusted-proxies"` + } `yaml:"api"` } type ChainConfig interface { diff --git a/types/message_state_test.go b/types/message_state_test.go index 2036c3b..348959c 100644 --- a/types/message_state_test.go +++ b/types/message_state_test.go @@ -12,16 +12,17 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethclient" "github.com/pascaldekloe/etherstream" + "github.com/strangelove-ventures/noble-cctp-relayer/cmd" ethinternal "github.com/strangelove-ventures/noble-cctp-relayer/ethereum" "github.com/strangelove-ventures/noble-cctp-relayer/types" "github.com/stretchr/testify/require" ) -var cfg types.Config +var cfg *types.Config func init() { var err error - cfg, err = types.Parse("../../.ignore/unit_tests.yaml") + cfg, err = cmd.Parse("../.ignore/unit_tests.yaml") if err != nil { panic(err) } @@ -29,7 +30,7 @@ func init() { func TestToMessageStateSuccess(t *testing.T) { - messageTransmitter, err := os.Open("../cmd/ethereum/abi/MessageTransmitter.json") + messageTransmitter, err := os.Open("../ethereum/abi/MessageTransmitter.json") require.Nil(t, err) messageTransmitterABI, err := abi.JSON(messageTransmitter) From 59e67f4cf5109f6d6c18a77bc45ce8f85c26d19b Mon Sep 17 00:00:00 2001 From: Dan Kanefsky Date: Tue, 23 Jan 2024 12:56:29 -0800 Subject: [PATCH 08/13] Add codec + fix context --- cmd/process.go | 10 +++------- cmd/root.go | 5 +++-- cosmos/codec.go | 5 ++--- ethereum/chain.go | 13 +++++++------ ethereum/listener_test.go | 5 ++++- go.mod | 2 +- main.go | 12 ++++++++++-- noble/chain.go | 28 ++++++++++++++++++++-------- noble/listener_test.go | 7 +++++-- types/chain.go | 2 -- 10 files changed, 55 insertions(+), 34 deletions(-) diff --git a/cmd/process.go b/cmd/process.go index f9be5de..1d6572d 100644 --- a/cmd/process.go +++ b/cmd/process.go @@ -4,8 +4,6 @@ import ( "context" "fmt" "os" - "os/signal" - "syscall" "time" "cosmossdk.io/log" @@ -29,11 +27,10 @@ var State = types.NewStateMap() var sequenceMap = types.NewSequenceMap() func Start(cmd *cobra.Command, args []string) { + // messageState processing queue var processingQueue = make(chan *types.TxState, 10000) - sigTerm := make(chan os.Signal, 1) - registeredDomains := make(map[types.Domain]types.Chain) for name, cfg := range Cfg.Chains { @@ -48,7 +45,7 @@ func Start(cmd *cobra.Command, args []string) { os.Exit(1) } - go c.StartListener(cmd.Context(), Logger, processingQueue, sigTerm) + go c.StartListener(cmd.Context(), Logger, processingQueue) if _, ok := registeredDomains[c.Domain()]; ok { Logger.Error("Duplicate domain found", "domain", c.Domain()) @@ -63,8 +60,7 @@ func Start(cmd *cobra.Command, args []string) { go StartProcessor(cmd.Context(), Cfg, Logger, registeredDomains, processingQueue, sequenceMap) } - signal.Notify(sigTerm, os.Interrupt, syscall.SIGTERM) - <-sigTerm + <-cmd.Context().Done() } // StartProcessor is the main processing pipeline. diff --git a/cmd/root.go b/cmd/root.go index f21a545..d98f8f5 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -1,6 +1,7 @@ package cmd import ( + "context" "fmt" "net/http" "os" @@ -30,8 +31,8 @@ var rootCmd = &cobra.Command{ Short: "A CLI tool for relaying CCTP messages", } -func Execute() { - if err := rootCmd.Execute(); err != nil { +func Execute(ctx context.Context) { + if err := rootCmd.ExecuteContext(ctx); err != nil { Logger.Error(err.Error()) os.Exit(1) } diff --git a/cosmos/codec.go b/cosmos/codec.go index 73e8ff0..6ac6f0d 100644 --- a/cosmos/codec.go +++ b/cosmos/codec.go @@ -7,15 +7,14 @@ import ( "github.com/cosmos/cosmos-sdk/codec/types" "github.com/cosmos/cosmos-sdk/std" "github.com/cosmos/cosmos-sdk/types/module" - - //"github.com/cosmos/cosmos-sdk/x/auth" + "github.com/cosmos/cosmos-sdk/x/auth" "github.com/cosmos/cosmos-sdk/x/auth/tx" // authz "github.com/cosmos/cosmos-sdk/x/authz/module" //"github.com/cosmos/cosmos-sdk/x/bank" ) var ModuleBasics = []module.AppModuleBasic{ - // auth.AppModuleBasic{}, + auth.AppModuleBasic{}, // authz.AppModuleBasic{}, // bank.AppModuleBasic{}, cctp.AppModuleBasic{}, diff --git a/ethereum/chain.go b/ethereum/chain.go index b228102..f5a8c9c 100644 --- a/ethereum/chain.go +++ b/ethereum/chain.go @@ -121,7 +121,6 @@ func (e *Ethereum) StartListener( ctx context.Context, logger log.Logger, processingQueue chan *types.TxState, - quit chan os.Signal, ) { logger = logger.With("chain", e.name, "chain_id", e.chainID, "domain", e.domain) @@ -138,19 +137,19 @@ func (e *Ethereum) StartListener( messageSent := messageTransmitterABI.Events["MessageSent"] - ethClient, err := ethclient.DialContext(context.Background(), e.wsURL) + ethClient, err := ethclient.DialContext(ctx, e.wsURL) if err != nil { logger.Error("unable to initialize ethereum client", "err", err) os.Exit(1) } - defer ethClient.Close() + // defer ethClient.Close() messageTransmitterAddress := common.HexToAddress(e.messageTransmitterAddress) etherReader := etherstream.Reader{Backend: ethClient} if e.startBlock == 0 { - header, err := ethClient.HeaderByNumber(context.Background(), nil) + header, err := ethClient.HeaderByNumber(ctx, nil) if err != nil { logger.Error("unable to retrieve latest eth block header", "err", err) os.Exit(1) @@ -172,7 +171,7 @@ func (e *Ethereum) StartListener( // websockets do not query history // https://github.com/ethereum/go-ethereum/issues/15063 - stream, sub, history, err := etherReader.QueryWithHistory(context.Background(), &query) + stream, sub, history, err := etherReader.QueryWithHistory(ctx, &query) if err != nil { logger.Error("unable to subscribe to logs", "err", err) os.Exit(1) @@ -199,10 +198,12 @@ func (e *Ethereum) StartListener( var txState *types.TxState for { select { - case <-quit: + case <-ctx.Done(): + ethClient.Close() return case err := <-sub.Err(): logger.Error("connection closed", "err", err) + ethClient.Close() os.Exit(1) case streamLog := <-stream: parsedMsg, err := types.EvmLogToMessageState(messageTransmitterABI, messageSent, &streamLog) diff --git a/ethereum/listener_test.go b/ethereum/listener_test.go index 4548a8c..32b076a 100644 --- a/ethereum/listener_test.go +++ b/ethereum/listener_test.go @@ -38,7 +38,10 @@ func TestStartListener(t *testing.T) { eth, err := ethCfg.Chain("ethereum") require.NoError(t, err) - go eth.StartListener(context.TODO(), logger, processingQueue, nil) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + go eth.StartListener(ctx, logger, processingQueue) time.Sleep(5 * time.Second) diff --git a/go.mod b/go.mod index adb5d77..9219d50 100644 --- a/go.mod +++ b/go.mod @@ -25,6 +25,7 @@ require ( github.com/gin-gonic/gin v1.8.1 github.com/pascaldekloe/etherstream v0.1.0 google.golang.org/grpc v1.57.0 + gopkg.in/yaml.v2 v2.4.0 ) require ( @@ -295,7 +296,6 @@ require ( google.golang.org/protobuf v1.31.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect honnef.co/go/tools v0.4.5 // indirect mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed // indirect mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b // indirect diff --git a/main.go b/main.go index 69c01be..109bf99 100644 --- a/main.go +++ b/main.go @@ -1,7 +1,15 @@ package main -import "github.com/strangelove-ventures/noble-cctp-relayer/cmd" +import ( + "context" + "os" + "os/signal" + + "github.com/strangelove-ventures/noble-cctp-relayer/cmd" +) func main() { - cmd.Execute() + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + cmd.Execute(ctx) } diff --git a/noble/chain.go b/noble/chain.go index 67ca19b..b026f59 100644 --- a/noble/chain.go +++ b/noble/chain.go @@ -7,7 +7,6 @@ import ( "errors" "fmt" "math/rand" - "os" "regexp" "strconv" "sync" @@ -166,7 +165,6 @@ func (n *Noble) StartListener( ctx context.Context, logger log.Logger, processingQueue chan *types.TxState, - quit chan os.Signal, ) { logger = logger.With("chain", n.Name(), "chain_id", n.chainID, "domain", n.Domain()) @@ -205,11 +203,13 @@ func (n *Noble) StartListener( // listen for new blocks go func() { + first := make(chan struct{}, 1) + first <- struct{}{} for { + timer := time.NewTimer(6 * time.Second) select { - case <-quit: - return - default: + case <-first: + timer.Stop() chainTip, err = n.chainTip(ctx) if err == nil { if chainTip >= currentBlock { @@ -219,7 +219,19 @@ func (n *Noble) StartListener( currentBlock = chainTip + 1 } } - time.Sleep(6 * time.Second) + case <-timer.C: + chainTip, err = n.chainTip(ctx) + if err == nil { + if chainTip >= currentBlock { + for i := currentBlock; i <= chainTip; i++ { + blockQueue <- i + } + currentBlock = chainTip + 1 + } + } + case <-ctx.Done(): + timer.Stop() + return } } }() @@ -229,7 +241,7 @@ func (n *Noble) StartListener( go func() { for { select { - case <-quit: + case <-ctx.Done(): return default: block := <-blockQueue @@ -255,7 +267,7 @@ func (n *Noble) StartListener( }() } - <-quit + <-ctx.Done() } func (n *Noble) chainTip(ctx context.Context) (uint64, error) { diff --git a/noble/listener_test.go b/noble/listener_test.go index 9de5bd1..459e5b2 100644 --- a/noble/listener_test.go +++ b/noble/listener_test.go @@ -20,7 +20,7 @@ var processingQueue chan *types.TxState func init() { var err error - cfg, err = cmd.Parse("../../.ignore/unit_tests.yaml") + cfg, err = cmd.Parse("../.ignore/testnet.yaml") if err != nil { panic(err) } @@ -35,7 +35,10 @@ func TestStartListener(t *testing.T) { n, err := cfg.Chains["noble"].(*noble.ChainConfig).Chain("noble") require.NoError(t, err) - go n.StartListener(context.TODO(), logger, processingQueue, nil) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + go n.StartListener(ctx, logger, processingQueue) time.Sleep(20 * time.Second) diff --git a/types/chain.go b/types/chain.go index 967b7b8..12dba4a 100644 --- a/types/chain.go +++ b/types/chain.go @@ -2,7 +2,6 @@ package types import ( "context" - "os" "cosmossdk.io/log" ) @@ -30,7 +29,6 @@ type Chain interface { ctx context.Context, logger log.Logger, processingQueue chan *TxState, - quit chan os.Signal, ) // Broadcast broadcasts CCTP mint messages to the chain. From 5ec07761571e657e596eebabac20050ffef619aa Mon Sep 17 00:00:00 2001 From: Dan Kanefsky Date: Tue, 23 Jan 2024 13:11:54 -0800 Subject: [PATCH 09/13] more context's --- cosmos/query.go | 2 +- noble/chain.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cosmos/query.go b/cosmos/query.go index 35073f8..448d728 100644 --- a/cosmos/query.go +++ b/cosmos/query.go @@ -46,7 +46,7 @@ func (cc *CosmosProvider) QueryUsedNonce(ctx context.Context, sourceDomain types Nonce: nonce, } - _, err := qc.UsedNonce(context.Background(), params) + _, err := qc.UsedNonce(ctx, params) if err != nil { if err.Error() == "rpc error: code = NotFound desc = rpc error: code = NotFound desc = not found: key not found" { return false, nil diff --git a/noble/chain.go b/noble/chain.go index b026f59..6fcf883 100644 --- a/noble/chain.go +++ b/noble/chain.go @@ -389,7 +389,7 @@ func (n *Noble) Broadcast( return fmt.Errorf("failed to proto encode tx: %w", err) } - rpcResponse, err := n.cc.RPCClient.BroadcastTxSync(context.Background(), txBytes) + rpcResponse, err := n.cc.RPCClient.BroadcastTxSync(ctx, txBytes) if err != nil || (rpcResponse != nil && rpcResponse.Code != 0) { // Log the error logger.Error(fmt.Sprintf("error during broadcast: %s", getErrorString(err, rpcResponse))) From 4c6f4f30e1397355e86c7ec18b56176e634eebed Mon Sep 17 00:00:00 2001 From: Dan Kanefsky Date: Wed, 24 Jan 2024 11:11:01 -0800 Subject: [PATCH 10/13] fix noble decoding --- noble/message_state.go | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/noble/message_state.go b/noble/message_state.go index bf99555..8866c6a 100644 --- a/noble/message_state.go +++ b/noble/message_state.go @@ -20,16 +20,26 @@ func txToMessageState(tx *ctypes.ResultTx) ([]*types.MessageState, error) { var messageStates []*types.MessageState - for i, event := range tx.TxResult.Events { + for _, event := range tx.TxResult.Events { if event.Type == "circle.cctp.v1.MessageSent" { //fmt.Printf("Saw cctp message %s - %d:%d\n", tx., i, j) var parsed bool var parseErrs error for _, attr := range event.Attributes { - if attr.Key == "message" { - fmt.Printf("Saw message attribute %s - %d\n", tx.Hash, i) - encoded := attr.Value[1 : len(attr.Value)-1] - rawMessageSentBytes, err := base64.StdEncoding.DecodeString(encoded) + decodedKey, err := base64.StdEncoding.DecodeString(attr.Key) + if err != nil { + parseErrs = errors.Join(parseErrs, fmt.Errorf("failed to decode attribue key: %w", err)) + } + if string(decodedKey) == "message" { + // fmt.Printf("Saw message attribute %s - %d\n", tx.Hash, i) + decodedValue, err := base64.StdEncoding.DecodeString(attr.Value) + if err != nil { + parseErrs = errors.Join(parseErrs, fmt.Errorf("error decoding attr.value: %w", err)) + continue + } + encoded := decodedValue[1 : len(decodedValue)-1] + // Because we are using cometBFT v0.38, we need to decode the value twice. + rawMessageSentBytes, err := base64.StdEncoding.DecodeString(string(encoded)) if err != nil { parseErrs = errors.Join(parseErrs, fmt.Errorf("failed to decode message: %w", err)) continue From 91db2e99d58f30782657d2558d1d9add8d7247bc Mon Sep 17 00:00:00 2001 From: Dan Kanefsky Date: Wed, 24 Jan 2024 11:14:35 -0800 Subject: [PATCH 11/13] add back domain --- ethereum/chain.go | 1 + ethereum/config.go | 2 ++ 2 files changed, 3 insertions(+) diff --git a/ethereum/chain.go b/ethereum/chain.go index f5a8c9c..ea06c32 100644 --- a/ethereum/chain.go +++ b/ethereum/chain.go @@ -52,6 +52,7 @@ type Ethereum struct { func NewChain( name string, + domain types.Domain, chainID int64, rpcURL string, wsURL string, diff --git a/ethereum/config.go b/ethereum/config.go index 1dc2558..74ae6fe 100644 --- a/ethereum/config.go +++ b/ethereum/config.go @@ -9,6 +9,7 @@ var _ types.ChainConfig = (*ChainConfig)(nil) type ChainConfig struct { RPC string `yaml:"rpc"` WS string `yaml:"ws"` + domain types.Domain ChainID int64 `yaml:"chain-id"` MessageTransmitter string `yaml:"message-transmitter"` @@ -25,6 +26,7 @@ type ChainConfig struct { func (c *ChainConfig) Chain(name string) (types.Chain, error) { return NewChain( name, + c.domain, c.ChainID, c.RPC, c.WS, From 7fc6b0790d72cfc2f7b8a161f89ca5a6ed284f75 Mon Sep 17 00:00:00 2001 From: Dan Kanefsky Date: Wed, 24 Jan 2024 11:25:33 -0800 Subject: [PATCH 12/13] update sample configs --- config/sample-app-config.yaml | 39 ---------------------- config/{sample.yaml => sample-config.yaml} | 14 ++++---- config/sample-integration-config.yaml | 13 ++++++-- 3 files changed, 17 insertions(+), 49 deletions(-) delete mode 100644 config/sample-app-config.yaml rename config/{sample.yaml => sample-config.yaml} (78%) diff --git a/config/sample-app-config.yaml b/config/sample-app-config.yaml deleted file mode 100644 index c1ececd..0000000 --- a/config/sample-app-config.yaml +++ /dev/null @@ -1,39 +0,0 @@ -networks: - source: - ethereum: - enabled: true - domain-id: 0 - rpc: "wss://goerli.infura.io/ws/v3/" - message-transmitter: "0x26413e8157CD32011E726065a5462e97dD4d03D9" - request-queue-size: 1000 - start-block: 0 # set to 0 to default to latest block - lookback-period: 20 # historical blocks to look back on launch - destination: - noble: - domain-id: 4 - api: "https://lcd.testnet.noble.strange.love:443" - rpc: "https://rpc.testnet.noble.strange.love:443" - chain-id: "grand-1" - gas-limit: 200000 - broadcast-retries: 5 # number of times to attempt the broadcast - broadcast-retry-interval: 5 # time between retries in seconds - filter-forwards-by-ibc-channel: false - forwarding-channel-whitelist: - - "channel-10" # osmo-test-5 - - "channel-15" # dydx-testnet-2 - # source domain id -> destination domain id - enabled-routes: - 0: 4 # ethereum to noble - # destination domain -> minter metadata - minters: - 4: - minter-address: "noble1...." - minter-mnemonic: "12345" # hex encoded, no prepended 0x -circle: - attestation-base-url: "https://iris-api-sandbox.circle.com/attestations/" - fetch-retries: 10 # additional times to fetch an attestation - fetch-retry-interval: 10 # time between retries in seconds -processor-worker-count: 16 -api: - trusted-proxies: - - "1.2.3.4" # add trusted proxy IPs here \ No newline at end of file diff --git a/config/sample.yaml b/config/sample-config.yaml similarity index 78% rename from config/sample.yaml rename to config/sample-config.yaml index 1c8bd1d..d6169e4 100644 --- a/config/sample.yaml +++ b/config/sample-config.yaml @@ -1,21 +1,22 @@ chains: ethereum: chain-id: 5 - rpc: "https://goerli.infura.io/v3/apiKey" - ws: "wss://goerli.infura.io/ws/v3/apiKey" + domain: 0 + rpc: # Ethereum RPC + ws: # Ethereum Websocket message-transmitter: "0x26413e8157CD32011E726065a5462e97dD4d03D9" - start-block: 9737196 + start-block: 0 # set to 0 to default to latest block lookback-period: 5 # historical blocks to look back on launch broadcast-retries: 5 # number of times to attempt the broadcast broadcast-retry-interval: 10 # time between retries in seconds - minter-private-key: "privateKey" + minter-private-key: # private key noble: - rpc: "https://rpc.testnet.noble.strange.love:443" + rpc: #noble RPC; for stability, use a reliable private node chain-id: "grand-1" start-block: 0 # set to 0 to default to latest block @@ -27,8 +28,7 @@ chains: broadcast-retries: 5 # number of times to attempt the broadcast broadcast-retry-interval: 5 # time between retries in seconds - # hex encoded - minter-private-key: "hexEncodedPrivateKey" + minter-private-key: # hex encoded privateKey # source domain id -> destination domain id enabled-routes: diff --git a/config/sample-integration-config.yaml b/config/sample-integration-config.yaml index b7ca432..467a0c1 100644 --- a/config/sample-integration-config.yaml +++ b/config/sample-integration-config.yaml @@ -1,5 +1,12 @@ +# This file is for integration testing. +# These extra wallets keep the relayer wallet separate from the wallet used to send test transactions + networks: ethereum: - rpc: "https://goerli.infura.io/v3/" - address: "noble1...." - private_key: "..." # hex encoded, no 0x prefix \ No newline at end of file + # Sepolia + address: + private_key: + + noble: + address: + private_key: \ No newline at end of file From 97af3585d0200c565811393b46e08bd5a614dab7 Mon Sep 17 00:00:00 2001 From: Dan Kanefsky Date: Thu, 25 Jan 2024 15:54:39 -0800 Subject: [PATCH 13/13] integration testing --- circle/attestation.go | 2 +- ethereum/chain.go | 2 +- integration/config.go | 88 ++++++++----- ...eth_burn_to_noble_mint_and_forward_test.go | 103 --------------- integration/eth_burn_to_noble_mint_test.go | 59 ++++++--- ...end_test.go => eth_multi_send_test.go.bak} | 0 integration/noble_burn_to_eth_mint_test.go | 121 ++++-------------- ...d_test.go => noble_multi_send_test.go.bak} | 0 integration/util.go | 86 +++++++++++-- 9 files changed, 196 insertions(+), 265 deletions(-) delete mode 100644 integration/eth_burn_to_noble_mint_and_forward_test.go rename integration/{eth_multi_send_test.go => eth_multi_send_test.go.bak} (100%) rename integration/{noble_multi_send_test.go => noble_multi_send_test.go.bak} (100%) diff --git a/circle/attestation.go b/circle/attestation.go index c51da8a..ca069fc 100644 --- a/circle/attestation.go +++ b/circle/attestation.go @@ -23,7 +23,7 @@ func CheckAttestation(attestationURL string, logger log.Logger, irisLookupId str return nil } if rawResponse.StatusCode != http.StatusOK { - logger.Debug("non 200 response received") + logger.Debug("non 200 response received from Circles attestation API") return nil } body, err := io.ReadAll(rawResponse.Body) diff --git a/ethereum/chain.go b/ethereum/chain.go index ea06c32..749006f 100644 --- a/ethereum/chain.go +++ b/ethereum/chain.go @@ -263,7 +263,7 @@ MsgLoop: for _, msg := range msgs { if msg.Status == types.Complete { - continue + continue MsgLoop } attestationBytes, err := hex.DecodeString(msg.Attestation[2:]) diff --git a/integration/config.go b/integration/config.go index cb6e600..ee30584 100644 --- a/integration/config.go +++ b/integration/config.go @@ -6,50 +6,63 @@ import ( "cosmossdk.io/log" "github.com/rs/zerolog" "github.com/strangelove-ventures/noble-cctp-relayer/cmd" - "github.com/strangelove-ventures/noble-cctp-relayer/cmd/noble" + "github.com/strangelove-ventures/noble-cctp-relayer/ethereum" + "github.com/strangelove-ventures/noble-cctp-relayer/noble" "github.com/strangelove-ventures/noble-cctp-relayer/types" - "gopkg.in/yaml.v3" ) -var testCfg *types.Config // for testing secrets -var cfg *types.Config // app config +var cfg *types.Config // app config +var integrationWallets *IntegrationWallets // for testing secrets + +var nobleCfg *noble.ChainConfig +var ethCfg *ethereum.ChainConfig + var logger log.Logger var err error -// goerli -const TokenMessengerAddress = "0xd0c3da58f55358142b8d3e06c1c30c5c6114efe8" -const TokenMessengerWithMetadataAddress = "0x1ae045d99236365cbdc1855acd2d2cfc232d04d1" -const UsdcAddress = "0x07865c6e87b9f70255377e024ace6630c1eaa37f" +var nobleChain types.Chain +var ethChain types.Chain + +// Sepolia +const TokenMessengerAddress = "0x9f3B8679c73C2Fef8b59B4f3444d4e156fb70AA5" +const UsdcAddress = "0x1c7D4B196Cb0C7B01d743Fbc6116a902379C7238" var sequenceMap *types.SequenceMap -func setupTest() func() { - // setup - testCfg, err = cmd.Parse("../.ignore/integration.yaml") - cfg, err = cmd.Parse("../.ignore/testnet.yaml") +func setupTestIntegration() func() { logger = log.NewLogger(os.Stdout, log.LevelOption(zerolog.DebugLevel)) - _, nextMinterSequence, err := noble.GetNobleAccountNumberSequence( - cfg.Networks.Destination.Noble.API, - cfg.Networks.Minters[4].MinterAddress) + // cctp relayer app config setup for sepolia netowrk + cfg, err = cmd.Parse("../.ignore/testnet.yaml") + if err != nil { + logger.Error("Error parsing relayer config") + os.Exit(1) + } + // extra wallets to keep relayer wallet separate from test transaction + // see config/sample-integration-config.yaml + err = ParseIntegration("../.ignore/integration.yaml") if err != nil { - logger.Error("Error retrieving account sequence") + logger.Error("Error parsing integration wallets") os.Exit(1) } + + nobleCfg = cfg.Chains["noble"].(*noble.ChainConfig) + ethCfg = cfg.Chains["ethereum"].(*ethereum.ChainConfig) + sequenceMap = types.NewSequenceMap() - sequenceMap.Put(uint32(4), nextMinterSequence) - - for i, minter := range cfg.Networks.Minters { - switch i { - case 0: - minter.MinterAddress = "0x971c54a6Eb782fAccD00bc3Ed5E934Cc5bD8e3Ef" - cfg.Networks.Minters[0] = minter - case 4: - minter.MinterAddress = "noble1ar2gaqww6aphxd9qve5qglj8kqq96je6a4yrhj" - cfg.Networks.Minters[4] = minter - } + + nobleChain, err = nobleCfg.Chain("noble") + if err != nil { + logger.Error("Error creating new chain", "err", err) + os.Exit(1) + } + + ethChain, err = ethCfg.Chain("eth") + if err != nil { + logger.Error("Error creating new chain", "err", err) + os.Exit(1) } return func() { @@ -57,22 +70,29 @@ func setupTest() func() { } } -type Config struct { +// Wallets used for integration testing +type IntegrationWallets struct { Networks struct { Ethereum struct { - RPC string `yaml:"rpc"` + Address string `yaml:"address"` PrivateKey string `yaml:"private_key"` } `yaml:"ethereum"` Noble struct { - RPC string `yaml:"rpc"` + Address string `yaml:"address"` PrivateKey string `yaml:"private_key"` } `yaml:"noble"` } `yaml:"networks"` } -func Parse(file string) (cfg Config) { - data, _ := os.ReadFile(file) - _ = yaml.Unmarshal(data, &cfg) +func ParseIntegration(file string) (err error) { + data, err := os.ReadFile(file) + if err != nil { + return err + } + err = yaml.Unmarshal(data, &integrationWallets) + if err != nil { + return err + } - return + return nil } diff --git a/integration/eth_burn_to_noble_mint_and_forward_test.go b/integration/eth_burn_to_noble_mint_and_forward_test.go deleted file mode 100644 index 3e8b332..0000000 --- a/integration/eth_burn_to_noble_mint_and_forward_test.go +++ /dev/null @@ -1,103 +0,0 @@ -package integration_testing - -import ( - "context" - "fmt" - "math/big" - "testing" - "time" - - "github.com/cosmos/cosmos-sdk/testutil/testdata" - "github.com/cosmos/cosmos-sdk/types/bech32" - "github.com/ethereum/go-ethereum/accounts/abi/bind" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethclient" - "github.com/strangelove-ventures/noble-cctp-relayer/cmd" - eth "github.com/strangelove-ventures/noble-cctp-relayer/cmd/ethereum" - "github.com/strangelove-ventures/noble-cctp-relayer/types" - "github.com/stretchr/testify/require" -) - -// TestEthBurnToNobleMintAndForward generates a depositForBurn on Ethereum Goerli and mints + forwards on Noble -func TestEthBurnToNobleMintAndForward(t *testing.T) { - setupTest() - - p := cmd.NewProcessor() - - // start up relayer - cfg.Networks.Source.Ethereum.StartBlock = getEthereumLatestBlockHeight(t) - cfg.Networks.Source.Ethereum.LookbackPeriod = 0 - - fmt.Println("Starting relayer...") - processingQueue := make(chan *types.MessageState, 10) - go eth.StartListener(cfg, logger, processingQueue) - go p.StartProcessor(context.TODO(), cfg, logger, processingQueue, sequenceMap) - - fmt.Println("Building Ethereum depositForBurnWithMetadata txn...") - _, _, cosmosAddress := testdata.KeyTestPubAddr() - nobleAddress, _ := bech32.ConvertAndEncode("noble", cosmosAddress) - fmt.Println("Intermediately minting on Noble to " + nobleAddress) - - _, _, cosmosAddress2 := testdata.KeyTestPubAddr() - dydxAddress, _ := bech32.ConvertAndEncode("dydx", cosmosAddress2) - fmt.Println("Forwarding funds to " + dydxAddress) - - // verify dydx usdc amount - originalDydx := getDydxBalance(dydxAddress) - - // deposit for burn with metadata - client, err := ethclient.Dial(testCfg.Networks.Ethereum.RPC) - require.Nil(t, err) - defer client.Close() - - privateKey, err := crypto.HexToECDSA(testCfg.Networks.Ethereum.PrivateKey) - require.Nil(t, err) - auth, err := bind.NewKeyedTransactorWithChainID(privateKey, big.NewInt(5)) - require.Nil(t, err) - - tokenMessengerWithMetadata, err := cmd.NewTokenMessengerWithMetadata(common.HexToAddress(TokenMessengerWithMetadataAddress), client) - require.Nil(t, err) - - mintRecipientPadded := append([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, cosmosAddress...) - require.Nil(t, err) - - erc20, err := NewERC20(common.HexToAddress(UsdcAddress), client) - _, err = erc20.Approve(auth, common.HexToAddress(TokenMessengerWithMetadataAddress), big.NewInt(99999)) - require.Nil(t, err) - - channel := uint64(20) - destinationBech32Prefix := - append([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, []byte("dydx")...) - destinationRecipient := append([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, cosmosAddress2...) - - var BurnAmount = big.NewInt(1) - - tx, err := tokenMessengerWithMetadata.DepositForBurn( - auth, - channel, // channel - [32]byte(destinationBech32Prefix), // destinationBech32Prefix - [32]byte(destinationRecipient), // destinationRecipient - BurnAmount, // amount - [32]byte(mintRecipientPadded), // mint recipient - common.HexToAddress(UsdcAddress), // burn token - []byte{}, // memo - ) - if err != nil { - logger.Error("Failed to update value: %v", err) - } - - time.Sleep(5 * time.Second) - fmt.Printf("Update pending: https://goerli.etherscan.io/tx/%s\n", tx.Hash().String()) - - fmt.Println("Checking dydx wallet...") - for i := 0; i < 250; i++ { - if originalDydx+BurnAmount.Uint64() == getDydxBalance(dydxAddress) { - fmt.Println("Successfully minted at https://testnet.mintscan.io/dydx-testnet/account/" + dydxAddress) - return - } - time.Sleep(1 * time.Second) - } - // verify dydx balance - require.Equal(t, originalDydx+BurnAmount.Uint64(), getDydxBalance(dydxAddress)) -} diff --git a/integration/eth_burn_to_noble_mint_test.go b/integration/eth_burn_to_noble_mint_test.go index 5b8d777..5c52194 100644 --- a/integration/eth_burn_to_noble_mint_test.go +++ b/integration/eth_burn_to_noble_mint_test.go @@ -14,52 +14,66 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethclient" "github.com/strangelove-ventures/noble-cctp-relayer/cmd" - eth "github.com/strangelove-ventures/noble-cctp-relayer/cmd/ethereum" + "github.com/strangelove-ventures/noble-cctp-relayer/cosmos" + "github.com/strangelove-ventures/noble-cctp-relayer/ethereum/contracts" "github.com/strangelove-ventures/noble-cctp-relayer/types" "github.com/stretchr/testify/require" ) +const uusdcDenom = "uusdc" + // TestEthBurnToNobleMint generates a depositForBurn on Ethereum Goerli and mints on Noble func TestEthBurnToNobleMint(t *testing.T) { - setupTest() + ctx := context.Background() + setupTestIntegration() + + fmt.Println("Starting relayer...") + processingQueue := make(chan *types.TxState, 10) - p := cmd.NewProcessor() + registeredDomains := make(map[types.Domain]types.Chain) + registeredDomains[0] = ethChain + registeredDomains[4] = nobleChain - // start up relayer - cfg.Networks.Source.Ethereum.StartBlock = getEthereumLatestBlockHeight(t) - cfg.Networks.Source.Ethereum.LookbackPeriod = 0 + nobleChain.InitializeBroadcaster(ctx, logger, sequenceMap) - fmt.Println("Starting relayer...") - processingQueue := make(chan *types.MessageState, 10) - go eth.StartListener(cfg, logger, processingQueue) - go p.StartProcessor(context.TODO(), cfg, logger, processingQueue, sequenceMap) + go ethChain.StartListener(ctx, logger, processingQueue) + go cmd.StartProcessor(ctx, cfg, logger, registeredDomains, processingQueue, sequenceMap) - fmt.Println("Building Ethereum depositForBurnWithMetadata txn...") + fmt.Println("Building Ethereum depositForBurn txn...") _, _, cosmosAddress := testdata.KeyTestPubAddr() nobleAddress, _ := bech32.ConvertAndEncode("noble", cosmosAddress) fmt.Println("Minting on Noble to https://testnet.mintscan.io/noble-testnet/account/" + nobleAddress) // verify noble usdc amount - originalNobleBalance := getNobleBalance(nobleAddress) + cc, err := cosmos.NewProvider(nobleCfg.RPC) + require.Nil(t, err) + // originalNobleBalance := getNobleBalance(nobleAddress) + originalNobleBalance, err := getNobleAccountBalance(ctx, cc, nobleAddress, uusdcDenom) + require.NoError(t, err) // deposit for burn with metadata - client, err := ethclient.Dial(testCfg.Networks.Ethereum.RPC) + client, err := ethclient.Dial(ethCfg.RPC) require.Nil(t, err) defer client.Close() - privateKey, err := crypto.HexToECDSA(testCfg.Networks.Ethereum.PrivateKey) + privateKey, err := crypto.HexToECDSA(integrationWallets.Networks.Ethereum.PrivateKey) + require.Nil(t, err) - auth, err := bind.NewKeyedTransactorWithChainID(privateKey, big.NewInt(5)) + + sepoliaChainID := big.NewInt(ethCfg.ChainID) + auth, err := bind.NewKeyedTransactorWithChainID(privateKey, sepoliaChainID) require.Nil(t, err) - tokenMessenger, err := cmd.NewTokenMessenger(common.HexToAddress(TokenMessengerAddress), client) + tokenMessenger, err := contracts.NewTokenMessenger(common.HexToAddress(TokenMessengerAddress), client) require.Nil(t, err) mintRecipientPadded := append([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, cosmosAddress...) require.Nil(t, err) erc20, err := NewERC20(common.HexToAddress(UsdcAddress), client) - _, err = erc20.Approve(auth, common.HexToAddress(TokenMessengerWithMetadataAddress), big.NewInt(99999)) + require.NoError(t, err) + + _, err = erc20.Approve(auth, common.HexToAddress(TokenMessengerAddress), big.NewInt(99999)) require.Nil(t, err) var burnAmount = big.NewInt(1) @@ -72,20 +86,23 @@ func TestEthBurnToNobleMint(t *testing.T) { common.HexToAddress(UsdcAddress), ) if err != nil { - logger.Error("Failed to update value: %v", err) + logger.Error("Failed to update value", "err", err) } time.Sleep(5 * time.Second) - fmt.Printf("Update pending: https://goerli.etherscan.io/tx/%s\n", tx.Hash().String()) + fmt.Printf("Update pending: https://sepolia.etherscan.io/tx/%s\n", tx.Hash().String()) + var newBalance uint64 fmt.Println("Checking noble wallet...") for i := 0; i < 250; i++ { - if originalNobleBalance+burnAmount.Uint64() == getNobleBalance(nobleAddress) { + newBalance, err = getNobleAccountBalance(ctx, cc, nobleAddress, uusdcDenom) + require.NoError(t, err) + if originalNobleBalance+burnAmount.Uint64() == newBalance { fmt.Println("Successfully minted at https://testnet.mintscan.io/noble-testnet/account/" + nobleAddress) return } time.Sleep(1 * time.Second) } // verify noble balance - require.Equal(t, originalNobleBalance+burnAmount.Uint64(), getNobleBalance(nobleAddress)) + require.Equal(t, originalNobleBalance+burnAmount.Uint64(), newBalance) } diff --git a/integration/eth_multi_send_test.go b/integration/eth_multi_send_test.go.bak similarity index 100% rename from integration/eth_multi_send_test.go rename to integration/eth_multi_send_test.go.bak diff --git a/integration/noble_burn_to_eth_mint_test.go b/integration/noble_burn_to_eth_mint_test.go index cfbe32c..38b2957 100644 --- a/integration/noble_burn_to_eth_mint_test.go +++ b/integration/noble_burn_to_eth_mint_test.go @@ -3,22 +3,12 @@ package integration_testing import ( "context" "encoding/hex" - "encoding/json" - "errors" "fmt" - "io" - "log" - "math/big" - "net/http" - "strconv" - "strings" "testing" "time" "cosmossdk.io/math" nobletypes "github.com/circlefin/noble-cctp/x/cctp/types" - rpchttp "github.com/cometbft/cometbft/rpc/client/http" - libclient "github.com/cometbft/cometbft/rpc/jsonrpc/client" sdkClient "github.com/cosmos/cosmos-sdk/client" clientTx "github.com/cosmos/cosmos-sdk/client/tx" "github.com/cosmos/cosmos-sdk/codec" @@ -28,38 +18,41 @@ import ( "github.com/cosmos/cosmos-sdk/types/tx/signing" xauthsigning "github.com/cosmos/cosmos-sdk/x/auth/signing" xauthtx "github.com/cosmos/cosmos-sdk/x/auth/tx" - "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethclient" "github.com/strangelove-ventures/noble-cctp-relayer/cmd" - "github.com/strangelove-ventures/noble-cctp-relayer/cmd/noble" + "github.com/strangelove-ventures/noble-cctp-relayer/cosmos" "github.com/strangelove-ventures/noble-cctp-relayer/types" + "github.com/stretchr/testify/require" ) // TestNobleBurnToEthMint generates and broadcasts a depositForBurn on Noble // and broadcasts on Ethereum Goerli func TestNobleBurnToEthMint(t *testing.T) { - setupTest() - p := cmd.NewProcessor() - - cfg.Networks.Source.Ethereum.Enabled = false + ctx := context.Background() - // start up relayer - cfg.Networks.Source.Noble.StartBlock = getNobleLatestBlockHeight() + setupTestIntegration() fmt.Println("Starting relayer...") - processingQueue := make(chan *types.MessageState, 10) - go noble.StartListener(cfg, logger, processingQueue) - go p.StartProcessor(context.TODO(), cfg, logger, processingQueue, sequenceMap) + processingQueue := make(chan *types.TxState, 10) + + registeredDomains := make(map[types.Domain]types.Chain) + registeredDomains[0] = ethChain + registeredDomains[4] = nobleChain + + err := ethChain.InitializeBroadcaster(ctx, logger, sequenceMap) + require.NoError(t, err) + + go nobleChain.StartListener(ctx, logger, processingQueue) + go cmd.StartProcessor(context.TODO(), cfg, logger, registeredDomains, processingQueue, sequenceMap) fmt.Println("Building Noble depositForBurn txn...") - ethDestinationAddress := "0x971c54a6Eb782fAccD00bc3Ed5E934Cc5bD8e3Ef" - fmt.Println("Minting on Ethereum to https://goerli.etherscan.io/address/" + ethDestinationAddress) + ethDestinationAddress := integrationWallets.Networks.Ethereum.Address + fmt.Println("Minting on Ethereum to https://sepolia.etherscan.io/address/" + ethDestinationAddress) // verify ethereum usdc amount - client, _ := ethclient.Dial(testCfg.Networks.Ethereum.RPC) + client, _ := ethclient.Dial(ethCfg.RPC) defer client.Close() originalEthBalance := getEthBalance(client, ethDestinationAddress) @@ -74,7 +67,7 @@ func TestNobleBurnToEthMint(t *testing.T) { } txBuilder := sdkContext.TxConfig.NewTxBuilder() // get priv key - keyBz, _ := hex.DecodeString(testCfg.Networks.Noble.PrivateKey) + keyBz, _ := hex.DecodeString(integrationWallets.Networks.Noble.PrivateKey) privKey := secp256k1.PrivKey{Key: keyBz} nobleAddress, err := bech32.ConvertAndEncode("noble", privKey.PubKey().Address()) require.Nil(t, err) @@ -94,13 +87,14 @@ func TestNobleBurnToEthMint(t *testing.T) { err = txBuilder.SetMsgs(burnMsg) require.Nil(t, err) - txBuilder.SetGasLimit(cfg.Networks.Destination.Noble.GasLimit) + txBuilder.SetGasLimit(nobleCfg.GasLimit) // sign + broadcast txn - rpcClient, err := NewRPCClient(testCfg.Networks.Noble.RPC, 10*time.Second) + cc, err := cosmos.NewProvider(nobleCfg.RPC) require.Nil(t, err) - accountNumber, accountSequence, err := GetNobleAccountNumberSequence(cfg.Networks.Destination.Noble.API, nobleAddress) + accountNumber, accountSequence, err := getNobleAccountNumberSequenceGRPC(cc, nobleAddress) + require.Nil(t, err) sigV2 := signing.SignatureV2{ @@ -113,7 +107,7 @@ func TestNobleBurnToEthMint(t *testing.T) { } signerData := xauthsigning.SignerData{ - ChainID: cfg.Networks.Destination.Noble.ChainId, + ChainID: nobleCfg.ChainID, AccountNumber: uint64(accountNumber), Sequence: uint64(accountSequence), } @@ -127,6 +121,7 @@ func TestNobleBurnToEthMint(t *testing.T) { sdkContext.TxConfig, uint64(accountSequence), ) + require.Nil(t, err) err = txBuilder.SetSignatures(sigV2) require.Nil(t, err) @@ -135,78 +130,18 @@ func TestNobleBurnToEthMint(t *testing.T) { txBytes, err := sdkContext.TxConfig.TxEncoder()(txBuilder.GetTx()) require.Nil(t, err) - rpcResponse, err := rpcClient.BroadcastTxSync(context.Background(), txBytes) + rpcResponse, err := cc.RPCClient.BroadcastTxSync(context.Background(), txBytes) require.Nil(t, err) fmt.Printf("Update pending: https://testnet.mintscan.io/noble-testnet/txs/%s\n", rpcResponse.Hash.String()) fmt.Println("Checking eth wallet...") for i := 0; i < 60; i++ { if originalEthBalance+burnAmount.Uint64() == getEthBalance(client, ethDestinationAddress) { - fmt.Println("Successfully minted at https://goerli.etherscan.io/address/" + ethDestinationAddress) + fmt.Println("Successfully minted at https://sepolia.etherscan.io/address/" + ethDestinationAddress) return } - time.Sleep(1 * time.Second) + time.Sleep(3 * time.Second) } // verify eth balance require.Equal(t, originalEthBalance+burnAmount.Uint64(), getEthBalance(client, ethDestinationAddress)) } - -func getEthBalance(client *ethclient.Client, address string) uint64 { - accountAddress := common.HexToAddress(address) - tokenAddress := common.HexToAddress("0x07865c6e87b9f70255377e024ace6630c1eaa37f") // USDC goerli - erc20ABI := `[{"constant":true,"inputs":[{"name":"_owner","type":"address"}],"name":"balanceOf","outputs":[{"name":"balance","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"}]` - parsedABI, err := abi.JSON(strings.NewReader(erc20ABI)) - if err != nil { - log.Fatalf("Failed to parse contract ABI: %v", err) - } - - data, err := parsedABI.Pack("balanceOf", accountAddress) - if err != nil { - log.Fatalf("Failed to pack data into ABI interface: %v", err) - } - - result, err := client.CallContract(context.Background(), ethereum.CallMsg{To: &tokenAddress, Data: data}, nil) - if err != nil { - log.Fatalf("Failed to call contract: %v", err) - } - - balance := new(big.Int) - err = parsedABI.UnpackIntoInterface(&balance, "balanceOf", result) - if err != nil { - log.Fatalf("Failed to unpack data from ABI interface: %v", err) - } - - // Convert to uint64 - return balance.Uint64() -} - -// NewRPCClient initializes a new tendermint RPC client connected to the specified address. -func NewRPCClient(addr string, timeout time.Duration) (*rpchttp.HTTP, error) { - httpClient, err := libclient.DefaultHTTPClient(addr) - if err != nil { - return nil, err - } - httpClient.Timeout = timeout - rpcClient, err := rpchttp.NewWithClient(addr, "/websocket", httpClient) - if err != nil { - return nil, err - } - return rpcClient, nil -} - -func GetNobleAccountNumberSequence(urlBase string, address string) (int64, int64, error) { - rawResp, err := http.Get(fmt.Sprintf("%s/cosmos/auth/v1beta1/accounts/%s", urlBase, address)) - if err != nil { - return 0, 0, errors.New("unable to fetch account number, sequence") - } - body, _ := io.ReadAll(rawResp.Body) - var resp types.AccountResp - err = json.Unmarshal(body, &resp) - if err != nil { - return 0, 0, errors.New("unable to parse account number, sequence") - } - accountNumber, _ := strconv.ParseInt(resp.AccountNumber, 10, 0) - accountSequence, _ := strconv.ParseInt(resp.Sequence, 10, 0) - - return accountNumber, accountSequence, nil -} diff --git a/integration/noble_multi_send_test.go b/integration/noble_multi_send_test.go.bak similarity index 100% rename from integration/noble_multi_send_test.go rename to integration/noble_multi_send_test.go.bak diff --git a/integration/util.go b/integration/util.go index 1b0421a..6af1569 100644 --- a/integration/util.go +++ b/integration/util.go @@ -4,14 +4,27 @@ import ( "context" "encoding/json" "fmt" - "github.com/ethereum/go-ethereum/ethclient" - "github.com/stretchr/testify/require" "io" + "log" + "math/big" "net/http" "strconv" + "strings" "testing" + + authtypes "github.com/cosmos/cosmos-sdk/x/auth/types" + bankTypes "github.com/cosmos/cosmos-sdk/x/bank/types" + ethereum "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/accounts/abi" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethclient" + "github.com/strangelove-ventures/noble-cctp-relayer/cosmos" + "github.com/stretchr/testify/require" ) +// USDC Token Address on Sepolia +const usdcTokenAddress = "0x1c7D4B196Cb0C7B01d743Fbc6116a902379C7238" + func getDydxBalance(address string) uint64 { rawResponse, _ := http.Get(fmt.Sprintf( "https://dydx-testnet-api.polkachu.com/cosmos/bank/v1beta1/balances/%s/by_denom?denom=ibc/8E27BA2D5493AF5636760E354E46004562C46AB7EC0CC4C1CA14E9E20E2545B5", address)) @@ -23,7 +36,7 @@ func getDydxBalance(address string) uint64 { } func getEthereumLatestBlockHeight(t *testing.T) uint64 { - client, err := ethclient.Dial(cfg.Networks.Source.Ethereum.RPC) + client, err := ethclient.Dial(ethCfg.RPC) require.Nil(t, err) header, err := client.HeaderByNumber(context.Background(), nil) @@ -31,15 +44,6 @@ func getEthereumLatestBlockHeight(t *testing.T) uint64 { return header.Number.Uint64() } -func getNobleBalance(address string) uint64 { - rawResponse, _ := http.Get(fmt.Sprintf("https://lcd.testnet.noble.strange.love/cosmos/bank/v1beta1/balances/%s/by_denom?denom=uusdc", address)) - body, _ := io.ReadAll(rawResponse.Body) - response := BalanceResponse{} - _ = json.Unmarshal(body, &response) - result, _ := strconv.ParseInt(response.Balance.Amount, 10, 0) - return uint64(result) -} - func getNobleLatestBlockHeight() uint64 { rawResponse, _ := http.Get("https://rpc.testnet.noble.strange.love/block") body, _ := io.ReadAll(rawResponse.Body) @@ -48,3 +52,61 @@ func getNobleLatestBlockHeight() uint64 { res, _ := strconv.ParseInt(response.Result.Block.Header.Height, 0, 0) return uint64(res) } + +func getNobleAccountBalance(ctx context.Context, cc *cosmos.CosmosProvider, address, denom string) (uint64, error) { + qc := bankTypes.NewQueryClient(cc) + res, err := qc.Balance(ctx, &bankTypes.QueryBalanceRequest{ + Address: address, + Denom: denom, + }) + if err != nil { + return 0, err + } + + return res.Balance.Amount.Uint64(), nil +} + +func getNobleAccountNumberSequenceGRPC(cc *cosmos.CosmosProvider, address string) (uint64, uint64, error) { + res, err := authtypes.NewQueryClient(cc).Account(context.Background(), &authtypes.QueryAccountRequest{ + Address: address, + }) + if err != nil { + return 0, 0, fmt.Errorf("unable to query account for noble: %w", err) + } + var acc authtypes.AccountI + if err := cc.Cdc.InterfaceRegistry.UnpackAny(res.Account, &acc); err != nil { + return 0, 0, fmt.Errorf("unable to unpack account for noble: %w", err) + } + + return acc.GetAccountNumber(), acc.GetSequence(), nil + +} + +func getEthBalance(client *ethclient.Client, address string) uint64 { + accountAddress := common.HexToAddress(address) + tokenAddress := common.HexToAddress(usdcTokenAddress) + erc20ABI := `[{"constant":true,"inputs":[{"name":"_owner","type":"address"}],"name":"balanceOf","outputs":[{"name":"balance","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"}]` + parsedABI, err := abi.JSON(strings.NewReader(erc20ABI)) + if err != nil { + log.Fatalf("Failed to parse contract ABI: %v", err) + } + + data, err := parsedABI.Pack("balanceOf", accountAddress) + if err != nil { + log.Fatalf("Failed to pack data into ABI interface: %v", err) + } + + result, err := client.CallContract(context.Background(), ethereum.CallMsg{To: &tokenAddress, Data: data}, nil) + if err != nil { + log.Fatalf("Failed to call contract: %v", err) + } + + balance := new(big.Int) + err = parsedABI.UnpackIntoInterface(&balance, "balanceOf", result) + if err != nil { + log.Fatalf("Failed to unpack data from ABI interface: %v", err) + } + + // Convert to uint64 + return balance.Uint64() +}