diff --git a/README.md b/README.md index 103698e..8fddda0 100644 --- a/README.md +++ b/README.md @@ -9,14 +9,23 @@ Installation ```shell git clone https://github.com/strangelove-ventures/noble-cctp-relayer cd noble-cctp-relayer -go install +make install ``` Running the relayer ```shell noble-cctp-relayer start --config ./config/sample-app-config.yaml ``` -Sample configs can be found in config/. +Sample configs can be found in [config](config). +### Promethius Metrics + +By default, metrics are exported at on port :2112/metrics (`http://localhost:2112/metrics`). You can customize the port using the `--metrics-port` flag. + +| **Exported Metric** | **Description** | **Type** | +|-----------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------|----------| +| cctp_relayer_wallet_balance | Current balance of a relayer wallet in Wei.

Noble balances are not currently exported b/c `MsgReceiveMessage` is free to submit on Noble. | Gauge | + + ### API Simple API to query message state cache ```shell @@ -50,7 +59,3 @@ abigen --abi ethereum/abi/MessageTransmitter.json --pkg contracts- --type Messag ### Useful links [Goerli USDC faucet](https://usdcfaucet.com/) - -[Goerli ETH faucet](https://goerlifaucet.com/) - - diff --git a/cmd/appstate.go b/cmd/appstate.go index 158d02a..732c6d3 100644 --- a/cmd/appstate.go +++ b/cmd/appstate.go @@ -14,7 +14,6 @@ type AppState struct { ConfigPath string - // Depreciated in favor of LogLevel Debug bool LogLevel string @@ -48,7 +47,7 @@ func (a *AppState) InitLogger() { level = zerolog.ErrorLevel } - // a.Debug is Depreciatred! + // a.Debug ovverrides a.loglevel if a.Debug { a.Logger = log.NewLogger(os.Stdout, log.LevelOption(zerolog.DebugLevel)) } else { diff --git a/cmd/flags.go b/cmd/flags.go index 866e45e..884d81e 100644 --- a/cmd/flags.go +++ b/cmd/flags.go @@ -1,22 +1,24 @@ package cmd import ( + "fmt" + "github.com/spf13/cobra" ) const ( - flagConfigPath = "config" - // depreciated - flagVerbose = "verbose" - flagLogLevel = "log-level" - flagJSON = "json" + flagConfigPath = "config" + flagVerbose = "verbose" + flagLogLevel = "log-level" + flagJSON = "json" + flagMetricsPort = "metrics-port" ) func addAppPersistantFlags(cmd *cobra.Command, a *AppState) *cobra.Command { cmd.PersistentFlags().StringVar(&a.ConfigPath, flagConfigPath, defaultConfigPath, "file path of config file") - cmd.PersistentFlags().BoolVarP(&a.Debug, flagVerbose, "v", false, "use this flag to set log level to `debug`") - cmd.PersistentFlags().MarkDeprecated(flagVerbose, "depericated") + cmd.PersistentFlags().BoolVarP(&a.Debug, flagVerbose, "v", false, fmt.Sprintf("use this flag to set log level to `debug` (overrides %s flag)", flagLogLevel)) cmd.PersistentFlags().StringVar(&a.LogLevel, flagLogLevel, "info", "log level (debug, info, warn, error)") + cmd.PersistentFlags().Int16P(flagMetricsPort, "p", 2112, "customize Prometheus metrics port") return cmd } diff --git a/cmd/process.go b/cmd/process.go index 045177a..b9c8125 100644 --- a/cmd/process.go +++ b/cmd/process.go @@ -16,6 +16,7 @@ import ( "github.com/strangelove-ventures/noble-cctp-relayer/circle" "github.com/strangelove-ventures/noble-cctp-relayer/ethereum" "github.com/strangelove-ventures/noble-cctp-relayer/noble" + "github.com/strangelove-ventures/noble-cctp-relayer/relayer" "github.com/strangelove-ventures/noble-cctp-relayer/types" ) @@ -47,6 +48,14 @@ func Start(a *AppState) *cobra.Command { registeredDomains := make(map[types.Domain]types.Chain) + port, err := cmd.Flags().GetInt16(flagMetricsPort) + if err != nil { + logger.Error("Invalid port", "error", err) + os.Exit(1) + } + + metrics := relayer.InitPromMetrics(port) + for name, cfg := range cfg.Chains { c, err := cfg.Chain(name) if err != nil { @@ -55,11 +64,12 @@ func Start(a *AppState) *cobra.Command { } if err := c.InitializeBroadcaster(cmd.Context(), logger, sequenceMap); err != nil { - logger.Error("Error initializing broadcaster", "err: ", err) + logger.Error("Error initializing broadcaster", "error", err) os.Exit(1) } go c.StartListener(cmd.Context(), logger, processingQueue) + go c.WalletBalanceMetric(cmd.Context(), a.Logger, metrics) if _, ok := registeredDomains[c.Domain()]; ok { logger.Error("Duplicate domain found", "domain", c.Domain(), "name:", c.Name()) diff --git a/config/sample-config.yaml b/config/sample-config.yaml index 5f2518a..d5ae11c 100644 --- a/config/sample-config.yaml +++ b/config/sample-config.yaml @@ -14,7 +14,7 @@ chains: block-queue-channel-size: 1000000 # 1000000 is a safe default, increase number if starting from a very early block - min-mint-amount: 0 # minamum transaction amount needed for relayer to broadcast the MsgReceive/burn for this chain. IE. if this chain is the destintation chain + min-mint-amount: 0 # minimum transaction amount needed for relayer to broadcast the MsgReceive/burn for this chain. IE. if this chain is the destination chain minter-private-key: # hex encoded privateKey @@ -31,7 +31,13 @@ chains: broadcast-retries: 5 # number of times to attempt the broadcast broadcast-retry-interval: 10 # time between retries in seconds - min-mint-amount: 10000000 # (10000000 = $10) minamum transaction amount needed for relayer to broadcast the MsgReceive/burn for this chain. IE. if this chain is the destintation chain + min-mint-amount: 10000000 # (10000000 = $10) minimum transaction amount needed for relayer to broadcast the MsgReceive/burn for this chain. IE. if this chain is the destination chain + + # Both metrics values are OPTIONAL and used solely for Prometheus metrics. + metrics-denom: "ETH" + # metrics-exponent is used to determine the correct denomination. Wallet balances are originally queried in Wei. To convert Wei to Eth use 18. + # Example `walletBalance*10^-18` + metrics-exponent: 18 minter-private-key: # private key @@ -50,6 +56,9 @@ chains: min-mint-amount: 10000000 + metrics-denom: "ETH" + metrics-exponent: 18 + minter-private-key: "" arbitrum: @@ -67,6 +76,9 @@ chains: min-mint-amount: 10000000 + metrics-denom: "ETH" + metrics-exponent: 18 + minter-private-key: "" avalanche: @@ -84,6 +96,9 @@ chains: min-mint-amount: 10000000 + metrics-denom: "AVAX" + metrics-exponent: 18 + minter-private-key: "" # source domain id -> []destination domain id diff --git a/config/sample-integration-config.yaml b/config/sample-integration-config.yaml index 7a0a352..5eef16f 100644 --- a/config/sample-integration-config.yaml +++ b/config/sample-integration-config.yaml @@ -60,6 +60,26 @@ testnet: token-messenger-address: "0x9f3B8679c73C2Fef8b59B4f3444d4e156fb70AA5" destination-caller: "" + polygon: + chain-id: "80001" + domain: 7 + address: "" + private-key: "" + rpc: "" + usdc-token-address: "0x9999f7fea5938fd3b1e26a12c3f2fb024e194f97" + token-messenger-address: "0x9f3B8679c73C2Fef8b59B4f3444d4e156fb70AA5" + destination-caller: "" + + base: + chain-id: "84532" + domain: 6 + address: "" + private-key: "" + rpc: "" + usdc-token-address: "0x036CbD53842c5426634e7929541eC2318f3dCF7e" + token-messenger-address: "0x9f3B8679c73C2Fef8b59B4f3444d4e156fb70AA5" + destination-caller: "" + mainnet: noble: chain-id: "noble-1" @@ -109,3 +129,22 @@ mainnet: token-messenger-address: "0x2B4069517957735bE00ceE0fadAE88a26365528f" destination-caller: "" + polygon: + chain-id: "137" + domain: 7 + address: "" + private-key: "" + rpc: "" + usdc-token-address: "0x3c499c542cef5e3811e1192ce70d8cc03d5c3359" + token-messenger-address: "0x9daF8c91AEFAE50b9c0E69629D3F6Ca40cA3B3FE" + destination-caller: "" + + base: + chain-id: "8453" + domain: 6 + address: "" + private-key: "" + rpc: "" + usdc-token-address: "0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913" + token-messenger-address: "0x1682Ae6375C4E4A97e4B583BC394c861A46D8962" + destination-caller: "" \ No newline at end of file diff --git a/ethereum/chain.go b/ethereum/chain.go index 10414c8..4bd8059 100644 --- a/ethereum/chain.go +++ b/ethereum/chain.go @@ -30,6 +30,8 @@ type Ethereum struct { maxRetries int retryIntervalSeconds int minAmount uint64 + MetricsDenom string + MetricsExponent int mu sync.Mutex } @@ -47,6 +49,8 @@ func NewChain( maxRetries int, retryIntervalSeconds int, minAmount uint64, + metricsDenom string, + metricsExponent int, ) (*Ethereum, error) { privEcdsaKey, ethereumAddress, err := GetEcdsaKeyAddress(privateKey) if err != nil { @@ -66,6 +70,8 @@ func NewChain( maxRetries: maxRetries, retryIntervalSeconds: retryIntervalSeconds, minAmount: minAmount, + MetricsDenom: metricsDenom, + MetricsExponent: metricsExponent, }, nil } diff --git a/ethereum/config.go b/ethereum/config.go index 9df3809..39fdcf4 100644 --- a/ethereum/config.go +++ b/ethereum/config.go @@ -21,6 +21,9 @@ type ChainConfig struct { MinMintAmount uint64 `yaml:"min-mint-amount"` + MetricsDenom string `yaml:"metrics-denom"` + MetricsExponent int `yaml:"metrics-exponent"` + // TODO move to keyring MinterPrivateKey string `yaml:"minter-private-key"` } @@ -39,5 +42,7 @@ func (c *ChainConfig) Chain(name string) (types.Chain, error) { c.BroadcastRetries, c.BroadcastRetryInterval, c.MinMintAmount, + c.MetricsDenom, + c.MetricsExponent, ) } diff --git a/ethereum/listener.go b/ethereum/listener.go index d604d37..9632dd2 100644 --- a/ethereum/listener.go +++ b/ethereum/listener.go @@ -6,6 +6,7 @@ import ( "fmt" "math/big" "os" + "time" "cosmossdk.io/log" ethereum "github.com/ethereum/go-ethereum" @@ -13,6 +14,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethclient" "github.com/pascaldekloe/etherstream" + "github.com/strangelove-ventures/noble-cctp-relayer/relayer" "github.com/strangelove-ventures/noble-cctp-relayer/types" ) @@ -42,8 +44,6 @@ func (e *Ethereum) StartListener( os.Exit(1) } - // defer ethClient.Close() - messageTransmitterAddress := common.HexToAddress(e.messageTransmitterAddress) etherReader := etherstream.Reader{Backend: ethClient} @@ -129,3 +129,80 @@ func (e *Ethereum) StartListener( } }() } + +func (e *Ethereum) WalletBalanceMetric(ctx context.Context, logger log.Logger, m *relayer.PromMetrics) { + logger = logger.With("metric", "wallet blance", "chain", e.name, "domain", e.domain) + queryRate := 30 * time.Second + + var err error + var client *ethclient.Client + + account := common.HexToAddress(e.minterAddress) + + exponent := big.NewInt(int64(e.MetricsExponent)) // ex: 18 + scaleFactor := new(big.Float).SetInt(new(big.Int).Exp(big.NewInt(10), exponent, nil)) // ex: 10^18 + + defer func() { + if client != nil { + client.Close() + } + }() + + first := make(chan struct{}, 1) + first <- struct{}{} + createClient := true + for { + timer := time.NewTimer(queryRate) + select { + // don't wait the "queryRate" amount of time if this is the first time running + case <-first: + timer.Stop() + if createClient { + client, err = ethclient.DialContext(ctx, e.rpcURL) + if err != nil { + logger.Error(fmt.Sprintf("error dialing eth client. Will try again in %d sec", queryRate), "error", err) + createClient = true + continue + } + } + balance, err := client.BalanceAt(ctx, account, nil) + if err != nil { + logger.Error(fmt.Sprintf("error querying balance. Will try again in %d sec", queryRate), "error", err) + createClient = true + continue + } + + balanceBigFloat := new(big.Float).SetInt(balance) + balanceScaled, _ := new(big.Float).Quo(balanceBigFloat, scaleFactor).Float64() + + m.SetWalletBalance(e.name, e.minterAddress, e.MetricsDenom, balanceScaled) + + createClient = false + case <-timer.C: + if createClient { + client, err = ethclient.DialContext(ctx, e.rpcURL) + if err != nil { + logger.Error(fmt.Sprintf("error dialing eth client. Will try again in %d sec", queryRate), "error", err) + createClient = true + continue + } + } + balance, err := client.BalanceAt(ctx, account, nil) + if err != nil { + logger.Error(fmt.Sprintf("error querying balance. Will try again in %d sec", queryRate), "error", err) + createClient = true + continue + } + + balanceBigFloat := new(big.Float).SetInt(balance) + balanceScaled, _ := new(big.Float).Quo(balanceBigFloat, scaleFactor).Float64() + + m.SetWalletBalance(e.name, e.minterAddress, e.MetricsDenom, balanceScaled) + + createClient = false + case <-ctx.Done(): + timer.Stop() + return + } + } +} diff --git a/go.mod b/go.mod index 3ab9da1..890a01a 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,7 @@ require ( github.com/gin-gonic/gin v1.9.1 github.com/joho/godotenv v1.5.1 github.com/pascaldekloe/etherstream v0.1.0 + github.com/prometheus/client_golang v1.14.0 google.golang.org/grpc v1.60.0 gopkg.in/yaml.v2 v2.4.0 ) @@ -127,7 +128,6 @@ require ( github.com/petermattis/goid v0.0.0-20230317030725-371a4b8eda08 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/client_golang v1.14.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect diff --git a/integration/deployed_relayer_test.go b/integration/deployed_relayer_test.go index 2fecdbd..d162036 100644 --- a/integration/deployed_relayer_test.go +++ b/integration/deployed_relayer_test.go @@ -58,6 +58,8 @@ func TestNobleBurnToEthDeployed(t *testing.T) { // ethConfig := cfg["arbitrum"] // ethConfig := cfg["avalanche"] // ethConfig := cfg["optimism"] + // ethConfig := cfg["polygon"] + // ethConfig := cfg["base"] destAddress := ethConfig.Address @@ -198,6 +200,8 @@ func TestEthBurnToNobleDeployed(t *testing.T) { // ethConfig := cfg["arbitrum"] // ethConfig := cfg["avalanche"] // ethConfig := cfg["optimism"] + // ethConfig := cfg["polygon"] + // ethConfig := cfg["base"] // -- DEST CHAIN -- nobleCfg := cfg["noble"] diff --git a/noble/listener.go b/noble/listener.go index 138b402..461f64f 100644 --- a/noble/listener.go +++ b/noble/listener.go @@ -6,6 +6,7 @@ import ( "time" "cosmossdk.io/log" + "github.com/strangelove-ventures/noble-cctp-relayer/relayer" "github.com/strangelove-ventures/noble-cctp-relayer/types" ) @@ -130,3 +131,7 @@ func (n *Noble) chainTip(ctx context.Context) (uint64, error) { } return uint64(res.SyncInfo.LatestBlockHeight), nil } + +func (n *Noble) WalletBalanceMetric(ctx context.Context, logger log.Logger, m *relayer.PromMetrics) { + // Relaying is free. No need to track noble balance. +} diff --git a/relayer/metrics.go b/relayer/metrics.go new file mode 100644 index 0000000..3b04cf7 --- /dev/null +++ b/relayer/metrics.go @@ -0,0 +1,44 @@ +package relayer + +import ( + "fmt" + "log" + "net/http" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +type PromMetrics struct { + WalletBalance *prometheus.GaugeVec +} + +func InitPromMetrics(port int16) *PromMetrics { + reg := prometheus.NewRegistry() + + // labels + var ( + walletLabels = []string{"chain", "address", "denom"} + ) + + m := &PromMetrics{ + WalletBalance: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "cctp_relayer_wallet_balance", + Help: "The current balance for a wallet", + }, walletLabels), + } + + reg.MustRegister(m.WalletBalance) + + // Expose /metrics HTTP endpoint + go func() { + http.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{Registry: reg})) + log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", port), nil)) + }() + + return m +} + +func (m *PromMetrics) SetWalletBalance(chain, address, denom string, balance float64) { + m.WalletBalance.WithLabelValues(chain, address, denom).Set(balance) +} diff --git a/types/chain.go b/types/chain.go index 388ed58..f8f05c3 100644 --- a/types/chain.go +++ b/types/chain.go @@ -4,6 +4,7 @@ import ( "context" "cosmossdk.io/log" + "github.com/strangelove-ventures/noble-cctp-relayer/relayer" ) // Chain is an interface for common CCTP source and destination chain operations. @@ -39,4 +40,10 @@ type Chain interface { msgs []*MessageState, sequenceMap *SequenceMap, ) error + + WalletBalanceMetric( + ctx context.Context, + logger log.Logger, + metric *relayer.PromMetrics, + ) }