Implement Prometheus metrics to track wallet balance (#53)

* balance metric

* ctx, port, and readme

* fix context

* use timer instead of sleep

* log error

* readme touch up

* integration tests for base and polygon

* denom and exponent

* feedback

README.md

@@ -9,14 +9,23 @@ Installation
 ```shell
 git clone https://github.com/strangelove-ventures/noble-cctp-relayer
 cd noble-cctp-relayer
-go install
+make install
 ```
 
 Running the relayer
 ```shell
 noble-cctp-relayer start --config ./config/sample-app-config.yaml
 ```
-Sample configs can be found in config/.
+Sample configs can be found in [config](config).
+### Promethius Metrics
+
+By default, metrics are exported at on port :2112/metrics (`http://localhost:2112/metrics`). You can customize the port using the `--metrics-port` flag. 
+
+| **Exported Metric**         | **Description**                                                                                                                                    | **Type** |
+|-----------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------|----------|
+| cctp_relayer_wallet_balance | Current balance of a relayer wallet in Wei.<br><br>Noble balances are not currently exported b/c `MsgReceiveMessage` is free to submit on Noble.   | Gauge    |
+
+
 ### API
 Simple API to query message state cache
 ```shell
@@ -50,7 +59,3 @@ abigen --abi ethereum/abi/MessageTransmitter.json --pkg contracts- --type Messag
 
 ### Useful links
 [Goerli USDC faucet](https://usdcfaucet.com/)
-
-[Goerli ETH faucet](https://goerlifaucet.com/)
-
-

cmd/appstate.go

@@ -14,7 +14,6 @@ type AppState struct {
 
 	ConfigPath string
 
-	// Depreciated in favor of LogLevel
 	Debug bool
 
 	LogLevel string
@@ -48,7 +47,7 @@ func (a *AppState) InitLogger() {
 		level = zerolog.ErrorLevel
 	}
 
-	// a.Debug is Depreciatred!
+	// a.Debug ovverrides a.loglevel
 	if a.Debug {
 		a.Logger = log.NewLogger(os.Stdout, log.LevelOption(zerolog.DebugLevel))
 	} else {

cmd/flags.go

@@ -1,22 +1,24 @@
 package cmd
 
 import (
+	"fmt"
+
 	"github.com/spf13/cobra"
 )
 
 const (
-	flagConfigPath = "config"
-	// depreciated
-	flagVerbose  = "verbose"
-	flagLogLevel = "log-level"
-	flagJSON     = "json"
+	flagConfigPath  = "config"
+	flagVerbose     = "verbose"
+	flagLogLevel    = "log-level"
+	flagJSON        = "json"
+	flagMetricsPort = "metrics-port"
 )
 
 func addAppPersistantFlags(cmd *cobra.Command, a *AppState) *cobra.Command {
 	cmd.PersistentFlags().StringVar(&a.ConfigPath, flagConfigPath, defaultConfigPath, "file path of config file")
-	cmd.PersistentFlags().BoolVarP(&a.Debug, flagVerbose, "v", false, "use this flag to set log level to `debug`")
-	cmd.PersistentFlags().MarkDeprecated(flagVerbose, "depericated")
+	cmd.PersistentFlags().BoolVarP(&a.Debug, flagVerbose, "v", false, fmt.Sprintf("use this flag to set log level to `debug` (overrides %s flag)", flagLogLevel))
 	cmd.PersistentFlags().StringVar(&a.LogLevel, flagLogLevel, "info", "log level (debug, info, warn, error)")
+	cmd.PersistentFlags().Int16P(flagMetricsPort, "p", 2112, "customize Prometheus metrics port")
 	return cmd
 
 }

cmd/process.go

@@ -16,6 +16,7 @@ import (
 	"github.com/strangelove-ventures/noble-cctp-relayer/circle"
 	"github.com/strangelove-ventures/noble-cctp-relayer/ethereum"
 	"github.com/strangelove-ventures/noble-cctp-relayer/noble"
+	"github.com/strangelove-ventures/noble-cctp-relayer/relayer"
 	"github.com/strangelove-ventures/noble-cctp-relayer/types"
 )
 
@@ -47,6 +48,14 @@ func Start(a *AppState) *cobra.Command {
 
 			registeredDomains := make(map[types.Domain]types.Chain)
 
+			port, err := cmd.Flags().GetInt16(flagMetricsPort)
+			if err != nil {
+				logger.Error("Invalid port", "error", err)
+				os.Exit(1)
+			}
+
+			metrics := relayer.InitPromMetrics(port)
+
 			for name, cfg := range cfg.Chains {
 				c, err := cfg.Chain(name)
 				if err != nil {
@@ -55,11 +64,12 @@ func Start(a *AppState) *cobra.Command {
 				}
 
 				if err := c.InitializeBroadcaster(cmd.Context(), logger, sequenceMap); err != nil {
-					logger.Error("Error initializing broadcaster", "err: ", err)
+					logger.Error("Error initializing broadcaster", "error", err)
 					os.Exit(1)
 				}
 
 				go c.StartListener(cmd.Context(), logger, processingQueue)
+				go c.WalletBalanceMetric(cmd.Context(), a.Logger, metrics)
 
 				if _, ok := registeredDomains[c.Domain()]; ok {
 					logger.Error("Duplicate domain found", "domain", c.Domain(), "name:", c.Name())

config/sample-config.yaml

@@ -14,7 +14,7 @@ chains:
 
     block-queue-channel-size: 1000000 # 1000000 is a safe default, increase number if starting from a very early block
 
-    min-mint-amount: 0 # minamum transaction amount needed for relayer to broadcast the MsgReceive/burn for this chain. IE. if this chain is the destintation chain
+    min-mint-amount: 0 # minimum transaction amount needed for relayer to broadcast the MsgReceive/burn for this chain. IE. if this chain is the destination chain
 
     minter-private-key: # hex encoded privateKey
 
@@ -31,7 +31,13 @@ chains:
     broadcast-retries: 5 # number of times to attempt the broadcast
     broadcast-retry-interval: 10 # time between retries in seconds
 
-    min-mint-amount: 10000000 # (10000000 = $10) minamum transaction amount needed for relayer to broadcast the MsgReceive/burn for this chain. IE. if this chain is the destintation chain
+    min-mint-amount: 10000000 # (10000000 = $10) minimum transaction amount needed for relayer to broadcast the MsgReceive/burn for this chain. IE. if this chain is the destination chain
+
+    # Both metrics values are OPTIONAL and used solely for Prometheus metrics.
+    metrics-denom: "ETH"
+    # metrics-exponent is used to determine the correct denomination. Wallet balances are originally queried in Wei. To convert Wei to Eth use 18.
+    # Example `walletBalance*10^-18`
+    metrics-exponent: 18
 
     minter-private-key: # private key
 
@@ -50,6 +56,9 @@ chains:
 
     min-mint-amount: 10000000
 
+    metrics-denom: "ETH"
+    metrics-exponent: 18
+
     minter-private-key: ""
 
   arbitrum:
@@ -67,6 +76,9 @@ chains:
 
     min-mint-amount: 10000000
 
+    metrics-denom: "ETH"
+    metrics-exponent: 18
+
     minter-private-key: ""
 
   avalanche:
@@ -84,6 +96,9 @@ chains:
 
     min-mint-amount: 10000000
 
+    metrics-denom: "AVAX"
+    metrics-exponent: 18
+
     minter-private-key: "" 
 
 # source domain id -> []destination domain id

config/sample-integration-config.yaml

@@ -60,6 +60,26 @@ testnet:
     token-messenger-address: "0x9f3B8679c73C2Fef8b59B4f3444d4e156fb70AA5"
     destination-caller: ""
 
+  polygon:
+    chain-id: "80001"
+    domain: 7
+    address: ""
+    private-key: ""
+    rpc: ""
+    usdc-token-address: "0x9999f7fea5938fd3b1e26a12c3f2fb024e194f97"
+    token-messenger-address: "0x9f3B8679c73C2Fef8b59B4f3444d4e156fb70AA5"
+    destination-caller: ""
+
+  base:
+    chain-id: "84532"
+    domain: 6
+    address: ""
+    private-key: ""
+    rpc: ""
+    usdc-token-address: "0x036CbD53842c5426634e7929541eC2318f3dCF7e"
+    token-messenger-address: "0x9f3B8679c73C2Fef8b59B4f3444d4e156fb70AA5"
+    destination-caller: ""
+
 mainnet:
   noble:
     chain-id: "noble-1"
@@ -109,3 +129,22 @@ mainnet:
     token-messenger-address: "0x2B4069517957735bE00ceE0fadAE88a26365528f"
     destination-caller: ""
 
+  polygon:
+    chain-id: "137"
+    domain: 7
+    address: ""
+    private-key: ""
+    rpc: ""
+    usdc-token-address: "0x3c499c542cef5e3811e1192ce70d8cc03d5c3359"
+    token-messenger-address: "0x9daF8c91AEFAE50b9c0E69629D3F6Ca40cA3B3FE"
+    destination-caller: ""
+
+  base:
+    chain-id: "8453"
+    domain: 6
+    address: ""
+    private-key: ""
+    rpc: ""
+    usdc-token-address: "0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913"
+    token-messenger-address: "0x1682Ae6375C4E4A97e4B583BC394c861A46D8962"
+    destination-caller: ""

ethereum/chain.go

@@ -30,6 +30,8 @@ type Ethereum struct {
 	maxRetries                int
 	retryIntervalSeconds      int
 	minAmount                 uint64
+	MetricsDenom              string
+	MetricsExponent           int
 
 	mu sync.Mutex
 }
@@ -47,6 +49,8 @@ func NewChain(
 	maxRetries int,
 	retryIntervalSeconds int,
 	minAmount uint64,
+	metricsDenom string,
+	metricsExponent int,
 ) (*Ethereum, error) {
 	privEcdsaKey, ethereumAddress, err := GetEcdsaKeyAddress(privateKey)
 	if err != nil {
@@ -66,6 +70,8 @@ func NewChain(
 		maxRetries:                maxRetries,
 		retryIntervalSeconds:      retryIntervalSeconds,
 		minAmount:                 minAmount,
+		MetricsDenom:              metricsDenom,
+		MetricsExponent:           metricsExponent,
 	}, nil
 }
 

ethereum/config.go

@@ -21,6 +21,9 @@ type ChainConfig struct {
 
 	MinMintAmount uint64 `yaml:"min-mint-amount"`
 
+	MetricsDenom    string `yaml:"metrics-denom"`
+	MetricsExponent int    `yaml:"metrics-exponent"`
+
 	// TODO move to keyring
 	MinterPrivateKey string `yaml:"minter-private-key"`
 }
@@ -39,5 +42,7 @@ func (c *ChainConfig) Chain(name string) (types.Chain, error) {
 		c.BroadcastRetries,
 		c.BroadcastRetryInterval,
 		c.MinMintAmount,
+		c.MetricsDenom,
+		c.MetricsExponent,
 	)
 }

ethereum/listener.go

@@ -6,13 +6,15 @@ import (
 	"fmt"
 	"math/big"
 	"os"
+	"time"
 
 	"cosmossdk.io/log"
 	ethereum "github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/accounts/abi"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/pascaldekloe/etherstream"
+	"github.com/strangelove-ventures/noble-cctp-relayer/relayer"
 	"github.com/strangelove-ventures/noble-cctp-relayer/types"
 )
 
@@ -42,8 +44,6 @@ func (e *Ethereum) StartListener(
 		os.Exit(1)
 	}
 
-	// defer ethClient.Close()
-
 	messageTransmitterAddress := common.HexToAddress(e.messageTransmitterAddress)
 	etherReader := etherstream.Reader{Backend: ethClient}
 
@@ -129,3 +129,80 @@ func (e *Ethereum) StartListener(
 		}
 	}()
 }
+
+func (e *Ethereum) WalletBalanceMetric(ctx context.Context, logger log.Logger, m *relayer.PromMetrics) {
+	logger = logger.With("metric", "wallet blance", "chain", e.name, "domain", e.domain)
+	queryRate := 30 * time.Second
+
+	var err error
+	var client *ethclient.Client
+
+	account := common.HexToAddress(e.minterAddress)
+
+	exponent := big.NewInt(int64(e.MetricsExponent))                                      // ex: 18
+	scaleFactor := new(big.Float).SetInt(new(big.Int).Exp(big.NewInt(10), exponent, nil)) // ex: 10^18
+
+	defer func() {
+		if client != nil {
+			client.Close()
+		}
+	}()
+
+	first := make(chan struct{}, 1)
+	first <- struct{}{}
+	createClient := true
+	for {
+		timer := time.NewTimer(queryRate)
+		select {
+		// don't wait the "queryRate" amount of time if this is the first time running
+		case <-first:
+			timer.Stop()
+			if createClient {
+				client, err = ethclient.DialContext(ctx, e.rpcURL)
+				if err != nil {
+					logger.Error(fmt.Sprintf("error dialing eth client. Will try again in %d sec", queryRate), "error", err)
+					createClient = true
+					continue
+				}
+			}
+			balance, err := client.BalanceAt(ctx, account, nil)
+			if err != nil {
+				logger.Error(fmt.Sprintf("error querying balance. Will try again in %d sec", queryRate), "error", err)
+				createClient = true
+				continue
+			}
+
+			balanceBigFloat := new(big.Float).SetInt(balance)
+			balanceScaled, _ := new(big.Float).Quo(balanceBigFloat, scaleFactor).Float64()
+
+			m.SetWalletBalance(e.name, e.minterAddress, e.MetricsDenom, balanceScaled)
+
+			createClient = false
+		case <-timer.C:
+			if createClient {
+				client, err = ethclient.DialContext(ctx, e.rpcURL)
+				if err != nil {
+					logger.Error(fmt.Sprintf("error dialing eth client. Will try again in %d sec", queryRate), "error", err)
+					createClient = true
+					continue
+				}
+			}
+			balance, err := client.BalanceAt(ctx, account, nil)
+			if err != nil {
+				logger.Error(fmt.Sprintf("error querying balance. Will try again in %d sec", queryRate), "error", err)
+				createClient = true
+				continue
+			}
+
+			balanceBigFloat := new(big.Float).SetInt(balance)
+			balanceScaled, _ := new(big.Float).Quo(balanceBigFloat, scaleFactor).Float64()
+
+			m.SetWalletBalance(e.name, e.minterAddress, e.MetricsDenom, balanceScaled)
+
+			createClient = false
+		case <-ctx.Done():
+			timer.Stop()
+			return
+		}
+	}
+}

go.mod

@@ -23,6 +23,7 @@ require (
 	github.com/gin-gonic/gin v1.9.1
 	github.com/joho/godotenv v1.5.1
 	github.com/pascaldekloe/etherstream v0.1.0
+	github.com/prometheus/client_golang v1.14.0
 	google.golang.org/grpc v1.60.0
 	gopkg.in/yaml.v2 v2.4.0
 )
@@ -127,7 +128,6 @@ require (
 	github.com/petermattis/goid v0.0.0-20230317030725-371a4b8eda08 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
-	github.com/prometheus/client_golang v1.14.0 // indirect
 	github.com/prometheus/client_model v0.3.0 // indirect
 	github.com/prometheus/common v0.42.0 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect