From fd47ac5ee4b106a0a6910d757cb40349e1080f63 Mon Sep 17 00:00:00 2001
From: Dan Kanefsky <56059752+boojamya@users.noreply.github.com>
Date: Thu, 14 Mar 2024 12:00:55 -0700
Subject: [PATCH] Implement Prometheus metrics to track wallet balance (#53)
* balance metric
* ctx, port, and readme
* fix context
* use timer instead of sleep
* log error
* readme touch up
* integration tests for base and polygon
* denom and exponent
* feedback
---
README.md | 17 ++++--
cmd/appstate.go | 3 +-
cmd/flags.go | 16 +++---
cmd/process.go | 12 +++-
config/sample-config.yaml | 19 ++++++-
config/sample-integration-config.yaml | 39 +++++++++++++
ethereum/chain.go | 6 ++
ethereum/config.go | 5 ++
ethereum/listener.go | 81 ++++++++++++++++++++++++++-
go.mod | 2 +-
integration/deployed_relayer_test.go | 4 ++
noble/listener.go | 5 ++
relayer/metrics.go | 44 +++++++++++++++
types/chain.go | 7 +++
14 files changed, 239 insertions(+), 21 deletions(-)
create mode 100644 relayer/metrics.go
diff --git a/README.md b/README.md
index 103698e..8fddda0 100644
--- a/README.md
+++ b/README.md
@@ -9,14 +9,23 @@ Installation
```shell
git clone https://github.com/strangelove-ventures/noble-cctp-relayer
cd noble-cctp-relayer
-go install
+make install
```
Running the relayer
```shell
noble-cctp-relayer start --config ./config/sample-app-config.yaml
```
-Sample configs can be found in config/.
+Sample configs can be found in [config](config).
+### Promethius Metrics
+
+By default, metrics are exported at on port :2112/metrics (`http://localhost:2112/metrics`). You can customize the port using the `--metrics-port` flag.
+
+| **Exported Metric** | **Description** | **Type** |
+|-----------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------|----------|
+| cctp_relayer_wallet_balance | Current balance of a relayer wallet in Wei.
Noble balances are not currently exported b/c `MsgReceiveMessage` is free to submit on Noble. | Gauge |
+
+
### API
Simple API to query message state cache
```shell
@@ -50,7 +59,3 @@ abigen --abi ethereum/abi/MessageTransmitter.json --pkg contracts- --type Messag
### Useful links
[Goerli USDC faucet](https://usdcfaucet.com/)
-
-[Goerli ETH faucet](https://goerlifaucet.com/)
-
-
diff --git a/cmd/appstate.go b/cmd/appstate.go
index 158d02a..732c6d3 100644
--- a/cmd/appstate.go
+++ b/cmd/appstate.go
@@ -14,7 +14,6 @@ type AppState struct {
ConfigPath string
- // Depreciated in favor of LogLevel
Debug bool
LogLevel string
@@ -48,7 +47,7 @@ func (a *AppState) InitLogger() {
level = zerolog.ErrorLevel
}
- // a.Debug is Depreciatred!
+ // a.Debug ovverrides a.loglevel
if a.Debug {
a.Logger = log.NewLogger(os.Stdout, log.LevelOption(zerolog.DebugLevel))
} else {
diff --git a/cmd/flags.go b/cmd/flags.go
index 866e45e..884d81e 100644
--- a/cmd/flags.go
+++ b/cmd/flags.go
@@ -1,22 +1,24 @@
package cmd
import (
+ "fmt"
+
"github.com/spf13/cobra"
)
const (
- flagConfigPath = "config"
- // depreciated
- flagVerbose = "verbose"
- flagLogLevel = "log-level"
- flagJSON = "json"
+ flagConfigPath = "config"
+ flagVerbose = "verbose"
+ flagLogLevel = "log-level"
+ flagJSON = "json"
+ flagMetricsPort = "metrics-port"
)
func addAppPersistantFlags(cmd *cobra.Command, a *AppState) *cobra.Command {
cmd.PersistentFlags().StringVar(&a.ConfigPath, flagConfigPath, defaultConfigPath, "file path of config file")
- cmd.PersistentFlags().BoolVarP(&a.Debug, flagVerbose, "v", false, "use this flag to set log level to `debug`")
- cmd.PersistentFlags().MarkDeprecated(flagVerbose, "depericated")
+ cmd.PersistentFlags().BoolVarP(&a.Debug, flagVerbose, "v", false, fmt.Sprintf("use this flag to set log level to `debug` (overrides %s flag)", flagLogLevel))
cmd.PersistentFlags().StringVar(&a.LogLevel, flagLogLevel, "info", "log level (debug, info, warn, error)")
+ cmd.PersistentFlags().Int16P(flagMetricsPort, "p", 2112, "customize Prometheus metrics port")
return cmd
}
diff --git a/cmd/process.go b/cmd/process.go
index 045177a..b9c8125 100644
--- a/cmd/process.go
+++ b/cmd/process.go
@@ -16,6 +16,7 @@ import (
"github.com/strangelove-ventures/noble-cctp-relayer/circle"
"github.com/strangelove-ventures/noble-cctp-relayer/ethereum"
"github.com/strangelove-ventures/noble-cctp-relayer/noble"
+ "github.com/strangelove-ventures/noble-cctp-relayer/relayer"
"github.com/strangelove-ventures/noble-cctp-relayer/types"
)
@@ -47,6 +48,14 @@ func Start(a *AppState) *cobra.Command {
registeredDomains := make(map[types.Domain]types.Chain)
+ port, err := cmd.Flags().GetInt16(flagMetricsPort)
+ if err != nil {
+ logger.Error("Invalid port", "error", err)
+ os.Exit(1)
+ }
+
+ metrics := relayer.InitPromMetrics(port)
+
for name, cfg := range cfg.Chains {
c, err := cfg.Chain(name)
if err != nil {
@@ -55,11 +64,12 @@ func Start(a *AppState) *cobra.Command {
}
if err := c.InitializeBroadcaster(cmd.Context(), logger, sequenceMap); err != nil {
- logger.Error("Error initializing broadcaster", "err: ", err)
+ logger.Error("Error initializing broadcaster", "error", err)
os.Exit(1)
}
go c.StartListener(cmd.Context(), logger, processingQueue)
+ go c.WalletBalanceMetric(cmd.Context(), a.Logger, metrics)
if _, ok := registeredDomains[c.Domain()]; ok {
logger.Error("Duplicate domain found", "domain", c.Domain(), "name:", c.Name())
diff --git a/config/sample-config.yaml b/config/sample-config.yaml
index 5f2518a..d5ae11c 100644
--- a/config/sample-config.yaml
+++ b/config/sample-config.yaml
@@ -14,7 +14,7 @@ chains:
block-queue-channel-size: 1000000 # 1000000 is a safe default, increase number if starting from a very early block
- min-mint-amount: 0 # minamum transaction amount needed for relayer to broadcast the MsgReceive/burn for this chain. IE. if this chain is the destintation chain
+ min-mint-amount: 0 # minimum transaction amount needed for relayer to broadcast the MsgReceive/burn for this chain. IE. if this chain is the destination chain
minter-private-key: # hex encoded privateKey
@@ -31,7 +31,13 @@ chains:
broadcast-retries: 5 # number of times to attempt the broadcast
broadcast-retry-interval: 10 # time between retries in seconds
- min-mint-amount: 10000000 # (10000000 = $10) minamum transaction amount needed for relayer to broadcast the MsgReceive/burn for this chain. IE. if this chain is the destintation chain
+ min-mint-amount: 10000000 # (10000000 = $10) minimum transaction amount needed for relayer to broadcast the MsgReceive/burn for this chain. IE. if this chain is the destination chain
+
+ # Both metrics values are OPTIONAL and used solely for Prometheus metrics.
+ metrics-denom: "ETH"
+ # metrics-exponent is used to determine the correct denomination. Wallet balances are originally queried in Wei. To convert Wei to Eth use 18.
+ # Example `walletBalance*10^-18`
+ metrics-exponent: 18
minter-private-key: # private key
@@ -50,6 +56,9 @@ chains:
min-mint-amount: 10000000
+ metrics-denom: "ETH"
+ metrics-exponent: 18
+
minter-private-key: ""
arbitrum:
@@ -67,6 +76,9 @@ chains:
min-mint-amount: 10000000
+ metrics-denom: "ETH"
+ metrics-exponent: 18
+
minter-private-key: ""
avalanche:
@@ -84,6 +96,9 @@ chains:
min-mint-amount: 10000000
+ metrics-denom: "AVAX"
+ metrics-exponent: 18
+
minter-private-key: ""
# source domain id -> []destination domain id
diff --git a/config/sample-integration-config.yaml b/config/sample-integration-config.yaml
index 7a0a352..5eef16f 100644
--- a/config/sample-integration-config.yaml
+++ b/config/sample-integration-config.yaml
@@ -60,6 +60,26 @@ testnet:
token-messenger-address: "0x9f3B8679c73C2Fef8b59B4f3444d4e156fb70AA5"
destination-caller: ""
+ polygon:
+ chain-id: "80001"
+ domain: 7
+ address: ""
+ private-key: ""
+ rpc: ""
+ usdc-token-address: "0x9999f7fea5938fd3b1e26a12c3f2fb024e194f97"
+ token-messenger-address: "0x9f3B8679c73C2Fef8b59B4f3444d4e156fb70AA5"
+ destination-caller: ""
+
+ base:
+ chain-id: "84532"
+ domain: 6
+ address: ""
+ private-key: ""
+ rpc: ""
+ usdc-token-address: "0x036CbD53842c5426634e7929541eC2318f3dCF7e"
+ token-messenger-address: "0x9f3B8679c73C2Fef8b59B4f3444d4e156fb70AA5"
+ destination-caller: ""
+
mainnet:
noble:
chain-id: "noble-1"
@@ -109,3 +129,22 @@ mainnet:
token-messenger-address: "0x2B4069517957735bE00ceE0fadAE88a26365528f"
destination-caller: ""
+ polygon:
+ chain-id: "137"
+ domain: 7
+ address: ""
+ private-key: ""
+ rpc: ""
+ usdc-token-address: "0x3c499c542cef5e3811e1192ce70d8cc03d5c3359"
+ token-messenger-address: "0x9daF8c91AEFAE50b9c0E69629D3F6Ca40cA3B3FE"
+ destination-caller: ""
+
+ base:
+ chain-id: "8453"
+ domain: 6
+ address: ""
+ private-key: ""
+ rpc: ""
+ usdc-token-address: "0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913"
+ token-messenger-address: "0x1682Ae6375C4E4A97e4B583BC394c861A46D8962"
+ destination-caller: ""
\ No newline at end of file
diff --git a/ethereum/chain.go b/ethereum/chain.go
index 10414c8..4bd8059 100644
--- a/ethereum/chain.go
+++ b/ethereum/chain.go
@@ -30,6 +30,8 @@ type Ethereum struct {
maxRetries int
retryIntervalSeconds int
minAmount uint64
+ MetricsDenom string
+ MetricsExponent int
mu sync.Mutex
}
@@ -47,6 +49,8 @@ func NewChain(
maxRetries int,
retryIntervalSeconds int,
minAmount uint64,
+ metricsDenom string,
+ metricsExponent int,
) (*Ethereum, error) {
privEcdsaKey, ethereumAddress, err := GetEcdsaKeyAddress(privateKey)
if err != nil {
@@ -66,6 +70,8 @@ func NewChain(
maxRetries: maxRetries,
retryIntervalSeconds: retryIntervalSeconds,
minAmount: minAmount,
+ MetricsDenom: metricsDenom,
+ MetricsExponent: metricsExponent,
}, nil
}
diff --git a/ethereum/config.go b/ethereum/config.go
index 9df3809..39fdcf4 100644
--- a/ethereum/config.go
+++ b/ethereum/config.go
@@ -21,6 +21,9 @@ type ChainConfig struct {
MinMintAmount uint64 `yaml:"min-mint-amount"`
+ MetricsDenom string `yaml:"metrics-denom"`
+ MetricsExponent int `yaml:"metrics-exponent"`
+
// TODO move to keyring
MinterPrivateKey string `yaml:"minter-private-key"`
}
@@ -39,5 +42,7 @@ func (c *ChainConfig) Chain(name string) (types.Chain, error) {
c.BroadcastRetries,
c.BroadcastRetryInterval,
c.MinMintAmount,
+ c.MetricsDenom,
+ c.MetricsExponent,
)
}
diff --git a/ethereum/listener.go b/ethereum/listener.go
index d604d37..9632dd2 100644
--- a/ethereum/listener.go
+++ b/ethereum/listener.go
@@ -6,6 +6,7 @@ import (
"fmt"
"math/big"
"os"
+ "time"
"cosmossdk.io/log"
ethereum "github.com/ethereum/go-ethereum"
@@ -13,6 +14,7 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/ethclient"
"github.com/pascaldekloe/etherstream"
+ "github.com/strangelove-ventures/noble-cctp-relayer/relayer"
"github.com/strangelove-ventures/noble-cctp-relayer/types"
)
@@ -42,8 +44,6 @@ func (e *Ethereum) StartListener(
os.Exit(1)
}
- // defer ethClient.Close()
-
messageTransmitterAddress := common.HexToAddress(e.messageTransmitterAddress)
etherReader := etherstream.Reader{Backend: ethClient}
@@ -129,3 +129,80 @@ func (e *Ethereum) StartListener(
}
}()
}
+
+func (e *Ethereum) WalletBalanceMetric(ctx context.Context, logger log.Logger, m *relayer.PromMetrics) {
+ logger = logger.With("metric", "wallet blance", "chain", e.name, "domain", e.domain)
+ queryRate := 30 * time.Second
+
+ var err error
+ var client *ethclient.Client
+
+ account := common.HexToAddress(e.minterAddress)
+
+ exponent := big.NewInt(int64(e.MetricsExponent)) // ex: 18
+ scaleFactor := new(big.Float).SetInt(new(big.Int).Exp(big.NewInt(10), exponent, nil)) // ex: 10^18
+
+ defer func() {
+ if client != nil {
+ client.Close()
+ }
+ }()
+
+ first := make(chan struct{}, 1)
+ first <- struct{}{}
+ createClient := true
+ for {
+ timer := time.NewTimer(queryRate)
+ select {
+ // don't wait the "queryRate" amount of time if this is the first time running
+ case <-first:
+ timer.Stop()
+ if createClient {
+ client, err = ethclient.DialContext(ctx, e.rpcURL)
+ if err != nil {
+ logger.Error(fmt.Sprintf("error dialing eth client. Will try again in %d sec", queryRate), "error", err)
+ createClient = true
+ continue
+ }
+ }
+ balance, err := client.BalanceAt(ctx, account, nil)
+ if err != nil {
+ logger.Error(fmt.Sprintf("error querying balance. Will try again in %d sec", queryRate), "error", err)
+ createClient = true
+ continue
+ }
+
+ balanceBigFloat := new(big.Float).SetInt(balance)
+ balanceScaled, _ := new(big.Float).Quo(balanceBigFloat, scaleFactor).Float64()
+
+ m.SetWalletBalance(e.name, e.minterAddress, e.MetricsDenom, balanceScaled)
+
+ createClient = false
+ case <-timer.C:
+ if createClient {
+ client, err = ethclient.DialContext(ctx, e.rpcURL)
+ if err != nil {
+ logger.Error(fmt.Sprintf("error dialing eth client. Will try again in %d sec", queryRate), "error", err)
+ createClient = true
+ continue
+ }
+ }
+ balance, err := client.BalanceAt(ctx, account, nil)
+ if err != nil {
+ logger.Error(fmt.Sprintf("error querying balance. Will try again in %d sec", queryRate), "error", err)
+ createClient = true
+ continue
+ }
+
+ balanceBigFloat := new(big.Float).SetInt(balance)
+ balanceScaled, _ := new(big.Float).Quo(balanceBigFloat, scaleFactor).Float64()
+
+ m.SetWalletBalance(e.name, e.minterAddress, e.MetricsDenom, balanceScaled)
+
+ createClient = false
+ case <-ctx.Done():
+ timer.Stop()
+ return
+ }
+ }
+}
diff --git a/go.mod b/go.mod
index 3ab9da1..890a01a 100644
--- a/go.mod
+++ b/go.mod
@@ -23,6 +23,7 @@ require (
github.com/gin-gonic/gin v1.9.1
github.com/joho/godotenv v1.5.1
github.com/pascaldekloe/etherstream v0.1.0
+ github.com/prometheus/client_golang v1.14.0
google.golang.org/grpc v1.60.0
gopkg.in/yaml.v2 v2.4.0
)
@@ -127,7 +128,6 @@ require (
github.com/petermattis/goid v0.0.0-20230317030725-371a4b8eda08 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
- github.com/prometheus/client_golang v1.14.0 // indirect
github.com/prometheus/client_model v0.3.0 // indirect
github.com/prometheus/common v0.42.0 // indirect
github.com/prometheus/procfs v0.12.0 // indirect
diff --git a/integration/deployed_relayer_test.go b/integration/deployed_relayer_test.go
index 2fecdbd..d162036 100644
--- a/integration/deployed_relayer_test.go
+++ b/integration/deployed_relayer_test.go
@@ -58,6 +58,8 @@ func TestNobleBurnToEthDeployed(t *testing.T) {
// ethConfig := cfg["arbitrum"]
// ethConfig := cfg["avalanche"]
// ethConfig := cfg["optimism"]
+ // ethConfig := cfg["polygon"]
+ // ethConfig := cfg["base"]
destAddress := ethConfig.Address
@@ -198,6 +200,8 @@ func TestEthBurnToNobleDeployed(t *testing.T) {
// ethConfig := cfg["arbitrum"]
// ethConfig := cfg["avalanche"]
// ethConfig := cfg["optimism"]
+ // ethConfig := cfg["polygon"]
+ // ethConfig := cfg["base"]
// -- DEST CHAIN --
nobleCfg := cfg["noble"]
diff --git a/noble/listener.go b/noble/listener.go
index 138b402..461f64f 100644
--- a/noble/listener.go
+++ b/noble/listener.go
@@ -6,6 +6,7 @@ import (
"time"
"cosmossdk.io/log"
+ "github.com/strangelove-ventures/noble-cctp-relayer/relayer"
"github.com/strangelove-ventures/noble-cctp-relayer/types"
)
@@ -130,3 +131,7 @@ func (n *Noble) chainTip(ctx context.Context) (uint64, error) {
}
return uint64(res.SyncInfo.LatestBlockHeight), nil
}
+
+func (n *Noble) WalletBalanceMetric(ctx context.Context, logger log.Logger, m *relayer.PromMetrics) {
+ // Relaying is free. No need to track noble balance.
+}
diff --git a/relayer/metrics.go b/relayer/metrics.go
new file mode 100644
index 0000000..3b04cf7
--- /dev/null
+++ b/relayer/metrics.go
@@ -0,0 +1,44 @@
+package relayer
+
+import (
+ "fmt"
+ "log"
+ "net/http"
+
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+type PromMetrics struct {
+ WalletBalance *prometheus.GaugeVec
+}
+
+func InitPromMetrics(port int16) *PromMetrics {
+ reg := prometheus.NewRegistry()
+
+ // labels
+ var (
+ walletLabels = []string{"chain", "address", "denom"}
+ )
+
+ m := &PromMetrics{
+ WalletBalance: prometheus.NewGaugeVec(prometheus.GaugeOpts{
+ Name: "cctp_relayer_wallet_balance",
+ Help: "The current balance for a wallet",
+ }, walletLabels),
+ }
+
+ reg.MustRegister(m.WalletBalance)
+
+ // Expose /metrics HTTP endpoint
+ go func() {
+ http.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{Registry: reg}))
+ log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", port), nil))
+ }()
+
+ return m
+}
+
+func (m *PromMetrics) SetWalletBalance(chain, address, denom string, balance float64) {
+ m.WalletBalance.WithLabelValues(chain, address, denom).Set(balance)
+}
diff --git a/types/chain.go b/types/chain.go
index 388ed58..f8f05c3 100644
--- a/types/chain.go
+++ b/types/chain.go
@@ -4,6 +4,7 @@ import (
"context"
"cosmossdk.io/log"
+ "github.com/strangelove-ventures/noble-cctp-relayer/relayer"
)
// Chain is an interface for common CCTP source and destination chain operations.
@@ -39,4 +40,10 @@ type Chain interface {
msgs []*MessageState,
sequenceMap *SequenceMap,
) error
+
+ WalletBalanceMetric(
+ ctx context.Context,
+ logger log.Logger,
+ metric *relayer.PromMetrics,
+ )
}