feat(observability): add prometheus metrics and detailed logging (#45)

* feat: install cobra and add version command

* fix: makefile targets

* feat: add release-snapshot command

* chore: add more detailed error logging

* fix: goreleaser commands

* fix: version

* fix: dockerfile

* feat: add prometheus counter

* feat: add aggregate price provider metric

* feat: add post prices prometheus counter

* chore: add metrics documentation

* chore: add metrics documentation

.goreleaser.yml

@@ -5,7 +5,7 @@ env:
 
 builds:
   - id: darwin
-    main: ./cmd/feeder
+    main: .
     binary: pricefeeder
     hooks:
       pre:
@@ -35,7 +35,7 @@ builds:
           - CC=oa64-clang
 
   - id: linux
-    main: ./cmd/feeder
+    main: .
     binary: pricefeeder
     hooks:
       pre:

Dockerfile

@@ -7,7 +7,7 @@ RUN go mod download
 COPY . .
 RUN --mount=type=cache,target=/root/.cache/go-build \
   --mount=type=cache,target=/go/pkg \
-  go build -o ./build/feeder ./cmd/feeder/
+  go build -o ./build/feeder .
 
 FROM gcr.io/distroless/static:nonroot
 

Makefile

@@ -11,10 +11,10 @@ test:
 	go test ./...
 
 run:
-	go run ./cmd/feeder/main.go
+	go run ./main.go
 
 run-debug:
-	go run ./cmd/feeder/main.go -debug true
+	go run ./main.go -debug true
 
 ###############################################################################
 ###                                Build                                    ###
@@ -45,3 +45,14 @@ release:
 		-e GITHUB_TOKEN=${GITHUB_TOKEN} \
 		goreleaser/goreleaser-cross:${GOLANG_CROSS_VERSION} \
 		release --rm-dist
+
+release-snapshot:
+	docker run \
+		--rm \
+		--platform=linux/amd64 \
+		-v "$(CURDIR)":/go/src/$(PACKAGE_NAME) \
+		-w /go/src/$(PACKAGE_NAME) \
+		-e CGO_ENABLED=1 \
+		-e GITHUB_TOKEN=${GITHUB_TOKEN} \
+		goreleaser/goreleaser-cross:${GOLANG_CROSS_VERSION} \
+		release --rm-dist --snapshot

README.md

@@ -36,7 +36,7 @@ This would allow you to run `pricefeeder` using a local instance of the network.
 ```bash
 git clone [email protected]:NibiruChain/nibiru.git
 cd nibiru
-git checkout v1.0.0
+git checkout v1.0.3
 make localnet
 ```
 

cmd/root.go

@@ -0,0 +1,83 @@
+package cmd
+
+import (
+	"flag"
+	"fmt"
+	"net/http"
+	"os"
+	"os/signal"
+
+	"github.com/NibiruChain/nibiru/app"
+	"github.com/NibiruChain/pricefeeder/config"
+	"github.com/NibiruChain/pricefeeder/feeder"
+	"github.com/NibiruChain/pricefeeder/feeder/eventstream"
+	"github.com/NibiruChain/pricefeeder/feeder/priceposter"
+	"github.com/NibiruChain/pricefeeder/feeder/priceprovider"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/rs/zerolog"
+	"github.com/spf13/cobra"
+)
+
+func setupLogger() zerolog.Logger {
+	zerolog.TimeFieldFormat = zerolog.TimeFormatUnix
+	debug := flag.Bool("debug", false, "sets log level to debug")
+	flag.Parse()
+	// Default level is INFO, unless debug flag is present
+	zerolog.SetGlobalLevel(zerolog.InfoLevel)
+	if *debug {
+		zerolog.SetGlobalLevel(zerolog.DebugLevel)
+	}
+
+	return zerolog.New(os.Stderr).With().Timestamp().Logger()
+}
+
+// handleInterrupt listens for SIGINT and gracefully shuts down the feeder.
+func handleInterrupt(logger zerolog.Logger, f *feeder.Feeder) {
+	interrupt := make(chan os.Signal, 1)
+	signal.Notify(interrupt, os.Interrupt)
+
+	go func() {
+		<-interrupt
+		logger.Info().Msg("shutting down gracefully")
+		f.Close()
+		os.Exit(1)
+	}()
+}
+
+var rootCmd = &cobra.Command{
+	Use:   "pricefeeder",
+	Short: "Pricefeeder daemon for posting prices to Nibiru Chain",
+	Run: func(cmd *cobra.Command, args []string) {
+		logger := setupLogger()
+		app.SetPrefixes(app.AccountAddressPrefix)
+
+		c := config.MustGet()
+
+		eventStream := eventstream.Dial(c.WebsocketEndpoint, c.GRPCEndpoint, c.EnableTLS, logger)
+		priceProvider := priceprovider.NewAggregatePriceProvider(c.ExchangesToPairToSymbolMap, c.DataSourceConfigMap, logger)
+		kb, valAddr, feederAddr := config.GetAuth(c.FeederMnemonic)
+
+		if c.ValidatorAddr != nil {
+			valAddr = *c.ValidatorAddr
+		}
+		pricePoster := priceposter.Dial(c.GRPCEndpoint, c.ChainID, c.EnableTLS, kb, valAddr, feederAddr, logger)
+
+		f := feeder.NewFeeder(eventStream, priceProvider, pricePoster, logger)
+		f.Run()
+		defer f.Close()
+
+		handleInterrupt(logger, f)
+
+		http.Handle("/metrics", promhttp.Handler())
+		http.ListenAndServe(":8080", nil)
+
+		select {}
+	},
+}
+
+func Execute() {
+	if err := rootCmd.Execute(); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}

cmd/version.go

@@ -0,0 +1,20 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	rootCmd.AddCommand(versionCmd)
+}
+
+var versionCmd = &cobra.Command{
+	Use:   "version",
+	Short: "Print the version number of Hugo",
+	Long:  `All software has versions. This is Hugo's`,
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("v1.0.3")
+	},
+}

feeder/priceposter/client.go

@@ -7,13 +7,16 @@ import (
 
 	"github.com/NibiruChain/nibiru/app"
 	oracletypes "github.com/NibiruChain/nibiru/x/oracle/types"
+	"github.com/NibiruChain/pricefeeder/metrics"
 	"github.com/NibiruChain/pricefeeder/types"
 	"github.com/cosmos/cosmos-sdk/client"
 	codectypes "github.com/cosmos/cosmos-sdk/codec/types"
 	"github.com/cosmos/cosmos-sdk/crypto/keyring"
 	sdk "github.com/cosmos/cosmos-sdk/types"
 	txservice "github.com/cosmos/cosmos-sdk/types/tx"
 	authtypes "github.com/cosmos/cosmos-sdk/x/auth/types"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/rs/zerolog"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
@@ -102,6 +105,12 @@ func (c *Client) Whoami() sdk.ValAddress {
 	return c.validator
 }
 
+var pricePosterCounter = promauto.NewCounterVec(prometheus.CounterOpts{
+	Namespace: metrics.PrometheusNamespace,
+	Name:      "prices_posted_total",
+	Help:      "The total number of price update txs sent to the chain, by success status",
+}, []string{"success"})
+
 func (c *Client) SendPrices(vp types.VotingPeriod, prices []types.Price) {
 	logger := c.logger.With().Uint64("voting-period-height", vp.Height).Logger()
 
@@ -112,11 +121,13 @@ func (c *Client) SendPrices(vp types.VotingPeriod, prices []types.Price) {
 	resp, err := vote(ctx, newPrevote, c.previousPrevote, c.validator, c.feeder, c.deps, logger)
 	if err != nil {
 		logger.Err(err).Msg("prevote failed")
+		pricePosterCounter.WithLabelValues("false").Inc()
 		return
 	}
 
 	c.previousPrevote = newPrevote
 	logger.Info().Str("tx-hash", resp.TxHash).Msg("successfully forwarded prices")
+	pricePosterCounter.WithLabelValues("true").Inc()
 }
 
 func (c *Client) Close() {

feeder/priceprovider/aggregateprovider.go

@@ -4,7 +4,10 @@ import (
 	"encoding/json"
 
 	"github.com/NibiruChain/nibiru/x/common/asset"
+	"github.com/NibiruChain/pricefeeder/metrics"
 	"github.com/NibiruChain/pricefeeder/types"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/rs/zerolog"
 )
 
@@ -37,6 +40,12 @@ func NewAggregatePriceProvider(
 	}
 }
 
+var aggregatePriceProvider = promauto.NewCounterVec(prometheus.CounterOpts{
+	Namespace: metrics.PrometheusNamespace,
+	Name:      "aggregate_prices_total",
+	Help:      "The total number prices provided by the aggregate price provider, by pair, source, and success status",
+}, []string{"pair", "source", "success"})
+
 // GetPrice fetches the first available and correct price from the wrapped PriceProviders.
 // Iteration is exhaustive and random.
 // If no correct PriceResponse is found, then an invalid PriceResponse is returned.
@@ -46,12 +55,14 @@ func (a AggregatePriceProvider) GetPrice(pair asset.Pair) types.Price {
 	for _, p := range a.providers {
 		price := p.GetPrice(pair)
 		if price.Valid {
+			aggregatePriceProvider.WithLabelValues(pair.String(), price.SourceName, "true").Inc()
 			return price
 		}
 	}
 
 	// if we reach here no valid symbols were found
 	a.logger.Warn().Str("pair", pair.String()).Msg("no valid price found")
+	aggregatePriceProvider.WithLabelValues(pair.String(), "missing", "false").Inc()
 	return types.Price{
 		SourceName: "missing",
 		Pair:       pair,

feeder/priceprovider/sources/binance.go

@@ -6,6 +6,7 @@ import (
 	"net/http"
 
 	"github.com/NibiruChain/nibiru/x/common/set"
+	"github.com/NibiruChain/pricefeeder/metrics"
 	"github.com/NibiruChain/pricefeeder/types"
 	"github.com/rs/zerolog"
 )
@@ -36,19 +37,25 @@ func BinancePriceUpdate(symbols set.Set[types.Symbol], logger zerolog.Logger) (r
 	url := "https://api.binance.us/api/v3/ticker/price?symbols=%5B" + BinanceSymbolCsv(symbols) + "%5D"
 	resp, err := http.Get(url)
 	if err != nil {
+		logger.Err(err).Msg("failed to fetch prices from Binance")
+		metrics.PriceSourceCounter.WithLabelValues(Binance, "false").Inc()
 		return nil, err
 	}
 	defer resp.Body.Close()
 
 	b, err := io.ReadAll(resp.Body)
 	if err != nil {
+		logger.Err(err).Msg("failed to read response body from Binance")
+		metrics.PriceSourceCounter.WithLabelValues(Binance, "false").Inc()
 		return nil, err
 	}
 
 	tickers := make([]BinanceTicker, len(symbols))
 
 	err = json.Unmarshal(b, &tickers)
 	if err != nil {
+		logger.Err(err).Msg("failed to unmarshal response body from Binance")
+		metrics.PriceSourceCounter.WithLabelValues(Binance, "false").Inc()
 		return nil, err
 	}
 
@@ -57,6 +64,7 @@ func BinancePriceUpdate(symbols set.Set[types.Symbol], logger zerolog.Logger) (r
 		rawPrices[types.Symbol(ticker.Symbol)] = ticker.Price
 		logger.Debug().Msgf("fetched price for %s on data source %s: %f", ticker.Symbol, Binance, ticker.Price)
 	}
+	metrics.PriceSourceCounter.WithLabelValues(Binance, "true").Inc()
 
 	return rawPrices, nil
 }