feat(metrics): collect txmgr metrics (#725)

Co-authored-by: Gavin Yu <[email protected]>

driver/chain_syncer/calldata/syncer.go

@@ -148,7 +148,7 @@ func (s *Syncer) processL1Blocks(ctx context.Context) error {
 	// If there is a L1 reorg, we don't update the L1Current cursor.
 	if !s.reorgDetectedFlag {
 		s.state.SetL1Current(l1End)
-		metrics.DriverL1CurrentHeightGauge.Update(s.state.GetL1Current().Number.Int64())
+		metrics.DriverL1CurrentHeightGauge.Set(float64(s.state.GetL1Current().Number.Uint64()))
 	}
 
 	return nil
@@ -300,7 +300,7 @@ func (s *Syncer) onBlockProposed(
 		"withdrawals", len(payloadData.Withdrawals),
 	)
 
-	metrics.DriverL1CurrentHeightGauge.Update(int64(event.Raw.BlockNumber))
+	metrics.DriverL1CurrentHeightGauge.Set(float64(event.Raw.BlockNumber))
 	s.lastInsertedBlockID = event.BlockId
 
 	if s.progressTracker.Triggered() {

driver/state/state.go

@@ -161,7 +161,7 @@ func (s *State) setL1Head(l1Head *types.Header) {
 	}
 
 	log.Debug("New L1 head", "height", l1Head.Number, "hash", l1Head.Hash(), "timestamp", l1Head.Time)
-	metrics.DriverL1HeadHeightGauge.Update(l1Head.Number.Int64())
+	metrics.DriverL1HeadHeightGauge.Set(float64(l1Head.Number.Int64()))
 
 	s.l1Head.Store(l1Head)
 }
@@ -179,7 +179,7 @@ func (s *State) setL2Head(l2Head *types.Header) {
 	}
 
 	log.Debug("New L2 head", "height", l2Head.Number, "hash", l2Head.Hash(), "timestamp", l2Head.Time)
-	metrics.DriverL2HeadHeightGauge.Update(l2Head.Number.Int64())
+	metrics.DriverL2HeadHeightGauge.Set(float64(l2Head.Number.Uint64()))
 
 	s.l2Head.Store(l2Head)
 }
@@ -192,7 +192,7 @@ func (s *State) GetL2Head() *types.Header {
 // setHeadBlockID sets the last pending block ID concurrent safely.
 func (s *State) setHeadBlockID(id *big.Int) {
 	log.Debug("New head block ID", "ID", id)
-	metrics.DriverL2HeadIDGauge.Update(id.Int64())
+	metrics.DriverL2HeadIDGauge.Set(float64(id.Uint64()))
 	s.l2HeadBlockID.Store(id)
 }
 

go.mod

@@ -14,6 +14,7 @@ require (
 	github.com/labstack/echo/v4 v4.11.1
 	github.com/modern-go/reflect2 v1.0.2
 	github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5
+	github.com/prometheus/client_golang v1.19.0
 	github.com/prysmaticlabs/prysm/v4 v4.2.0
 	github.com/stretchr/testify v1.9.0
 	github.com/swaggo/swag v1.16.2
@@ -174,7 +175,6 @@ require (
 	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_golang v1.19.0 // indirect
 	github.com/prometheus/client_model v0.5.0 // indirect
 	github.com/prometheus/common v0.48.0 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect

internal/docker/nodes/docker-compose.yml

@@ -3,6 +3,7 @@ services:
     container_name: l1_node
     image: ghcr.io/foundry-rs/foundry:nightly
     restart: unless-stopped
+    platform: linux/amd64
     pull_policy: always
     ports:
       - "8545"

internal/metrics/metrics.go

@@ -2,46 +2,58 @@ package metrics
 
 import (
 	"context"
-	"net"
-	"net/http"
-	"strconv"
-	"time"
 
+	opMetrics "github.com/ethereum-optimism/optimism/op-service/metrics"
+	"github.com/ethereum-optimism/optimism/op-service/opio"
+	txmgrMetrics "github.com/ethereum-optimism/optimism/op-service/txmgr/metrics"
 	"github.com/ethereum/go-ethereum/log"
-	"github.com/ethereum/go-ethereum/metrics"
-	"github.com/ethereum/go-ethereum/metrics/prometheus"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/urfave/cli/v2"
 
 	"github.com/taikoxyz/taiko-client/cmd/flags"
 )
 
 // Metrics
 var (
+	registry = opMetrics.NewRegistry()
+	factory  = opMetrics.With(registry)
+
 	// Driver
-	DriverL1HeadHeightGauge     = metrics.NewRegisteredGauge("driver/l1Head/height", nil)
-	DriverL2HeadHeightGauge     = metrics.NewRegisteredGauge("driver/l2Head/height", nil)
-	DriverL1CurrentHeightGauge  = metrics.NewRegisteredGauge("driver/l1Current/height", nil)
-	DriverL2HeadIDGauge         = metrics.NewRegisteredGauge("driver/l2Head/id", nil)
-	DriverL2VerifiedHeightGauge = metrics.NewRegisteredGauge("driver/l2Verified/id", nil)
+	DriverL1HeadHeightGauge     = factory.NewGauge(prometheus.GaugeOpts{Name: "driver_l1Head_height"})
+	DriverL2HeadHeightGauge     = factory.NewGauge(prometheus.GaugeOpts{Name: "driver_l2Head_height"})
+	DriverL1CurrentHeightGauge  = factory.NewGauge(prometheus.GaugeOpts{Name: "driver_l1Current_height"})
+	DriverL2HeadIDGauge         = factory.NewGauge(prometheus.GaugeOpts{Name: "driver_l2Head_id"})
+	DriverL2VerifiedHeightGauge = factory.NewGauge(prometheus.GaugeOpts{Name: "driver_l2Verified_id"})
 
 	// Proposer
-	ProposerProposeEpochCounter    = metrics.NewRegisteredCounter("proposer/epoch", nil)
-	ProposerProposedTxListsCounter = metrics.NewRegisteredCounter("proposer/proposed/txLists", nil)
-	ProposerProposedTxsCounter     = metrics.NewRegisteredCounter("proposer/proposed/txs", nil)
+	ProposerProposeEpochCounter    = factory.NewCounter(prometheus.CounterOpts{Name: "proposer_epoch"})
+	ProposerProposedTxListsCounter = factory.NewCounter(prometheus.CounterOpts{Name: "proposer_proposed_txLists"})
+	ProposerProposedTxsCounter     = factory.NewCounter(prometheus.CounterOpts{Name: "proposer_proposed_txs"})
 
 	// Prover
-	ProverLatestVerifiedIDGauge      = metrics.NewRegisteredGauge("prover/latestVerified/id", nil)
-	ProverLatestProvenBlockIDGauge   = metrics.NewRegisteredGauge("prover/latestProven/id", nil)
-	ProverQueuedProofCounter         = metrics.NewRegisteredCounter("prover/proof/all/queued", nil)
-	ProverReceivedProofCounter       = metrics.NewRegisteredCounter("prover/proof/all/received", nil)
-	ProverSentProofCounter           = metrics.NewRegisteredCounter("prover/proof/all/sent", nil)
-	ProverProofsAssigned             = metrics.NewRegisteredCounter("prover/proof/assigned", nil)
-	ProverReceivedProposedBlockGauge = metrics.NewRegisteredGauge("prover/proposed/received", nil)
-	ProverReceivedProvenBlockGauge   = metrics.NewRegisteredGauge("prover/proven/received", nil)
-	ProverSubmissionAcceptedCounter  = metrics.NewRegisteredCounter("prover/proof/submission/accepted", nil)
-	ProverSubmissionErrorCounter     = metrics.NewRegisteredCounter("prover/proof/submission/error", nil)
-	ProverSgxProofGeneratedCounter   = metrics.NewRegisteredCounter("prover/proof/sgx/generated", nil)
-	ProverSubmissionRevertedCounter  = metrics.NewRegisteredCounter("prover/proof/submission/reverted", nil)
+	ProverLatestVerifiedIDGauge      = factory.NewGauge(prometheus.GaugeOpts{Name: "prover_latestVerified_id"})
+	ProverLatestProvenBlockIDGauge   = factory.NewGauge(prometheus.GaugeOpts{Name: "prover_latestProven_id"})
+	ProverQueuedProofCounter         = factory.NewCounter(prometheus.CounterOpts{Name: "prover_proof_all_queued"})
+	ProverReceivedProofCounter       = factory.NewCounter(prometheus.CounterOpts{Name: "prover_proof_all_received"})
+	ProverSentProofCounter           = factory.NewCounter(prometheus.CounterOpts{Name: "prover_proof_all_sent"})
+	ProverProofsAssigned             = factory.NewCounter(prometheus.CounterOpts{Name: "prover_proof_assigned"})
+	ProverReceivedProposedBlockGauge = factory.NewGauge(prometheus.GaugeOpts{Name: "prover_proposed_received"})
+	ProverReceivedProvenBlockGauge   = factory.NewGauge(prometheus.GaugeOpts{Name: "prover_proven_received"})
+	ProverSubmissionAcceptedCounter  = factory.NewCounter(prometheus.CounterOpts{
+		Name: "prover_proof_submission_accepted",
+	})
+	ProverSubmissionErrorCounter = factory.NewCounter(prometheus.CounterOpts{
+		Name: "prover_proof_submission_error",
+	})
+	ProverSgxProofGeneratedCounter = factory.NewCounter(prometheus.CounterOpts{
+		Name: "prover_proof_sgx_generated",
+	})
+	ProverSubmissionRevertedCounter = factory.NewCounter(prometheus.CounterOpts{
+		Name: "prover_proof_submission_reverted",
+	})
+
+	// TxManager
+	TxMgrMetrics = txmgrMetrics.MakeTxMetrics("client", factory)
 )
 
 // Serve starts the metrics server on the given address, will be closed when the given
@@ -51,25 +63,28 @@ func Serve(ctx context.Context, c *cli.Context) error {
 		return nil
 	}
 
-	address := net.JoinHostPort(
-		c.String(flags.MetricsAddr.Name),
-		strconv.Itoa(c.Int(flags.MetricsPort.Name)),
+	log.Info(
+		"Starting metrics server",
+		"host", c.String(flags.MetricsAddr.Name),
+		"port", c.Int(flags.MetricsPort.Name),
 	)
 
-	server := http.Server{
-		ReadHeaderTimeout: time.Minute,
-		Addr:              address,
-		Handler:           prometheus.Handler(metrics.DefaultRegistry),
+	server, err := opMetrics.StartServer(
+		registry,
+		c.String(flags.MetricsAddr.Name),
+		c.Int(flags.MetricsPort.Name),
+	)
+	if err != nil {
+		return err
 	}
 
-	go func() {
-		<-ctx.Done()
-		if err := server.Close(); err != nil {
+	defer func() {
+		if err := server.Stop(ctx); err != nil {
 			log.Error("Failed to close metrics server", "error", err)
 		}
 	}()
 
-	log.Info("Starting metrics server", "address", address)
+	opio.BlockOnInterruptsContext(ctx)
 
-	return server.ListenAndServe()
+	return nil
 }

proposer/proposer.go

@@ -10,7 +10,6 @@ import (
 
 	"github.com/ethereum-optimism/optimism/op-challenger/sender"
 	"github.com/ethereum-optimism/optimism/op-service/txmgr"
-	txmgrMetrics "github.com/ethereum-optimism/optimism/op-service/txmgr/metrics"
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
@@ -107,7 +106,7 @@ func (p *Proposer) InitFromConfig(ctx context.Context, cfg *Config) (err error)
 	txmgr, err := txmgr.NewSimpleTxManager(
 		"proposer",
 		log.Root(),
-		new(txmgrMetrics.NoopTxMetrics),
+		&metrics.TxMgrMetrics,
 		*cfg.TxmgrConfigs,
 	)
 	if err != nil {
@@ -182,7 +181,7 @@ func (p *Proposer) eventLoop() {
 			return
 		// proposing interval timer has been reached
 		case <-p.proposingTimer.C:
-			metrics.ProposerProposeEpochCounter.Inc(1)
+			metrics.ProposerProposeEpochCounter.Add(1)
 
 			// Attempt a proposing operation
 			if err := p.ProposeOp(p.ctx); err != nil {
@@ -231,6 +230,11 @@ func (p *Proposer) fetchPoolContent(filterPoolContent bool) ([]types.Transaction
 	}
 	// If the pool content is empty and the checkPoolContent flag is not set, return an empty list.
 	if !filterPoolContent && len(txLists) == 0 {
+		log.Info(
+			"Pool content is empty, proposing an empty block",
+			"lastProposedAt", p.lastProposedAt,
+			"minProposingInternal", p.MinProposingInternal,
+		)
 		txLists = append(txLists, types.Transactions{})
 	}
 
@@ -294,6 +298,11 @@ func (p *Proposer) ProposeOp(ctx context.Context) error {
 		return err
 	}
 
+	// If the pool content is empty, return.
+	if len(txLists) == 0 {
+		return nil
+	}
+
 	// Propose all L2 transactions lists.
 	for i, txs := range txLists {
 		if i >= int(p.MaxProposedTxListsPerEpoch) {
@@ -314,8 +323,8 @@ func (p *Proposer) ProposeOp(ctx context.Context) error {
 			continue
 		}
 
-		metrics.ProposerProposedTxListsCounter.Inc(1)
-		metrics.ProposerProposedTxsCounter.Inc(int64(len(txLists[i])))
+		metrics.ProposerProposedTxListsCounter.Add(1)
+		metrics.ProposerProposedTxsCounter.Add(float64(len(txLists[i])))
 
 		log.Info("📝 Propose transactions succeeded", "txs", len(txLists[i]))
 		p.lastProposedAt = time.Now()

prover/event_handler/block_proposed.go

@@ -124,7 +124,7 @@ func (h *BlockProposedEventHandler) Handle(
 		"minTier", e.Meta.MinTier,
 		"blobUsed", e.Meta.BlobUsed,
 	)
-	metrics.ProverReceivedProposedBlockGauge.Update(e.BlockId.Int64())
+	metrics.ProverReceivedProposedBlockGauge.Set(float64(e.BlockId.Uint64()))
 
 	// Move l1Current cursor.
 	newL1Current, err := h.rpc.L1.HeaderByHash(ctx, e.Raw.BlockHash)
@@ -323,7 +323,7 @@ func (h *BlockProposedEventHandler) checkExpirationAndSubmitProof(
 		"tier", tier,
 	)
 
-	metrics.ProverProofsAssigned.Inc(1)
+	metrics.ProverProofsAssigned.Add(1)
 
 	h.proofSubmissionCh <- &proofProducer.ProofRequestBody{Tier: tier, Event: e}
 

prover/event_handler/block_verified.go

@@ -12,7 +12,7 @@ type BlockVerifiedEventHandler struct{}
 
 // Handle handles the BlockVerified event.
 func (h *BlockVerifiedEventHandler) Handle(e *bindings.TaikoL1ClientBlockVerified) {
-	metrics.ProverLatestVerifiedIDGauge.Update(e.BlockId.Int64())
+	metrics.ProverLatestVerifiedIDGauge.Set(float64(e.BlockId.Uint64()))
 
 	log.Info(
 		"New verified block",

prover/event_handler/transition_proved.go

@@ -33,7 +33,7 @@ func (h *TransitionProvedEventHandler) Handle(
 	ctx context.Context,
 	e *bindings.TaikoL1ClientTransitionProved,
 ) error {
-	metrics.ProverReceivedProvenBlockGauge.Update(e.BlockId.Int64())
+	metrics.ProverReceivedProvenBlockGauge.Set(float64(e.BlockId.Uint64()))
 
 	// If this prover is in contest mode, we check the validity of this proof and if it's invalid,
 	// contest it with a higher tier proof.

prover/proof_producer/sgx_producer.go

@@ -99,7 +99,7 @@ func (s *SGXProofProducer) RequestProof(
 		return nil, err
 	}
 
-	metrics.ProverSgxProofGeneratedCounter.Inc(1)
+	metrics.ProverSgxProofGeneratedCounter.Add(1)
 
 	return &ProofWithHeader{
 		BlockID: blockID,

prover/proof_submitter/proof_submitter.go

@@ -114,7 +114,7 @@ func (s *ProofSubmitter) RequestProof(ctx context.Context, event *bindings.Taiko
 	}
 	s.resultCh <- result
 
-	metrics.ProverQueuedProofCounter.Inc(1)
+	metrics.ProverQueuedProofCounter.Add(1)
 
 	return nil
 }
@@ -135,7 +135,7 @@ func (s *ProofSubmitter) SubmitProof(
 		"tier", proofWithHeader.Tier,
 	)
 
-	metrics.ProverReceivedProofCounter.Inc(1)
+	metrics.ProverReceivedProofCounter.Add(1)
 
 	// Get the corresponding L2 block.
 	block, err := s.rpc.L2.BlockByHash(ctx, proofWithHeader.Header.Hash())
@@ -176,12 +176,12 @@ func (s *ProofSubmitter) SubmitProof(
 		if err.Error() == transaction.ErrUnretryableSubmission.Error() {
 			return nil
 		}
-		metrics.ProverSubmissionErrorCounter.Inc(1)
+		metrics.ProverSubmissionErrorCounter.Add(1)
 		return err
 	}
 
-	metrics.ProverSentProofCounter.Inc(1)
-	metrics.ProverLatestProvenBlockIDGauge.Update(proofWithHeader.BlockID.Int64())
+	metrics.ProverSentProofCounter.Add(1)
+	metrics.ProverLatestProvenBlockIDGauge.Set(float64(proofWithHeader.BlockID.Uint64()))
 
 	return nil
 }

prover/proof_submitter/transaction/sender.go

@@ -66,7 +66,7 @@ func (s *Sender) Send(
 			"tier", proofWithHeader.Tier,
 			"txHash", receipt.TxHash,
 		)
-		metrics.ProverSubmissionRevertedCounter.Inc(1)
+		metrics.ProverSubmissionRevertedCounter.Add(1)
 		return ErrUnretryableSubmission
 	}
 
@@ -81,7 +81,7 @@ func (s *Sender) Send(
 		"isContest", len(proofWithHeader.Proof) == 0,
 	)
 
-	metrics.ProverSubmissionAcceptedCounter.Inc(1)
+	metrics.ProverSubmissionAcceptedCounter.Add(1)
 
 	return nil
 }

prover/prover.go

@@ -12,7 +12,6 @@ import (
 
 	"github.com/cenkalti/backoff/v4"
 	"github.com/ethereum-optimism/optimism/op-service/txmgr"
-	"github.com/ethereum-optimism/optimism/op-service/txmgr/metrics"
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/vm"
@@ -21,6 +20,7 @@ import (
 
 	"github.com/taikoxyz/taiko-client/bindings"
 	"github.com/taikoxyz/taiko-client/bindings/encoding"
+	"github.com/taikoxyz/taiko-client/internal/metrics"
 	"github.com/taikoxyz/taiko-client/internal/version"
 	eventIterator "github.com/taikoxyz/taiko-client/pkg/chain_iterator/event_iterator"
 	"github.com/taikoxyz/taiko-client/pkg/rpc"
@@ -148,7 +148,7 @@ func InitFromConfig(ctx context.Context, p *Prover, cfg *Config) (err error) {
 	if p.txmgr, err = txmgr.NewSimpleTxManager(
 		"prover",
 		log.Root(),
-		new(metrics.NoopTxMetrics),
+		&metrics.TxMgrMetrics,
 		*cfg.TxmgrConfigs,
 	); err != nil {
 		return err