From b4a91aee3308e57336fdcfa53222f1a4bf3224c0 Mon Sep 17 00:00:00 2001 From: Aaron Lu <50029043+aalu1418@users.noreply.github.com> Date: Wed, 5 Jun 2024 11:46:54 -0600 Subject: [PATCH] SOM: network fees (#728) --- cmd/monitoring/main.go | 12 ++- pkg/monitoring/chain_reader.go | 16 +++ pkg/monitoring/exporter/networkfees.go | 104 ++++++++++++++++++++ pkg/monitoring/exporter/networkfees_test.go | 62 ++++++++++++ pkg/monitoring/metrics/metrics.go | 12 +++ pkg/monitoring/metrics/mocks/NetworkFees.go | 38 +++++++ pkg/monitoring/metrics/networkfees.go | 70 +++++++++++++ pkg/monitoring/metrics/networkfees_test.go | 50 ++++++++++ pkg/monitoring/mocks/ChainReader.go | 26 +++++ pkg/monitoring/source_networkfees.go | 49 +++++++++ pkg/monitoring/source_networkfees_test.go | 39 ++++++++ pkg/monitoring/types/types.go | 3 + 12 files changed, 480 insertions(+), 1 deletion(-) create mode 100644 pkg/monitoring/exporter/networkfees.go create mode 100644 pkg/monitoring/exporter/networkfees_test.go create mode 100644 pkg/monitoring/metrics/mocks/NetworkFees.go create mode 100644 pkg/monitoring/metrics/networkfees.go create mode 100644 pkg/monitoring/metrics/networkfees_test.go create mode 100644 pkg/monitoring/source_networkfees.go create mode 100644 pkg/monitoring/source_networkfees_test.go diff --git a/cmd/monitoring/main.go b/cmd/monitoring/main.go index 219df2fca..e5630e666 100644 --- a/cmd/monitoring/main.go +++ b/cmd/monitoring/main.go @@ -77,11 +77,16 @@ func main() { ) slotHeightSourceFactory := monitoring.NewSlotHeightSourceFactory( chainReader, - logger.With(log, "component", "souce-slot-height"), + logger.With(log, "component", "source-slot-height"), + ) + networkFeesSourceFactory := monitoring.NewNetworkFeesSourceFactory( + chainReader, + logger.With(log, "component", "source-network-fees"), ) monitor.NetworkSourceFactories = append(monitor.NetworkSourceFactories, nodeBalancesSourceFactory, slotHeightSourceFactory, + networkFeesSourceFactory, ) // exporter names @@ -121,9 +126,14 @@ func main() { logger.With(log, "component", promExporter), metrics.NewSlotHeight(logger.With(log, "component", promMetrics)), ) + networkFeesExporterFactory := exporter.NewNetworkFeesFactory( + logger.With(log, "component", promExporter), + metrics.NewNetworkFees(logger.With(log, "component", promMetrics)), + ) monitor.NetworkExporterFactories = append(monitor.NetworkExporterFactories, nodeBalancesExporterFactory, slotHeightExporterFactory, + networkFeesExporterFactory, ) monitor.Run() diff --git a/pkg/monitoring/chain_reader.go b/pkg/monitoring/chain_reader.go index ffefd4f59..eb4d4b8e5 100644 --- a/pkg/monitoring/chain_reader.go +++ b/pkg/monitoring/chain_reader.go @@ -19,6 +19,7 @@ type ChainReader interface { GetSignaturesForAddressWithOpts(ctx context.Context, account solana.PublicKey, opts *rpc.GetSignaturesForAddressOpts) (out []*rpc.TransactionSignature, err error) GetTransaction(ctx context.Context, txSig solana.Signature, opts *rpc.GetTransactionOpts) (out *rpc.GetTransactionResult, err error) GetSlot(ctx context.Context) (slot uint64, err error) + GetLatestBlock(ctx context.Context, commitment rpc.CommitmentType) (*rpc.GetBlockResult, error) } func NewChainReader(client *rpc.Client) ChainReader { @@ -56,3 +57,18 @@ func (c *chainReader) GetTransaction(ctx context.Context, txSig solana.Signature func (c *chainReader) GetSlot(ctx context.Context) (uint64, error) { return c.client.GetSlot(ctx, rpc.CommitmentProcessed) // get latest height } + +func (c *chainReader) GetLatestBlock(ctx context.Context, commitment rpc.CommitmentType) (*rpc.GetBlockResult, error) { + // get slot based on confirmation + slot, err := c.client.GetSlot(ctx, commitment) + if err != nil { + return nil, err + } + + // get block based on slot + version := uint64(0) // pull all tx types (legacy + v0) + return c.client.GetBlockWithOpts(ctx, slot, &rpc.GetBlockOpts{ + Commitment: commitment, + MaxSupportedTransactionVersion: &version, + }) +} diff --git a/pkg/monitoring/exporter/networkfees.go b/pkg/monitoring/exporter/networkfees.go new file mode 100644 index 000000000..7e8b63f48 --- /dev/null +++ b/pkg/monitoring/exporter/networkfees.go @@ -0,0 +1,104 @@ +package exporter + +import ( + "context" + "errors" + "slices" + + commonMonitoring "github.com/smartcontractkit/chainlink-common/pkg/monitoring" + "github.com/smartcontractkit/chainlink-common/pkg/utils/mathutil" + "golang.org/x/exp/constraints" + + "github.com/smartcontractkit/chainlink-solana/pkg/monitoring/metrics" + "github.com/smartcontractkit/chainlink-solana/pkg/solana/fees" +) + +func NewNetworkFeesFactory( + lgr commonMonitoring.Logger, + metrics metrics.NetworkFees, +) commonMonitoring.ExporterFactory { + return &networkFeesFactory{ + metrics, + lgr, + } +} + +type networkFeesFactory struct { + metrics metrics.NetworkFees + lgr commonMonitoring.Logger +} + +func (p *networkFeesFactory) NewExporter( + params commonMonitoring.ExporterParams, +) (commonMonitoring.Exporter, error) { + return &networkFees{ + params.ChainConfig.GetNetworkName(), + p.metrics, + p.lgr, + }, nil +} + +type networkFees struct { + chain string + metrics metrics.NetworkFees + lgr commonMonitoring.Logger +} + +func (p *networkFees) Export(ctx context.Context, data interface{}) { + blockData, ok := data.(fees.BlockData) + if !ok { + return // skip if input could not be parsed + } + + input := metrics.NetworkFeesInput{} + if err := aggregateFees(input, "computeUnitPrice", blockData.Prices); err != nil { + p.lgr.Errorw("failed to calculate computeUnitPrice", "error", err) + return + } + if err := aggregateFees(input, "totalFee", blockData.Fees); err != nil { + p.lgr.Errorw("failed to calculate totalFee", "error", err) + return + } + + p.metrics.Set(input, p.chain) +} + +func (p *networkFees) Cleanup(_ context.Context) { + p.metrics.Cleanup() +} + +func aggregateFees[V constraints.Integer](input metrics.NetworkFeesInput, name string, data []V) error { + // skip if empty list + if len(data) == 0 { + return nil + } + + slices.Sort(data) // ensure sorted + + // calculate median / avg + medianPrice, medianPriceErr := mathutil.Median(data...) + input.Set(name, "median", uint64(medianPrice)) + avgPrice, avgPriceErr := mathutil.Avg(data...) + input.Set(name, "avg", uint64(avgPrice)) + + // calculate lower / upper quartile + var lowerData, upperData []V + l := len(data) + if l%2 == 0 { + lowerData = data[:l/2] + upperData = data[l/2:] + } else { + lowerData = data[:l/2] + upperData = data[l/2+1:] + } + lowerQuartilePrice, lowerQuartilePriceErr := mathutil.Median(lowerData...) + input.Set(name, "lowerQuartile", uint64(lowerQuartilePrice)) + upperQuartilePrice, upperQuartilePriceErr := mathutil.Median(upperData...) + input.Set(name, "upperQuartile", uint64(upperQuartilePrice)) + + // calculate min/max + input.Set(name, "max", uint64(slices.Max(data))) + input.Set(name, "min", uint64(slices.Min(data))) + + return errors.Join(medianPriceErr, avgPriceErr, lowerQuartilePriceErr, upperQuartilePriceErr) +} diff --git a/pkg/monitoring/exporter/networkfees_test.go b/pkg/monitoring/exporter/networkfees_test.go new file mode 100644 index 000000000..3f8d85e0a --- /dev/null +++ b/pkg/monitoring/exporter/networkfees_test.go @@ -0,0 +1,62 @@ +package exporter + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + commonMonitoring "github.com/smartcontractkit/chainlink-common/pkg/monitoring" + "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" + + "github.com/smartcontractkit/chainlink-solana/pkg/monitoring/metrics" + "github.com/smartcontractkit/chainlink-solana/pkg/monitoring/metrics/mocks" + "github.com/smartcontractkit/chainlink-solana/pkg/monitoring/testutils" + "github.com/smartcontractkit/chainlink-solana/pkg/solana/fees" +) + +func TestNetworkFees(t *testing.T) { + ctx := tests.Context(t) + m := mocks.NewNetworkFees(t) + m.On("Set", mock.Anything, mock.Anything).Once() + m.On("Cleanup").Once() + + factory := NewNetworkFeesFactory(logger.Test(t), m) + + chainConfig := testutils.GenerateChainConfig() + exporter, err := factory.NewExporter(commonMonitoring.ExporterParams{ChainConfig: chainConfig}) + require.NoError(t, err) + + // happy path + exporter.Export(ctx, fees.BlockData{}) + exporter.Cleanup(ctx) + + // test passing uint64 instead of NetworkFees - should not call mock + // NetworkFees alias of uint64 + exporter.Export(ctx, uint64(10)) +} + +func TestAggregateFees(t *testing.T) { + input := metrics.NetworkFeesInput{} + v0 := []int{10, 12, 3, 4, 1, 2} + v1 := []int{5, 1, 10, 2, 3, 12, 4} + + require.NoError(t, aggregateFees(input, "0", v0)) + require.NoError(t, aggregateFees(input, "1", v1)) + + assert.Equal(t, uint64(3), input["0"]["median"]) + assert.Equal(t, uint64(5), input["0"]["avg"]) + assert.Equal(t, uint64(1), input["0"]["min"]) + assert.Equal(t, uint64(12), input["0"]["max"]) + assert.Equal(t, uint64(2), input["0"]["lowerQuartile"]) + assert.Equal(t, uint64(10), input["0"]["upperQuartile"]) + + assert.Equal(t, uint64(4), input["1"]["median"]) + assert.Equal(t, uint64(5), input["1"]["avg"]) + assert.Equal(t, uint64(1), input["1"]["min"]) + assert.Equal(t, uint64(12), input["1"]["max"]) + assert.Equal(t, uint64(2), input["1"]["lowerQuartile"]) + assert.Equal(t, uint64(10), input["1"]["upperQuartile"]) +} diff --git a/pkg/monitoring/metrics/metrics.go b/pkg/monitoring/metrics/metrics.go index b108736eb..810f9c360 100644 --- a/pkg/monitoring/metrics/metrics.go +++ b/pkg/monitoring/metrics/metrics.go @@ -87,6 +87,18 @@ func init() { }, []string{"chain", "url"}, ) + + // init gauge for network fees + gauges[types.NetworkFeesMetric] = promauto.NewGaugeVec( + prometheus.GaugeOpts{ + Name: types.NetworkFeesMetric, + }, + []string{ + "type", // compute budget price, total fee + "operation", // avg, median, upper/lower quartile, min, max + "chain", + }, + ) } type FeedInput struct { diff --git a/pkg/monitoring/metrics/mocks/NetworkFees.go b/pkg/monitoring/metrics/mocks/NetworkFees.go new file mode 100644 index 000000000..187ff358e --- /dev/null +++ b/pkg/monitoring/metrics/mocks/NetworkFees.go @@ -0,0 +1,38 @@ +// Code generated by mockery v2.20.0. DO NOT EDIT. + +package mocks + +import ( + metrics "github.com/smartcontractkit/chainlink-solana/pkg/monitoring/metrics" + mock "github.com/stretchr/testify/mock" +) + +// NetworkFees is an autogenerated mock type for the NetworkFees type +type NetworkFees struct { + mock.Mock +} + +// Cleanup provides a mock function with given fields: +func (_m *NetworkFees) Cleanup() { + _m.Called() +} + +// Set provides a mock function with given fields: slot, chain +func (_m *NetworkFees) Set(slot metrics.NetworkFeesInput, chain string) { + _m.Called(slot, chain) +} + +type mockConstructorTestingTNewNetworkFees interface { + mock.TestingT + Cleanup(func()) +} + +// NewNetworkFees creates a new instance of NetworkFees. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewNetworkFees(t mockConstructorTestingTNewNetworkFees) *NetworkFees { + mock := &NetworkFees{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/pkg/monitoring/metrics/networkfees.go b/pkg/monitoring/metrics/networkfees.go new file mode 100644 index 000000000..9700529e3 --- /dev/null +++ b/pkg/monitoring/metrics/networkfees.go @@ -0,0 +1,70 @@ +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + commonMonitoring "github.com/smartcontractkit/chainlink-common/pkg/monitoring" + + "github.com/smartcontractkit/chainlink-solana/pkg/monitoring/types" +) + +//go:generate mockery --name NetworkFees --output ./mocks/ + +type NetworkFees interface { + Set(slot NetworkFeesInput, chain string) + Cleanup() +} + +var _ NetworkFees = (*networkFees)(nil) + +type networkFees struct { + simpleGauge + labels []prometheus.Labels +} + +func NewNetworkFees(log commonMonitoring.Logger) *networkFees { + return &networkFees{ + simpleGauge: newSimpleGauge(log, types.NetworkFeesMetric), + } +} + +func (sh *networkFees) Set(input NetworkFeesInput, chain string) { + for feeType, opMap := range input { + for operation, value := range opMap { + label := prometheus.Labels{ + "type": feeType, + "operation": operation, + "chain": chain, + } + sh.set(float64(value), label) + } + } + sh.labels = input.Labels(chain) +} + +func (sh *networkFees) Cleanup() { + for _, l := range sh.labels { + sh.delete(l) + } +} + +type NetworkFeesInput map[string]map[string]uint64 + +func (i NetworkFeesInput) Set(feeType, operation string, value uint64) { + if _, exists := i[feeType]; !exists { + i[feeType] = map[string]uint64{} + } + i[feeType][operation] = value +} + +func (i NetworkFeesInput) Labels(chain string) (l []prometheus.Labels) { + for feeType, opMap := range i { + for operation := range opMap { + l = append(l, prometheus.Labels{ + "type": feeType, + "operation": operation, + "chain": chain, + }) + } + } + return +} diff --git a/pkg/monitoring/metrics/networkfees_test.go b/pkg/monitoring/metrics/networkfees_test.go new file mode 100644 index 000000000..0d9f1e6a2 --- /dev/null +++ b/pkg/monitoring/metrics/networkfees_test.go @@ -0,0 +1,50 @@ +package metrics + +import ( + "slices" + "testing" + + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + + "github.com/smartcontractkit/chainlink-solana/pkg/monitoring/types" +) + +func TestNetworkFees(t *testing.T) { + lgr := logger.Test(t) + m := NewNetworkFees(lgr) + + // fetching gauges + g, ok := gauges[types.NetworkFeesMetric] + require.True(t, ok) + + input := NetworkFeesInput{} + chain := t.Name() + "_chain" + ind := 0 + for _, t := range []string{"fee", "computeUnitPrice"} { + for _, o := range []string{"median", "avg"} { + ind++ + input.Set(t, o, uint64(ind)) // 1..4 + } + } + + // set gauge + var values []int + assert.NotPanics(t, func() { m.Set(input, chain) }) + // check values + for _, l := range input.Labels(chain) { + promBal := testutil.ToFloat64(g.With(l)) + values = append(values, int(promBal)) + } + for i := 1; i <= ind; i++ { + assert.True(t, slices.Contains(values, i)) + } + + // cleanup gauges + assert.Equal(t, ind, testutil.CollectAndCount(g)) + assert.NotPanics(t, func() { m.Cleanup() }) + assert.Equal(t, 0, testutil.CollectAndCount(g)) +} diff --git a/pkg/monitoring/mocks/ChainReader.go b/pkg/monitoring/mocks/ChainReader.go index 609625c10..f2bf34ade 100644 --- a/pkg/monitoring/mocks/ChainReader.go +++ b/pkg/monitoring/mocks/ChainReader.go @@ -45,6 +45,32 @@ func (_m *ChainReader) GetBalance(ctx context.Context, account solana.PublicKey, return r0, r1 } +// GetLatestBlock provides a mock function with given fields: ctx, commitment +func (_m *ChainReader) GetLatestBlock(ctx context.Context, commitment rpc.CommitmentType) (*rpc.GetBlockResult, error) { + ret := _m.Called(ctx, commitment) + + var r0 *rpc.GetBlockResult + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, rpc.CommitmentType) (*rpc.GetBlockResult, error)); ok { + return rf(ctx, commitment) + } + if rf, ok := ret.Get(0).(func(context.Context, rpc.CommitmentType) *rpc.GetBlockResult); ok { + r0 = rf(ctx, commitment) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*rpc.GetBlockResult) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, rpc.CommitmentType) error); ok { + r1 = rf(ctx, commitment) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GetLatestTransmission provides a mock function with given fields: ctx, account, commitment func (_m *ChainReader) GetLatestTransmission(ctx context.Context, account solana.PublicKey, commitment rpc.CommitmentType) (pkgsolana.Answer, uint64, error) { ret := _m.Called(ctx, account, commitment) diff --git a/pkg/monitoring/source_networkfees.go b/pkg/monitoring/source_networkfees.go new file mode 100644 index 000000000..9952e76eb --- /dev/null +++ b/pkg/monitoring/source_networkfees.go @@ -0,0 +1,49 @@ +package monitoring + +import ( + "context" + + "github.com/gagliardetto/solana-go/rpc" + commonMonitoring "github.com/smartcontractkit/chainlink-common/pkg/monitoring" + + "github.com/smartcontractkit/chainlink-solana/pkg/monitoring/types" + "github.com/smartcontractkit/chainlink-solana/pkg/solana/fees" +) + +func NewNetworkFeesSourceFactory( + client ChainReader, + log commonMonitoring.Logger, +) commonMonitoring.NetworkSourceFactory { + return &networkFeesSourceFactory{ + client, + log, + } +} + +type networkFeesSourceFactory struct { + client ChainReader + log commonMonitoring.Logger +} + +func (s *networkFeesSourceFactory) NewSource( + _ commonMonitoring.ChainConfig, + _ []commonMonitoring.NodeConfig, +) (commonMonitoring.Source, error) { + return &networkFeesSource{s.client}, nil +} + +func (s *networkFeesSourceFactory) GetType() string { + return types.NetworkFeesType +} + +type networkFeesSource struct { + client ChainReader +} + +func (t *networkFeesSource) Fetch(ctx context.Context) (interface{}, error) { + block, err := t.client.GetLatestBlock(ctx, rpc.CommitmentConfirmed) + if err != nil { + return nil, err + } + return fees.ParseBlock(block) // return fees.BlockData, err +} diff --git a/pkg/monitoring/source_networkfees_test.go b/pkg/monitoring/source_networkfees_test.go new file mode 100644 index 000000000..05bc1e984 --- /dev/null +++ b/pkg/monitoring/source_networkfees_test.go @@ -0,0 +1,39 @@ +package monitoring + +import ( + "testing" + + "github.com/gagliardetto/solana-go/rpc" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" + + "github.com/smartcontractkit/chainlink-solana/pkg/monitoring/mocks" + "github.com/smartcontractkit/chainlink-solana/pkg/monitoring/types" + "github.com/smartcontractkit/chainlink-solana/pkg/solana/fees" +) + +func TestNetworkFeesSource(t *testing.T) { + cr := mocks.NewChainReader(t) + lgr := logger.Test(t) + ctx := tests.Context(t) + + factory := NewNetworkFeesSourceFactory(cr, lgr) + assert.Equal(t, types.NetworkFeesType, factory.GetType()) + + // generate source + source, err := factory.NewSource(nil, nil) + require.NoError(t, err) + cr.On("GetLatestBlock", mock.Anything, mock.Anything).Return(&rpc.GetBlockResult{}, nil).Once() + + // happy path + out, err := source.Fetch(ctx) + require.NoError(t, err) + slot, ok := out.(fees.BlockData) + require.True(t, ok) + assert.Equal(t, 0, len(slot.Fees)) + assert.Equal(t, 0, len(slot.Prices)) +} diff --git a/pkg/monitoring/types/types.go b/pkg/monitoring/types/types.go index 665a11b0f..db04f6037 100644 --- a/pkg/monitoring/types/types.go +++ b/pkg/monitoring/types/types.go @@ -4,6 +4,9 @@ package types const ( SlotHeightType = "slot_height" SlotHeightMetric = "sol_" + SlotHeightType + + NetworkFeesType = "network_fees" + NetworkFeesMetric = "sol_" + NetworkFeesType ) // SlotHeight type wraps the uint64 type returned by the RPC call