From 7c053119a13dc085a0417d6f1912c60213dab60c Mon Sep 17 00:00:00 2001 From: Owen Date: Tue, 4 Jul 2023 09:29:38 -0400 Subject: [PATCH] [EVM-696] eth_FeeHistory (#1669) Implement feeHistory into gasprice package, add eth endpoint --- gasprice/feehistory.go | 172 ++++++++++++++++++++++ gasprice/feehistory_test.go | 278 ++++++++++++++++++++++++++++++++++++ gasprice/gasprice.go | 16 ++- gasprice/gasprice_test.go | 12 +- jsonrpc/eth_endpoint.go | 42 ++++++ jsonrpc/types.go | 34 +++++ server/server.go | 5 +- 7 files changed, 552 insertions(+), 7 deletions(-) create mode 100644 gasprice/feehistory.go create mode 100644 gasprice/feehistory_test.go diff --git a/gasprice/feehistory.go b/gasprice/feehistory.go new file mode 100644 index 0000000000..89b941b3f9 --- /dev/null +++ b/gasprice/feehistory.go @@ -0,0 +1,172 @@ +package gasprice + +import ( + "encoding/binary" + "errors" + "math" + "math/big" + "sort" +) + +var ( + ErrInvalidPercentile = errors.New("invalid percentile") + ErrBlockCount = errors.New("blockCount must be greater than 0") + ErrBlockNotFound = errors.New("could not find block") +) + +const ( + maxBlockRequest = 1024 +) + +type cacheKey struct { + number uint64 + percentiles string +} + +// processedFees contains the results of a processed block. +type processedFees struct { + reward []uint64 + baseFee uint64 + gasUsedRatio float64 +} + +type txGasAndReward struct { + gasUsed *big.Int + reward *big.Int +} + +type FeeHistoryReturn struct { + OldestBlock uint64 + BaseFeePerGas []uint64 + GasUsedRatio []float64 + Reward [][]uint64 +} + +func (g *GasHelper) FeeHistory(blockCount uint64, newestBlock uint64, rewardPercentiles []float64) ( + *FeeHistoryReturn, error) { + if blockCount < 1 { + return &FeeHistoryReturn{0, nil, nil, nil}, ErrBlockCount + } + + if newestBlock > g.backend.Header().Number { + newestBlock = g.backend.Header().Number + } + + if blockCount > maxBlockRequest { + blockCount = maxBlockRequest + } + + if blockCount > newestBlock { + blockCount = newestBlock + } + + for i, p := range rewardPercentiles { + if p < 0 || p > 100 { + return &FeeHistoryReturn{0, nil, nil, nil}, ErrInvalidPercentile + } + + if i > 0 && p < rewardPercentiles[i-1] { + return &FeeHistoryReturn{0, nil, nil, nil}, ErrInvalidPercentile + } + } + + var ( + oldestBlock = newestBlock - blockCount + 1 + baseFeePerGas = make([]uint64, blockCount+1) + gasUsedRatio = make([]float64, blockCount) + reward = make([][]uint64, blockCount) + ) + + if oldestBlock < 1 { + oldestBlock = 1 + } + + percentileKey := make([]byte, 8*len(rewardPercentiles)) + for i, p := range rewardPercentiles { + binary.LittleEndian.PutUint64(percentileKey[i*8:(i+1)*8], math.Float64bits(p)) + } + + for i := oldestBlock; i <= newestBlock; i++ { + cacheKey := cacheKey{number: i, percentiles: string(percentileKey)} + //cache is hit, load from cache and continue to next block + if p, ok := g.historyCache.Get(cacheKey); ok { + processedFee, isOk := p.(*processedFees) + if !isOk { + return &FeeHistoryReturn{0, nil, nil, nil}, errors.New("could not convert catched processed fee") + } + + baseFeePerGas[i-oldestBlock] = processedFee.baseFee + gasUsedRatio[i-oldestBlock] = processedFee.gasUsedRatio + reward[i-oldestBlock] = processedFee.reward + + continue + } + + block, ok := g.backend.GetBlockByNumber(i, true) + if !ok { + return &FeeHistoryReturn{0, nil, nil, nil}, ErrBlockNotFound + } + + baseFeePerGas[i-oldestBlock] = block.Header.BaseFee + gasUsedRatio[i-oldestBlock] = float64(block.Header.GasUsed) / float64(block.Header.GasLimit) + + if math.IsNaN(gasUsedRatio[i-oldestBlock]) { + //gasUsedRatio is NaN, set to 0 + gasUsedRatio[i-oldestBlock] = 0 + } + + if len(rewardPercentiles) == 0 { + //reward percentiles not requested, skip rest of this loop + continue + } + + reward[i-oldestBlock] = make([]uint64, len(rewardPercentiles)) + if len(block.Transactions) == 0 { + for j := range reward[i-oldestBlock] { + reward[i-oldestBlock][j] = 0 + } + //no transactions in block, set rewards to 0 and move to next block + continue + } + + sorter := make([]*txGasAndReward, len(block.Transactions)) + + for j, tx := range block.Transactions { + cost := tx.Cost() + sorter[j] = &txGasAndReward{ + gasUsed: cost.Sub(cost, tx.Value), + reward: tx.EffectiveTip(block.Header.BaseFee), + } + } + + sort.Slice(sorter, func(i, j int) bool { + return sorter[i].reward.Cmp(sorter[j].reward) < 0 + }) + + var txIndex int + + sumGasUsed := sorter[0].gasUsed.Uint64() + + // calculate reward for each percentile + for c, v := range rewardPercentiles { + thresholdGasUsed := uint64(float64(block.Header.GasUsed) * v / 100) + for sumGasUsed < thresholdGasUsed && txIndex < len(block.Transactions)-1 { + txIndex++ + sumGasUsed += sorter[txIndex].gasUsed.Uint64() + } + + reward[i-oldestBlock][c] = sorter[txIndex].reward.Uint64() + } + + blockFees := &processedFees{ + reward: reward[i-oldestBlock], + baseFee: block.Header.BaseFee, + gasUsedRatio: gasUsedRatio[i-oldestBlock], + } + g.historyCache.Add(cacheKey, blockFees) + } + + baseFeePerGas[blockCount] = g.backend.Header().BaseFee + + return &FeeHistoryReturn{oldestBlock, baseFeePerGas, gasUsedRatio, reward}, nil +} diff --git a/gasprice/feehistory_test.go b/gasprice/feehistory_test.go new file mode 100644 index 0000000000..124869c9e2 --- /dev/null +++ b/gasprice/feehistory_test.go @@ -0,0 +1,278 @@ +package gasprice + +import ( + "math/rand" + "testing" + "time" + + "github.com/0xPolygon/polygon-edge/chain" + "github.com/0xPolygon/polygon-edge/crypto" + "github.com/0xPolygon/polygon-edge/helper/tests" + "github.com/0xPolygon/polygon-edge/types" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/umbracle/ethgo" +) + +func TestGasHelper_FeeHistory(t *testing.T) { + t.Parallel() + + var cases = []struct { + Name string + ExpectedOldestBlock uint64 + ExpectedBaseFeePerGas []uint64 + ExpectedGasUsedRatio []float64 + ExpectedRewards [][]uint64 + BlockRange uint64 + NewestBlock uint64 + RewardPercentiles []float64 + Error bool + GetBackend func() Blockchain + }{ + { + Name: "Block does not exist", + Error: true, + BlockRange: 10, + NewestBlock: 30, + RewardPercentiles: []float64{15, 20}, + GetBackend: func() Blockchain { + header := &types.Header{ + Number: 1, + Hash: types.StringToHash("some header"), + } + backend := new(backendMock) + backend.On("Header").Return(header) + backend.On("GetBlockByNumber", mock.Anything, mock.Anything).Return(&types.Block{}, false) + + return backend + }, + }, + { + Name: "blockRange < 1", + Error: true, + BlockRange: 0, + NewestBlock: 30, + RewardPercentiles: []float64{10, 15}, + GetBackend: func() Blockchain { + backend := createTestBlocks(t, 30) + + return backend + }, + }, + { + Name: "Invalid rewardPercentile", + Error: true, + BlockRange: 10, + NewestBlock: 30, + RewardPercentiles: []float64{101, 0}, + GetBackend: func() Blockchain { + backend := createTestBlocks(t, 50) + createTestTxs(t, backend, 1, 200) + + return backend + }, + }, + { + Name: "rewardPercentile not set", + BlockRange: 5, + NewestBlock: 30, + RewardPercentiles: []float64{}, + ExpectedOldestBlock: 26, + ExpectedBaseFeePerGas: []uint64{ + chain.GenesisBaseFee, + chain.GenesisBaseFee, + chain.GenesisBaseFee, + chain.GenesisBaseFee, + chain.GenesisBaseFee, + chain.GenesisBaseFee, + }, + ExpectedGasUsedRatio: []float64{ + 0, 0, 0, 0, 0, + }, + ExpectedRewards: [][]uint64{ + nil, nil, nil, nil, nil, + }, + GetBackend: func() Blockchain { + backend := createTestBlocks(t, 30) + createTestTxs(t, backend, 5, 500) + + return backend + }, + }, + { + Name: "blockRange > newestBlock", + BlockRange: 20, + NewestBlock: 10, + RewardPercentiles: []float64{}, + ExpectedOldestBlock: 1, + ExpectedBaseFeePerGas: []uint64{ + chain.GenesisBaseFee, + chain.GenesisBaseFee, + chain.GenesisBaseFee, + chain.GenesisBaseFee, + chain.GenesisBaseFee, + chain.GenesisBaseFee, + chain.GenesisBaseFee, + chain.GenesisBaseFee, + chain.GenesisBaseFee, + chain.GenesisBaseFee, + chain.GenesisBaseFee, + }, + ExpectedGasUsedRatio: []float64{ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + }, + ExpectedRewards: [][]uint64{ + nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + }, + GetBackend: func() Blockchain { + backend := createTestBlocks(t, 30) + createTestTxs(t, backend, 5, 500) + + return backend + }, + }, + { + Name: "blockRange == newestBlock", + BlockRange: 10, + NewestBlock: 10, + RewardPercentiles: []float64{}, + ExpectedOldestBlock: 1, + ExpectedBaseFeePerGas: []uint64{ + chain.GenesisBaseFee, chain.GenesisBaseFee, chain.GenesisBaseFee, + chain.GenesisBaseFee, chain.GenesisBaseFee, chain.GenesisBaseFee, + chain.GenesisBaseFee, chain.GenesisBaseFee, chain.GenesisBaseFee, + chain.GenesisBaseFee, chain.GenesisBaseFee, + }, + ExpectedGasUsedRatio: []float64{ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + }, + ExpectedRewards: [][]uint64{ + nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + }, + GetBackend: func() Blockchain { + backend := createTestBlocks(t, 30) + createTestTxs(t, backend, 5, 500) + + return backend + }, + }, + { + Name: "rewardPercentile requested", + BlockRange: 5, + NewestBlock: 10, + RewardPercentiles: []float64{10, 25}, + ExpectedOldestBlock: 6, + ExpectedBaseFeePerGas: []uint64{ + chain.GenesisBaseFee, + chain.GenesisBaseFee, + chain.GenesisBaseFee, + chain.GenesisBaseFee, + chain.GenesisBaseFee, + chain.GenesisBaseFee, + }, + ExpectedGasUsedRatio: []float64{ + 0, 0, 0, 0, 0, + }, + ExpectedRewards: [][]uint64{ + {0x2e90edd000, 0x2e90edd000}, + {0x2e90edd000, 0x2e90edd000}, + {0x2e90edd000, 0x2e90edd000}, + {0x2e90edd000, 0x2e90edd000}, + {0x2e90edd000, 0x2e90edd000}, + }, + GetBackend: func() Blockchain { + backend := createTestBlocks(t, 10) + rand.Seed(time.Now().UTC().UnixNano()) + + senderKey, sender := tests.GenerateKeyAndAddr(t) + + for _, b := range backend.blocksByNumber { + signer := crypto.NewSigner(backend.Config().Forks.At(b.Number()), + uint64(backend.Config().ChainID)) + + b.Transactions = make([]*types.Transaction, 3) + b.Header.Miner = sender.Bytes() + + for i := 0; i < 3; i++ { + tx := &types.Transaction{ + From: sender, + Value: ethgo.Ether(1), + To: &types.ZeroAddress, + Type: types.DynamicFeeTx, + GasTipCap: ethgo.Gwei(uint64(200)), + GasFeeCap: ethgo.Gwei(uint64(200 + 200)), + } + + tx, err := signer.SignTx(tx, senderKey) + require.NoError(t, err) + b.Transactions[i] = tx + } + } + + return backend + }, + }, + { + Name: "BaseFeePerGas sanity check", + BlockRange: 5, + NewestBlock: 10, + RewardPercentiles: []float64{}, + ExpectedOldestBlock: 6, + ExpectedBaseFeePerGas: []uint64{ + chain.GenesisBaseFee, + chain.GenesisBaseFee, + chain.GenesisBaseFee, + chain.GenesisBaseFee, + chain.GenesisBaseFee, + chain.GenesisBaseFee, + }, + ExpectedGasUsedRatio: []float64{ + 0, 0, 0, 0, 0, + }, + ExpectedRewards: [][]uint64{ + nil, nil, nil, nil, nil, + }, + GetBackend: func() Blockchain { + backend := createTestBlocks(t, 10) + + return backend + }, + }, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.Name, func(t *testing.T) { + t.Parallel() + + backend := tc.GetBackend() + gasHelper, err := NewGasHelper(DefaultGasHelperConfig, backend) + require.NoError(t, err) + history, err := gasHelper.FeeHistory(tc.BlockRange, tc.NewestBlock, tc.RewardPercentiles) + + if tc.Error { + require.Error(t, err) + } else { + require.NoError(t, err) + require.Equal(t, tc.ExpectedOldestBlock, history.OldestBlock) + require.Equal(t, tc.ExpectedBaseFeePerGas, history.BaseFeePerGas) + require.Equal(t, tc.ExpectedGasUsedRatio, history.GasUsedRatio) + require.Equal(t, tc.ExpectedRewards, history.Reward) + } + }) + } +} + +var _ Blockchain = (*backendMock)(nil) + +func (b *backendMock) GetBlockByNumber(number uint64, full bool) (*types.Block, bool) { + if len(b.blocks) == 0 { + args := b.Called(number, full) + + return args.Get(0).(*types.Block), args.Get(1).(bool) //nolint:forcetypeassert + } + + block, exists := b.blocksByNumber[number] + + return block, exists +} diff --git a/gasprice/gasprice.go b/gasprice/gasprice.go index a800a26002..ed6f3680b6 100644 --- a/gasprice/gasprice.go +++ b/gasprice/gasprice.go @@ -9,6 +9,7 @@ import ( "github.com/0xPolygon/polygon-edge/chain" "github.com/0xPolygon/polygon-edge/crypto" "github.com/0xPolygon/polygon-edge/types" + lru "github.com/hashicorp/golang-lru" "github.com/umbracle/ethgo" ) @@ -44,6 +45,7 @@ type Config struct { // Blockchain is the interface representing blockchain type Blockchain interface { + GetBlockByNumber(number uint64, full bool) (*types.Block, bool) GetBlockByHash(hash types.Hash, full bool) (*types.Block, bool) Header() *types.Header Config() *chain.Params @@ -53,6 +55,8 @@ type Blockchain interface { type GasStore interface { // MaxPriorityFeePerGas calculates the priority fee needed for transaction to be included in a block MaxPriorityFeePerGas() (*big.Int, error) + // FeeHistory returns the collection of historical gas information + FeeHistory(uint64, uint64, []float64) (*FeeHistoryReturn, error) } var _ GasStore = (*GasHelper)(nil) @@ -78,15 +82,22 @@ type GasHelper struct { lastHeaderHash types.Hash lock sync.Mutex + + historyCache *lru.Cache } // NewGasHelper is the constructor function for GasHelper struct -func NewGasHelper(config *Config, backend Blockchain) *GasHelper { +func NewGasHelper(config *Config, backend Blockchain) (*GasHelper, error) { pricePercentile := config.PricePercentile if pricePercentile > 100 { pricePercentile = 100 } + cache, err := lru.New(100) + if err != nil { + return nil, err + } + return &GasHelper{ numOfBlocksToCheck: config.NumOfBlocksToCheck, pricePercentile: pricePercentile, @@ -95,7 +106,8 @@ func NewGasHelper(config *Config, backend Blockchain) *GasHelper { lastPrice: config.LastPrice, maxPrice: config.MaxPrice, backend: backend, - } + historyCache: cache, + }, nil } // MaxPriorityFeePerGas calculates the priority fee needed for transaction to be included in a block diff --git a/gasprice/gasprice_test.go b/gasprice/gasprice_test.go index f4ff7bbde3..387e48c5e6 100644 --- a/gasprice/gasprice_test.go +++ b/gasprice/gasprice_test.go @@ -154,7 +154,8 @@ func TestGasHelper_MaxPriorityFeePerGas(t *testing.T) { t.Parallel() backend := tc.GetBackend() - gasHelper := NewGasHelper(DefaultGasHelperConfig, backend) + gasHelper, err := NewGasHelper(DefaultGasHelperConfig, backend) + require.NoError(t, err) price, err := gasHelper.MaxPriorityFeePerGas() if tc.Error { @@ -170,7 +171,7 @@ func TestGasHelper_MaxPriorityFeePerGas(t *testing.T) { func createTestBlocks(t *testing.T, numOfBlocks int) *backendMock { t.Helper() - backend := &backendMock{blocks: make(map[types.Hash]*types.Block)} + backend := &backendMock{blocks: make(map[types.Hash]*types.Block), blocksByNumber: make(map[uint64]*types.Block)} genesis := &types.Block{ Header: &types.Header{ Number: 0, @@ -180,6 +181,7 @@ func createTestBlocks(t *testing.T, numOfBlocks int) *backendMock { }, } backend.blocks[genesis.Hash()] = genesis + backend.blocksByNumber[genesis.Number()] = genesis currentBlock := genesis @@ -190,9 +192,10 @@ func createTestBlocks(t *testing.T, numOfBlocks int) *backendMock { Hash: types.BytesToHash([]byte(fmt.Sprintf("Block %d", i))), Miner: types.ZeroAddress.Bytes(), ParentHash: currentBlock.Hash(), + BaseFee: chain.GenesisBaseFee, }, } - + backend.blocksByNumber[block.Number()] = block backend.blocks[block.Hash()] = block currentBlock = block } @@ -237,7 +240,8 @@ var _ Blockchain = (*backendMock)(nil) type backendMock struct { mock.Mock - blocks map[types.Hash]*types.Block + blocks map[types.Hash]*types.Block + blocksByNumber map[uint64]*types.Block } func (b *backendMock) Header() *types.Header { diff --git a/jsonrpc/eth_endpoint.go b/jsonrpc/eth_endpoint.go index cbea344587..d8f510db44 100644 --- a/jsonrpc/eth_endpoint.go +++ b/jsonrpc/eth_endpoint.go @@ -802,3 +802,45 @@ func (e *Eth) MaxPriorityFeePerGas() (interface{}, error) { return argBigPtr(priorityFee), nil } + +func (e *Eth) FeeHistory(blockCount uint64, newestBlock uint64, rewardPercentiles []float64) (interface{}, error) { + // Retrieve oldestBlock, baseFeePerGas, gasUsedRatio, and reward synchronously + history, err := e.store.FeeHistory(blockCount, newestBlock, rewardPercentiles) + if err != nil { + return nil, err + } + + // Create channels to receive the processed slices asynchronously + baseFeePerGasCh := make(chan []argUint64) + gasUsedRatioCh := make(chan []argUint64) + rewardCh := make(chan [][]argUint64) + + // Process baseFeePerGas asynchronously + go func() { + baseFeePerGasCh <- convertToArgUint64Slice(history.BaseFeePerGas) + }() + + // Process gasUsedRatio asynchronously + go func() { + gasUsedRatioCh <- convertFloat64SliceToArgUint64Slice(history.GasUsedRatio) + }() + + // Process reward asynchronously + go func() { + rewardCh <- convertToArgUint64SliceSlice(history.Reward) + }() + + // Wait for the processed slices from goroutines + baseFeePerGasResult := <-baseFeePerGasCh + gasUsedRatioResult := <-gasUsedRatioCh + rewardResult := <-rewardCh + + result := &feeHistoryResult{ + OldestBlock: *argUintPtr(history.OldestBlock), + BaseFeePerGas: baseFeePerGasResult, + GasUsedRatio: gasUsedRatioResult, + Reward: rewardResult, + } + + return result, nil +} diff --git a/jsonrpc/types.go b/jsonrpc/types.go index 5db313a59c..1e289b9ba2 100644 --- a/jsonrpc/types.go +++ b/jsonrpc/types.go @@ -344,3 +344,37 @@ type progression struct { CurrentBlock argUint64 `json:"currentBlock"` HighestBlock argUint64 `json:"highestBlock"` } + +type feeHistoryResult struct { + OldestBlock argUint64 + BaseFeePerGas []argUint64 + GasUsedRatio []argUint64 + Reward [][]argUint64 +} + +func convertToArgUint64Slice(slice []uint64) []argUint64 { + argSlice := make([]argUint64, len(slice)) + for i, value := range slice { + argSlice[i] = argUint64(value) + } + + return argSlice +} + +func convertToArgUint64SliceSlice(slice [][]uint64) [][]argUint64 { + argSlice := make([][]argUint64, len(slice)) + for i, value := range slice { + argSlice[i] = convertToArgUint64Slice(value) + } + + return argSlice +} + +func convertFloat64SliceToArgUint64Slice(slice []float64) []argUint64 { + argSlice := make([]argUint64, len(slice)) + for i, value := range slice { + argSlice[i] = argUint64(value) + } + + return argSlice +} diff --git a/server/server.go b/server/server.go index c1872955e3..4081c52877 100644 --- a/server/server.go +++ b/server/server.go @@ -342,7 +342,10 @@ func NewServer(config *Config) (*Server, error) { } // here we can provide some other configuration - m.gasHelper = gasprice.NewGasHelper(gasprice.DefaultGasHelperConfig, m.blockchain) + m.gasHelper, err = gasprice.NewGasHelper(gasprice.DefaultGasHelperConfig, m.blockchain) + if err != nil { + return nil, err + } m.executor.GetHash = m.blockchain.GetHashHelper