From 619e5791fef2748fcaee787d60e1fecb3e0ddda6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Fri, 29 Nov 2024 11:26:19 +0200 Subject: [PATCH 1/3] Sketch mempool host. Move "GetTransferredValue()". --- dataRetriever/factory/dataPoolFactory.go | 3 +- dataRetriever/txpool/argShardedTxPool.go | 8 +- dataRetriever/txpool/interface.go | 8 + .../txpool/memorytests/memory_test.go | 2 + dataRetriever/txpool/mempoolHost.go | 108 +++++++++++++ dataRetriever/txpool/shardedTxPool.go | 14 +- dataRetriever/txpool/shardedTxPool_test.go | 12 ++ .../apiTransactionProcessor_test.go | 10 +- process/block/preprocess/selectionSession.go | 75 --------- .../block/preprocess/selectionSession_test.go | 152 ------------------ process/block/preprocess/transactions.go | 1 - storage/txcache/txcache.go | 10 +- storage/txcache/txcache_test.go | 4 +- testscommon/dataRetriever/poolFactory.go | 1 + testscommon/dataRetriever/poolsHolderMock.go | 2 + testscommon/txcachemocks/mempoolHostMock.go | 41 +++++ 16 files changed, 205 insertions(+), 246 deletions(-) create mode 100644 dataRetriever/txpool/mempoolHost.go create mode 100644 testscommon/txcachemocks/mempoolHostMock.go diff --git a/dataRetriever/factory/dataPoolFactory.go b/dataRetriever/factory/dataPoolFactory.go index 28f4b819f21..f48dc3d3c37 100644 --- a/dataRetriever/factory/dataPoolFactory.go +++ b/dataRetriever/factory/dataPoolFactory.go @@ -64,9 +64,10 @@ func NewDataPoolFromConfig(args ArgsDataPool) (dataRetriever.PoolsHolder, error) txPool, err := txpool.NewShardedTxPool(txpool.ArgShardedTxPool{ Config: factory.GetCacherFromConfig(mainConfig.TxDataPool), + TxGasHandler: args.EconomicsData, + Marshalizer: args.Marshalizer, NumberOfShards: args.ShardCoordinator.NumberOfShards(), SelfShardID: args.ShardCoordinator.SelfId(), - TxGasHandler: args.EconomicsData, }) if err != nil { return nil, fmt.Errorf("%w while creating the cache for the transactions", err) diff --git a/dataRetriever/txpool/argShardedTxPool.go b/dataRetriever/txpool/argShardedTxPool.go index ddf26b04343..dca1efa56bd 100644 --- a/dataRetriever/txpool/argShardedTxPool.go +++ b/dataRetriever/txpool/argShardedTxPool.go @@ -4,15 +4,16 @@ import ( "fmt" "github.com/multiversx/mx-chain-core-go/core/check" + "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/storage/storageunit" - "github.com/multiversx/mx-chain-go/storage/txcache" ) // ArgShardedTxPool is the argument for ShardedTxPool's constructor type ArgShardedTxPool struct { Config storageunit.CacheConfig - TxGasHandler txcache.TxGasHandler + TxGasHandler txGasHandler + Marshalizer marshal.Marshalizer NumberOfShards uint32 SelfShardID uint32 } @@ -39,6 +40,9 @@ func (args *ArgShardedTxPool) verify() error { if check.IfNil(args.TxGasHandler) { return fmt.Errorf("%w: TxGasHandler is not valid", dataRetriever.ErrNilTxGasHandler) } + if check.IfNil(args.Marshalizer) { + return fmt.Errorf("%w: Marshalizer is not valid", dataRetriever.ErrNilMarshalizer) + } if args.NumberOfShards == 0 { return fmt.Errorf("%w: NumberOfShards is not valid", dataRetriever.ErrCacheConfigInvalidSharding) } diff --git a/dataRetriever/txpool/interface.go b/dataRetriever/txpool/interface.go index 6579659d692..ee55a246a48 100644 --- a/dataRetriever/txpool/interface.go +++ b/dataRetriever/txpool/interface.go @@ -1,6 +1,9 @@ package txpool import ( + "math/big" + + "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/storage/txcache" ) @@ -17,3 +20,8 @@ type txCache interface { Diagnose(deep bool) GetTransactionsPoolForSender(sender string) []*txcache.WrappedTransaction } + +type txGasHandler interface { + ComputeTxFee(tx data.TransactionWithFeeHandler) *big.Int + IsInterfaceNil() bool +} diff --git a/dataRetriever/txpool/memorytests/memory_test.go b/dataRetriever/txpool/memorytests/memory_test.go index 2f26e574830..1359ae8fb5f 100644 --- a/dataRetriever/txpool/memorytests/memory_test.go +++ b/dataRetriever/txpool/memorytests/memory_test.go @@ -12,6 +12,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/data/transaction" + "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/dataRetriever/txpool" "github.com/multiversx/mx-chain-go/storage/storageunit" @@ -112,6 +113,7 @@ func newPool() dataRetriever.ShardedDataCacherNotifier { args := txpool.ArgShardedTxPool{ Config: config, TxGasHandler: txcachemocks.NewTxGasHandlerMock(), + Marshalizer: &marshal.GogoProtoMarshalizer{}, NumberOfShards: 2, SelfShardID: 0, } diff --git a/dataRetriever/txpool/mempoolHost.go b/dataRetriever/txpool/mempoolHost.go new file mode 100644 index 00000000000..7c0b2166287 --- /dev/null +++ b/dataRetriever/txpool/mempoolHost.go @@ -0,0 +1,108 @@ +package txpool + +import ( + "bytes" + "math/big" + + "github.com/multiversx/mx-chain-core-go/core" + "github.com/multiversx/mx-chain-core-go/core/check" + "github.com/multiversx/mx-chain-core-go/data" + "github.com/multiversx/mx-chain-core-go/marshal" + "github.com/multiversx/mx-chain-go/process" + vmcommon "github.com/multiversx/mx-chain-vm-common-go" + "github.com/multiversx/mx-chain-vm-common-go/parsers" +) + +type argsMempoolHost struct { + txGasHandler txGasHandler + marshalizer marshal.Marshalizer +} + +type mempoolHost struct { + txGasHandler txGasHandler + callArgumentsParser process.CallArgumentsParser + esdtTransferParser vmcommon.ESDTTransferParser +} + +func newMempoolHost(args argsMempoolHost) (*mempoolHost, error) { + if check.IfNil(args.marshalizer) { + return nil, process.ErrNilMarshalizer + } + + argsParser := parsers.NewCallArgsParser() + + esdtTransferParser, err := parsers.NewESDTTransferParser(args.marshalizer) + if err != nil { + return nil, err + } + + return &mempoolHost{ + txGasHandler: args.txGasHandler, + callArgumentsParser: argsParser, + esdtTransferParser: esdtTransferParser, + }, nil +} + +// ComputeTxFee computes the fee for a transaction. +func (host *mempoolHost) ComputeTxFee(tx data.TransactionWithFeeHandler) *big.Int { + return host.txGasHandler.ComputeTxFee(tx) +} + +// GetTransferredValue returns the value transferred by a transaction. +func (host *mempoolHost) GetTransferredValue(tx data.TransactionHandler) *big.Int { + value := tx.GetValue() + hasValue := value != nil && value.Sign() != 0 + if hasValue { + // Early exit (optimization): a transaction can either bear a regular value or be a "MultiESDTNFTTransfer". + return value + } + + data := tx.GetData() + hasData := len(data) > 0 + if !hasData { + // Early exit (optimization): no "MultiESDTNFTTransfer" to parse. + return tx.GetValue() + } + + maybeMultiTransfer := bytes.HasPrefix(data, []byte(core.BuiltInFunctionMultiESDTNFTTransfer)) + if !maybeMultiTransfer { + // Early exit (optimization). + return nil + } + + function, args, err := host.callArgumentsParser.ParseData(string(data)) + if err != nil { + return nil + } + + if function != core.BuiltInFunctionMultiESDTNFTTransfer { + // Early exit (optimization). + return nil + } + + esdtTransfers, err := host.esdtTransferParser.ParseESDTTransfers(tx.GetSndAddr(), tx.GetRcvAddr(), function, args) + if err != nil { + return nil + } + + accumulatedNativeValue := big.NewInt(0) + + for _, transfer := range esdtTransfers.ESDTTransfers { + if transfer.ESDTTokenNonce != 0 { + continue + } + if string(transfer.ESDTTokenName) != vmcommon.EGLDIdentifier { + // We only care about native transfers. + continue + } + + _ = accumulatedNativeValue.Add(accumulatedNativeValue, transfer.ESDTValue) + } + + return accumulatedNativeValue +} + +// IsInterfaceNil returns true if there is no value under the interface +func (host *mempoolHost) IsInterfaceNil() bool { + return host == nil +} diff --git a/dataRetriever/txpool/shardedTxPool.go b/dataRetriever/txpool/shardedTxPool.go index c4ec79285f7..0f40817893d 100644 --- a/dataRetriever/txpool/shardedTxPool.go +++ b/dataRetriever/txpool/shardedTxPool.go @@ -27,7 +27,7 @@ type shardedTxPool struct { configPrototypeDestinationMe txcache.ConfigDestinationMe configPrototypeSourceMe txcache.ConfigSourceMe selfShardID uint32 - txGasHandler txcache.TxGasHandler + host txcache.MempoolHost } type txPoolShard struct { @@ -45,6 +45,14 @@ func NewShardedTxPool(args ArgShardedTxPool) (*shardedTxPool, error) { return nil, err } + mempoolHost, err := newMempoolHost(argsMempoolHost{ + txGasHandler: args.TxGasHandler, + marshalizer: args.Marshalizer, + }) + if err != nil { + return nil, err + } + halfOfSizeInBytes := args.Config.SizeInBytes / 2 halfOfCapacity := args.Config.Capacity / 2 @@ -77,7 +85,7 @@ func NewShardedTxPool(args ArgShardedTxPool) (*shardedTxPool, error) { configPrototypeDestinationMe: configPrototypeDestinationMe, configPrototypeSourceMe: configPrototypeSourceMe, selfShardID: args.SelfShardID, - txGasHandler: args.TxGasHandler, + host: mempoolHost, } return shardedTxPoolObject, nil @@ -134,7 +142,7 @@ func (txPool *shardedTxPool) createTxCache(cacheID string) txCache { if isForSenderMe { config := txPool.configPrototypeSourceMe config.Name = cacheID - cache, err := txcache.NewTxCache(config, txPool.txGasHandler) + cache, err := txcache.NewTxCache(config, txPool.host) if err != nil { log.Error("shardedTxPool.createTxCache()", "err", err) return txcache.NewDisabledCache() diff --git a/dataRetriever/txpool/shardedTxPool_test.go b/dataRetriever/txpool/shardedTxPool_test.go index 90638faff1f..1b3ab585dc3 100644 --- a/dataRetriever/txpool/shardedTxPool_test.go +++ b/dataRetriever/txpool/shardedTxPool_test.go @@ -10,6 +10,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/transaction" + "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/storage/storageunit" "github.com/multiversx/mx-chain-go/testscommon/txcachemocks" @@ -34,6 +35,7 @@ func Test_NewShardedTxPool_WhenBadConfig(t *testing.T) { Shards: 16, }, TxGasHandler: txcachemocks.NewTxGasHandlerMock(), + Marshalizer: &marshal.GogoProtoMarshalizer{}, NumberOfShards: 1, } @@ -79,6 +81,13 @@ func Test_NewShardedTxPool_WhenBadConfig(t *testing.T) { require.NotNil(t, err) require.Errorf(t, err, dataRetriever.ErrNilTxGasHandler.Error()) + args = goodArgs + args.Marshalizer = nil + pool, err = NewShardedTxPool(args) + require.Nil(t, pool) + require.NotNil(t, err) + require.Errorf(t, err, dataRetriever.ErrNilMarshalizer.Error()) + args = goodArgs args.NumberOfShards = 0 pool, err = NewShardedTxPool(args) @@ -92,6 +101,7 @@ func Test_NewShardedTxPool_ComputesCacheConfig(t *testing.T) { args := ArgShardedTxPool{ Config: config, TxGasHandler: txcachemocks.NewTxGasHandlerMock(), + Marshalizer: &marshal.GogoProtoMarshalizer{}, NumberOfShards: 2, } @@ -374,6 +384,7 @@ func Test_routeToCacheUnions(t *testing.T) { args := ArgShardedTxPool{ Config: config, TxGasHandler: txcachemocks.NewTxGasHandlerMock(), + Marshalizer: &marshal.GogoProtoMarshalizer{}, NumberOfShards: 4, SelfShardID: 42, } @@ -414,6 +425,7 @@ func newTxPoolToTest() (dataRetriever.ShardedDataCacherNotifier, error) { args := ArgShardedTxPool{ Config: config, TxGasHandler: txcachemocks.NewTxGasHandlerMock(), + Marshalizer: &marshal.GogoProtoMarshalizer{}, NumberOfShards: 4, SelfShardID: 0, } diff --git a/node/external/transactionAPI/apiTransactionProcessor_test.go b/node/external/transactionAPI/apiTransactionProcessor_test.go index 3c7cf049d1d..40357484880 100644 --- a/node/external/transactionAPI/apiTransactionProcessor_test.go +++ b/node/external/transactionAPI/apiTransactionProcessor_test.go @@ -884,7 +884,7 @@ func TestApiTransactionProcessor_GetTransactionsPoolForSender(t *testing.T) { CountThreshold: math.MaxUint32, CountPerSenderThreshold: math.MaxUint32, NumItemsToPreemptivelyEvict: 1, - }, txcachemocks.NewTxGasHandlerMock()) + }, txcachemocks.NewMempoolHostMock()) require.NoError(t, err) @@ -901,7 +901,7 @@ func TestApiTransactionProcessor_GetTransactionsPoolForSender(t *testing.T) { CountThreshold: math.MaxUint32, CountPerSenderThreshold: math.MaxUint32, NumItemsToPreemptivelyEvict: 1, - }, txcachemocks.NewTxGasHandlerMock()) + }, txcachemocks.NewMempoolHostMock()) txCacheWithMeta.AddTx(createTx(txHash3, sender, 4)) txCacheWithMeta.AddTx(createTx(txHash4, sender, 5)) @@ -974,7 +974,7 @@ func TestApiTransactionProcessor_GetLastPoolNonceForSender(t *testing.T) { CountThreshold: math.MaxUint32, CountPerSenderThreshold: math.MaxUint32, NumItemsToPreemptivelyEvict: 1, - }, txcachemocks.NewTxGasHandlerMock()) + }, txcachemocks.NewMempoolHostMock()) txCacheIntraShard.AddTx(createTx(txHash2, sender, 3)) txCacheIntraShard.AddTx(createTx(txHash0, sender, 1)) txCacheIntraShard.AddTx(createTx(txHash1, sender, 2)) @@ -1026,7 +1026,7 @@ func TestApiTransactionProcessor_GetTransactionsPoolNonceGapsForSender(t *testin CountThreshold: math.MaxUint32, CountPerSenderThreshold: math.MaxUint32, NumItemsToPreemptivelyEvict: 1, - }, txcachemocks.NewTxGasHandlerMock()) + }, txcachemocks.NewMempoolHostMock()) require.NoError(t, err) @@ -1038,7 +1038,7 @@ func TestApiTransactionProcessor_GetTransactionsPoolNonceGapsForSender(t *testin CountThreshold: math.MaxUint32, CountPerSenderThreshold: math.MaxUint32, NumItemsToPreemptivelyEvict: 1, - }, txcachemocks.NewTxGasHandlerMock()) + }, txcachemocks.NewMempoolHostMock()) require.NoError(t, err) diff --git a/process/block/preprocess/selectionSession.go b/process/block/preprocess/selectionSession.go index 6d0f8a6d397..425428a7d31 100644 --- a/process/block/preprocess/selectionSession.go +++ b/process/block/preprocess/selectionSession.go @@ -1,33 +1,24 @@ package preprocess import ( - "bytes" "errors" - "math/big" - "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/transaction" - "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/state" "github.com/multiversx/mx-chain-go/storage/txcache" - vmcommon "github.com/multiversx/mx-chain-vm-common-go" - "github.com/multiversx/mx-chain-vm-common-go/parsers" ) type selectionSession struct { accountsAdapter state.AccountsAdapter transactionsProcessor process.TransactionProcessor - callArgumentsParser process.CallArgumentsParser - esdtTransferParser vmcommon.ESDTTransferParser } type argsSelectionSession struct { accountsAdapter state.AccountsAdapter transactionsProcessor process.TransactionProcessor - marshalizer marshal.Marshalizer } func newSelectionSession(args argsSelectionSession) (*selectionSession, error) { @@ -37,22 +28,10 @@ func newSelectionSession(args argsSelectionSession) (*selectionSession, error) { if check.IfNil(args.transactionsProcessor) { return nil, process.ErrNilTxProcessor } - if check.IfNil(args.marshalizer) { - return nil, process.ErrNilMarshalizer - } - - argsParser := parsers.NewCallArgsParser() - - esdtTransferParser, err := parsers.NewESDTTransferParser(args.marshalizer) - if err != nil { - return nil, err - } return &selectionSession{ accountsAdapter: args.accountsAdapter, transactionsProcessor: args.transactionsProcessor, - callArgumentsParser: argsParser, - esdtTransferParser: esdtTransferParser, }, nil } @@ -99,60 +78,6 @@ func (session *selectionSession) IsIncorrectlyGuarded(tx data.TransactionHandler return errors.Is(err, process.ErrTransactionNotExecutable) } -// GetTransferredValue returns the value transferred by a transaction. -func (session *selectionSession) GetTransferredValue(tx data.TransactionHandler) *big.Int { - value := tx.GetValue() - hasValue := value != nil && value.Sign() != 0 - if hasValue { - // Early exit (optimization): a transaction can either bear a regular value or be a "MultiESDTNFTTransfer". - return value - } - - data := tx.GetData() - hasData := len(data) > 0 - if !hasData { - // Early exit (optimization): no "MultiESDTNFTTransfer" to parse. - return tx.GetValue() - } - - maybeMultiTransfer := bytes.HasPrefix(data, []byte(core.BuiltInFunctionMultiESDTNFTTransfer)) - if !maybeMultiTransfer { - // Early exit (optimization). - return nil - } - - function, args, err := session.callArgumentsParser.ParseData(string(data)) - if err != nil { - return nil - } - - if function != core.BuiltInFunctionMultiESDTNFTTransfer { - // Early exit (optimization). - return nil - } - - esdtTransfers, err := session.esdtTransferParser.ParseESDTTransfers(tx.GetSndAddr(), tx.GetRcvAddr(), function, args) - if err != nil { - return nil - } - - accumulatedNativeValue := big.NewInt(0) - - for _, transfer := range esdtTransfers.ESDTTransfers { - if transfer.ESDTTokenNonce != 0 { - continue - } - if string(transfer.ESDTTokenName) != vmcommon.EGLDIdentifier { - // We only care about native transfers. - continue - } - - _ = accumulatedNativeValue.Add(accumulatedNativeValue, transfer.ESDTValue) - } - - return accumulatedNativeValue -} - // IsInterfaceNil returns true if there is no value under the interface func (session *selectionSession) IsInterfaceNil() bool { return session == nil diff --git a/process/block/preprocess/selectionSession_test.go b/process/block/preprocess/selectionSession_test.go index 9f69a994f0c..df6199bf555 100644 --- a/process/block/preprocess/selectionSession_test.go +++ b/process/block/preprocess/selectionSession_test.go @@ -2,14 +2,10 @@ package preprocess import ( "bytes" - "encoding/hex" "fmt" - "math/big" "testing" - "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/data/transaction" - "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/state" "github.com/multiversx/mx-chain-go/testscommon" @@ -24,7 +20,6 @@ func TestNewSelectionSession(t *testing.T) { session, err := newSelectionSession(argsSelectionSession{ accountsAdapter: nil, transactionsProcessor: &testscommon.TxProcessorStub{}, - marshalizer: &marshal.GogoProtoMarshalizer{}, }) require.Nil(t, session) require.ErrorIs(t, err, process.ErrNilAccountsAdapter) @@ -32,7 +27,6 @@ func TestNewSelectionSession(t *testing.T) { session, err = newSelectionSession(argsSelectionSession{ accountsAdapter: &stateMock.AccountsStub{}, transactionsProcessor: nil, - marshalizer: &marshal.GogoProtoMarshalizer{}, }) require.Nil(t, session) require.ErrorIs(t, err, process.ErrNilTxProcessor) @@ -40,7 +34,6 @@ func TestNewSelectionSession(t *testing.T) { session, err = newSelectionSession(argsSelectionSession{ accountsAdapter: &stateMock.AccountsStub{}, transactionsProcessor: &testscommon.TxProcessorStub{}, - marshalizer: &marshal.GogoProtoMarshalizer{}, }) require.NoError(t, err) require.NotNil(t, session) @@ -76,7 +69,6 @@ func TestSelectionSession_GetAccountState(t *testing.T) { session, err := newSelectionSession(argsSelectionSession{ accountsAdapter: accounts, transactionsProcessor: processor, - marshalizer: &marshal.GogoProtoMarshalizer{}, }) require.NoError(t, err) require.NotNil(t, session) @@ -122,7 +114,6 @@ func TestSelectionSession_IsIncorrectlyGuarded(t *testing.T) { session, err := newSelectionSession(argsSelectionSession{ accountsAdapter: accounts, transactionsProcessor: processor, - marshalizer: &marshal.GogoProtoMarshalizer{}, }) require.NoError(t, err) require.NotNil(t, session) @@ -139,146 +130,3 @@ func TestSelectionSession_IsIncorrectlyGuarded(t *testing.T) { isIncorrectlyGuarded = session.IsIncorrectlyGuarded(&transaction.Transaction{Nonce: 45, SndAddr: []byte("bob")}) require.True(t, isIncorrectlyGuarded) } - -func TestSelectionSession_GetTransferredValue(t *testing.T) { - t.Parallel() - - session, err := newSelectionSession(argsSelectionSession{ - accountsAdapter: &stateMock.AccountsStub{}, - transactionsProcessor: &testscommon.TxProcessorStub{}, - marshalizer: &marshal.GogoProtoMarshalizer{}, - }) - require.NoError(t, err) - require.NotNil(t, session) - - t.Run("with value", func(t *testing.T) { - value := session.GetTransferredValue(&transaction.Transaction{ - Value: big.NewInt(1000000000000000000), - }) - require.Equal(t, big.NewInt(1000000000000000000), value) - }) - - t.Run("with value and data", func(t *testing.T) { - value := session.GetTransferredValue(&transaction.Transaction{ - Value: big.NewInt(1000000000000000000), - Data: []byte("data"), - }) - require.Equal(t, big.NewInt(1000000000000000000), value) - }) - - t.Run("native transfer within MultiESDTNFTTransfer", func(t *testing.T) { - value := session.GetTransferredValue(&transaction.Transaction{ - SndAddr: testscommon.TestPubKeyAlice, - RcvAddr: testscommon.TestPubKeyAlice, - Data: []byte("MultiESDTNFTTransfer@8049d639e5a6980d1cd2392abcce41029cda74a1563523a202f09641cc2618f8@03@4e46542d313233343536@0a@01@544553542d393837363534@01@01@45474c442d303030303030@@0de0b6b3a7640000"), - }) - require.Equal(t, big.NewInt(1000000000000000000), value) - }) - - t.Run("native transfer within MultiESDTNFTTransfer; transfer & execute", func(t *testing.T) { - value := session.GetTransferredValue(&transaction.Transaction{ - SndAddr: testscommon.TestPubKeyAlice, - RcvAddr: testscommon.TestPubKeyAlice, - Data: []byte("MultiESDTNFTTransfer@00000000000000000500b9353fe8407f87310c87e12fa1ac807f0485da39d152@03@4e46542d313233343536@01@01@4e46542d313233343536@2a@01@45474c442d303030303030@@0de0b6b3a7640000@64756d6d79@07"), - }) - require.Equal(t, big.NewInt(1000000000000000000), value) - }) -} - -func TestBenchmarkSelectionSession_GetTransferredValue(t *testing.T) { - session, err := newSelectionSession(argsSelectionSession{ - accountsAdapter: &stateMock.AccountsStub{}, - transactionsProcessor: &testscommon.TxProcessorStub{}, - marshalizer: &marshal.GogoProtoMarshalizer{}, - }) - require.NoError(t, err) - require.NotNil(t, session) - - sw := core.NewStopWatch() - - valueMultiplier := int64(1_000_000_000_000) - - t.Run("numTransactions = 5_000", func(t *testing.T) { - numTransactions := 5_000 - transactions := createMultiESDTNFTTransfersWithNativeTransfer(numTransactions, valueMultiplier) - - sw.Start(t.Name()) - - for i := 0; i < numTransactions; i++ { - tx := transactions[i] - value := session.GetTransferredValue(tx) - require.Equal(t, big.NewInt(int64(i)*valueMultiplier), value) - } - - sw.Stop(t.Name()) - }) - - t.Run("numTransactions = 10_000", func(t *testing.T) { - numTransactions := 10_000 - transactions := createMultiESDTNFTTransfersWithNativeTransfer(numTransactions, valueMultiplier) - - sw.Start(t.Name()) - - for i := 0; i < numTransactions; i++ { - tx := transactions[i] - value := session.GetTransferredValue(tx) - require.Equal(t, big.NewInt(int64(i)*valueMultiplier), value) - } - - sw.Stop(t.Name()) - }) - - t.Run("numTransactions = 20_000", func(t *testing.T) { - numTransactions := 20_000 - transactions := createMultiESDTNFTTransfersWithNativeTransfer(numTransactions, valueMultiplier) - - sw.Start(t.Name()) - - for i := 0; i < numTransactions; i++ { - tx := transactions[i] - value := session.GetTransferredValue(tx) - require.Equal(t, big.NewInt(int64(i)*valueMultiplier), value) - } - - sw.Stop(t.Name()) - }) - - for name, measurement := range sw.GetMeasurementsMap() { - fmt.Printf("%fs (%s)\n", measurement, name) - } - - // (1) - // Vendor ID: GenuineIntel - // Model name: 11th Gen Intel(R) Core(TM) i7-1165G7 @ 2.80GHz - // CPU family: 6 - // Model: 140 - // Thread(s) per core: 2 - // Core(s) per socket: 4 - // - // NOTE: 20% is also due to the require() / assert() calls. - // 0.012993s (TestBenchmarkSelectionSession_GetTransferredValue/numTransactions_=_5_000) - // 0.024580s (TestBenchmarkSelectionSession_GetTransferredValue/numTransactions_=_10_000) - // 0.048808s (TestBenchmarkSelectionSession_GetTransferredValue/numTransactions_=_20_000) -} - -func createMultiESDTNFTTransfersWithNativeTransfer(numTransactions int, valueMultiplier int64) []*transaction.Transaction { - transactions := make([]*transaction.Transaction, 0, numTransactions) - - for i := 0; i < numTransactions; i++ { - nativeValue := big.NewInt(int64(i) * valueMultiplier) - data := fmt.Sprintf( - "MultiESDTNFTTransfer@8049d639e5a6980d1cd2392abcce41029cda74a1563523a202f09641cc2618f8@03@4e46542d313233343536@0a@01@544553542d393837363534@01@01@45474c442d303030303030@@%s", - hex.EncodeToString(nativeValue.Bytes()), - ) - - tx := &transaction.Transaction{ - SndAddr: testscommon.TestPubKeyAlice, - RcvAddr: testscommon.TestPubKeyAlice, - Data: []byte(data), - } - - transactions = append(transactions, tx) - } - - return transactions -} diff --git a/process/block/preprocess/transactions.go b/process/block/preprocess/transactions.go index 3ebf1528ac9..8578a97e92d 100644 --- a/process/block/preprocess/transactions.go +++ b/process/block/preprocess/transactions.go @@ -1414,7 +1414,6 @@ func (txs *transactions) computeSortedTxs( session, err := newSelectionSession(argsSelectionSession{ accountsAdapter: txs.accounts, transactionsProcessor: txs.txProcessor, - marshalizer: txs.marshalizer, }) if err != nil { return nil, nil, err diff --git a/storage/txcache/txcache.go b/storage/txcache/txcache.go index b70a6091c20..01e9212ea15 100644 --- a/storage/txcache/txcache.go +++ b/storage/txcache/txcache.go @@ -11,10 +11,10 @@ type WrappedTransaction = txcache.WrappedTransaction // AccountState represents the state of an account (as seen by the mempool) type AccountState = types.AccountState -// TxGasHandler handles a transaction gas and gas cost -type TxGasHandler = txcache.TxGasHandler +// MempoolHost provides blockchain information for mempool operations +type MempoolHost = txcache.MempoolHost -// SelectionSession provides provides blockchain information for transaction selection +// SelectionSession provides blockchain information for transaction selection type SelectionSession = txcache.SelectionSession // ForEachTransaction is an iterator callback @@ -36,8 +36,8 @@ type DisabledCache = txcache.DisabledCache type CrossTxCache = txcache.CrossTxCache // NewTxCache creates a new transaction cache -func NewTxCache(config ConfigSourceMe, txGasHandler TxGasHandler) (*TxCache, error) { - return txcache.NewTxCache(config, txGasHandler) +func NewTxCache(config ConfigSourceMe, host MempoolHost) (*TxCache, error) { + return txcache.NewTxCache(config, host) } // NewDisabledCache creates a new disabled cache diff --git a/storage/txcache/txcache_test.go b/storage/txcache/txcache_test.go index f113216ce92..5b84daba2d5 100644 --- a/storage/txcache/txcache_test.go +++ b/storage/txcache/txcache_test.go @@ -27,7 +27,7 @@ func TestNewTxCache(t *testing.T) { cache, err := NewTxCache(cfg, nil) assert.Nil(t, cache) - assert.ErrorContains(t, err, "nil tx gas handler") + assert.ErrorContains(t, err, "nil mempool host") }) t.Run("should work", func(t *testing.T) { t.Parallel() @@ -42,7 +42,7 @@ func TestNewTxCache(t *testing.T) { NumItemsToPreemptivelyEvict: 1, } - cache, err := NewTxCache(cfg, txcachemocks.NewTxGasHandlerMock()) + cache, err := NewTxCache(cfg, txcachemocks.NewMempoolHostMock()) assert.NotNil(t, cache) assert.Nil(t, err) }) diff --git a/testscommon/dataRetriever/poolFactory.go b/testscommon/dataRetriever/poolFactory.go index b621c9245b9..71a27a718f2 100644 --- a/testscommon/dataRetriever/poolFactory.go +++ b/testscommon/dataRetriever/poolFactory.go @@ -41,6 +41,7 @@ func CreateTxPool(numShards uint32, selfShard uint32) (dataRetriever.ShardedData NumberOfShards: numShards, SelfShardID: selfShard, TxGasHandler: txcachemocks.NewTxGasHandlerMock(), + Marshalizer: &marshal.GogoProtoMarshalizer{}, }, ) } diff --git a/testscommon/dataRetriever/poolsHolderMock.go b/testscommon/dataRetriever/poolsHolderMock.go index 6167b1eac6b..75321b6854c 100644 --- a/testscommon/dataRetriever/poolsHolderMock.go +++ b/testscommon/dataRetriever/poolsHolderMock.go @@ -4,6 +4,7 @@ import ( "time" "github.com/multiversx/mx-chain-core-go/core/check" + "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/config" "github.com/multiversx/mx-chain-go/dataRetriever" @@ -50,6 +51,7 @@ func NewPoolsHolderMock() *PoolsHolderMock { Shards: 16, }, TxGasHandler: txcachemocks.NewTxGasHandlerMock(), + Marshalizer: &marshal.GogoProtoMarshalizer{}, NumberOfShards: 1, }, ) diff --git a/testscommon/txcachemocks/mempoolHostMock.go b/testscommon/txcachemocks/mempoolHostMock.go new file mode 100644 index 00000000000..3e9ecc995ec --- /dev/null +++ b/testscommon/txcachemocks/mempoolHostMock.go @@ -0,0 +1,41 @@ +package txcachemocks + +import ( + "math/big" + + "github.com/multiversx/mx-chain-core-go/data" +) + +// MempoolHostMock - +type MempoolHostMock struct { + ComputeTxFeeCalled func(tx data.TransactionWithFeeHandler) *big.Int + GetTransferredValueCalled func(tx data.TransactionHandler) *big.Int +} + +// NewMempoolHostMock - +func NewMempoolHostMock() *MempoolHostMock { + return &MempoolHostMock{} +} + +// ComputeTxFee - +func (mock *MempoolHostMock) ComputeTxFee(tx data.TransactionWithFeeHandler) *big.Int { + if mock.ComputeTxFeeCalled != nil { + return mock.ComputeTxFeeCalled(tx) + } + + return big.NewInt(0) +} + +// GetTransferredValue - +func (mock *MempoolHostMock) GetTransferredValue(tx data.TransactionHandler) *big.Int { + if mock.GetTransferredValueCalled != nil { + return mock.GetTransferredValueCalled(tx) + } + + return tx.GetValue() +} + +// IsInterfaceNil - +func (mock *MempoolHostMock) IsInterfaceNil() bool { + return mock == nil +} From 33bf019f668a2eeac2940bb4e9c0f1df425ff7f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Fri, 29 Nov 2024 11:54:34 +0200 Subject: [PATCH 2/3] Add tests, reference storage-go. --- dataRetriever/txpool/mempoolHost.go | 6 +- dataRetriever/txpool/mempoolHost_test.go | 182 +++++++++++++++++++++++ go.mod | 2 +- go.sum | 4 +- 4 files changed, 190 insertions(+), 4 deletions(-) create mode 100644 dataRetriever/txpool/mempoolHost_test.go diff --git a/dataRetriever/txpool/mempoolHost.go b/dataRetriever/txpool/mempoolHost.go index 7c0b2166287..94a54d5ccba 100644 --- a/dataRetriever/txpool/mempoolHost.go +++ b/dataRetriever/txpool/mempoolHost.go @@ -8,6 +8,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/marshal" + "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/process" vmcommon "github.com/multiversx/mx-chain-vm-common-go" "github.com/multiversx/mx-chain-vm-common-go/parsers" @@ -25,8 +26,11 @@ type mempoolHost struct { } func newMempoolHost(args argsMempoolHost) (*mempoolHost, error) { + if check.IfNil(args.txGasHandler) { + return nil, dataRetriever.ErrNilTxGasHandler + } if check.IfNil(args.marshalizer) { - return nil, process.ErrNilMarshalizer + return nil, dataRetriever.ErrNilMarshalizer } argsParser := parsers.NewCallArgsParser() diff --git a/dataRetriever/txpool/mempoolHost_test.go b/dataRetriever/txpool/mempoolHost_test.go new file mode 100644 index 00000000000..a013a88fa19 --- /dev/null +++ b/dataRetriever/txpool/mempoolHost_test.go @@ -0,0 +1,182 @@ +package txpool + +import ( + "encoding/hex" + "fmt" + "math/big" + "testing" + + "github.com/multiversx/mx-chain-core-go/core" + "github.com/multiversx/mx-chain-core-go/data/transaction" + "github.com/multiversx/mx-chain-core-go/marshal" + "github.com/multiversx/mx-chain-go/dataRetriever" + "github.com/multiversx/mx-chain-go/testscommon" + "github.com/multiversx/mx-chain-go/testscommon/txcachemocks" + "github.com/stretchr/testify/require" +) + +func TestNewMempoolHost(t *testing.T) { + t.Parallel() + + host, err := newMempoolHost(argsMempoolHost{ + txGasHandler: nil, + marshalizer: &marshal.GogoProtoMarshalizer{}, + }) + require.Nil(t, host) + require.ErrorIs(t, err, dataRetriever.ErrNilTxGasHandler) + + host, err = newMempoolHost(argsMempoolHost{ + txGasHandler: txcachemocks.NewTxGasHandlerMock(), + marshalizer: nil, + }) + require.Nil(t, host) + require.ErrorIs(t, err, dataRetriever.ErrNilMarshalizer) + + host, err = newMempoolHost(argsMempoolHost{ + txGasHandler: txcachemocks.NewTxGasHandlerMock(), + marshalizer: &marshal.GogoProtoMarshalizer{}, + }) + require.NoError(t, err) + require.NotNil(t, host) +} + +func TestMempoolHost_GetTransferredValue(t *testing.T) { + t.Parallel() + + host, err := newMempoolHost(argsMempoolHost{ + txGasHandler: txcachemocks.NewTxGasHandlerMock(), + marshalizer: &marshal.GogoProtoMarshalizer{}, + }) + require.NoError(t, err) + require.NotNil(t, host) + + t.Run("with value", func(t *testing.T) { + value := host.GetTransferredValue(&transaction.Transaction{ + Value: big.NewInt(1000000000000000000), + }) + require.Equal(t, big.NewInt(1000000000000000000), value) + }) + + t.Run("with value and data", func(t *testing.T) { + value := host.GetTransferredValue(&transaction.Transaction{ + Value: big.NewInt(1000000000000000000), + Data: []byte("data"), + }) + require.Equal(t, big.NewInt(1000000000000000000), value) + }) + + t.Run("native transfer within MultiESDTNFTTransfer", func(t *testing.T) { + value := host.GetTransferredValue(&transaction.Transaction{ + SndAddr: testscommon.TestPubKeyAlice, + RcvAddr: testscommon.TestPubKeyAlice, + Data: []byte("MultiESDTNFTTransfer@8049d639e5a6980d1cd2392abcce41029cda74a1563523a202f09641cc2618f8@03@4e46542d313233343536@0a@01@544553542d393837363534@01@01@45474c442d303030303030@@0de0b6b3a7640000"), + }) + require.Equal(t, big.NewInt(1000000000000000000), value) + }) + + t.Run("native transfer within MultiESDTNFTTransfer; transfer & execute", func(t *testing.T) { + value := host.GetTransferredValue(&transaction.Transaction{ + SndAddr: testscommon.TestPubKeyAlice, + RcvAddr: testscommon.TestPubKeyAlice, + Data: []byte("MultiESDTNFTTransfer@00000000000000000500b9353fe8407f87310c87e12fa1ac807f0485da39d152@03@4e46542d313233343536@01@01@4e46542d313233343536@2a@01@45474c442d303030303030@@0de0b6b3a7640000@64756d6d79@07"), + }) + require.Equal(t, big.NewInt(1000000000000000000), value) + }) +} + +func TestBenchmarkMempoolHost_GetTransferredValue(t *testing.T) { + host, err := newMempoolHost(argsMempoolHost{ + txGasHandler: txcachemocks.NewTxGasHandlerMock(), + marshalizer: &marshal.GogoProtoMarshalizer{}, + }) + require.NoError(t, err) + require.NotNil(t, host) + + sw := core.NewStopWatch() + + valueMultiplier := int64(1_000_000_000_000) + + t.Run("numTransactions = 5_000", func(t *testing.T) { + numTransactions := 5_000 + transactions := createMultiESDTNFTTransfersWithNativeTransfer(numTransactions, valueMultiplier) + + sw.Start(t.Name()) + + for i := 0; i < numTransactions; i++ { + tx := transactions[i] + value := host.GetTransferredValue(tx) + require.Equal(t, big.NewInt(int64(i)*valueMultiplier), value) + } + + sw.Stop(t.Name()) + }) + + t.Run("numTransactions = 10_000", func(t *testing.T) { + numTransactions := 10_000 + transactions := createMultiESDTNFTTransfersWithNativeTransfer(numTransactions, valueMultiplier) + + sw.Start(t.Name()) + + for i := 0; i < numTransactions; i++ { + tx := transactions[i] + value := host.GetTransferredValue(tx) + require.Equal(t, big.NewInt(int64(i)*valueMultiplier), value) + } + + sw.Stop(t.Name()) + }) + + t.Run("numTransactions = 20_000", func(t *testing.T) { + numTransactions := 20_000 + transactions := createMultiESDTNFTTransfersWithNativeTransfer(numTransactions, valueMultiplier) + + sw.Start(t.Name()) + + for i := 0; i < numTransactions; i++ { + tx := transactions[i] + value := host.GetTransferredValue(tx) + require.Equal(t, big.NewInt(int64(i)*valueMultiplier), value) + } + + sw.Stop(t.Name()) + }) + + for name, measurement := range sw.GetMeasurementsMap() { + fmt.Printf("%fs (%s)\n", measurement, name) + } + + // (1) + // Vendor ID: GenuineIntel + // Model name: 11th Gen Intel(R) Core(TM) i7-1165G7 @ 2.80GHz + // CPU family: 6 + // Model: 140 + // Thread(s) per core: 2 + // Core(s) per socket: 4 + // + // NOTE: 20% is also due to the require() / assert() calls. + // 0.012993s (TestBenchmarkMempoolHost_GetTransferredValue/numTransactions_=_5_000) + // 0.024580s (TestBenchmarkMempoolHost_GetTransferredValue/numTransactions_=_10_000) + // 0.048808s (TestBenchmarkMempoolHost_GetTransferredValue/numTransactions_=_20_000) +} + +func createMultiESDTNFTTransfersWithNativeTransfer(numTransactions int, valueMultiplier int64) []*transaction.Transaction { + transactions := make([]*transaction.Transaction, 0, numTransactions) + + for i := 0; i < numTransactions; i++ { + nativeValue := big.NewInt(int64(i) * valueMultiplier) + data := fmt.Sprintf( + "MultiESDTNFTTransfer@8049d639e5a6980d1cd2392abcce41029cda74a1563523a202f09641cc2618f8@03@4e46542d313233343536@0a@01@544553542d393837363534@01@01@45474c442d303030303030@@%s", + hex.EncodeToString(nativeValue.Bytes()), + ) + + tx := &transaction.Transaction{ + SndAddr: testscommon.TestPubKeyAlice, + RcvAddr: testscommon.TestPubKeyAlice, + Data: []byte(data), + } + + transactions = append(transactions, tx) + } + + return transactions +} diff --git a/go.mod b/go.mod index 0a330e9cef2..252d2c11f83 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/multiversx/mx-chain-es-indexer-go v1.7.10 github.com/multiversx/mx-chain-logger-go v1.0.15 github.com/multiversx/mx-chain-scenario-go v1.4.4 - github.com/multiversx/mx-chain-storage-go v1.0.18-0.20241128110709-156b1244e04f + github.com/multiversx/mx-chain-storage-go v1.0.18-0.20241128201607-ea3ee2dfd9eb github.com/multiversx/mx-chain-vm-common-go v1.5.16 github.com/multiversx/mx-chain-vm-go v1.5.37 github.com/multiversx/mx-chain-vm-v1_2-go v1.2.68 diff --git a/go.sum b/go.sum index 5b6066d2161..18f41bcc459 100644 --- a/go.sum +++ b/go.sum @@ -397,8 +397,8 @@ github.com/multiversx/mx-chain-logger-go v1.0.15 h1:HlNdK8etyJyL9NQ+6mIXyKPEBo+w github.com/multiversx/mx-chain-logger-go v1.0.15/go.mod h1:t3PRKaWB1M+i6gUfD27KXgzLJJC+mAQiN+FLlL1yoGQ= github.com/multiversx/mx-chain-scenario-go v1.4.4 h1:DVE2V+FPeyD/yWoC+KEfPK3jsFzHeruelESfpTlf460= github.com/multiversx/mx-chain-scenario-go v1.4.4/go.mod h1:kI+TWR3oIEgUkbwkHCPo2CQ3VjIge+ezGTibiSGwMxo= -github.com/multiversx/mx-chain-storage-go v1.0.18-0.20241128110709-156b1244e04f h1:sRPekt5fzNr+c7w2IzwufOeqANTT3Du6ciD3FX5mCvI= -github.com/multiversx/mx-chain-storage-go v1.0.18-0.20241128110709-156b1244e04f/go.mod h1:eFDEOrG7Wiyk5I/ObpwcN2eoBlOnnfeEMTvTer1cymk= +github.com/multiversx/mx-chain-storage-go v1.0.18-0.20241128201607-ea3ee2dfd9eb h1:1yFG5WEHIxsyONn2j+GhUSEOwm5tXbAggqOwEVuj/g0= +github.com/multiversx/mx-chain-storage-go v1.0.18-0.20241128201607-ea3ee2dfd9eb/go.mod h1:eFDEOrG7Wiyk5I/ObpwcN2eoBlOnnfeEMTvTer1cymk= github.com/multiversx/mx-chain-vm-common-go v1.5.16 h1:g1SqYjxl7K66Y1O/q6tvDJ37fzpzlxCSfRzSm/woQQY= github.com/multiversx/mx-chain-vm-common-go v1.5.16/go.mod h1:1rSkXreUZNXyPTTdhj47M+Fy62yjxbu3aAsXEtKN3UY= github.com/multiversx/mx-chain-vm-go v1.5.37 h1:Iy3KCvM+DOq1f9UPA7uYK/rI3ZbBOXc2CVNO2/vm5zw= From 9428366c6a7206ffc3250f2166bcf895c71970ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Fri, 29 Nov 2024 15:29:29 +0200 Subject: [PATCH 3/3] Fix after review. --- dataRetriever/txpool/mempoolHost.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dataRetriever/txpool/mempoolHost.go b/dataRetriever/txpool/mempoolHost.go index 94a54d5ccba..916dd436cd7 100644 --- a/dataRetriever/txpool/mempoolHost.go +++ b/dataRetriever/txpool/mempoolHost.go @@ -71,22 +71,22 @@ func (host *mempoolHost) GetTransferredValue(tx data.TransactionHandler) *big.In maybeMultiTransfer := bytes.HasPrefix(data, []byte(core.BuiltInFunctionMultiESDTNFTTransfer)) if !maybeMultiTransfer { // Early exit (optimization). - return nil + return big.NewInt(0) } function, args, err := host.callArgumentsParser.ParseData(string(data)) if err != nil { - return nil + return big.NewInt(0) } if function != core.BuiltInFunctionMultiESDTNFTTransfer { // Early exit (optimization). - return nil + return big.NewInt(0) } esdtTransfers, err := host.esdtTransferParser.ParseESDTTransfers(tx.GetSndAddr(), tx.GetRcvAddr(), function, args) if err != nil { - return nil + return big.NewInt(0) } accumulatedNativeValue := big.NewInt(0)