diff --git a/api/groups/networkGroup.go b/api/groups/networkGroup.go index ae0cca6846d..8a8e7854f9d 100644 --- a/api/groups/networkGroup.go +++ b/api/groups/networkGroup.go @@ -117,7 +117,7 @@ func NewNetworkGroup(facade networkFacadeHandler) (*networkGroup, error) { { Path: getNFTsPath, Method: http.MethodGet, - Handler: ng.getHandlerFuncForEsdt(core.NonFungibleESDT), + Handler: ng.getHandlerFuncForEsdt(core.NonFungibleESDTv2), }, { Path: directStakedInfoPath, diff --git a/cmd/node/config/config.toml b/cmd/node/config/config.toml index 6e1205d5f7e..4da40dfde30 100644 --- a/cmd/node/config/config.toml +++ b/cmd/node/config/config.toml @@ -386,9 +386,9 @@ [TxDataPool] Name = "TxDataPool" Capacity = 600000 - SizePerSender = 20000 + SizePerSender = 5001 SizeInBytes = 419430400 #400MB - SizeInBytesPerSender = 12288000 + SizeInBytesPerSender = 12288000 #12MB Type = "TxCache" Shards = 16 diff --git a/cmd/node/main.go b/cmd/node/main.go index c7cc3c1085c..5a812bc2f45 100644 --- a/cmd/node/main.go +++ b/cmd/node/main.go @@ -290,6 +290,7 @@ func attachFileLogger(log logger.Logger, flagsConfig *config.ContextFlagsConfig) logger.ToggleCorrelation(flagsConfig.EnableLogCorrelation) logger.ToggleLoggerName(flagsConfig.EnableLogName) logLevelFlagValue := flagsConfig.LogLevel + err = logger.SetLogLevel(logLevelFlagValue) if err != nil { return nil, err diff --git a/consensus/broadcast/delayedBroadcast.go b/consensus/broadcast/delayedBroadcast.go index 955a81f0f73..511ac6d79e6 100644 --- a/consensus/broadcast/delayedBroadcast.go +++ b/consensus/broadcast/delayedBroadcast.go @@ -644,7 +644,7 @@ func (dbb *delayedBlockBroadcaster) interceptedHeader(_ string, headerHash []byt ) alarmsToCancel := make([]string, 0) - dbb.mutDataForBroadcast.RLock() + dbb.mutDataForBroadcast.Lock() for i, broadcastData := range dbb.valHeaderBroadcastData { samePrevRandSeed := bytes.Equal(broadcastData.header.GetPrevRandSeed(), headerHandler.GetPrevRandSeed()) sameRound := broadcastData.header.GetRound() == headerHandler.GetRound() @@ -663,7 +663,7 @@ func (dbb *delayedBlockBroadcaster) interceptedHeader(_ string, headerHash []byt } } - dbb.mutDataForBroadcast.RUnlock() + dbb.mutDataForBroadcast.Unlock() for _, alarmID := range alarmsToCancel { dbb.alarm.Cancel(alarmID) diff --git a/dataRetriever/constants.go b/dataRetriever/constants.go index ef8ceaca082..198e2ea4ea0 100644 --- a/dataRetriever/constants.go +++ b/dataRetriever/constants.go @@ -1,8 +1,5 @@ package dataRetriever -// TxPoolNumSendersToPreemptivelyEvict instructs tx pool eviction algorithm to remove this many senders when eviction takes place -const TxPoolNumSendersToPreemptivelyEvict = uint32(100) - // UnsignedTxPoolName defines the name of the unsigned transactions pool const UnsignedTxPoolName = "uTxPool" diff --git a/dataRetriever/errors.go b/dataRetriever/errors.go index a015e6e10ed..8b7b2f2e3dc 100644 --- a/dataRetriever/errors.go +++ b/dataRetriever/errors.go @@ -116,9 +116,6 @@ var ErrCacheConfigInvalidSize = errors.New("cache parameter [size] is not valid, // ErrCacheConfigInvalidShards signals that the cache parameter "shards" is invalid var ErrCacheConfigInvalidShards = errors.New("cache parameter [shards] is not valid, it must be a positive number") -// ErrCacheConfigInvalidEconomics signals that an economics parameter required by the cache is invalid -var ErrCacheConfigInvalidEconomics = errors.New("cache-economics parameter is not valid") - // ErrCacheConfigInvalidSharding signals that a sharding parameter required by the cache is invalid var ErrCacheConfigInvalidSharding = errors.New("cache-sharding parameter is not valid") diff --git a/dataRetriever/factory/dataPoolFactory.go b/dataRetriever/factory/dataPoolFactory.go index 6e1415ddfd8..f48dc3d3c37 100644 --- a/dataRetriever/factory/dataPoolFactory.go +++ b/dataRetriever/factory/dataPoolFactory.go @@ -53,6 +53,9 @@ func NewDataPoolFromConfig(args ArgsDataPool) (dataRetriever.PoolsHolder, error) if check.IfNil(args.ShardCoordinator) { return nil, dataRetriever.ErrNilShardCoordinator } + if check.IfNil(args.Marshalizer) { + return nil, dataRetriever.ErrNilMarshalizer + } if check.IfNil(args.PathManager) { return nil, dataRetriever.ErrNilPathManager } @@ -61,9 +64,10 @@ func NewDataPoolFromConfig(args ArgsDataPool) (dataRetriever.PoolsHolder, error) txPool, err := txpool.NewShardedTxPool(txpool.ArgShardedTxPool{ Config: factory.GetCacherFromConfig(mainConfig.TxDataPool), + TxGasHandler: args.EconomicsData, + Marshalizer: args.Marshalizer, NumberOfShards: args.ShardCoordinator.NumberOfShards(), SelfShardID: args.ShardCoordinator.SelfId(), - TxGasHandler: args.EconomicsData, }) if err != nil { return nil, fmt.Errorf("%w while creating the cache for the transactions", err) diff --git a/dataRetriever/factory/dataPoolFactory_test.go b/dataRetriever/factory/dataPoolFactory_test.go index c9ae8b60c43..b4896244ad9 100644 --- a/dataRetriever/factory/dataPoolFactory_test.go +++ b/dataRetriever/factory/dataPoolFactory_test.go @@ -41,6 +41,12 @@ func TestNewDataPoolFromConfig_MissingDependencyShouldErr(t *testing.T) { require.Nil(t, holder) require.Equal(t, dataRetriever.ErrNilShardCoordinator, err) + args = getGoodArgs() + args.Marshalizer = nil + holder, err = NewDataPoolFromConfig(args) + require.Nil(t, holder) + require.Equal(t, dataRetriever.ErrNilMarshalizer, err) + args = getGoodArgs() args.PathManager = nil holder, err = NewDataPoolFromConfig(args) @@ -76,7 +82,6 @@ func TestNewDataPoolFromConfig_BadConfigShouldErr(t *testing.T) { args.Config.HeadersPoolConfig.MaxHeadersPerShard = 0 holder, err = NewDataPoolFromConfig(args) require.Nil(t, holder) - fmt.Println(err) require.True(t, errors.Is(err, headersCache.ErrInvalidHeadersCacheParameter)) require.True(t, strings.Contains(err.Error(), "the cache for the headers")) @@ -84,7 +89,6 @@ func TestNewDataPoolFromConfig_BadConfigShouldErr(t *testing.T) { args.Config.TxBlockBodyDataPool.Capacity = 0 holder, err = NewDataPoolFromConfig(args) require.Nil(t, holder) - fmt.Println(err) require.NotNil(t, err) require.True(t, strings.Contains(err.Error(), "must provide a positive size while creating the cache for the miniblocks")) @@ -92,7 +96,6 @@ func TestNewDataPoolFromConfig_BadConfigShouldErr(t *testing.T) { args.Config.PeerBlockBodyDataPool.Capacity = 0 holder, err = NewDataPoolFromConfig(args) require.Nil(t, holder) - fmt.Println(err) require.NotNil(t, err) require.True(t, strings.Contains(err.Error(), "must provide a positive size while creating the cache for the peer mini block body")) @@ -100,19 +103,9 @@ func TestNewDataPoolFromConfig_BadConfigShouldErr(t *testing.T) { args.Config.TrieSyncStorage.Capacity = 0 holder, err = NewDataPoolFromConfig(args) require.Nil(t, holder) - fmt.Println(err) require.True(t, errors.Is(err, storage.ErrCacheSizeInvalid)) require.True(t, strings.Contains(err.Error(), "the cache for the trie nodes")) - args = getGoodArgs() - args.Config.TrieSyncStorage.EnableDB = true - args.Config.TrieSyncStorage.DB.Type = "invalid DB type" - holder, err = NewDataPoolFromConfig(args) - require.Nil(t, holder) - fmt.Println(err) - require.True(t, errors.Is(err, storage.ErrNotSupportedDBType)) - require.True(t, strings.Contains(err.Error(), "the db for the trie nodes")) - args = getGoodArgs() args.Config.TrieNodesChunksDataPool.Type = "invalid cache type" holder, err = NewDataPoolFromConfig(args) diff --git a/dataRetriever/requestHandlers/requestHandler.go b/dataRetriever/requestHandlers/requestHandler.go index 7166715dd3c..91e4992aee3 100644 --- a/dataRetriever/requestHandlers/requestHandler.go +++ b/dataRetriever/requestHandlers/requestHandler.go @@ -14,7 +14,7 @@ import ( "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/epochStart" "github.com/multiversx/mx-chain-go/process/factory" - "github.com/multiversx/mx-chain-logger-go" + logger "github.com/multiversx/mx-chain-logger-go" ) var _ epochStart.RequestHandler = (*resolverRequestHandler)(nil) @@ -571,10 +571,12 @@ func (rrh *resolverRequestHandler) RequestValidatorInfo(hash []byte) { return } + epoch := rrh.getEpoch() + log.Debug("requesting validator info messages from network", "topic", common.ValidatorInfoTopic, "hash", hash, - "epoch", rrh.epoch, + "epoch", epoch, ) requester, err := rrh.requestersFinder.MetaChainRequester(common.ValidatorInfoTopic) @@ -583,20 +585,20 @@ func (rrh *resolverRequestHandler) RequestValidatorInfo(hash []byte) { "error", err.Error(), "topic", common.ValidatorInfoTopic, "hash", hash, - "epoch", rrh.epoch, + "epoch", epoch, ) return } rrh.whiteList.Add([][]byte{hash}) - err = requester.RequestDataFromHash(hash, rrh.epoch) + err = requester.RequestDataFromHash(hash, epoch) if err != nil { log.Debug("RequestValidatorInfo.RequestDataFromHash", "error", err.Error(), "topic", common.ValidatorInfoTopic, "hash", hash, - "epoch", rrh.epoch, + "epoch", epoch, ) return } @@ -611,10 +613,12 @@ func (rrh *resolverRequestHandler) RequestValidatorsInfo(hashes [][]byte) { return } + epoch := rrh.getEpoch() + log.Debug("requesting validator info messages from network", "topic", common.ValidatorInfoTopic, "num hashes", len(unrequestedHashes), - "epoch", rrh.epoch, + "epoch", epoch, ) requester, err := rrh.requestersFinder.MetaChainRequester(common.ValidatorInfoTopic) @@ -623,7 +627,7 @@ func (rrh *resolverRequestHandler) RequestValidatorsInfo(hashes [][]byte) { "error", err.Error(), "topic", common.ValidatorInfoTopic, "num hashes", len(unrequestedHashes), - "epoch", rrh.epoch, + "epoch", epoch, ) return } @@ -636,13 +640,13 @@ func (rrh *resolverRequestHandler) RequestValidatorsInfo(hashes [][]byte) { rrh.whiteList.Add(unrequestedHashes) - err = validatorInfoRequester.RequestDataFromHashArray(unrequestedHashes, rrh.epoch) + err = validatorInfoRequester.RequestDataFromHashArray(unrequestedHashes, epoch) if err != nil { log.Debug("RequestValidatorInfo.RequestDataFromHash", "error", err.Error(), "topic", common.ValidatorInfoTopic, "num hashes", len(unrequestedHashes), - "epoch", rrh.epoch, + "epoch", epoch, ) return } @@ -827,11 +831,13 @@ func (rrh *resolverRequestHandler) GetNumPeersToQuery(key string) (int, int, err // RequestPeerAuthenticationsByHashes asks for peer authentication messages from specific peers hashes func (rrh *resolverRequestHandler) RequestPeerAuthenticationsByHashes(destShardID uint32, hashes [][]byte) { + epoch := rrh.getEpoch() + log.Debug("requesting peer authentication messages from network", "topic", common.PeerAuthenticationTopic, "shard", destShardID, "num hashes", len(hashes), - "epoch", rrh.epoch, + "epoch", epoch, ) requester, err := rrh.requestersFinder.MetaChainRequester(common.PeerAuthenticationTopic) @@ -840,7 +846,7 @@ func (rrh *resolverRequestHandler) RequestPeerAuthenticationsByHashes(destShardI "error", err.Error(), "topic", common.PeerAuthenticationTopic, "shard", destShardID, - "epoch", rrh.epoch, + "epoch", epoch, ) return } @@ -851,13 +857,13 @@ func (rrh *resolverRequestHandler) RequestPeerAuthenticationsByHashes(destShardI return } - err = peerAuthRequester.RequestDataFromHashArray(hashes, rrh.epoch) + err = peerAuthRequester.RequestDataFromHashArray(hashes, epoch) if err != nil { log.Debug("RequestPeerAuthenticationsByHashes.RequestDataFromHashArray", "error", err.Error(), "topic", common.PeerAuthenticationTopic, "shard", destShardID, - "epoch", rrh.epoch, + "epoch", epoch, ) } } diff --git a/dataRetriever/shardedData/shardedData.go b/dataRetriever/shardedData/shardedData.go index 2800bb3352b..785998164b9 100644 --- a/dataRetriever/shardedData/shardedData.go +++ b/dataRetriever/shardedData/shardedData.go @@ -4,6 +4,7 @@ import ( "fmt" "sync" + "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/counting" "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/dataRetriever" @@ -52,7 +53,7 @@ func NewShardedData(name string, config storageunit.CacheConfig) (*shardedData, NumChunks: config.Shards, MaxNumItems: config.Capacity, MaxNumBytes: uint32(config.SizeInBytes), - NumItemsToPreemptivelyEvict: storage.TxPoolNumTxsToPreemptivelyEvict, + NumItemsToPreemptivelyEvict: storage.ShardedDataNumItemsToPreemptivelyEvict, } err := configPrototype.Verify() @@ -161,6 +162,9 @@ func (sd *shardedData) RemoveSetOfDataFromPool(keys [][]byte, cacheID string) { return } + stopWatch := core.NewStopWatch() + stopWatch.Start("removal") + numRemoved := 0 for _, key := range keys { if store.cache.RemoveWithResult(key) { @@ -168,7 +172,15 @@ func (sd *shardedData) RemoveSetOfDataFromPool(keys [][]byte, cacheID string) { } } - log.Trace("shardedData.removeTxBulk()", "name", sd.name, "cacheID", cacheID, "numToRemove", len(keys), "numRemoved", numRemoved) + stopWatch.Stop("removal") + + log.Debug("shardedData.removeTxBulk", + "name", sd.name, + "cacheID", cacheID, + "numToRemove", len(keys), + "numRemoved", numRemoved, + "duration", stopWatch.GetMeasurement("removal"), + ) } // ImmunizeSetOfDataAgainstEviction marks the items as non-evictable diff --git a/dataRetriever/txpool/argShardedTxPool.go b/dataRetriever/txpool/argShardedTxPool.go index 8b12dbddf7a..dca1efa56bd 100644 --- a/dataRetriever/txpool/argShardedTxPool.go +++ b/dataRetriever/txpool/argShardedTxPool.go @@ -1,19 +1,19 @@ package txpool import ( - "encoding/json" "fmt" "github.com/multiversx/mx-chain-core-go/core/check" + "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/storage/storageunit" - "github.com/multiversx/mx-chain-go/storage/txcache" ) // ArgShardedTxPool is the argument for ShardedTxPool's constructor type ArgShardedTxPool struct { Config storageunit.CacheConfig - TxGasHandler txcache.TxGasHandler + TxGasHandler txGasHandler + Marshalizer marshal.Marshalizer NumberOfShards uint32 SelfShardID uint32 } @@ -40,8 +40,8 @@ func (args *ArgShardedTxPool) verify() error { if check.IfNil(args.TxGasHandler) { return fmt.Errorf("%w: TxGasHandler is not valid", dataRetriever.ErrNilTxGasHandler) } - if args.TxGasHandler.MinGasPrice() == 0 { - return fmt.Errorf("%w: MinGasPrice is not valid", dataRetriever.ErrCacheConfigInvalidEconomics) + if check.IfNil(args.Marshalizer) { + return fmt.Errorf("%w: Marshalizer is not valid", dataRetriever.ErrNilMarshalizer) } if args.NumberOfShards == 0 { return fmt.Errorf("%w: NumberOfShards is not valid", dataRetriever.ErrCacheConfigInvalidSharding) @@ -49,13 +49,3 @@ func (args *ArgShardedTxPool) verify() error { return nil } - -// String returns a readable representation of the object -func (args *ArgShardedTxPool) String() string { - bytes, err := json.Marshal(args) - if err != nil { - log.Error("ArgShardedTxPool.String()", "err", err) - } - - return string(bytes) -} diff --git a/dataRetriever/txpool/interface.go b/dataRetriever/txpool/interface.go index 6579659d692..ee55a246a48 100644 --- a/dataRetriever/txpool/interface.go +++ b/dataRetriever/txpool/interface.go @@ -1,6 +1,9 @@ package txpool import ( + "math/big" + + "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/storage/txcache" ) @@ -17,3 +20,8 @@ type txCache interface { Diagnose(deep bool) GetTransactionsPoolForSender(sender string) []*txcache.WrappedTransaction } + +type txGasHandler interface { + ComputeTxFee(tx data.TransactionWithFeeHandler) *big.Int + IsInterfaceNil() bool +} diff --git a/dataRetriever/txpool/memorytests/memory_test.go b/dataRetriever/txpool/memorytests/memory_test.go index a0484f016b8..1359ae8fb5f 100644 --- a/dataRetriever/txpool/memorytests/memory_test.go +++ b/dataRetriever/txpool/memorytests/memory_test.go @@ -12,6 +12,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/data/transaction" + "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/dataRetriever/txpool" "github.com/multiversx/mx-chain-go/storage/storageunit" @@ -35,18 +36,18 @@ func TestShardedTxPool_MemoryFootprint(t *testing.T) { journals = append(journals, runScenario(t, newScenario(200, 1, core.MegabyteSize, "0"), memoryAssertion{200, 200}, memoryAssertion{0, 1})) journals = append(journals, runScenario(t, newScenario(10, 1000, 20480, "0"), memoryAssertion{190, 205}, memoryAssertion{1, 4})) - journals = append(journals, runScenario(t, newScenario(10000, 1, 1024, "0"), memoryAssertion{10, 16}, memoryAssertion{4, 10})) - journals = append(journals, runScenario(t, newScenario(1, 60000, 256, "0"), memoryAssertion{30, 40}, memoryAssertion{10, 16})) - journals = append(journals, runScenario(t, newScenario(10, 10000, 100, "0"), memoryAssertion{36, 52}, memoryAssertion{16, 24})) - journals = append(journals, runScenario(t, newScenario(100000, 1, 1024, "0"), memoryAssertion{120, 138}, memoryAssertion{56, 60})) + journals = append(journals, runScenario(t, newScenario(10000, 1, 1024, "0"), memoryAssertion{10, 16}, memoryAssertion{0, 10})) + journals = append(journals, runScenario(t, newScenario(1, 60000, 256, "0"), memoryAssertion{30, 40}, memoryAssertion{10, 24})) + journals = append(journals, runScenario(t, newScenario(10, 10000, 100, "0"), memoryAssertion{36, 52}, memoryAssertion{16, 36})) + journals = append(journals, runScenario(t, newScenario(100000, 1, 1024, "0"), memoryAssertion{120, 138}, memoryAssertion{32, 60})) // With larger memory footprint - journals = append(journals, runScenario(t, newScenario(100000, 3, 650, "0"), memoryAssertion{290, 335}, memoryAssertion{95, 120})) - journals = append(journals, runScenario(t, newScenario(150000, 2, 650, "0"), memoryAssertion{290, 335}, memoryAssertion{120, 140})) - journals = append(journals, runScenario(t, newScenario(300000, 1, 650, "0"), memoryAssertion{290, 335}, memoryAssertion{170, 190})) - journals = append(journals, runScenario(t, newScenario(30, 10000, 650, "0"), memoryAssertion{290, 335}, memoryAssertion{60, 75})) - journals = append(journals, runScenario(t, newScenario(300, 1000, 650, "0"), memoryAssertion{290, 335}, memoryAssertion{60, 80})) + journals = append(journals, runScenario(t, newScenario(100000, 3, 650, "0"), memoryAssertion{290, 335}, memoryAssertion{80, 148})) + journals = append(journals, runScenario(t, newScenario(150000, 2, 650, "0"), memoryAssertion{290, 335}, memoryAssertion{90, 160})) + journals = append(journals, runScenario(t, newScenario(300000, 1, 650, "0"), memoryAssertion{290, 335}, memoryAssertion{100, 190})) + journals = append(journals, runScenario(t, newScenario(30, 10000, 650, "0"), memoryAssertion{290, 335}, memoryAssertion{60, 132})) + journals = append(journals, runScenario(t, newScenario(300, 1000, 650, "0"), memoryAssertion{290, 335}, memoryAssertion{60, 148})) // Scenarios where destination == me @@ -110,12 +111,9 @@ func newPool() dataRetriever.ShardedDataCacherNotifier { } args := txpool.ArgShardedTxPool{ - Config: config, - TxGasHandler: &txcachemocks.TxGasHandlerMock{ - MinimumGasMove: 50000, - MinimumGasPrice: 200000000000, - GasProcessingDivisor: 100, - }, + Config: config, + TxGasHandler: txcachemocks.NewTxGasHandlerMock(), + Marshalizer: &marshal.GogoProtoMarshalizer{}, NumberOfShards: 2, SelfShardID: 0, } @@ -187,9 +185,10 @@ func createTxWithPayload(senderTag int, nonce int, payloadLength int) *dummyTx { return &dummyTx{ Transaction: transaction.Transaction{ - SndAddr: sender, - Nonce: uint64(nonce), - Data: make([]byte, payloadLength), + SndAddr: sender, + Nonce: uint64(nonce), + Data: make([]byte, payloadLength), + GasLimit: uint64(50000 + 1500*payloadLength), }, hash: hash, } diff --git a/dataRetriever/txpool/mempoolHost.go b/dataRetriever/txpool/mempoolHost.go new file mode 100644 index 00000000000..916dd436cd7 --- /dev/null +++ b/dataRetriever/txpool/mempoolHost.go @@ -0,0 +1,112 @@ +package txpool + +import ( + "bytes" + "math/big" + + "github.com/multiversx/mx-chain-core-go/core" + "github.com/multiversx/mx-chain-core-go/core/check" + "github.com/multiversx/mx-chain-core-go/data" + "github.com/multiversx/mx-chain-core-go/marshal" + "github.com/multiversx/mx-chain-go/dataRetriever" + "github.com/multiversx/mx-chain-go/process" + vmcommon "github.com/multiversx/mx-chain-vm-common-go" + "github.com/multiversx/mx-chain-vm-common-go/parsers" +) + +type argsMempoolHost struct { + txGasHandler txGasHandler + marshalizer marshal.Marshalizer +} + +type mempoolHost struct { + txGasHandler txGasHandler + callArgumentsParser process.CallArgumentsParser + esdtTransferParser vmcommon.ESDTTransferParser +} + +func newMempoolHost(args argsMempoolHost) (*mempoolHost, error) { + if check.IfNil(args.txGasHandler) { + return nil, dataRetriever.ErrNilTxGasHandler + } + if check.IfNil(args.marshalizer) { + return nil, dataRetriever.ErrNilMarshalizer + } + + argsParser := parsers.NewCallArgsParser() + + esdtTransferParser, err := parsers.NewESDTTransferParser(args.marshalizer) + if err != nil { + return nil, err + } + + return &mempoolHost{ + txGasHandler: args.txGasHandler, + callArgumentsParser: argsParser, + esdtTransferParser: esdtTransferParser, + }, nil +} + +// ComputeTxFee computes the fee for a transaction. +func (host *mempoolHost) ComputeTxFee(tx data.TransactionWithFeeHandler) *big.Int { + return host.txGasHandler.ComputeTxFee(tx) +} + +// GetTransferredValue returns the value transferred by a transaction. +func (host *mempoolHost) GetTransferredValue(tx data.TransactionHandler) *big.Int { + value := tx.GetValue() + hasValue := value != nil && value.Sign() != 0 + if hasValue { + // Early exit (optimization): a transaction can either bear a regular value or be a "MultiESDTNFTTransfer". + return value + } + + data := tx.GetData() + hasData := len(data) > 0 + if !hasData { + // Early exit (optimization): no "MultiESDTNFTTransfer" to parse. + return tx.GetValue() + } + + maybeMultiTransfer := bytes.HasPrefix(data, []byte(core.BuiltInFunctionMultiESDTNFTTransfer)) + if !maybeMultiTransfer { + // Early exit (optimization). + return big.NewInt(0) + } + + function, args, err := host.callArgumentsParser.ParseData(string(data)) + if err != nil { + return big.NewInt(0) + } + + if function != core.BuiltInFunctionMultiESDTNFTTransfer { + // Early exit (optimization). + return big.NewInt(0) + } + + esdtTransfers, err := host.esdtTransferParser.ParseESDTTransfers(tx.GetSndAddr(), tx.GetRcvAddr(), function, args) + if err != nil { + return big.NewInt(0) + } + + accumulatedNativeValue := big.NewInt(0) + + for _, transfer := range esdtTransfers.ESDTTransfers { + if transfer.ESDTTokenNonce != 0 { + continue + } + if string(transfer.ESDTTokenName) != vmcommon.EGLDIdentifier { + // We only care about native transfers. + continue + } + + _ = accumulatedNativeValue.Add(accumulatedNativeValue, transfer.ESDTValue) + } + + return accumulatedNativeValue +} + +// IsInterfaceNil returns true if there is no value under the interface +func (host *mempoolHost) IsInterfaceNil() bool { + return host == nil +} diff --git a/dataRetriever/txpool/mempoolHost_test.go b/dataRetriever/txpool/mempoolHost_test.go new file mode 100644 index 00000000000..a013a88fa19 --- /dev/null +++ b/dataRetriever/txpool/mempoolHost_test.go @@ -0,0 +1,182 @@ +package txpool + +import ( + "encoding/hex" + "fmt" + "math/big" + "testing" + + "github.com/multiversx/mx-chain-core-go/core" + "github.com/multiversx/mx-chain-core-go/data/transaction" + "github.com/multiversx/mx-chain-core-go/marshal" + "github.com/multiversx/mx-chain-go/dataRetriever" + "github.com/multiversx/mx-chain-go/testscommon" + "github.com/multiversx/mx-chain-go/testscommon/txcachemocks" + "github.com/stretchr/testify/require" +) + +func TestNewMempoolHost(t *testing.T) { + t.Parallel() + + host, err := newMempoolHost(argsMempoolHost{ + txGasHandler: nil, + marshalizer: &marshal.GogoProtoMarshalizer{}, + }) + require.Nil(t, host) + require.ErrorIs(t, err, dataRetriever.ErrNilTxGasHandler) + + host, err = newMempoolHost(argsMempoolHost{ + txGasHandler: txcachemocks.NewTxGasHandlerMock(), + marshalizer: nil, + }) + require.Nil(t, host) + require.ErrorIs(t, err, dataRetriever.ErrNilMarshalizer) + + host, err = newMempoolHost(argsMempoolHost{ + txGasHandler: txcachemocks.NewTxGasHandlerMock(), + marshalizer: &marshal.GogoProtoMarshalizer{}, + }) + require.NoError(t, err) + require.NotNil(t, host) +} + +func TestMempoolHost_GetTransferredValue(t *testing.T) { + t.Parallel() + + host, err := newMempoolHost(argsMempoolHost{ + txGasHandler: txcachemocks.NewTxGasHandlerMock(), + marshalizer: &marshal.GogoProtoMarshalizer{}, + }) + require.NoError(t, err) + require.NotNil(t, host) + + t.Run("with value", func(t *testing.T) { + value := host.GetTransferredValue(&transaction.Transaction{ + Value: big.NewInt(1000000000000000000), + }) + require.Equal(t, big.NewInt(1000000000000000000), value) + }) + + t.Run("with value and data", func(t *testing.T) { + value := host.GetTransferredValue(&transaction.Transaction{ + Value: big.NewInt(1000000000000000000), + Data: []byte("data"), + }) + require.Equal(t, big.NewInt(1000000000000000000), value) + }) + + t.Run("native transfer within MultiESDTNFTTransfer", func(t *testing.T) { + value := host.GetTransferredValue(&transaction.Transaction{ + SndAddr: testscommon.TestPubKeyAlice, + RcvAddr: testscommon.TestPubKeyAlice, + Data: []byte("MultiESDTNFTTransfer@8049d639e5a6980d1cd2392abcce41029cda74a1563523a202f09641cc2618f8@03@4e46542d313233343536@0a@01@544553542d393837363534@01@01@45474c442d303030303030@@0de0b6b3a7640000"), + }) + require.Equal(t, big.NewInt(1000000000000000000), value) + }) + + t.Run("native transfer within MultiESDTNFTTransfer; transfer & execute", func(t *testing.T) { + value := host.GetTransferredValue(&transaction.Transaction{ + SndAddr: testscommon.TestPubKeyAlice, + RcvAddr: testscommon.TestPubKeyAlice, + Data: []byte("MultiESDTNFTTransfer@00000000000000000500b9353fe8407f87310c87e12fa1ac807f0485da39d152@03@4e46542d313233343536@01@01@4e46542d313233343536@2a@01@45474c442d303030303030@@0de0b6b3a7640000@64756d6d79@07"), + }) + require.Equal(t, big.NewInt(1000000000000000000), value) + }) +} + +func TestBenchmarkMempoolHost_GetTransferredValue(t *testing.T) { + host, err := newMempoolHost(argsMempoolHost{ + txGasHandler: txcachemocks.NewTxGasHandlerMock(), + marshalizer: &marshal.GogoProtoMarshalizer{}, + }) + require.NoError(t, err) + require.NotNil(t, host) + + sw := core.NewStopWatch() + + valueMultiplier := int64(1_000_000_000_000) + + t.Run("numTransactions = 5_000", func(t *testing.T) { + numTransactions := 5_000 + transactions := createMultiESDTNFTTransfersWithNativeTransfer(numTransactions, valueMultiplier) + + sw.Start(t.Name()) + + for i := 0; i < numTransactions; i++ { + tx := transactions[i] + value := host.GetTransferredValue(tx) + require.Equal(t, big.NewInt(int64(i)*valueMultiplier), value) + } + + sw.Stop(t.Name()) + }) + + t.Run("numTransactions = 10_000", func(t *testing.T) { + numTransactions := 10_000 + transactions := createMultiESDTNFTTransfersWithNativeTransfer(numTransactions, valueMultiplier) + + sw.Start(t.Name()) + + for i := 0; i < numTransactions; i++ { + tx := transactions[i] + value := host.GetTransferredValue(tx) + require.Equal(t, big.NewInt(int64(i)*valueMultiplier), value) + } + + sw.Stop(t.Name()) + }) + + t.Run("numTransactions = 20_000", func(t *testing.T) { + numTransactions := 20_000 + transactions := createMultiESDTNFTTransfersWithNativeTransfer(numTransactions, valueMultiplier) + + sw.Start(t.Name()) + + for i := 0; i < numTransactions; i++ { + tx := transactions[i] + value := host.GetTransferredValue(tx) + require.Equal(t, big.NewInt(int64(i)*valueMultiplier), value) + } + + sw.Stop(t.Name()) + }) + + for name, measurement := range sw.GetMeasurementsMap() { + fmt.Printf("%fs (%s)\n", measurement, name) + } + + // (1) + // Vendor ID: GenuineIntel + // Model name: 11th Gen Intel(R) Core(TM) i7-1165G7 @ 2.80GHz + // CPU family: 6 + // Model: 140 + // Thread(s) per core: 2 + // Core(s) per socket: 4 + // + // NOTE: 20% is also due to the require() / assert() calls. + // 0.012993s (TestBenchmarkMempoolHost_GetTransferredValue/numTransactions_=_5_000) + // 0.024580s (TestBenchmarkMempoolHost_GetTransferredValue/numTransactions_=_10_000) + // 0.048808s (TestBenchmarkMempoolHost_GetTransferredValue/numTransactions_=_20_000) +} + +func createMultiESDTNFTTransfersWithNativeTransfer(numTransactions int, valueMultiplier int64) []*transaction.Transaction { + transactions := make([]*transaction.Transaction, 0, numTransactions) + + for i := 0; i < numTransactions; i++ { + nativeValue := big.NewInt(int64(i) * valueMultiplier) + data := fmt.Sprintf( + "MultiESDTNFTTransfer@8049d639e5a6980d1cd2392abcce41029cda74a1563523a202f09641cc2618f8@03@4e46542d313233343536@0a@01@544553542d393837363534@01@01@45474c442d303030303030@@%s", + hex.EncodeToString(nativeValue.Bytes()), + ) + + tx := &transaction.Transaction{ + SndAddr: testscommon.TestPubKeyAlice, + RcvAddr: testscommon.TestPubKeyAlice, + Data: []byte(data), + } + + transactions = append(transactions, tx) + } + + return transactions +} diff --git a/dataRetriever/txpool/shardedTxPool.go b/dataRetriever/txpool/shardedTxPool.go index af09753bd52..0f40817893d 100644 --- a/dataRetriever/txpool/shardedTxPool.go +++ b/dataRetriever/txpool/shardedTxPool.go @@ -27,7 +27,7 @@ type shardedTxPool struct { configPrototypeDestinationMe txcache.ConfigDestinationMe configPrototypeSourceMe txcache.ConfigSourceMe selfShardID uint32 - txGasHandler txcache.TxGasHandler + host txcache.MempoolHost } type txPoolShard struct { @@ -38,24 +38,32 @@ type txPoolShard struct { // NewShardedTxPool creates a new sharded tx pool // Implements "dataRetriever.TxPool" func NewShardedTxPool(args ArgShardedTxPool) (*shardedTxPool, error) { - log.Debug("NewShardedTxPool", "args", args.String()) + log.Debug("NewShardedTxPool", "args.SelfShardID", args.SelfShardID) err := args.verify() if err != nil { return nil, err } + mempoolHost, err := newMempoolHost(argsMempoolHost{ + txGasHandler: args.TxGasHandler, + marshalizer: args.Marshalizer, + }) + if err != nil { + return nil, err + } + halfOfSizeInBytes := args.Config.SizeInBytes / 2 halfOfCapacity := args.Config.Capacity / 2 configPrototypeSourceMe := txcache.ConfigSourceMe{ - NumChunks: args.Config.Shards, - EvictionEnabled: true, - NumBytesThreshold: uint32(halfOfSizeInBytes), - CountThreshold: halfOfCapacity, - NumBytesPerSenderThreshold: args.Config.SizeInBytesPerSender, - CountPerSenderThreshold: args.Config.SizePerSender, - NumSendersToPreemptivelyEvict: dataRetriever.TxPoolNumSendersToPreemptivelyEvict, + NumChunks: args.Config.Shards, + EvictionEnabled: true, + NumBytesThreshold: uint32(halfOfSizeInBytes), + CountThreshold: halfOfCapacity, + NumBytesPerSenderThreshold: args.Config.SizeInBytesPerSender, + CountPerSenderThreshold: args.Config.SizePerSender, + NumItemsToPreemptivelyEvict: storage.TxPoolSourceMeNumItemsToPreemptivelyEvict, } // We do not reserve cross tx cache capacity for [metachain] -> [me] (no transactions), [me] -> me (already reserved above). @@ -66,7 +74,7 @@ func NewShardedTxPool(args ArgShardedTxPool) (*shardedTxPool, error) { NumChunks: args.Config.Shards, MaxNumBytes: uint32(halfOfSizeInBytes) / numCrossTxCaches, MaxNumItems: halfOfCapacity / numCrossTxCaches, - NumItemsToPreemptivelyEvict: storage.TxPoolNumTxsToPreemptivelyEvict, + NumItemsToPreemptivelyEvict: storage.TxPoolDestinationMeNumItemsToPreemptivelyEvict, } shardedTxPoolObject := &shardedTxPool{ @@ -77,7 +85,7 @@ func NewShardedTxPool(args ArgShardedTxPool) (*shardedTxPool, error) { configPrototypeDestinationMe: configPrototypeDestinationMe, configPrototypeSourceMe: configPrototypeSourceMe, selfShardID: args.SelfShardID, - txGasHandler: args.TxGasHandler, + host: mempoolHost, } return shardedTxPoolObject, nil @@ -134,7 +142,7 @@ func (txPool *shardedTxPool) createTxCache(cacheID string) txCache { if isForSenderMe { config := txPool.configPrototypeSourceMe config.Name = cacheID - cache, err := txcache.NewTxCache(config, txPool.txGasHandler) + cache, err := txcache.NewTxCache(config, txPool.host) if err != nil { log.Error("shardedTxPool.createTxCache()", "err", err) return txcache.NewDisabledCache() @@ -188,6 +196,7 @@ func (txPool *shardedTxPool) AddData(key []byte, value interface{}, sizeInBytes func (txPool *shardedTxPool) addTx(tx *txcache.WrappedTransaction, cacheID string) { shard := txPool.getOrCreateShard(cacheID) cache := shard.Cache + _, added := cache.AddTx(tx) if added { txPool.onAdded(tx.TxHash, tx) @@ -228,13 +237,8 @@ func (txPool *shardedTxPool) searchFirstTx(txHash []byte) (tx data.TransactionHa // RemoveData removes the transaction from the pool func (txPool *shardedTxPool) RemoveData(key []byte, cacheID string) { - txPool.removeTx(key, cacheID) -} - -// removeTx removes the transaction from the pool -func (txPool *shardedTxPool) removeTx(txHash []byte, cacheID string) bool { shard := txPool.getOrCreateShard(cacheID) - return shard.Cache.RemoveTxByHash(txHash) + _ = shard.Cache.RemoveTxByHash(key) } // RemoveSetOfDataFromPool removes a bunch of transactions from the pool @@ -244,14 +248,27 @@ func (txPool *shardedTxPool) RemoveSetOfDataFromPool(keys [][]byte, cacheID stri // removeTxBulk removes a bunch of transactions from the pool func (txPool *shardedTxPool) removeTxBulk(txHashes [][]byte, cacheID string) { + shard := txPool.getOrCreateShard(cacheID) + + stopWatch := core.NewStopWatch() + stopWatch.Start("removal") + numRemoved := 0 for _, key := range txHashes { - if txPool.removeTx(key, cacheID) { + if shard.Cache.RemoveTxByHash(key) { numRemoved++ } } - log.Trace("shardedTxPool.removeTxBulk()", "name", cacheID, "numToRemove", len(txHashes), "numRemoved", numRemoved) + stopWatch.Stop("removal") + + // Transactions with lower / equal nonce are also removed, but the counter does not reflect that. + log.Debug("shardedTxPool.removeTxBulk", + "cacheID", cacheID, + "numToRemove", len(txHashes), + "numRemoved", numRemoved, + "duration", stopWatch.GetMeasurement("removal"), + ) } // RemoveDataFromAllShards removes the transaction from the pool (it searches in all shards) diff --git a/dataRetriever/txpool/shardedTxPool_test.go b/dataRetriever/txpool/shardedTxPool_test.go index 6fdaa4676ad..1b3ab585dc3 100644 --- a/dataRetriever/txpool/shardedTxPool_test.go +++ b/dataRetriever/txpool/shardedTxPool_test.go @@ -10,6 +10,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/transaction" + "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/storage/storageunit" "github.com/multiversx/mx-chain-go/testscommon/txcachemocks" @@ -33,11 +34,8 @@ func Test_NewShardedTxPool_WhenBadConfig(t *testing.T) { SizeInBytesPerSender: 40960, Shards: 16, }, - TxGasHandler: &txcachemocks.TxGasHandlerMock{ - MinimumGasMove: 50000, - MinimumGasPrice: 1000000000, - GasProcessingDivisor: 100, - }, + TxGasHandler: txcachemocks.NewTxGasHandlerMock(), + Marshalizer: &marshal.GogoProtoMarshalizer{}, NumberOfShards: 1, } @@ -84,15 +82,11 @@ func Test_NewShardedTxPool_WhenBadConfig(t *testing.T) { require.Errorf(t, err, dataRetriever.ErrNilTxGasHandler.Error()) args = goodArgs - args.TxGasHandler = &txcachemocks.TxGasHandlerMock{ - MinimumGasMove: 50000, - MinimumGasPrice: 0, - GasProcessingDivisor: 1, - } + args.Marshalizer = nil pool, err = NewShardedTxPool(args) require.Nil(t, pool) require.NotNil(t, err) - require.Errorf(t, err, dataRetriever.ErrCacheConfigInvalidEconomics.Error()) + require.Errorf(t, err, dataRetriever.ErrNilMarshalizer.Error()) args = goodArgs args.NumberOfShards = 0 @@ -105,12 +99,9 @@ func Test_NewShardedTxPool_WhenBadConfig(t *testing.T) { func Test_NewShardedTxPool_ComputesCacheConfig(t *testing.T) { config := storageunit.CacheConfig{SizeInBytes: 419430400, SizeInBytesPerSender: 614400, Capacity: 600000, SizePerSender: 1000, Shards: 1} args := ArgShardedTxPool{ - Config: config, - TxGasHandler: &txcachemocks.TxGasHandlerMock{ - MinimumGasMove: 50000, - MinimumGasPrice: 1000000000, - GasProcessingDivisor: 1, - }, + Config: config, + TxGasHandler: txcachemocks.NewTxGasHandlerMock(), + Marshalizer: &marshal.GogoProtoMarshalizer{}, NumberOfShards: 2, } @@ -121,7 +112,6 @@ func Test_NewShardedTxPool_ComputesCacheConfig(t *testing.T) { require.Equal(t, 209715200, int(pool.configPrototypeSourceMe.NumBytesThreshold)) require.Equal(t, 614400, int(pool.configPrototypeSourceMe.NumBytesPerSenderThreshold)) require.Equal(t, 1000, int(pool.configPrototypeSourceMe.CountPerSenderThreshold)) - require.Equal(t, 100, int(pool.configPrototypeSourceMe.NumSendersToPreemptivelyEvict)) require.Equal(t, 300000, int(pool.configPrototypeSourceMe.CountThreshold)) require.Equal(t, 300000, int(pool.configPrototypeDestinationMe.MaxNumItems)) @@ -392,12 +382,9 @@ func Test_routeToCacheUnions(t *testing.T) { Shards: 1, } args := ArgShardedTxPool{ - Config: config, - TxGasHandler: &txcachemocks.TxGasHandlerMock{ - MinimumGasMove: 50000, - MinimumGasPrice: 200000000000, - GasProcessingDivisor: 100, - }, + Config: config, + TxGasHandler: txcachemocks.NewTxGasHandlerMock(), + Marshalizer: &marshal.GogoProtoMarshalizer{}, NumberOfShards: 4, SelfShardID: 42, } @@ -414,8 +401,9 @@ func Test_routeToCacheUnions(t *testing.T) { func createTx(sender string, nonce uint64) data.TransactionHandler { return &transaction.Transaction{ - SndAddr: []byte(sender), - Nonce: nonce, + SndAddr: []byte(sender), + Nonce: nonce, + GasLimit: 50000, } } @@ -435,12 +423,9 @@ func newTxPoolToTest() (dataRetriever.ShardedDataCacherNotifier, error) { Shards: 1, } args := ArgShardedTxPool{ - Config: config, - TxGasHandler: &txcachemocks.TxGasHandlerMock{ - MinimumGasMove: 50000, - MinimumGasPrice: 200000000000, - GasProcessingDivisor: 100, - }, + Config: config, + TxGasHandler: txcachemocks.NewTxGasHandlerMock(), + Marshalizer: &marshal.GogoProtoMarshalizer{}, NumberOfShards: 4, SelfShardID: 0, } diff --git a/epochStart/bootstrap/process_test.go b/epochStart/bootstrap/process_test.go index 552148003d6..ca3fd78a5b8 100644 --- a/epochStart/bootstrap/process_test.go +++ b/epochStart/bootstrap/process_test.go @@ -818,16 +818,18 @@ func TestEpochStartBootstrap_BootstrapStartInEpochNotEnabled(t *testing.T) { coreComp, cryptoComp := createComponentsForEpochStart() args := createMockEpochStartBootstrapArgs(coreComp, cryptoComp) - err := errors.New("localErr") + localErr := errors.New("localErr") args.LatestStorageDataProvider = &mock.LatestStorageDataProviderStub{ GetCalled: func() (storage.LatestDataFromStorage, error) { - return storage.LatestDataFromStorage{}, err + return storage.LatestDataFromStorage{}, localErr }, } - epochStartProvider, _ := NewEpochStartBootstrap(args) + + epochStartProvider, err := NewEpochStartBootstrap(args) + assert.NoError(t, err) params, err := epochStartProvider.Bootstrap() - assert.Nil(t, err) + assert.NoError(t, err) assert.NotNil(t, params) } @@ -868,7 +870,8 @@ func TestPrepareForEpochZero(t *testing.T) { coreComp, cryptoComp := createComponentsForEpochStart() args := createMockEpochStartBootstrapArgs(coreComp, cryptoComp) - epochStartProvider, _ := NewEpochStartBootstrap(args) + epochStartProvider, err := NewEpochStartBootstrap(args) + assert.Nil(t, err) params, err := epochStartProvider.prepareEpochZero() assert.Nil(t, err) @@ -904,7 +907,8 @@ func TestPrepareForEpochZero_NodeInGenesisShouldNotAlterShardID(t *testing.T) { }, } - epochStartProvider, _ := NewEpochStartBootstrap(args) + epochStartProvider, err := NewEpochStartBootstrap(args) + assert.NoError(t, err) params, err := epochStartProvider.prepareEpochZero() assert.NoError(t, err) @@ -939,7 +943,8 @@ func TestPrepareForEpochZero_NodeNotInGenesisShouldAlterShardID(t *testing.T) { }, } - epochStartProvider, _ := NewEpochStartBootstrap(args) + epochStartProvider, err := NewEpochStartBootstrap(args) + assert.NoError(t, err) params, err := epochStartProvider.prepareEpochZero() assert.NoError(t, err) diff --git a/factory/disabled/txProcessor.go b/factory/disabled/txProcessor.go index 950add6c732..16df5db2d89 100644 --- a/factory/disabled/txProcessor.go +++ b/factory/disabled/txProcessor.go @@ -2,6 +2,7 @@ package disabled import ( "github.com/multiversx/mx-chain-core-go/data/transaction" + "github.com/multiversx/mx-chain-go/state" vmcommon "github.com/multiversx/mx-chain-vm-common-go" ) @@ -19,6 +20,11 @@ func (txProc *TxProcessor) VerifyTransaction(_ *transaction.Transaction) error { return nil } +// VerifyGuardian does nothing as it is disabled +func (txProc *TxProcessor) VerifyGuardian(_ *transaction.Transaction, _ state.UserAccountHandler) error { + return nil +} + // IsInterfaceNil returns true if there is no value under the interface func (txProc *TxProcessor) IsInterfaceNil() bool { return txProc == nil diff --git a/factory/processing/blockProcessorCreator_test.go b/factory/processing/blockProcessorCreator_test.go index 099fec4a82d..e80b62491d2 100644 --- a/factory/processing/blockProcessorCreator_test.go +++ b/factory/processing/blockProcessorCreator_test.go @@ -162,7 +162,8 @@ func Test_newBlockProcessorCreatorForMeta(t *testing.T) { componentsMock.SetShardCoordinator(t, args.BootstrapComponents, shardC) - pcf, _ := processComp.NewProcessComponentsFactory(args) + pcf, err := processComp.NewProcessComponentsFactory(args) + require.NoError(t, err) require.NotNil(t, pcf) _, err = pcf.Create() diff --git a/factory/processing/txSimulatorProcessComponents_test.go b/factory/processing/txSimulatorProcessComponents_test.go index 0c919b0ba95..e6fd1f90f6a 100644 --- a/factory/processing/txSimulatorProcessComponents_test.go +++ b/factory/processing/txSimulatorProcessComponents_test.go @@ -25,30 +25,33 @@ func TestManagedProcessComponents_createAPITransactionEvaluator(t *testing.T) { t.Run("invalid VMOutputCacher config should error", func(t *testing.T) { processArgs := components.GetProcessComponentsFactoryArgs(shardCoordinatorForShardID2) processArgs.Config.VMOutputCacher.Type = "invalid" - pcf, _ := processing.NewProcessComponentsFactory(processArgs) + pcf, err := processing.NewProcessComponentsFactory(processArgs) + require.Nil(t, err) apiTransactionEvaluator, vmContainerFactory, err := pcf.CreateAPITransactionEvaluator() - assert.NotNil(t, err) + require.NotNil(t, err) assert.True(t, check.IfNil(apiTransactionEvaluator)) assert.True(t, check.IfNil(vmContainerFactory)) assert.Contains(t, err.Error(), "not supported cache type") }) t.Run("should work for shard", func(t *testing.T) { processArgs := components.GetProcessComponentsFactoryArgs(shardCoordinatorForShardID2) - pcf, _ := processing.NewProcessComponentsFactory(processArgs) + pcf, err := processing.NewProcessComponentsFactory(processArgs) + require.Nil(t, err) apiTransactionEvaluator, vmContainerFactory, err := pcf.CreateAPITransactionEvaluator() - assert.Nil(t, err) + require.Nil(t, err) assert.False(t, check.IfNil(apiTransactionEvaluator)) assert.False(t, check.IfNil(vmContainerFactory)) require.NoError(t, vmContainerFactory.Close()) }) t.Run("should work for metachain", func(t *testing.T) { processArgs := components.GetProcessComponentsFactoryArgs(shardCoordinatorForMetachain) - pcf, _ := processing.NewProcessComponentsFactory(processArgs) + pcf, err := processing.NewProcessComponentsFactory(processArgs) + require.Nil(t, err) apiTransactionEvaluator, vmContainerFactory, err := pcf.CreateAPITransactionEvaluator() - assert.Nil(t, err) + require.Nil(t, err) assert.False(t, check.IfNil(apiTransactionEvaluator)) assert.False(t, check.IfNil(vmContainerFactory)) require.NoError(t, vmContainerFactory.Close()) diff --git a/go.mod b/go.mod index 083ce5d42fd..cc5f4259b96 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/multiversx/mx-chain-es-indexer-go v1.7.10 github.com/multiversx/mx-chain-logger-go v1.0.15 github.com/multiversx/mx-chain-scenario-go v1.4.4 - github.com/multiversx/mx-chain-storage-go v1.0.16 + github.com/multiversx/mx-chain-storage-go v1.0.18 github.com/multiversx/mx-chain-vm-common-go v1.5.16 github.com/multiversx/mx-chain-vm-go v1.5.37 github.com/multiversx/mx-chain-vm-v1_2-go v1.2.68 diff --git a/go.sum b/go.sum index b7402b7c8e4..b4ebd7201e0 100644 --- a/go.sum +++ b/go.sum @@ -397,8 +397,8 @@ github.com/multiversx/mx-chain-logger-go v1.0.15 h1:HlNdK8etyJyL9NQ+6mIXyKPEBo+w github.com/multiversx/mx-chain-logger-go v1.0.15/go.mod h1:t3PRKaWB1M+i6gUfD27KXgzLJJC+mAQiN+FLlL1yoGQ= github.com/multiversx/mx-chain-scenario-go v1.4.4 h1:DVE2V+FPeyD/yWoC+KEfPK3jsFzHeruelESfpTlf460= github.com/multiversx/mx-chain-scenario-go v1.4.4/go.mod h1:kI+TWR3oIEgUkbwkHCPo2CQ3VjIge+ezGTibiSGwMxo= -github.com/multiversx/mx-chain-storage-go v1.0.16 h1:l2lJq+EAN3YwLbjJrnoKfFd1/1Xmo9DcAUECND2obLs= -github.com/multiversx/mx-chain-storage-go v1.0.16/go.mod h1:uM/z7YyqTOD3wgyH8TfapyEl5sb+7x/Jaxne4cfG4HI= +github.com/multiversx/mx-chain-storage-go v1.0.18 h1:DA33o5COEjnCKclCeCvzXXI0zIgFp2QqZK32UTVvDes= +github.com/multiversx/mx-chain-storage-go v1.0.18/go.mod h1:eFDEOrG7Wiyk5I/ObpwcN2eoBlOnnfeEMTvTer1cymk= github.com/multiversx/mx-chain-vm-common-go v1.5.16 h1:g1SqYjxl7K66Y1O/q6tvDJ37fzpzlxCSfRzSm/woQQY= github.com/multiversx/mx-chain-vm-common-go v1.5.16/go.mod h1:1rSkXreUZNXyPTTdhj47M+Fy62yjxbu3aAsXEtKN3UY= github.com/multiversx/mx-chain-vm-go v1.5.37 h1:Iy3KCvM+DOq1f9UPA7uYK/rI3ZbBOXc2CVNO2/vm5zw= diff --git a/integrationTests/factory/bootstrapComponents/bootstrapComponents_test.go b/integrationTests/factory/bootstrapComponents/bootstrapComponents_test.go index 6c525ff9f12..03601ec46b1 100644 --- a/integrationTests/factory/bootstrapComponents/bootstrapComponents_test.go +++ b/integrationTests/factory/bootstrapComponents/bootstrapComponents_test.go @@ -36,7 +36,12 @@ func TestBootstrapComponents_Create_Close_ShouldWork(t *testing.T) { require.Nil(t, err) managedNetworkComponents, err := nr.CreateManagedNetworkComponents(managedCoreComponents, managedStatusCoreComponents, managedCryptoComponents) require.Nil(t, err) - managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents) + managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents( + managedStatusCoreComponents, + managedCoreComponents, + managedCryptoComponents, + managedNetworkComponents, + ) require.Nil(t, err) require.NotNil(t, managedBootstrapComponents) diff --git a/integrationTests/factory/consensusComponents/consensusComponents_test.go b/integrationTests/factory/consensusComponents/consensusComponents_test.go index f560f099705..b68e9dd95cc 100644 --- a/integrationTests/factory/consensusComponents/consensusComponents_test.go +++ b/integrationTests/factory/consensusComponents/consensusComponents_test.go @@ -39,9 +39,19 @@ func TestConsensusComponents_Close_ShouldWork(t *testing.T) { require.Nil(t, err) managedNetworkComponents, err := nr.CreateManagedNetworkComponents(managedCoreComponents, managedStatusCoreComponents, managedCryptoComponents) require.Nil(t, err) - managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents) + managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents( + managedStatusCoreComponents, + managedCoreComponents, + managedCryptoComponents, + managedNetworkComponents, + ) require.Nil(t, err) - managedDataComponents, err := nr.CreateManagedDataComponents(managedStatusCoreComponents, managedCoreComponents, managedBootstrapComponents, managedCryptoComponents) + managedDataComponents, err := nr.CreateManagedDataComponents( + managedStatusCoreComponents, + managedCoreComponents, + managedBootstrapComponents, + managedCryptoComponents, + ) require.Nil(t, err) managedStateComponents, err := nr.CreateManagedStateComponents(managedCoreComponents, managedDataComponents, managedStatusCoreComponents) require.Nil(t, err) diff --git a/integrationTests/factory/dataComponents/dataComponents_test.go b/integrationTests/factory/dataComponents/dataComponents_test.go index c28a41c6543..d4727818994 100644 --- a/integrationTests/factory/dataComponents/dataComponents_test.go +++ b/integrationTests/factory/dataComponents/dataComponents_test.go @@ -36,9 +36,19 @@ func TestDataComponents_Create_Close_ShouldWork(t *testing.T) { require.Nil(t, err) managedNetworkComponents, err := nr.CreateManagedNetworkComponents(managedCoreComponents, managedStatusCoreComponents, managedCryptoComponents) require.Nil(t, err) - managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents) + managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents( + managedStatusCoreComponents, + managedCoreComponents, + managedCryptoComponents, + managedNetworkComponents, + ) require.Nil(t, err) - managedDataComponents, err := nr.CreateManagedDataComponents(managedStatusCoreComponents, managedCoreComponents, managedBootstrapComponents, managedCryptoComponents) + managedDataComponents, err := nr.CreateManagedDataComponents( + managedStatusCoreComponents, + managedCoreComponents, + managedBootstrapComponents, + managedCryptoComponents, + ) require.Nil(t, err) require.NotNil(t, managedDataComponents) diff --git a/integrationTests/factory/heartbeatComponents/heartbeatComponents_test.go b/integrationTests/factory/heartbeatComponents/heartbeatComponents_test.go index 9082ce63c06..dd0a07ad91f 100644 --- a/integrationTests/factory/heartbeatComponents/heartbeatComponents_test.go +++ b/integrationTests/factory/heartbeatComponents/heartbeatComponents_test.go @@ -39,9 +39,19 @@ func TestHeartbeatComponents_Close_ShouldWork(t *testing.T) { require.Nil(t, err) managedNetworkComponents, err := nr.CreateManagedNetworkComponents(managedCoreComponents, managedStatusCoreComponents, managedCryptoComponents) require.Nil(t, err) - managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents) + managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents( + managedStatusCoreComponents, + managedCoreComponents, + managedCryptoComponents, + managedNetworkComponents, + ) require.Nil(t, err) - managedDataComponents, err := nr.CreateManagedDataComponents(managedStatusCoreComponents, managedCoreComponents, managedBootstrapComponents, managedCryptoComponents) + managedDataComponents, err := nr.CreateManagedDataComponents( + managedStatusCoreComponents, + managedCoreComponents, + managedBootstrapComponents, + managedCryptoComponents, + ) require.Nil(t, err) managedStateComponents, err := nr.CreateManagedStateComponents(managedCoreComponents, managedDataComponents, managedStatusCoreComponents) require.Nil(t, err) diff --git a/integrationTests/factory/processComponents/processComponents_test.go b/integrationTests/factory/processComponents/processComponents_test.go index 2f2c859bc94..17860520ea9 100644 --- a/integrationTests/factory/processComponents/processComponents_test.go +++ b/integrationTests/factory/processComponents/processComponents_test.go @@ -40,9 +40,19 @@ func TestProcessComponents_Close_ShouldWork(t *testing.T) { require.Nil(t, err) managedNetworkComponents, err := nr.CreateManagedNetworkComponents(managedCoreComponents, managedStatusCoreComponents, managedCryptoComponents) require.Nil(t, err) - managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents) + managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents( + managedStatusCoreComponents, + managedCoreComponents, + managedCryptoComponents, + managedNetworkComponents, + ) require.Nil(t, err) - managedDataComponents, err := nr.CreateManagedDataComponents(managedStatusCoreComponents, managedCoreComponents, managedBootstrapComponents, managedCryptoComponents) + managedDataComponents, err := nr.CreateManagedDataComponents( + managedStatusCoreComponents, + managedCoreComponents, + managedBootstrapComponents, + managedCryptoComponents, + ) require.Nil(t, err) managedStateComponents, err := nr.CreateManagedStateComponents(managedCoreComponents, managedDataComponents, managedStatusCoreComponents) require.Nil(t, err) diff --git a/integrationTests/factory/stateComponents/stateComponents_test.go b/integrationTests/factory/stateComponents/stateComponents_test.go index 3c942f54e53..18984a82bde 100644 --- a/integrationTests/factory/stateComponents/stateComponents_test.go +++ b/integrationTests/factory/stateComponents/stateComponents_test.go @@ -36,9 +36,20 @@ func TestStateComponents_Create_Close_ShouldWork(t *testing.T) { require.Nil(t, err) managedNetworkComponents, err := nr.CreateManagedNetworkComponents(managedCoreComponents, managedStatusCoreComponents, managedCryptoComponents) require.Nil(t, err) - managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents) + + managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents( + managedStatusCoreComponents, + managedCoreComponents, + managedCryptoComponents, + managedNetworkComponents, + ) require.Nil(t, err) - managedDataComponents, err := nr.CreateManagedDataComponents(managedStatusCoreComponents, managedCoreComponents, managedBootstrapComponents, managedCryptoComponents) + managedDataComponents, err := nr.CreateManagedDataComponents( + managedStatusCoreComponents, + managedCoreComponents, + managedBootstrapComponents, + managedCryptoComponents, + ) require.Nil(t, err) managedStateComponents, err := nr.CreateManagedStateComponents(managedCoreComponents, managedDataComponents, managedStatusCoreComponents) require.Nil(t, err) diff --git a/integrationTests/factory/statusComponents/statusComponents_test.go b/integrationTests/factory/statusComponents/statusComponents_test.go index 62e2ad1e289..dc5d3575b8c 100644 --- a/integrationTests/factory/statusComponents/statusComponents_test.go +++ b/integrationTests/factory/statusComponents/statusComponents_test.go @@ -40,9 +40,19 @@ func TestStatusComponents_Create_Close_ShouldWork(t *testing.T) { require.Nil(t, err) managedNetworkComponents, err := nr.CreateManagedNetworkComponents(managedCoreComponents, managedStatusCoreComponents, managedCryptoComponents) require.Nil(t, err) - managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents) + managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents( + managedStatusCoreComponents, + managedCoreComponents, + managedCryptoComponents, + managedNetworkComponents, + ) require.Nil(t, err) - managedDataComponents, err := nr.CreateManagedDataComponents(managedStatusCoreComponents, managedCoreComponents, managedBootstrapComponents, managedCryptoComponents) + managedDataComponents, err := nr.CreateManagedDataComponents( + managedStatusCoreComponents, + managedCoreComponents, + managedBootstrapComponents, + managedCryptoComponents, + ) require.Nil(t, err) managedStateComponents, err := nr.CreateManagedStateComponents(managedCoreComponents, managedDataComponents, managedStatusCoreComponents) require.Nil(t, err) diff --git a/integrationTests/multiShard/relayedTx/relayedTx_test.go b/integrationTests/multiShard/relayedTx/relayedTx_test.go index 41ece5b81eb..70df89b466c 100644 --- a/integrationTests/multiShard/relayedTx/relayedTx_test.go +++ b/integrationTests/multiShard/relayedTx/relayedTx_test.go @@ -188,7 +188,7 @@ func testRelayedTransactionInMultiShardEnvironmentWithSmartContractTX( } time.Sleep(time.Second) - roundToPropagateMultiShard := int64(25) + roundToPropagateMultiShard := int64(40) for i := int64(0); i <= roundToPropagateMultiShard; i++ { round, nonce = integrationTests.ProposeAndSyncOneBlock(t, nodes, idxProposers, round, nonce) integrationTests.AddSelfNotarizedHeaderByMetachain(nodes) @@ -200,7 +200,7 @@ func testRelayedTransactionInMultiShardEnvironmentWithSmartContractTX( finalBalance.Mul(finalBalance, sendValue) checkSCBalance(t, ownerNode, scAddress, receiverAddress1, finalBalance) - checkSCBalance(t, ownerNode, scAddress, receiverAddress1, finalBalance) + checkSCBalance(t, ownerNode, scAddress, receiverAddress2, finalBalance) checkPlayerBalances(t, nodes, players) @@ -436,7 +436,7 @@ func checkSCBalance(t *testing.T, node *integrationTests.TestProcessorNode, scAd }) assert.Nil(t, err) actualBalance := big.NewInt(0).SetBytes(vmOutput.ReturnData[0]) - assert.Equal(t, 0, actualBalance.Cmp(balance)) + assert.Equal(t, balance.String(), actualBalance.String()) } func checkPlayerBalances( diff --git a/integrationTests/realcomponents/processorRunner.go b/integrationTests/realcomponents/processorRunner.go index f788de20f84..3f3f4837201 100644 --- a/integrationTests/realcomponents/processorRunner.go +++ b/integrationTests/realcomponents/processorRunner.go @@ -72,6 +72,9 @@ func NewProcessorRunner(tb testing.TB, config config.Configs) *ProcessorRunner { } func (pr *ProcessorRunner) createComponents(tb testing.TB) { + var err error + require.Nil(tb, err) + pr.createCoreComponents(tb) pr.createCryptoComponents(tb) pr.createStatusCoreComponents(tb) diff --git a/integrationTests/singleShard/block/executingMiniblocks/executingMiniblocks_test.go b/integrationTests/singleShard/block/executingMiniblocks/executingMiniblocks_test.go index 8d6c00af8ae..2c7bb0f7a7c 100644 --- a/integrationTests/singleShard/block/executingMiniblocks/executingMiniblocks_test.go +++ b/integrationTests/singleShard/block/executingMiniblocks/executingMiniblocks_test.go @@ -82,12 +82,11 @@ func TestShardShouldNotProposeAndExecuteTwoBlocksInSameRound(t *testing.T) { } // TestShardShouldProposeBlockContainingInvalidTransactions tests the following scenario: -// 1. generate 3 move balance transactions: one that can be executed, one that can not be executed but the account has -// the balance for the fee and one that is completely invalid (no balance left for it) +// 1. generate 3 move balance transactions: one that can be executed, one to be processed as invalid, and one that isn't executable (no balance left for fee). // 2. proposer will have those 3 transactions in its pools and will propose a block // 3. another node will be able to sync the proposed block (and request - receive) the 2 transactions that // will end up in the block (one valid and one invalid) -// 4. the non-executable transaction will be removed from the proposer's pool +// 4. the non-executable transaction will not be immediately removed from the proposer's pool. See MX-16200. func TestShardShouldProposeBlockContainingInvalidTransactions(t *testing.T) { if testing.Short() { t.Skip("this is not a short test") @@ -195,7 +194,18 @@ func testStateOnNodes(t *testing.T, nodes []*integrationTests.TestProcessorNode, testTxIsInMiniblock(t, proposer, hashes[txValidIdx], block.TxBlock) testTxIsInMiniblock(t, proposer, hashes[txInvalidIdx], block.InvalidBlock) testTxIsInNotInBody(t, proposer, hashes[txDeletedIdx]) - testTxHashNotPresentInPool(t, proposer, hashes[txDeletedIdx]) + + // Removed from mempool. + _, ok := proposer.DataPool.Transactions().SearchFirstData(hashes[txValidIdx]) + assert.False(t, ok) + + // Removed from mempool. + _, ok = proposer.DataPool.Transactions().SearchFirstData(hashes[txInvalidIdx]) + assert.False(t, ok) + + // Not removed from mempool (see MX-16200). + _, ok = proposer.DataPool.Transactions().SearchFirstData(hashes[txDeletedIdx]) + assert.True(t, ok) } func testSameBlockHeight(t *testing.T, nodes []*integrationTests.TestProcessorNode, idxProposer int, expectedHeight uint64) { @@ -208,11 +218,6 @@ func testSameBlockHeight(t *testing.T, nodes []*integrationTests.TestProcessorNo } } -func testTxHashNotPresentInPool(t *testing.T, proposer *integrationTests.TestProcessorNode, hash []byte) { - txCache := proposer.DataPool.Transactions() - _, ok := txCache.SearchFirstData(hash) - assert.False(t, ok) -} func testTxIsInMiniblock(t *testing.T, proposer *integrationTests.TestProcessorNode, hash []byte, bt block.Type) { hdrHandler := proposer.BlockChain.GetCurrentBlockHeader() diff --git a/integrationTests/testProcessorNode.go b/integrationTests/testProcessorNode.go index dc828291384..80ba584c6b4 100644 --- a/integrationTests/testProcessorNode.go +++ b/integrationTests/testProcessorNode.go @@ -1689,6 +1689,7 @@ func (tpn *TestProcessorNode) initInnerProcessors(gasMap map[string]map[string]u txTypeHandler, _ := coordinator.NewTxTypeHandler(argsTxTypeHandler) tpn.GasHandler, _ = preprocess.NewGasComputation(tpn.EconomicsData, txTypeHandler, tpn.EnableEpochsHandler) badBlocksHandler, _ := tpn.InterimProcContainer.Get(dataBlock.InvalidBlock) + guardianChecker := &guardianMocks.GuardedAccountHandlerStub{} argsNewScProcessor := scrCommon.ArgsNewSmartContractProcessor{ VmContainer: tpn.VMContainer, @@ -1734,7 +1735,7 @@ func (tpn *TestProcessorNode) initInnerProcessors(gasMap map[string]map[string]u ScrForwarder: tpn.ScrForwarder, EnableRoundsHandler: tpn.EnableRoundsHandler, EnableEpochsHandler: tpn.EnableEpochsHandler, - GuardianChecker: &guardianMocks.GuardedAccountHandlerStub{}, + GuardianChecker: guardianChecker, TxVersionChecker: &testscommon.TxVersionCheckerStub{}, TxLogsProcessor: tpn.TransactionLogProcessor, } @@ -2816,9 +2817,13 @@ func (tpn *TestProcessorNode) WhiteListBody(nodes []*TestProcessorNode, bodyHand } } -// CommitBlock commits the block and body +// CommitBlock commits the block and body. +// This isn't entirely correct, since there's not state rollback if the commit fails. func (tpn *TestProcessorNode) CommitBlock(body data.BodyHandler, header data.HeaderHandler) { - _ = tpn.BlockProcessor.CommitBlock(header, body) + err := tpn.BlockProcessor.CommitBlock(header, body) + if err != nil { + log.Error("TestProcessorNode.CommitBlock", "error", err.Error()) + } } // GetShardHeader returns the first *dataBlock.Header stored in datapools having the nonce provided as parameter diff --git a/integrationTests/vm/esdt/process/esdtProcess_test.go b/integrationTests/vm/esdt/process/esdtProcess_test.go index 8fa9fd04101..4c4b900b3f3 100644 --- a/integrationTests/vm/esdt/process/esdtProcess_test.go +++ b/integrationTests/vm/esdt/process/esdtProcess_test.go @@ -1214,7 +1214,7 @@ func TestScACallsScBWithExecOnDestScAPerformsAsyncCall_NoCallbackInScB(t *testin ) time.Sleep(time.Second) - _, _ = integrationTests.WaitOperationToBeDone(t, nodes, 2, nonce, round, idxProposers) + nonce, round = integrationTests.WaitOperationToBeDone(t, nodes, 2, nonce, round, idxProposers) time.Sleep(time.Second) // issue ESDT by calling exec on dest context on child contract diff --git a/node/chainSimulator/components/testOnlyProcessingNode.go b/node/chainSimulator/components/testOnlyProcessingNode.go index 28256c4820f..efa4c12102c 100644 --- a/node/chainSimulator/components/testOnlyProcessingNode.go +++ b/node/chainSimulator/components/testOnlyProcessingNode.go @@ -175,10 +175,17 @@ func NewTestOnlyProcessingNode(args ArgsTestOnlyProcessingNode) (*testOnlyProces return nil, err } - err = instance.createDataPool(args) + instance.DataPool, err = dataRetrieverFactory.NewDataPoolFromConfig(dataRetrieverFactory.ArgsDataPool{ + Config: args.Configs.GeneralConfig, + EconomicsData: instance.CoreComponentsHolder.EconomicsData(), + ShardCoordinator: instance.BootstrapComponentsHolder.ShardCoordinator(), + Marshalizer: instance.CoreComponentsHolder.InternalMarshalizer(), + PathManager: instance.CoreComponentsHolder.PathHandler(), + }) if err != nil { return nil, err } + err = instance.createNodesCoordinator(args.Configs.PreferencesConfig.Preferences, *args.Configs.GeneralConfig) if err != nil { return nil, err @@ -261,22 +268,6 @@ func (node *testOnlyProcessingNode) createBlockChain(selfShardID uint32) error { return err } -func (node *testOnlyProcessingNode) createDataPool(args ArgsTestOnlyProcessingNode) error { - var err error - - argsDataPool := dataRetrieverFactory.ArgsDataPool{ - Config: args.Configs.GeneralConfig, - EconomicsData: node.CoreComponentsHolder.EconomicsData(), - ShardCoordinator: node.BootstrapComponentsHolder.ShardCoordinator(), - Marshalizer: node.CoreComponentsHolder.InternalMarshalizer(), - PathManager: node.CoreComponentsHolder.PathHandler(), - } - - node.DataPool, err = dataRetrieverFactory.NewDataPoolFromConfig(argsDataPool) - - return err -} - func (node *testOnlyProcessingNode) createNodesCoordinator(pref config.PreferencesConfig, generalConfig config.Config) error { nodesShufflerOut, err := bootstrapComp.CreateNodesShuffleOut( node.CoreComponentsHolder.GenesisNodesSetup(), diff --git a/node/external/transactionAPI/apiTransactionProcessor.go b/node/external/transactionAPI/apiTransactionProcessor.go index c67ad1cb445..a12d5efb974 100644 --- a/node/external/transactionAPI/apiTransactionProcessor.go +++ b/node/external/transactionAPI/apiTransactionProcessor.go @@ -401,8 +401,11 @@ func (atp *apiTransactionProcessor) getFieldGettersForTx(wrappedTx *txcache.Wrap guardedTx, isGuardedTx := wrappedTx.Tx.(data.GuardedTransactionHandler) if isGuardedTx { fieldGetters[signatureField] = hex.EncodeToString(guardedTx.GetSignature()) - fieldGetters[guardianField] = atp.addressPubKeyConverter.SilentEncode(guardedTx.GetGuardianAddr(), log) - fieldGetters[guardianSignatureField] = hex.EncodeToString(guardedTx.GetGuardianSignature()) + + if len(guardedTx.GetGuardianAddr()) > 0 { + fieldGetters[guardianField] = atp.addressPubKeyConverter.SilentEncode(guardedTx.GetGuardianAddr(), log) + fieldGetters[guardianSignatureField] = hex.EncodeToString(guardedTx.GetGuardianSignature()) + } } return fieldGetters diff --git a/node/external/transactionAPI/apiTransactionProcessor_test.go b/node/external/transactionAPI/apiTransactionProcessor_test.go index e6a7040fe87..40357484880 100644 --- a/node/external/transactionAPI/apiTransactionProcessor_test.go +++ b/node/external/transactionAPI/apiTransactionProcessor_test.go @@ -876,34 +876,37 @@ func TestApiTransactionProcessor_GetTransactionsPoolForSender(t *testing.T) { txHash0, txHash1, txHash2 := []byte("txHash0"), []byte("txHash1"), []byte("txHash2") sender := "alice" - txCacheIntraShard, _ := txcache.NewTxCache(txcache.ConfigSourceMe{ - Name: "test", - NumChunks: 4, - NumBytesPerSenderThreshold: 1_048_576, // 1 MB - CountPerSenderThreshold: math.MaxUint32, - }, &txcachemocks.TxGasHandlerMock{ - MinimumGasMove: 1, - MinimumGasPrice: 1, - GasProcessingDivisor: 1, - }) + txCacheIntraShard, err := txcache.NewTxCache(txcache.ConfigSourceMe{ + Name: "test", + NumChunks: 4, + NumBytesThreshold: 1_048_576, // 1 MB + NumBytesPerSenderThreshold: 1_048_576, // 1 MB + CountThreshold: math.MaxUint32, + CountPerSenderThreshold: math.MaxUint32, + NumItemsToPreemptivelyEvict: 1, + }, txcachemocks.NewMempoolHostMock()) + + require.NoError(t, err) + txCacheIntraShard.AddTx(createTx(txHash2, sender, 3)) txCacheIntraShard.AddTx(createTx(txHash0, sender, 1)) txCacheIntraShard.AddTx(createTx(txHash1, sender, 2)) txHash3, txHash4 := []byte("txHash3"), []byte("txHash4") - txCacheWithMeta, _ := txcache.NewTxCache(txcache.ConfigSourceMe{ - Name: "test-meta", - NumChunks: 4, - NumBytesPerSenderThreshold: 1_048_576, // 1 MB - CountPerSenderThreshold: math.MaxUint32, - }, &txcachemocks.TxGasHandlerMock{ - MinimumGasMove: 1, - MinimumGasPrice: 1, - GasProcessingDivisor: 1, - }) + txCacheWithMeta, err := txcache.NewTxCache(txcache.ConfigSourceMe{ + Name: "test-meta", + NumChunks: 4, + NumBytesThreshold: 1_048_576, // 1 MB + NumBytesPerSenderThreshold: 1_048_576, // 1 MB + CountThreshold: math.MaxUint32, + CountPerSenderThreshold: math.MaxUint32, + NumItemsToPreemptivelyEvict: 1, + }, txcachemocks.NewMempoolHostMock()) txCacheWithMeta.AddTx(createTx(txHash3, sender, 4)) txCacheWithMeta.AddTx(createTx(txHash4, sender, 5)) + require.NoError(t, err) + args := createMockArgAPITransactionProcessor() args.DataPool = &dataRetrieverMock.PoolsHolderStub{ TransactionsCalled: func() dataRetriever.ShardedDataCacherNotifier { @@ -964,15 +967,14 @@ func TestApiTransactionProcessor_GetLastPoolNonceForSender(t *testing.T) { sender := "alice" lastNonce := uint64(10) txCacheIntraShard, _ := txcache.NewTxCache(txcache.ConfigSourceMe{ - Name: "test", - NumChunks: 4, - NumBytesPerSenderThreshold: 1_048_576, // 1 MB - CountPerSenderThreshold: math.MaxUint32, - }, &txcachemocks.TxGasHandlerMock{ - MinimumGasMove: 1, - MinimumGasPrice: 1, - GasProcessingDivisor: 1, - }) + Name: "test", + NumChunks: 4, + NumBytesThreshold: 1_048_576, // 1 MB + NumBytesPerSenderThreshold: 1_048_576, // 1 MB + CountThreshold: math.MaxUint32, + CountPerSenderThreshold: math.MaxUint32, + NumItemsToPreemptivelyEvict: 1, + }, txcachemocks.NewMempoolHostMock()) txCacheIntraShard.AddTx(createTx(txHash2, sender, 3)) txCacheIntraShard.AddTx(createTx(txHash0, sender, 1)) txCacheIntraShard.AddTx(createTx(txHash1, sender, 2)) @@ -1016,27 +1018,29 @@ func TestApiTransactionProcessor_GetTransactionsPoolNonceGapsForSender(t *testin txHash1, txHash2, txHash3, txHash4 := []byte("txHash1"), []byte("txHash2"), []byte("txHash3"), []byte("txHash4") sender := "alice" - txCacheIntraShard, _ := txcache.NewTxCache(txcache.ConfigSourceMe{ - Name: "test", - NumChunks: 4, - NumBytesPerSenderThreshold: 1_048_576, // 1 MB - CountPerSenderThreshold: math.MaxUint32, - }, &txcachemocks.TxGasHandlerMock{ - MinimumGasMove: 1, - MinimumGasPrice: 1, - GasProcessingDivisor: 1, - }) + txCacheIntraShard, err := txcache.NewTxCache(txcache.ConfigSourceMe{ + Name: "test", + NumChunks: 4, + NumBytesThreshold: 1_048_576, // 1 MB + NumBytesPerSenderThreshold: 1_048_576, // 1 MB + CountThreshold: math.MaxUint32, + CountPerSenderThreshold: math.MaxUint32, + NumItemsToPreemptivelyEvict: 1, + }, txcachemocks.NewMempoolHostMock()) - txCacheWithMeta, _ := txcache.NewTxCache(txcache.ConfigSourceMe{ - Name: "test-meta", - NumChunks: 4, - NumBytesPerSenderThreshold: 1_048_576, // 1 MB - CountPerSenderThreshold: math.MaxUint32, - }, &txcachemocks.TxGasHandlerMock{ - MinimumGasMove: 1, - MinimumGasPrice: 1, - GasProcessingDivisor: 1, - }) + require.NoError(t, err) + + txCacheWithMeta, err := txcache.NewTxCache(txcache.ConfigSourceMe{ + Name: "test-meta", + NumChunks: 4, + NumBytesThreshold: 1_048_576, // 1 MB + NumBytesPerSenderThreshold: 1_048_576, // 1 MB + CountThreshold: math.MaxUint32, + CountPerSenderThreshold: math.MaxUint32, + NumItemsToPreemptivelyEvict: 1, + }, txcachemocks.NewMempoolHostMock()) + + require.NoError(t, err) accountNonce := uint64(20) // expected nonce gaps: 21-31, 33-33, 36-38 diff --git a/node/node.go b/node/node.go index a652e80be60..09ccebfa761 100644 --- a/node/node.go +++ b/node/node.go @@ -251,7 +251,7 @@ func (n *Node) GetAllIssuedESDTs(tokenType string, ctx context.Context) ([]strin continue } - if bytes.Equal(esdtToken.TokenType, []byte(tokenType)) { + if tokenTypeEquals(esdtToken.TokenType, tokenType) { tokens = append(tokens, tokenName) } } @@ -268,6 +268,19 @@ func (n *Node) GetAllIssuedESDTs(tokenType string, ctx context.Context) ([]strin return tokens, nil } +func tokenTypeEquals(tokenType []byte, providedTokenType string) bool { + if providedTokenType == core.NonFungibleESDTv2 || + providedTokenType == core.NonFungibleESDT { + return bytes.Equal(tokenType, []byte(core.NonFungibleESDTv2)) || bytes.Equal(tokenType, []byte(core.NonFungibleESDT)) || bytes.Equal(tokenType, []byte(core.DynamicNFTESDT)) + } + + if providedTokenType == core.SemiFungibleESDT { + return bytes.Equal(tokenType, []byte(core.SemiFungibleESDT)) || bytes.Equal(tokenType, []byte(core.DynamicSFTESDT)) + } + + return bytes.Equal(tokenType, []byte(providedTokenType)) +} + func (n *Node) getEsdtDataFromLeaf(leaf core.KeyValueHolder) (*systemSmartContracts.ESDTDataV2, bool) { esdtToken := &systemSmartContracts.ESDTDataV2{} diff --git a/node/nodeRunner.go b/node/nodeRunner.go index 54ffe84b4e3..1837c78b427 100644 --- a/node/nodeRunner.go +++ b/node/nodeRunner.go @@ -321,7 +321,12 @@ func (nr *nodeRunner) executeOneComponentCreationCycle( } log.Debug("creating bootstrap components") - managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents(managedStatusCoreComponents, managedCoreComponents, managedCryptoComponents, managedNetworkComponents) + managedBootstrapComponents, err := nr.CreateManagedBootstrapComponents( + managedStatusCoreComponents, + managedCoreComponents, + managedCryptoComponents, + managedNetworkComponents, + ) if err != nil { return true, err } @@ -329,7 +334,12 @@ func (nr *nodeRunner) executeOneComponentCreationCycle( nr.logInformation(managedCoreComponents, managedCryptoComponents, managedBootstrapComponents) log.Debug("creating data components") - managedDataComponents, err := nr.CreateManagedDataComponents(managedStatusCoreComponents, managedCoreComponents, managedBootstrapComponents, managedCryptoComponents) + managedDataComponents, err := nr.CreateManagedDataComponents( + managedStatusCoreComponents, + managedCoreComponents, + managedBootstrapComponents, + managedCryptoComponents, + ) if err != nil { return true, err } diff --git a/node/node_test.go b/node/node_test.go index 319890a5d0d..a39c1499e21 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -1159,7 +1159,10 @@ func TestNode_GetAllIssuedESDTs(t *testing.T) { acc := createAcc([]byte("newaddress")) esdtToken := []byte("TCK-RANDOM") sftToken := []byte("SFT-RANDOM") + sftTokenDynamic := []byte("SFT-Dynamic") nftToken := []byte("NFT-RANDOM") + nftTokenV2 := []byte("NFT-RANDOM-V2") + nftTokenDynamic := []byte("NFT-Dynamic") esdtData := &systemSmartContracts.ESDTDataV2{TokenName: []byte("fungible"), TokenType: []byte(core.FungibleESDT)} marshalledData, _ := getMarshalizer().Marshal(esdtData) @@ -1169,13 +1172,28 @@ func TestNode_GetAllIssuedESDTs(t *testing.T) { sftMarshalledData, _ := getMarshalizer().Marshal(sftData) _ = acc.SaveKeyValue(sftToken, sftMarshalledData) + sftDataDynamic := &systemSmartContracts.ESDTDataV2{TokenName: []byte("semi fungible dynamic"), TokenType: []byte(core.DynamicSFTESDT)} + sftMarshalledDataDynamic, _ := getMarshalizer().Marshal(sftDataDynamic) + _ = acc.SaveKeyValue(sftTokenDynamic, sftMarshalledDataDynamic) + nftData := &systemSmartContracts.ESDTDataV2{TokenName: []byte("non fungible"), TokenType: []byte(core.NonFungibleESDT)} nftMarshalledData, _ := getMarshalizer().Marshal(nftData) _ = acc.SaveKeyValue(nftToken, nftMarshalledData) + nftData2 := &systemSmartContracts.ESDTDataV2{TokenName: []byte("non fungible v2"), TokenType: []byte(core.NonFungibleESDT)} + nftMarshalledData2, _ := getMarshalizer().Marshal(nftData2) + _ = acc.SaveKeyValue(nftTokenV2, nftMarshalledData2) + + nftDataDynamic := &systemSmartContracts.ESDTDataV2{TokenName: []byte("non fungible dynamic"), TokenType: []byte(core.DynamicNFTESDT)} + nftMarshalledDataDyamic, _ := getMarshalizer().Marshal(nftDataDynamic) + _ = acc.SaveKeyValue(nftTokenDynamic, nftMarshalledDataDyamic) + esdtSuffix := append(esdtToken, acc.AddressBytes()...) nftSuffix := append(nftToken, acc.AddressBytes()...) + nftSuffix2 := append(nftTokenV2, acc.AddressBytes()...) + nftDynamicSuffix := append(nftTokenDynamic, acc.AddressBytes()...) sftSuffix := append(sftToken, acc.AddressBytes()...) + sftDynamicSuffix := append(sftTokenDynamic, acc.AddressBytes()...) acc.SetDataTrie( &trieMock.TrieStub{ @@ -1186,9 +1204,16 @@ func TestNode_GetAllIssuedESDTs(t *testing.T) { trieLeaf, _ = tlp.ParseLeaf(sftToken, append(sftMarshalledData, sftSuffix...), core.NotSpecified) leavesChannels.LeavesChan <- trieLeaf + trieLeaf, _ = tlp.ParseLeaf(sftTokenDynamic, append(sftMarshalledDataDynamic, sftDynamicSuffix...), core.NotSpecified) + leavesChannels.LeavesChan <- trieLeaf trieLeaf, _ = tlp.ParseLeaf(nftToken, append(nftMarshalledData, nftSuffix...), core.NotSpecified) leavesChannels.LeavesChan <- trieLeaf + trieLeaf, _ = tlp.ParseLeaf(nftTokenV2, append(nftMarshalledData2, nftSuffix2...), core.NotSpecified) + leavesChannels.LeavesChan <- trieLeaf + trieLeaf, _ = tlp.ParseLeaf(nftTokenDynamic, append(nftMarshalledDataDyamic, nftDynamicSuffix...), core.NotSpecified) + leavesChannels.LeavesChan <- trieLeaf + close(leavesChannels.LeavesChan) leavesChannels.ErrChan.Close() }() @@ -1236,17 +1261,27 @@ func TestNode_GetAllIssuedESDTs(t *testing.T) { value, err = n.GetAllIssuedESDTs(core.SemiFungibleESDT, context.Background()) assert.Nil(t, err) - assert.Equal(t, 1, len(value)) + assert.Equal(t, 2, len(value)) assert.Equal(t, string(sftToken), value[0]) + assert.Equal(t, string(sftTokenDynamic), value[1]) value, err = n.GetAllIssuedESDTs(core.NonFungibleESDT, context.Background()) assert.Nil(t, err) - assert.Equal(t, 1, len(value)) + assert.Equal(t, 3, len(value)) // for both versions + assert.Equal(t, string(nftToken), value[0]) + assert.Equal(t, string(nftTokenV2), value[1]) + assert.Equal(t, string(nftTokenDynamic), value[2]) + + value, err = n.GetAllIssuedESDTs(core.NonFungibleESDTv2, context.Background()) + assert.Nil(t, err) + assert.Equal(t, 3, len(value)) // for both versions assert.Equal(t, string(nftToken), value[0]) + assert.Equal(t, string(nftTokenV2), value[1]) + assert.Equal(t, string(nftTokenDynamic), value[2]) value, err = n.GetAllIssuedESDTs("", context.Background()) assert.Nil(t, err) - assert.Equal(t, 3, len(value)) + assert.Equal(t, 6, len(value)) } func TestNode_GetESDTsWithRole(t *testing.T) { diff --git a/process/block/preprocess/interfaces.go b/process/block/preprocess/interfaces.go index fbc155138ad..b98f2271308 100644 --- a/process/block/preprocess/interfaces.go +++ b/process/block/preprocess/interfaces.go @@ -2,21 +2,20 @@ package preprocess import ( "math/big" + "time" "github.com/multiversx/mx-chain-go/storage/txcache" ) // SortedTransactionsProvider defines the public API of the transactions cache type SortedTransactionsProvider interface { - GetSortedTransactions() []*txcache.WrappedTransaction - NotifyAccountNonce(accountKey []byte, nonce uint64) + GetSortedTransactions(session txcache.SelectionSession) []*txcache.WrappedTransaction IsInterfaceNil() bool } // TxCache defines the functionality for the transactions cache type TxCache interface { - SelectTransactionsWithBandwidth(numRequested int, batchSizePerSender int, bandwidthPerSender uint64) []*txcache.WrappedTransaction - NotifyAccountNonce(accountKey []byte, nonce uint64) + SelectTransactions(session txcache.SelectionSession, gasRequested uint64, maxNum int, selectionLoopMaximumDuration time.Duration) ([]*txcache.WrappedTransaction, uint64) IsInterfaceNil() bool } diff --git a/process/block/preprocess/miniBlockBuilder.go b/process/block/preprocess/miniBlockBuilder.go index a1a2e2bc82e..d10e6ba6ee5 100644 --- a/process/block/preprocess/miniBlockBuilder.go +++ b/process/block/preprocess/miniBlockBuilder.go @@ -22,7 +22,6 @@ import ( type miniBlocksBuilderArgs struct { gasTracker gasTracker accounts state.AccountsAdapter - accountTxsShards *accountTxsShards blockSizeComputation BlockSizeComputationHandler balanceComputationHandler BalanceComputationHandler haveTime func() bool @@ -50,7 +49,6 @@ type miniBlockBuilderStats struct { type miniBlocksBuilder struct { gasTracker accounts state.AccountsAdapter - accountTxsShards *accountTxsShards balanceComputationHandler BalanceComputationHandler blockSizeComputation BlockSizeComputationHandler gasConsumedInReceiverShard map[uint32]uint64 @@ -75,7 +73,6 @@ func newMiniBlockBuilder(args miniBlocksBuilderArgs) (*miniBlocksBuilder, error) return &miniBlocksBuilder{ gasTracker: args.gasTracker, accounts: args.accounts, - accountTxsShards: args.accountTxsShards, balanceComputationHandler: args.balanceComputationHandler, blockSizeComputation: args.blockSizeComputation, miniBlocks: initializeMiniBlocksMap(args.gasTracker.shardCoordinator), @@ -117,9 +114,6 @@ func checkMiniBlocksBuilderArgs(args miniBlocksBuilderArgs) error { if check.IfNil(args.txPool) { return process.ErrNilTransactionPool } - if args.accountTxsShards == nil { - return process.ErrNilAccountTxsPerShard - } if args.haveTime == nil { return process.ErrNilHaveTimeHandler } @@ -136,15 +130,6 @@ func checkMiniBlocksBuilderArgs(args miniBlocksBuilderArgs) error { return nil } -func (mbb *miniBlocksBuilder) updateAccountShardsInfo(tx *transaction.Transaction, wtx *txcache.WrappedTransaction) { - mbb.accountTxsShards.Lock() - mbb.accountTxsShards.accountsInfo[string(tx.GetSndAddr())] = &txShardInfo{ - senderShardID: wtx.SenderShardID, - receiverShardID: wtx.ReceiverShardID, - } - mbb.accountTxsShards.Unlock() -} - // checkAddTransaction method returns a set of actions which could be done afterwards, by checking the given transaction func (mbb *miniBlocksBuilder) checkAddTransaction(wtx *txcache.WrappedTransaction) (*processingActions, *transaction.Transaction) { tx, ok := wtx.Tx.(*transaction.Transaction) diff --git a/process/block/preprocess/miniBlockBuilder_test.go b/process/block/preprocess/miniBlockBuilder_test.go index d3a04147864..3ba49aa00a3 100644 --- a/process/block/preprocess/miniBlockBuilder_test.go +++ b/process/block/preprocess/miniBlockBuilder_test.go @@ -4,7 +4,6 @@ import ( "encoding/hex" "errors" "math/big" - "sync" "testing" "github.com/multiversx/mx-chain-core-go/data" @@ -82,16 +81,6 @@ func Test_checkMiniBlocksBuilderArgsNilBlockSizeComputationHandlerShouldErr(t *t require.Equal(t, process.ErrNilBlockSizeComputationHandler, err) } -func Test_checkMiniBlocksBuilderArgsNilAccountsTxsPerShardsShouldErr(t *testing.T) { - t.Parallel() - - args := createDefaultMiniBlockBuilderArgs() - args.accountTxsShards = nil - - err := checkMiniBlocksBuilderArgs(args) - require.Equal(t, process.ErrNilAccountTxsPerShard, err) -} - func Test_checkMiniBlocksBuilderArgsNilBalanceComputationHandlerShouldErr(t *testing.T) { t.Parallel() @@ -151,29 +140,6 @@ func Test_checkMiniBlocksBuilderArgsOK(t *testing.T) { require.Nil(t, err) } -func Test_MiniBlocksBuilderUpdateAccountShardsInfo(t *testing.T) { - t.Parallel() - - args := createDefaultMiniBlockBuilderArgs() - - mbb, _ := newMiniBlockBuilder(args) - senderAddr := []byte("senderAddr") - receiverAddr := []byte("receiverAddr") - tx := createDefaultTx(senderAddr, receiverAddr, 50000) - - senderShardID := uint32(0) - receiverShardID := uint32(0) - wtx := createWrappedTransaction(tx, senderShardID, receiverShardID) - - mbb.updateAccountShardsInfo(tx, wtx) - - addrShardInfo, ok := mbb.accountTxsShards.accountsInfo[string(tx.SndAddr)] - require.True(t, ok) - - require.Equal(t, senderShardID, addrShardInfo.senderShardID) - require.Equal(t, receiverShardID, addrShardInfo.receiverShardID) -} - func Test_MiniBlocksBuilderHandleGasRefundIntraShard(t *testing.T) { t.Parallel() @@ -659,12 +625,11 @@ func Test_MiniBlocksBuilderCheckAddTransactionWrongTypeAssertion(t *testing.T) { t.Parallel() wtx := &txcache.WrappedTransaction{ - Tx: nil, - TxHash: nil, - SenderShardID: 0, - ReceiverShardID: 0, - Size: 0, - TxFeeScoreNormalized: 0, + Tx: nil, + TxHash: nil, + SenderShardID: 0, + ReceiverShardID: 0, + Size: 0, } args := createDefaultMiniBlockBuilderArgs() @@ -882,11 +847,7 @@ func createDefaultMiniBlockBuilderArgs() miniBlocksBuilderArgs { }, }, }, - accounts: &stateMock.AccountsStub{}, - accountTxsShards: &accountTxsShards{ - accountsInfo: make(map[string]*txShardInfo), - RWMutex: sync.RWMutex{}, - }, + accounts: &stateMock.AccountsStub{}, blockSizeComputation: &testscommon.BlockSizeComputationStub{}, balanceComputationHandler: &testscommon.BalanceComputationStub{}, haveTime: haveTimeTrue, @@ -911,12 +872,14 @@ func createWrappedTransaction( txMarshalled, _ := marshaller.Marshal(tx) txHash := hasher.Compute(string(txMarshalled)) - return &txcache.WrappedTransaction{ - Tx: tx, - TxHash: txHash, - SenderShardID: senderShardID, - ReceiverShardID: receiverShardID, - Size: int64(len(txMarshalled)), - TxFeeScoreNormalized: 10, + wrappedTx := &txcache.WrappedTransaction{ + Tx: tx, + TxHash: txHash, + SenderShardID: senderShardID, + ReceiverShardID: receiverShardID, + Size: int64(len(txMarshalled)), } + + wrappedTx.PricePerUnit = 1_000_000_000 + return wrappedTx } diff --git a/process/block/preprocess/selectionSession.go b/process/block/preprocess/selectionSession.go new file mode 100644 index 00000000000..c5c16a56cde --- /dev/null +++ b/process/block/preprocess/selectionSession.go @@ -0,0 +1,105 @@ +package preprocess + +import ( + "errors" + + "github.com/multiversx/mx-chain-core-go/core/check" + "github.com/multiversx/mx-chain-core-go/data" + "github.com/multiversx/mx-chain-core-go/data/transaction" + "github.com/multiversx/mx-chain-go/process" + "github.com/multiversx/mx-chain-go/state" + "github.com/multiversx/mx-chain-go/storage/txcache" + vmcommon "github.com/multiversx/mx-chain-vm-common-go" +) + +type selectionSession struct { + accountsAdapter state.AccountsAdapter + transactionsProcessor process.TransactionProcessor + + // Cache of accounts, held in the scope of a single selection session. + // Not concurrency-safe, but never accessed concurrently. + ephemeralAccountsCache map[string]vmcommon.AccountHandler +} + +type argsSelectionSession struct { + accountsAdapter state.AccountsAdapter + transactionsProcessor process.TransactionProcessor +} + +func newSelectionSession(args argsSelectionSession) (*selectionSession, error) { + if check.IfNil(args.accountsAdapter) { + return nil, process.ErrNilAccountsAdapter + } + if check.IfNil(args.transactionsProcessor) { + return nil, process.ErrNilTxProcessor + } + + return &selectionSession{ + accountsAdapter: args.accountsAdapter, + transactionsProcessor: args.transactionsProcessor, + ephemeralAccountsCache: make(map[string]vmcommon.AccountHandler), + }, nil +} + +// GetAccountState returns the state of an account. +// Will be called by mempool during transaction selection. +func (session *selectionSession) GetAccountState(address []byte) (*txcache.AccountState, error) { + account, err := session.getExistingAccount(address) + if err != nil { + return nil, err + } + + userAccount, ok := account.(state.UserAccountHandler) + if !ok { + return nil, process.ErrWrongTypeAssertion + } + + return &txcache.AccountState{ + Nonce: userAccount.GetNonce(), + Balance: userAccount.GetBalance(), + }, nil +} + +func (session *selectionSession) getExistingAccount(address []byte) (vmcommon.AccountHandler, error) { + account, ok := session.ephemeralAccountsCache[string(address)] + if ok { + return account, nil + } + + account, err := session.accountsAdapter.GetExistingAccount(address) + if err != nil { + return nil, err + } + + session.ephemeralAccountsCache[string(address)] = account + return account, nil +} + +// IsIncorrectlyGuarded checks if a transaction is incorrectly guarded (not executable). +// Will be called by mempool during transaction selection. +func (session *selectionSession) IsIncorrectlyGuarded(tx data.TransactionHandler) bool { + address := tx.GetSndAddr() + account, err := session.getExistingAccount(address) + if err != nil { + return false + } + + userAccount, ok := account.(state.UserAccountHandler) + if !ok { + // On this branch, we are (approximately) mirroring the behavior of "transactionsProcessor.VerifyGuardian()". + return true + } + + txTyped, ok := tx.(*transaction.Transaction) + if !ok { + return false + } + + err = session.transactionsProcessor.VerifyGuardian(txTyped, userAccount) + return errors.Is(err, process.ErrTransactionNotExecutable) +} + +// IsInterfaceNil returns true if there is no value under the interface +func (session *selectionSession) IsInterfaceNil() bool { + return session == nil +} diff --git a/process/block/preprocess/selectionSession_test.go b/process/block/preprocess/selectionSession_test.go new file mode 100644 index 00000000000..fc46d5ced15 --- /dev/null +++ b/process/block/preprocess/selectionSession_test.go @@ -0,0 +1,171 @@ +package preprocess + +import ( + "bytes" + "fmt" + "testing" + + "github.com/multiversx/mx-chain-core-go/data/transaction" + "github.com/multiversx/mx-chain-go/process" + "github.com/multiversx/mx-chain-go/state" + "github.com/multiversx/mx-chain-go/testscommon" + stateMock "github.com/multiversx/mx-chain-go/testscommon/state" + vmcommon "github.com/multiversx/mx-chain-vm-common-go" + "github.com/stretchr/testify/require" +) + +func TestNewSelectionSession(t *testing.T) { + t.Parallel() + + session, err := newSelectionSession(argsSelectionSession{ + accountsAdapter: nil, + transactionsProcessor: &testscommon.TxProcessorStub{}, + }) + require.Nil(t, session) + require.ErrorIs(t, err, process.ErrNilAccountsAdapter) + + session, err = newSelectionSession(argsSelectionSession{ + accountsAdapter: &stateMock.AccountsStub{}, + transactionsProcessor: nil, + }) + require.Nil(t, session) + require.ErrorIs(t, err, process.ErrNilTxProcessor) + + session, err = newSelectionSession(argsSelectionSession{ + accountsAdapter: &stateMock.AccountsStub{}, + transactionsProcessor: &testscommon.TxProcessorStub{}, + }) + require.NoError(t, err) + require.NotNil(t, session) +} + +func TestSelectionSession_GetAccountState(t *testing.T) { + t.Parallel() + + accounts := &stateMock.AccountsStub{} + processor := &testscommon.TxProcessorStub{} + + accounts.GetExistingAccountCalled = func(address []byte) (vmcommon.AccountHandler, error) { + if bytes.Equal(address, []byte("alice")) { + return &stateMock.UserAccountStub{ + Address: []byte("alice"), + Nonce: 42, + }, nil + } + + if bytes.Equal(address, []byte("bob")) { + return &stateMock.UserAccountStub{ + Address: []byte("bob"), + Nonce: 7, + IsGuardedCalled: func() bool { + return true + }, + }, nil + } + + return nil, fmt.Errorf("account not found: %s", address) + } + + session, err := newSelectionSession(argsSelectionSession{ + accountsAdapter: accounts, + transactionsProcessor: processor, + }) + require.NoError(t, err) + require.NotNil(t, session) + + state, err := session.GetAccountState([]byte("alice")) + require.NoError(t, err) + require.Equal(t, uint64(42), state.Nonce) + + state, err = session.GetAccountState([]byte("bob")) + require.NoError(t, err) + require.Equal(t, uint64(7), state.Nonce) + + state, err = session.GetAccountState([]byte("carol")) + require.ErrorContains(t, err, "account not found: carol") + require.Nil(t, state) +} + +func TestSelectionSession_IsIncorrectlyGuarded(t *testing.T) { + t.Parallel() + + accounts := &stateMock.AccountsStub{} + processor := &testscommon.TxProcessorStub{} + + accounts.GetExistingAccountCalled = func(address []byte) (vmcommon.AccountHandler, error) { + if bytes.Equal(address, []byte("bob")) { + return &stateMock.BaseAccountMock{}, nil + } + + return &stateMock.UserAccountStub{}, nil + } + + processor.VerifyGuardianCalled = func(tx *transaction.Transaction, account state.UserAccountHandler) error { + if tx.Nonce == 43 { + return process.ErrTransactionNotExecutable + } + if tx.Nonce == 44 { + return fmt.Errorf("arbitrary processing error") + } + + return nil + } + + session, err := newSelectionSession(argsSelectionSession{ + accountsAdapter: accounts, + transactionsProcessor: processor, + }) + require.NoError(t, err) + require.NotNil(t, session) + + isIncorrectlyGuarded := session.IsIncorrectlyGuarded(&transaction.Transaction{Nonce: 42, SndAddr: []byte("alice")}) + require.False(t, isIncorrectlyGuarded) + + isIncorrectlyGuarded = session.IsIncorrectlyGuarded(&transaction.Transaction{Nonce: 43, SndAddr: []byte("alice")}) + require.True(t, isIncorrectlyGuarded) + + isIncorrectlyGuarded = session.IsIncorrectlyGuarded(&transaction.Transaction{Nonce: 44, SndAddr: []byte("alice")}) + require.False(t, isIncorrectlyGuarded) + + isIncorrectlyGuarded = session.IsIncorrectlyGuarded(&transaction.Transaction{Nonce: 45, SndAddr: []byte("bob")}) + require.True(t, isIncorrectlyGuarded) +} + +func TestSelectionSession_ephemeralAccountsCache_IsSharedAmongCalls(t *testing.T) { + t.Parallel() + + accounts := &stateMock.AccountsStub{} + processor := &testscommon.TxProcessorStub{} + + numCallsGetExistingAccount := 0 + + accounts.GetExistingAccountCalled = func(_ []byte) (vmcommon.AccountHandler, error) { + numCallsGetExistingAccount++ + return &stateMock.UserAccountStub{}, nil + } + + session, err := newSelectionSession(argsSelectionSession{ + accountsAdapter: accounts, + transactionsProcessor: processor, + }) + require.NoError(t, err) + require.NotNil(t, session) + + _, _ = session.GetAccountState([]byte("alice")) + require.Equal(t, 1, numCallsGetExistingAccount) + + _, _ = session.GetAccountState([]byte("alice")) + require.Equal(t, 1, numCallsGetExistingAccount) + + _ = session.IsIncorrectlyGuarded(&transaction.Transaction{Nonce: 42, SndAddr: []byte("alice")}) + require.Equal(t, 1, numCallsGetExistingAccount) + + _, _ = session.GetAccountState([]byte("bob")) + require.Equal(t, 2, numCallsGetExistingAccount) + + _, _ = session.GetAccountState([]byte("bob")) + require.Equal(t, 2, numCallsGetExistingAccount) + + _ = session.IsIncorrectlyGuarded(&transaction.Transaction{Nonce: 42, SndAddr: []byte("bob")}) + require.Equal(t, 2, numCallsGetExistingAccount) +} diff --git a/process/block/preprocess/sortedTransactionsProvider.go b/process/block/preprocess/sortedTransactionsProvider.go index 8c2613b7aa7..f30cb912892 100644 --- a/process/block/preprocess/sortedTransactionsProvider.go +++ b/process/block/preprocess/sortedTransactionsProvider.go @@ -32,16 +32,11 @@ func newAdapterTxCacheToSortedTransactionsProvider(txCache TxCache) *adapterTxCa } // GetSortedTransactions gets the transactions from the cache -func (adapter *adapterTxCacheToSortedTransactionsProvider) GetSortedTransactions() []*txcache.WrappedTransaction { - txs := adapter.txCache.SelectTransactionsWithBandwidth(process.MaxNumOfTxsToSelect, process.NumTxPerSenderBatchForFillingMiniblock, process.MaxGasBandwidthPerBatchPerSender) +func (adapter *adapterTxCacheToSortedTransactionsProvider) GetSortedTransactions(session txcache.SelectionSession) []*txcache.WrappedTransaction { + txs, _ := adapter.txCache.SelectTransactions(session, process.TxCacheSelectionGasRequested, process.TxCacheSelectionMaxNumTxs, process.TxCacheSelectionLoopMaximumDuration) return txs } -// NotifyAccountNonce notifies the cache about the current nonce of an account -func (adapter *adapterTxCacheToSortedTransactionsProvider) NotifyAccountNonce(accountKey []byte, nonce uint64) { - adapter.txCache.NotifyAccountNonce(accountKey, nonce) -} - // IsInterfaceNil returns true if there is no value under the interface func (adapter *adapterTxCacheToSortedTransactionsProvider) IsInterfaceNil() bool { return adapter == nil @@ -52,14 +47,10 @@ type disabledSortedTransactionsProvider struct { } // GetSortedTransactions returns an empty slice -func (adapter *disabledSortedTransactionsProvider) GetSortedTransactions() []*txcache.WrappedTransaction { +func (adapter *disabledSortedTransactionsProvider) GetSortedTransactions(_ txcache.SelectionSession) []*txcache.WrappedTransaction { return make([]*txcache.WrappedTransaction, 0) } -// NotifyAccountNonce does nothing -func (adapter *disabledSortedTransactionsProvider) NotifyAccountNonce(_ []byte, _ uint64) { -} - // IsInterfaceNil returns true if there is no value under the interface func (adapter *disabledSortedTransactionsProvider) IsInterfaceNil() bool { return adapter == nil diff --git a/process/block/preprocess/transactions.go b/process/block/preprocess/transactions.go index eb24585a55b..8578a97e92d 100644 --- a/process/block/preprocess/transactions.go +++ b/process/block/preprocess/transactions.go @@ -16,7 +16,6 @@ import ( "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" logger "github.com/multiversx/mx-chain-logger-go" - vmcommon "github.com/multiversx/mx-chain-vm-common-go" "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/dataRetriever" @@ -34,15 +33,10 @@ var _ process.PreProcessor = (*transactions)(nil) var log = logger.GetOrCreate("process/block/preprocess") // 200% bandwidth to allow 100% overshooting estimations -const selectionGasBandwidthIncreasePercent = 200 +const selectionGasBandwidthIncreasePercent = 400 // 130% to allow 30% overshooting estimations for scheduled SC calls -const selectionGasBandwidthIncreaseScheduledPercent = 130 - -type accountTxsShards struct { - accountsInfo map[string]*txShardInfo - sync.RWMutex -} +const selectionGasBandwidthIncreaseScheduledPercent = 260 // TODO: increase code coverage with unit test @@ -59,7 +53,6 @@ type transactions struct { mutOrderedTxs sync.RWMutex blockTracker BlockTracker blockType block.Type - accountTxsShards accountTxsShards emptyAddress []byte txTypeHandler process.TxTypeHandler scheduledTxsExecutionHandler process.ScheduledTxsExecutionHandler @@ -196,7 +189,6 @@ func NewTransactionPreprocessor( txs.txsForCurrBlock.txHashAndInfo = make(map[string]*txInfo) txs.orderedTxs = make(map[string][]data.TransactionHandler) txs.orderedTxHashes = make(map[string][][]byte) - txs.accountTxsShards.accountsInfo = make(map[string]*txShardInfo) txs.emptyAddress = make([]byte, txs.pubkeyConverter.Len()) @@ -719,7 +711,6 @@ func (txs *transactions) createAndProcessScheduledMiniBlocksFromMeAsValidator( mapSCTxs map[string]struct{}, randomness []byte, ) (block.MiniBlockSlice, error) { - if !txs.enableEpochsHandler.IsFlagEnabled(common.ScheduledMiniBlocksFlag) { return make(block.MiniBlockSlice, 0), nil } @@ -798,10 +789,6 @@ func (txs *transactions) CreateBlockStarted() { txs.orderedTxHashes = make(map[string][][]byte) txs.mutOrderedTxs.Unlock() - txs.accountTxsShards.Lock() - txs.accountTxsShards.accountsInfo = make(map[string]*txShardInfo) - txs.accountTxsShards.Unlock() - txs.scheduledTxsExecutionHandler.Init() } @@ -917,41 +904,6 @@ func (txs *transactions) processAndRemoveBadTransaction( return err } -func (txs *transactions) notifyTransactionProviderIfNeeded() { - txs.accountTxsShards.RLock() - for senderAddress, txShardInfoValue := range txs.accountTxsShards.accountsInfo { - if txShardInfoValue.senderShardID != txs.shardCoordinator.SelfId() { - continue - } - - account, err := txs.getAccountForAddress([]byte(senderAddress)) - if err != nil { - log.Debug("notifyTransactionProviderIfNeeded.getAccountForAddress", "error", err) - continue - } - - strCache := process.ShardCacherIdentifier(txShardInfoValue.senderShardID, txShardInfoValue.receiverShardID) - txShardPool := txs.txPool.ShardDataStore(strCache) - if check.IfNil(txShardPool) { - log.Trace("notifyTransactionProviderIfNeeded", "error", process.ErrNilTxDataPool) - continue - } - - sortedTransactionsProvider := createSortedTransactionsProvider(txShardPool) - sortedTransactionsProvider.NotifyAccountNonce([]byte(senderAddress), account.GetNonce()) - } - txs.accountTxsShards.RUnlock() -} - -func (txs *transactions) getAccountForAddress(address []byte) (vmcommon.AccountHandler, error) { - account, err := txs.accounts.GetExistingAccount(address) - if err != nil { - return nil, err - } - - return account, nil -} - // RequestTransactionsForMiniBlock requests missing transactions for a certain miniblock func (txs *transactions) RequestTransactionsForMiniBlock(miniBlock *block.MiniBlock) int { if miniBlock == nil { @@ -1183,7 +1135,6 @@ func (txs *transactions) createAndProcessMiniBlocksFromMeV1( args := miniBlocksBuilderArgs{ gasTracker: txs.gasTracker, accounts: txs.accounts, - accountTxsShards: &txs.accountTxsShards, balanceComputationHandler: txs.balanceComputation, blockSizeComputation: txs.blockSizeComputation, haveTime: haveTime, @@ -1199,10 +1150,6 @@ func (txs *transactions) createAndProcessMiniBlocksFromMeV1( return nil, nil, err } - defer func() { - go txs.notifyTransactionProviderIfNeeded() - }() - remainingTxs := make([]*txcache.WrappedTransaction, 0) for idx, wtx := range sortedTxs { actions, tx := mbBuilder.checkAddTransaction(wtx) @@ -1286,7 +1233,6 @@ func (txs *transactions) processMiniBlockBuilderTx( ) elapsedTime := time.Since(startTime) mb.stats.totalProcessingTime += elapsedTime - mb.updateAccountShardsInfo(tx, wtx) if err != nil && !errors.Is(err, process.ErrFailedTransaction) { txs.handleBadTransaction(err, wtx, tx, mb, snapshot) @@ -1464,7 +1410,16 @@ func (txs *transactions) computeSortedTxs( sortedTransactionsProvider := createSortedTransactionsProvider(txShardPool) log.Debug("computeSortedTxs.GetSortedTransactions") - sortedTxs := sortedTransactionsProvider.GetSortedTransactions() + + session, err := newSelectionSession(argsSelectionSession{ + accountsAdapter: txs.accounts, + transactionsProcessor: txs.txProcessor, + }) + if err != nil { + return nil, nil, err + } + + sortedTxs := sortedTransactionsProvider.GetSortedTransactions(session) // TODO: this could be moved to SortedTransactionsProvider selectedTxs, remainingTxs := txs.preFilterTransactionsWithMoveBalancePriority(sortedTxs, gasBandwidth) diff --git a/process/block/preprocess/transactionsV2.go b/process/block/preprocess/transactionsV2.go index 6391987983a..1eb46683894 100644 --- a/process/block/preprocess/transactionsV2.go +++ b/process/block/preprocess/transactionsV2.go @@ -25,10 +25,6 @@ func (txs *transactions) createAndProcessMiniBlocksFromMeV2( log.Debug("createAndProcessMiniBlocksFromMeV2", "totalGasConsumedInSelfShard", mbInfo.gasInfo.totalGasConsumedInSelfShard) - defer func() { - go txs.notifyTransactionProviderIfNeeded() - }() - remainingTxs := make([]*txcache.WrappedTransaction, 0) for index := range sortedTxs { if !haveTime() { @@ -175,10 +171,6 @@ func (txs *transactions) processTransaction( elapsedTime = time.Since(startTime) mbInfo.processingInfo.totalTimeUsedForProcess += elapsedTime - txs.accountTxsShards.Lock() - txs.accountTxsShards.accountsInfo[string(tx.GetSndAddr())] = &txShardInfo{senderShardID: senderShardID, receiverShardID: receiverShardID} - txs.accountTxsShards.Unlock() - if err != nil && !errors.Is(err, process.ErrFailedTransaction) { if errors.Is(err, process.ErrHigherNonceInTransaction) { mbInfo.senderAddressToSkip = tx.GetSndAddr() @@ -379,10 +371,6 @@ func (txs *transactions) verifyTransaction( elapsedTime = time.Since(startTime) mbInfo.schedulingInfo.totalTimeUsedForScheduledVerify += elapsedTime - txs.accountTxsShards.Lock() - txs.accountTxsShards.accountsInfo[string(tx.GetSndAddr())] = &txShardInfo{senderShardID: senderShardID, receiverShardID: receiverShardID} - txs.accountTxsShards.Unlock() - if err != nil { isTxTargetedForDeletion := errors.Is(err, process.ErrLowerNonceInTransaction) || errors.Is(err, process.ErrInsufficientFee) || errors.Is(err, process.ErrTransactionNotExecutable) if isTxTargetedForDeletion { diff --git a/process/block/preprocess/transactions_test.go b/process/block/preprocess/transactions_test.go index 67a5b312994..cf1bcca2ec8 100644 --- a/process/block/preprocess/transactions_test.go +++ b/process/block/preprocess/transactions_test.go @@ -446,6 +446,16 @@ func TestTxsPreprocessor_NewTransactionPreprocessorNilProcessedMiniBlocksTracker assert.Equal(t, process.ErrNilProcessedMiniBlocksTracker, err) } +func TestTxsPreprocessor_NewTransactionPreprocessorNilTxExecutionOrderHandler(t *testing.T) { + t.Parallel() + + args := createDefaultTransactionsProcessorArgs() + args.TxExecutionOrderHandler = nil + txs, err := NewTransactionPreprocessor(args) + assert.Nil(t, txs) + assert.Equal(t, process.ErrNilTxExecutionOrderHandler, err) +} + func TestTxsPreprocessor_NewTransactionPreprocessorOkValsShouldWork(t *testing.T) { t.Parallel() @@ -660,6 +670,14 @@ func TestTransactions_CreateAndProcessMiniBlockCrossShardGasLimitAddAll(t *testi return 0 }, } + args.Accounts = &stateMock.AccountsStub{ + GetExistingAccountCalled: func(_ []byte) (vmcommon.AccountHandler, error) { + return &stateMock.UserAccountStub{ + Nonce: 42, + Balance: big.NewInt(1000000000000000000), + }, nil + }, + } txs, _ := NewTransactionPreprocessor(args) assert.NotNil(t, txs) @@ -670,7 +688,7 @@ func TestTransactions_CreateAndProcessMiniBlockCrossShardGasLimitAddAll(t *testi addedTxs := make([]*transaction.Transaction, 0) for i := 0; i < 10; i++ { - newTx := &transaction.Transaction{GasLimit: uint64(i)} + newTx := &transaction.Transaction{GasLimit: uint64(i), Nonce: 42 + uint64(i)} txHash, _ := core.CalculateHash(args.Marshalizer, args.Hasher, newTx) args.TxDataPool.AddData(txHash, newTx, newTx.Size(), strCache) @@ -714,6 +732,14 @@ func TestTransactions_CreateAndProcessMiniBlockCrossShardGasLimitAddAllAsNoSCCal return 0 }, } + args.Accounts = &stateMock.AccountsStub{ + GetExistingAccountCalled: func(_ []byte) (vmcommon.AccountHandler, error) { + return &stateMock.UserAccountStub{ + Nonce: 42, + Balance: big.NewInt(1000000000000000000), + }, nil + }, + } args.TxDataPool, _ = dataRetrieverMock.CreateTxPool(2, 0) txs, _ := NewTransactionPreprocessor(args) assert.NotNil(t, txs) @@ -726,7 +752,7 @@ func TestTransactions_CreateAndProcessMiniBlockCrossShardGasLimitAddAllAsNoSCCal addedTxs := make([]*transaction.Transaction, 0) for i := 0; i < 10; i++ { - newTx := &transaction.Transaction{GasLimit: gasLimit, GasPrice: uint64(i), RcvAddr: []byte("012345678910")} + newTx := &transaction.Transaction{GasLimit: gasLimit, GasPrice: uint64(i), RcvAddr: []byte("012345678910"), Nonce: 42 + uint64(i)} txHash, _ := core.CalculateHash(args.Marshalizer, args.Hasher, newTx) args.TxDataPool.AddData(txHash, newTx, newTx.Size(), strCache) @@ -779,6 +805,14 @@ func TestTransactions_CreateAndProcessMiniBlockCrossShardGasLimitAddOnly5asSCCal RemoveGasRefundedCalled: func(hashes [][]byte) { }, } + args.Accounts = &stateMock.AccountsStub{ + GetExistingAccountCalled: func(_ []byte) (vmcommon.AccountHandler, error) { + return &stateMock.UserAccountStub{ + Nonce: 42, + Balance: big.NewInt(1000000000000000000), + }, nil + }, + } txs, _ := NewTransactionPreprocessor(args) @@ -790,7 +824,7 @@ func TestTransactions_CreateAndProcessMiniBlockCrossShardGasLimitAddOnly5asSCCal scAddress, _ := hex.DecodeString("000000000000000000005fed9c659422cd8429ce92f8973bba2a9fb51e0eb3a1") for i := 0; i < 10; i++ { - newTx := &transaction.Transaction{GasLimit: gasLimit, GasPrice: uint64(i), RcvAddr: scAddress} + newTx := &transaction.Transaction{GasLimit: gasLimit, GasPrice: uint64(i), RcvAddr: scAddress, Nonce: 42 + uint64(i)} txHash, _ := core.CalculateHash(args.Marshalizer, args.Hasher, newTx) args.TxDataPool.AddData(txHash, newTx, newTx.Size(), strCache) diff --git a/process/block/shardblock_test.go b/process/block/shardblock_test.go index 39797f8db0c..d029b44e65b 100644 --- a/process/block/shardblock_test.go +++ b/process/block/shardblock_test.go @@ -3053,6 +3053,12 @@ func TestShardProcessor_CreateMiniBlocksShouldWorkWithIntraShardTxs(t *testing.T JournalLenCalled: func() int { return 0 }, + GetExistingAccountCalled: func(_ []byte) (vmcommon.AccountHandler, error) { + return &stateMock.UserAccountStub{ + Nonce: 45, + Balance: big.NewInt(1000000000000000000), + }, nil + }, } totalGasProvided := uint64(0) diff --git a/process/constants.go b/process/constants.go index f75e7b882ee..997e0a2a458 100644 --- a/process/constants.go +++ b/process/constants.go @@ -2,6 +2,7 @@ package process import ( "fmt" + "time" ) // BlockHeaderState specifies which is the state of the block header received @@ -79,11 +80,6 @@ const EpochChangeGracePeriod = 1 // in one round, when node processes a received block const MaxHeaderRequestsAllowed = 20 -// NumTxPerSenderBatchForFillingMiniblock defines the number of transactions to be drawn -// from the transactions pool, for a specific sender, in a single pass. -// Drawing transactions for a miniblock happens in multiple passes, until "MaxItemsInBlock" are drawn. -const NumTxPerSenderBatchForFillingMiniblock = 10 - // NonceDifferenceWhenSynced defines the difference between probable highest nonce seen from network and node's last // committed block nonce, after which, node is considered himself not synced const NonceDifferenceWhenSynced = 0 @@ -135,12 +131,6 @@ const MaxShardHeadersAllowedInOneMetaBlock = 60 // which would be included in one meta block if they are available const MinShardHeadersFromSameShardInOneMetaBlock = 10 -// MaxNumOfTxsToSelect defines the maximum number of transactions that should be selected from the cache -const MaxNumOfTxsToSelect = 30000 - -// MaxGasBandwidthPerBatchPerSender defines the maximum gas bandwidth that should be selected for a sender per batch from the cache -const MaxGasBandwidthPerBatchPerSender = 5000000 - // MaxHeadersToWhitelistInAdvance defines the maximum number of headers whose miniblocks will be whitelisted in advance const MaxHeadersToWhitelistInAdvance = 300 @@ -148,3 +138,12 @@ const MaxHeadersToWhitelistInAdvance = 300 // the real gas used, after which the transaction will be considered an attack and all the gas will be consumed and // nothing will be refunded to the sender const MaxGasFeeHigherFactorAccepted = 10 + +// TxCacheSelectionGasRequested defines the maximum total gas for transactions that should be selected from the cache. +const TxCacheSelectionGasRequested = 10_000_000_000 + +// TxCacheSelectionMaxNumTxs defines the maximum number of transactions that should be selected from the cache. +const TxCacheSelectionMaxNumTxs = 30_000 + +// TxCacheSelectionLoopMaximumDuration defines the maximum duration for the loop that selects transactions from the cache. +const TxCacheSelectionLoopMaximumDuration = 250 * time.Millisecond diff --git a/process/coordinator/process_test.go b/process/coordinator/process_test.go index d1dff667cb7..85eeabf1008 100644 --- a/process/coordinator/process_test.go +++ b/process/coordinator/process_test.go @@ -32,6 +32,7 @@ import ( "github.com/multiversx/mx-chain-go/process/factory" "github.com/multiversx/mx-chain-go/process/factory/shard" "github.com/multiversx/mx-chain-go/process/mock" + "github.com/multiversx/mx-chain-go/state" "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/storage/database" "github.com/multiversx/mx-chain-go/storage/storageunit" @@ -586,6 +587,7 @@ func createInterimProcessorContainer() process.IntermediateProcessorContainer { func createPreProcessorContainerWithDataPool( dataPool dataRetriever.PoolsHolder, feeHandler process.FeeHandler, + accounts state.AccountsAdapter, ) process.PreProcessorsContainer { totalGasProvided := uint64(0) @@ -596,7 +598,7 @@ func createPreProcessorContainerWithDataPool( &hashingMocks.HasherMock{}, dataPool, createMockPubkeyConverter(), - &stateMock.AccountsStub{}, + accounts, &testscommon.RequestHandlerStub{}, &testscommon.TxProcessorMock{ ProcessTransactionCalled: func(transaction *transaction.Transaction) (vmcommon.ReturnCode, error) { @@ -1247,7 +1249,7 @@ func TestTransactionCoordinator_CreateMbsAndProcessTransactionsFromMeNoTime(t *t tdp := initDataPool(txHash) argsTransactionCoordinator := createMockTransactionCoordinatorArguments() argsTransactionCoordinator.MiniBlockPool = tdp.MiniBlocks() - argsTransactionCoordinator.PreProcessors = createPreProcessorContainerWithDataPool(tdp, FeeHandlerMock()) + argsTransactionCoordinator.PreProcessors = createPreProcessorContainerWithDataPool(tdp, FeeHandlerMock(), argsTransactionCoordinator.Accounts) tc, err := NewTransactionCoordinator(argsTransactionCoordinator) assert.Nil(t, err) assert.NotNil(t, tc) @@ -1266,7 +1268,7 @@ func TestTransactionCoordinator_CreateMbsAndProcessTransactionsFromMeNoSpace(t * tdp := initDataPool(txHash) argsTransactionCoordinator := createMockTransactionCoordinatorArguments() argsTransactionCoordinator.MiniBlockPool = tdp.MiniBlocks() - argsTransactionCoordinator.PreProcessors = createPreProcessorContainerWithDataPool(tdp, FeeHandlerMock()) + argsTransactionCoordinator.PreProcessors = createPreProcessorContainerWithDataPool(tdp, FeeHandlerMock(), argsTransactionCoordinator.Accounts) argsTransactionCoordinator.GasHandler = &testscommon.GasHandlerStub{ TotalGasProvidedCalled: func() uint64 { return totalGasProvided @@ -1295,9 +1297,18 @@ func TestTransactionCoordinator_CreateMbsAndProcessTransactionsFromMe(t *testing } argsTransactionCoordinator := createMockTransactionCoordinatorArguments() + argsTransactionCoordinator.Accounts = &stateMock.AccountsStub{ + GetExistingAccountCalled: func(_ []byte) (vmcommon.AccountHandler, error) { + return &stateMock.UserAccountStub{ + Nonce: 42, + Balance: big.NewInt(1000000000000000000), + }, nil + }, + } argsTransactionCoordinator.ShardCoordinator = mock.NewMultiShardsCoordinatorMock(nrShards) argsTransactionCoordinator.MiniBlockPool = tdp.MiniBlocks() - argsTransactionCoordinator.PreProcessors = createPreProcessorContainerWithDataPool(tdp, FeeHandlerMock()) + argsTransactionCoordinator.PreProcessors = createPreProcessorContainerWithDataPool(tdp, FeeHandlerMock(), argsTransactionCoordinator.Accounts) + tc, err := NewTransactionCoordinator(argsTransactionCoordinator) assert.Nil(t, err) assert.NotNil(t, tc) @@ -1310,7 +1321,7 @@ func TestTransactionCoordinator_CreateMbsAndProcessTransactionsFromMe(t *testing hasher := &hashingMocks.HasherMock{} for shId := uint32(0); shId < nrShards; shId++ { strCache := process.ShardCacherIdentifier(0, shId) - newTx := &transaction.Transaction{GasLimit: uint64(shId)} + newTx := &transaction.Transaction{GasLimit: uint64(shId), Nonce: 42 + uint64(shId)} computedTxHash, _ := core.CalculateHash(marshalizer, hasher, newTx) txPool.AddData(computedTxHash, newTx, newTx.Size(), strCache) @@ -1333,9 +1344,17 @@ func TestTransactionCoordinator_CreateMbsAndProcessTransactionsFromMeMultipleMin } argsTransactionCoordinator := createMockTransactionCoordinatorArguments() + argsTransactionCoordinator.Accounts = &stateMock.AccountsStub{ + GetExistingAccountCalled: func(_ []byte) (vmcommon.AccountHandler, error) { + return &stateMock.UserAccountStub{ + Nonce: 0, + Balance: big.NewInt(1000000000000000000), + }, nil + }, + } argsTransactionCoordinator.ShardCoordinator = mock.NewMultiShardsCoordinatorMock(nrShards) argsTransactionCoordinator.MiniBlockPool = tdp.MiniBlocks() - argsTransactionCoordinator.PreProcessors = createPreProcessorContainerWithDataPool(tdp, FeeHandlerMock()) + argsTransactionCoordinator.PreProcessors = createPreProcessorContainerWithDataPool(tdp, FeeHandlerMock(), argsTransactionCoordinator.Accounts) tc, err := NewTransactionCoordinator(argsTransactionCoordinator) assert.Nil(t, err) assert.NotNil(t, tc) @@ -1386,6 +1405,14 @@ func TestTransactionCoordinator_CreateMbsAndProcessTransactionsFromMeMultipleMin } argsTransactionCoordinator := createMockTransactionCoordinatorArguments() + argsTransactionCoordinator.Accounts = &stateMock.AccountsStub{ + GetExistingAccountCalled: func(_ []byte) (vmcommon.AccountHandler, error) { + return &stateMock.UserAccountStub{ + Nonce: 0, + Balance: big.NewInt(1000000000000000000), + }, nil + }, + } argsTransactionCoordinator.ShardCoordinator = mock.NewMultiShardsCoordinatorMock(nrShards) argsTransactionCoordinator.MiniBlockPool = tdp.MiniBlocks() argsTransactionCoordinator.PreProcessors = createPreProcessorContainerWithDataPool( @@ -1403,7 +1430,7 @@ func TestTransactionCoordinator_CreateMbsAndProcessTransactionsFromMeMultipleMin ComputeGasLimitCalled: func(tx data.TransactionWithFeeHandler) uint64 { return gasLimit / uint64(numMiniBlocks) }, - }) + }, argsTransactionCoordinator.Accounts) tc, err := NewTransactionCoordinator(argsTransactionCoordinator) assert.Nil(t, err) assert.NotNil(t, tc) @@ -1449,6 +1476,14 @@ func TestTransactionCoordinator_CompactAndExpandMiniblocksShouldWork(t *testing. } argsTransactionCoordinator := createMockTransactionCoordinatorArguments() + argsTransactionCoordinator.Accounts = &stateMock.AccountsStub{ + GetExistingAccountCalled: func(_ []byte) (vmcommon.AccountHandler, error) { + return &stateMock.UserAccountStub{ + Nonce: 0, + Balance: big.NewInt(1000000000000000000), + }, nil + }, + } argsTransactionCoordinator.ShardCoordinator = mock.NewMultiShardsCoordinatorMock(nrShards) argsTransactionCoordinator.MiniBlockPool = tdp.MiniBlocks() argsTransactionCoordinator.PreProcessors = createPreProcessorContainerWithDataPool( @@ -1466,7 +1501,7 @@ func TestTransactionCoordinator_CompactAndExpandMiniblocksShouldWork(t *testing. ComputeGasLimitCalled: func(tx data.TransactionWithFeeHandler) uint64 { return 0 }, - }) + }, argsTransactionCoordinator.Accounts) tc, err := NewTransactionCoordinator(argsTransactionCoordinator) assert.Nil(t, err) assert.NotNil(t, tc) @@ -1513,9 +1548,17 @@ func TestTransactionCoordinator_GetAllCurrentUsedTxs(t *testing.T) { } argsTransactionCoordinator := createMockTransactionCoordinatorArguments() + argsTransactionCoordinator.Accounts = &stateMock.AccountsStub{ + GetExistingAccountCalled: func(_ []byte) (vmcommon.AccountHandler, error) { + return &stateMock.UserAccountStub{ + Nonce: 42, + Balance: big.NewInt(1000000000000000000), + }, nil + }, + } argsTransactionCoordinator.ShardCoordinator = mock.NewMultiShardsCoordinatorMock(nrShards) argsTransactionCoordinator.MiniBlockPool = tdp.MiniBlocks() - argsTransactionCoordinator.PreProcessors = createPreProcessorContainerWithDataPool(tdp, FeeHandlerMock()) + argsTransactionCoordinator.PreProcessors = createPreProcessorContainerWithDataPool(tdp, FeeHandlerMock(), argsTransactionCoordinator.Accounts) argsTransactionCoordinator.GasHandler = &testscommon.GasHandlerStub{ ComputeGasProvidedByTxCalled: func(txSndShId uint32, txRcvShId uint32, txHandler data.TransactionHandler) (uint64, uint64, error) { return 0, 0, nil @@ -1537,7 +1580,7 @@ func TestTransactionCoordinator_GetAllCurrentUsedTxs(t *testing.T) { hasher := &hashingMocks.HasherMock{} for i := uint32(0); i < nrShards; i++ { strCache := process.ShardCacherIdentifier(0, i) - newTx := &transaction.Transaction{GasLimit: uint64(i)} + newTx := &transaction.Transaction{GasLimit: uint64(i), Nonce: 42 + uint64(i)} computedTxHash, _ := core.CalculateHash(marshalizer, hasher, newTx) txPool.AddData(computedTxHash, newTx, newTx.Size(), strCache) @@ -1558,7 +1601,7 @@ func TestTransactionCoordinator_RequestBlockTransactionsNilBody(t *testing.T) { argsTransactionCoordinator := createMockTransactionCoordinatorArguments() argsTransactionCoordinator.ShardCoordinator = mock.NewMultiShardsCoordinatorMock(nrShards) argsTransactionCoordinator.MiniBlockPool = tdp.MiniBlocks() - argsTransactionCoordinator.PreProcessors = createPreProcessorContainerWithDataPool(tdp, FeeHandlerMock()) + argsTransactionCoordinator.PreProcessors = createPreProcessorContainerWithDataPool(tdp, FeeHandlerMock(), argsTransactionCoordinator.Accounts) tc, err := NewTransactionCoordinator(argsTransactionCoordinator) assert.Nil(t, err) assert.NotNil(t, tc) @@ -1580,7 +1623,7 @@ func TestTransactionCoordinator_RequestBlockTransactionsRequestOne(t *testing.T) argsTransactionCoordinator := createMockTransactionCoordinatorArguments() argsTransactionCoordinator.ShardCoordinator = mock.NewMultiShardsCoordinatorMock(nrShards) argsTransactionCoordinator.MiniBlockPool = tdp.MiniBlocks() - argsTransactionCoordinator.PreProcessors = createPreProcessorContainerWithDataPool(tdp, FeeHandlerMock()) + argsTransactionCoordinator.PreProcessors = createPreProcessorContainerWithDataPool(tdp, FeeHandlerMock(), argsTransactionCoordinator.Accounts) tc, err := NewTransactionCoordinator(argsTransactionCoordinator) assert.Nil(t, err) assert.NotNil(t, tc) @@ -1610,7 +1653,7 @@ func TestTransactionCoordinator_IsDataPreparedForProcessing(t *testing.T) { argsTransactionCoordinator := createMockTransactionCoordinatorArguments() argsTransactionCoordinator.ShardCoordinator = mock.NewMultiShardsCoordinatorMock(nrShards) argsTransactionCoordinator.MiniBlockPool = tdp.MiniBlocks() - argsTransactionCoordinator.PreProcessors = createPreProcessorContainerWithDataPool(tdp, FeeHandlerMock()) + argsTransactionCoordinator.PreProcessors = createPreProcessorContainerWithDataPool(tdp, FeeHandlerMock(), argsTransactionCoordinator.Accounts) tc, err := NewTransactionCoordinator(argsTransactionCoordinator) assert.Nil(t, err) assert.NotNil(t, tc) @@ -1630,7 +1673,7 @@ func TestTransactionCoordinator_SaveTxsToStorage(t *testing.T) { argsTransactionCoordinator.ShardCoordinator = mock.NewMultiShardsCoordinatorMock(3) argsTransactionCoordinator.Accounts = initAccountsMock() argsTransactionCoordinator.MiniBlockPool = tdp.MiniBlocks() - argsTransactionCoordinator.PreProcessors = createPreProcessorContainerWithDataPool(tdp, FeeHandlerMock()) + argsTransactionCoordinator.PreProcessors = createPreProcessorContainerWithDataPool(tdp, FeeHandlerMock(), argsTransactionCoordinator.Accounts) tc, err := NewTransactionCoordinator(argsTransactionCoordinator) assert.Nil(t, err) assert.NotNil(t, tc) @@ -1666,7 +1709,7 @@ func TestTransactionCoordinator_RestoreBlockDataFromStorage(t *testing.T) { argsTransactionCoordinator.ShardCoordinator = mock.NewMultiShardsCoordinatorMock(3) argsTransactionCoordinator.Accounts = initAccountsMock() argsTransactionCoordinator.MiniBlockPool = tdp.MiniBlocks() - argsTransactionCoordinator.PreProcessors = createPreProcessorContainerWithDataPool(tdp, FeeHandlerMock()) + argsTransactionCoordinator.PreProcessors = createPreProcessorContainerWithDataPool(tdp, FeeHandlerMock(), argsTransactionCoordinator.Accounts) tc, err := NewTransactionCoordinator(argsTransactionCoordinator) assert.Nil(t, err) assert.NotNil(t, tc) @@ -1704,7 +1747,7 @@ func TestTransactionCoordinator_RemoveBlockDataFromPool(t *testing.T) { argsTransactionCoordinator.ShardCoordinator = mock.NewMultiShardsCoordinatorMock(3) argsTransactionCoordinator.Accounts = initAccountsMock() argsTransactionCoordinator.MiniBlockPool = dataPool.MiniBlocks() - argsTransactionCoordinator.PreProcessors = createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock()) + argsTransactionCoordinator.PreProcessors = createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock(), argsTransactionCoordinator.Accounts) tc, err := NewTransactionCoordinator(argsTransactionCoordinator) assert.Nil(t, err) assert.NotNil(t, tc) @@ -1811,7 +1854,7 @@ func TestTransactionCoordinator_ProcessBlockTransaction(t *testing.T) { argsTransactionCoordinator.ShardCoordinator = mock.NewMultiShardsCoordinatorMock(3) argsTransactionCoordinator.Accounts = initAccountsMock() argsTransactionCoordinator.MiniBlockPool = dataPool.MiniBlocks() - argsTransactionCoordinator.PreProcessors = createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock()) + argsTransactionCoordinator.PreProcessors = createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock(), argsTransactionCoordinator.Accounts) tc, err := NewTransactionCoordinator(argsTransactionCoordinator) assert.Nil(t, err) assert.NotNil(t, tc) @@ -2366,7 +2409,7 @@ func TestTransactionCoordinator_SaveTxsToStorageCallsSaveIntermediate(t *testing argsTransactionCoordinator.ShardCoordinator = mock.NewMultiShardsCoordinatorMock(3) argsTransactionCoordinator.Accounts = initAccountsMock() argsTransactionCoordinator.MiniBlockPool = tdp.MiniBlocks() - argsTransactionCoordinator.PreProcessors = createPreProcessorContainerWithDataPool(tdp, FeeHandlerMock()) + argsTransactionCoordinator.PreProcessors = createPreProcessorContainerWithDataPool(tdp, FeeHandlerMock(), argsTransactionCoordinator.Accounts) argsTransactionCoordinator.InterProcessors = &mock.InterimProcessorContainerMock{ KeysCalled: func() []block.Type { return []block.Type{block.SmartContractResultBlock} @@ -2404,7 +2447,7 @@ func TestTransactionCoordinator_PreprocessorsHasToBeOrderedRewardsAreLast(t *tes argsTransactionCoordinator.ShardCoordinator = mock.NewMultiShardsCoordinatorMock(3) argsTransactionCoordinator.Accounts = initAccountsMock() argsTransactionCoordinator.MiniBlockPool = dataPool.MiniBlocks() - argsTransactionCoordinator.PreProcessors = createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock()) + argsTransactionCoordinator.PreProcessors = createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock(), argsTransactionCoordinator.Accounts) argsTransactionCoordinator.InterProcessors = createInterimProcessorContainer() tc, err := NewTransactionCoordinator(argsTransactionCoordinator) assert.Nil(t, err) @@ -2588,14 +2631,16 @@ func TestTransactionCoordinator_VerifyCreatedMiniBlocksShouldReturnWhenEpochIsNo t.Parallel() dataPool := initDataPool(txHash) + accounts := initAccountsMock() + txCoordinatorArgs := ArgTransactionCoordinator{ Hasher: &hashingMocks.HasherMock{}, Marshalizer: &mock.MarshalizerMock{}, ShardCoordinator: mock.NewMultiShardsCoordinatorMock(3), - Accounts: initAccountsMock(), + Accounts: accounts, MiniBlockPool: dataPool.MiniBlocks(), RequestHandler: &testscommon.RequestHandlerStub{}, - PreProcessors: createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock()), + PreProcessors: createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock(), accounts), InterProcessors: createInterimProcessorContainer(), GasHandler: &testscommon.GasHandlerStub{}, FeeHandler: &mock.FeeAccumulatorStub{}, @@ -2633,14 +2678,16 @@ func TestTransactionCoordinator_VerifyCreatedMiniBlocksShouldErrMaxGasLimitPerMi maxGasLimitPerBlock := uint64(1500000000) dataPool := initDataPool(txHash) + accounts := initAccountsMock() + txCoordinatorArgs := ArgTransactionCoordinator{ Hasher: &hashingMocks.HasherMock{}, Marshalizer: &mock.MarshalizerMock{}, ShardCoordinator: mock.NewMultiShardsCoordinatorMock(3), - Accounts: initAccountsMock(), + Accounts: accounts, MiniBlockPool: dataPool.MiniBlocks(), RequestHandler: &testscommon.RequestHandlerStub{}, - PreProcessors: createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock()), + PreProcessors: createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock(), accounts), InterProcessors: createInterimProcessorContainer(), GasHandler: &testscommon.GasHandlerStub{}, FeeHandler: &mock.FeeAccumulatorStub{}, @@ -2699,14 +2746,16 @@ func TestTransactionCoordinator_VerifyCreatedMiniBlocksShouldErrMaxAccumulatedFe maxGasLimitPerBlock := uint64(1500000000) dataPool := initDataPool(txHash) + accounts := initAccountsMock() + txCoordinatorArgs := ArgTransactionCoordinator{ Hasher: &hashingMocks.HasherMock{}, Marshalizer: &mock.MarshalizerMock{}, ShardCoordinator: mock.NewMultiShardsCoordinatorMock(3), - Accounts: initAccountsMock(), + Accounts: accounts, MiniBlockPool: dataPool.MiniBlocks(), RequestHandler: &testscommon.RequestHandlerStub{}, - PreProcessors: createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock()), + PreProcessors: createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock(), accounts), InterProcessors: createInterimProcessorContainer(), GasHandler: &testscommon.GasHandlerStub{}, FeeHandler: &mock.FeeAccumulatorStub{}, @@ -2776,14 +2825,16 @@ func TestTransactionCoordinator_VerifyCreatedMiniBlocksShouldErrMaxDeveloperFees maxGasLimitPerBlock := uint64(1500000000) dataPool := initDataPool(txHash) + accounts := initAccountsMock() + txCoordinatorArgs := ArgTransactionCoordinator{ Hasher: &hashingMocks.HasherMock{}, Marshalizer: &mock.MarshalizerMock{}, ShardCoordinator: mock.NewMultiShardsCoordinatorMock(3), - Accounts: initAccountsMock(), + Accounts: accounts, MiniBlockPool: dataPool.MiniBlocks(), RequestHandler: &testscommon.RequestHandlerStub{}, - PreProcessors: createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock()), + PreProcessors: createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock(), accounts), InterProcessors: createInterimProcessorContainer(), GasHandler: &testscommon.GasHandlerStub{}, FeeHandler: &mock.FeeAccumulatorStub{}, @@ -2853,14 +2904,16 @@ func TestTransactionCoordinator_VerifyCreatedMiniBlocksShouldWork(t *testing.T) maxGasLimitPerBlock := uint64(1500000000) dataPool := initDataPool(txHash) + accounts := initAccountsMock() + txCoordinatorArgs := ArgTransactionCoordinator{ Hasher: &hashingMocks.HasherMock{}, Marshalizer: &mock.MarshalizerMock{}, ShardCoordinator: mock.NewMultiShardsCoordinatorMock(3), - Accounts: initAccountsMock(), + Accounts: accounts, MiniBlockPool: dataPool.MiniBlocks(), RequestHandler: &testscommon.RequestHandlerStub{}, - PreProcessors: createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock()), + PreProcessors: createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock(), accounts), InterProcessors: createInterimProcessorContainer(), GasHandler: &testscommon.GasHandlerStub{}, FeeHandler: &mock.FeeAccumulatorStub{}, @@ -2929,14 +2982,16 @@ func TestTransactionCoordinator_GetAllTransactionsShouldWork(t *testing.T) { t.Parallel() dataPool := initDataPool(txHash) + accounts := initAccountsMock() + txCoordinatorArgs := ArgTransactionCoordinator{ Hasher: &hashingMocks.HasherMock{}, Marshalizer: &mock.MarshalizerMock{}, ShardCoordinator: mock.NewMultiShardsCoordinatorMock(3), - Accounts: initAccountsMock(), + Accounts: accounts, MiniBlockPool: dataPool.MiniBlocks(), RequestHandler: &testscommon.RequestHandlerStub{}, - PreProcessors: createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock()), + PreProcessors: createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock(), accounts), InterProcessors: createInterimProcessorContainer(), GasHandler: &testscommon.GasHandlerStub{}, FeeHandler: &mock.FeeAccumulatorStub{}, @@ -3002,14 +3057,16 @@ func TestTransactionCoordinator_VerifyGasLimitShouldErrMaxGasLimitPerMiniBlockIn tx3GasLimit := uint64(300000001) dataPool := initDataPool(txHash) + accounts := initAccountsMock() + txCoordinatorArgs := ArgTransactionCoordinator{ Hasher: &hashingMocks.HasherMock{}, Marshalizer: &mock.MarshalizerMock{}, ShardCoordinator: mock.NewMultiShardsCoordinatorMock(3), - Accounts: initAccountsMock(), + Accounts: accounts, MiniBlockPool: dataPool.MiniBlocks(), RequestHandler: &testscommon.RequestHandlerStub{}, - PreProcessors: createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock()), + PreProcessors: createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock(), accounts), InterProcessors: createInterimProcessorContainer(), GasHandler: &testscommon.GasHandlerStub{}, FeeHandler: &mock.FeeAccumulatorStub{}, @@ -3095,14 +3152,16 @@ func TestTransactionCoordinator_VerifyGasLimitShouldWork(t *testing.T) { tx3GasLimit := uint64(300) dataPool := initDataPool(txHash) + accounts := initAccountsMock() + txCoordinatorArgs := ArgTransactionCoordinator{ Hasher: &hashingMocks.HasherMock{}, Marshalizer: &mock.MarshalizerMock{}, ShardCoordinator: mock.NewMultiShardsCoordinatorMock(3), - Accounts: initAccountsMock(), + Accounts: accounts, MiniBlockPool: dataPool.MiniBlocks(), RequestHandler: &testscommon.RequestHandlerStub{}, - PreProcessors: createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock()), + PreProcessors: createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock(), accounts), InterProcessors: createInterimProcessorContainer(), GasHandler: &testscommon.GasHandlerStub{}, FeeHandler: &mock.FeeAccumulatorStub{}, @@ -3184,14 +3243,16 @@ func TestTransactionCoordinator_CheckGasProvidedByMiniBlockInReceiverShardShould t.Parallel() dataPool := initDataPool(txHash) + accounts := initAccountsMock() + txCoordinatorArgs := ArgTransactionCoordinator{ Hasher: &hashingMocks.HasherMock{}, Marshalizer: &mock.MarshalizerMock{}, ShardCoordinator: mock.NewMultiShardsCoordinatorMock(3), - Accounts: initAccountsMock(), + Accounts: accounts, MiniBlockPool: dataPool.MiniBlocks(), RequestHandler: &testscommon.RequestHandlerStub{}, - PreProcessors: createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock()), + PreProcessors: createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock(), accounts), InterProcessors: createInterimProcessorContainer(), GasHandler: &testscommon.GasHandlerStub{}, FeeHandler: &mock.FeeAccumulatorStub{}, @@ -3226,14 +3287,16 @@ func TestTransactionCoordinator_CheckGasProvidedByMiniBlockInReceiverShardShould tx1GasLimit := uint64(100) dataPool := initDataPool(txHash) + accounts := initAccountsMock() + txCoordinatorArgs := ArgTransactionCoordinator{ Hasher: &hashingMocks.HasherMock{}, Marshalizer: &mock.MarshalizerMock{}, ShardCoordinator: mock.NewMultiShardsCoordinatorMock(3), - Accounts: initAccountsMock(), + Accounts: accounts, MiniBlockPool: dataPool.MiniBlocks(), RequestHandler: &testscommon.RequestHandlerStub{}, - PreProcessors: createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock()), + PreProcessors: createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock(), accounts), InterProcessors: createInterimProcessorContainer(), GasHandler: &testscommon.GasHandlerStub{}, FeeHandler: &mock.FeeAccumulatorStub{}, @@ -3283,14 +3346,16 @@ func TestTransactionCoordinator_CheckGasProvidedByMiniBlockInReceiverShardShould tx2GasLimit := uint64(1) dataPool := initDataPool(txHash) + accounts := initAccountsMock() + txCoordinatorArgs := ArgTransactionCoordinator{ Hasher: &hashingMocks.HasherMock{}, Marshalizer: &mock.MarshalizerMock{}, ShardCoordinator: mock.NewMultiShardsCoordinatorMock(3), - Accounts: initAccountsMock(), + Accounts: accounts, MiniBlockPool: dataPool.MiniBlocks(), RequestHandler: &testscommon.RequestHandlerStub{}, - PreProcessors: createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock()), + PreProcessors: createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock(), accounts), InterProcessors: createInterimProcessorContainer(), GasHandler: &testscommon.GasHandlerStub{}, FeeHandler: &mock.FeeAccumulatorStub{}, @@ -3345,14 +3410,16 @@ func TestTransactionCoordinator_CheckGasProvidedByMiniBlockInReceiverShardShould tx3GasLimit := uint64(300000001) dataPool := initDataPool(txHash) + accounts := initAccountsMock() + txCoordinatorArgs := ArgTransactionCoordinator{ Hasher: &hashingMocks.HasherMock{}, Marshalizer: &mock.MarshalizerMock{}, ShardCoordinator: mock.NewMultiShardsCoordinatorMock(3), - Accounts: initAccountsMock(), + Accounts: accounts, MiniBlockPool: dataPool.MiniBlocks(), RequestHandler: &testscommon.RequestHandlerStub{}, - PreProcessors: createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock()), + PreProcessors: createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock(), accounts), InterProcessors: createInterimProcessorContainer(), GasHandler: &testscommon.GasHandlerStub{}, FeeHandler: &mock.FeeAccumulatorStub{}, @@ -3412,14 +3479,16 @@ func TestTransactionCoordinator_CheckGasProvidedByMiniBlockInReceiverShardShould tx3GasLimit := uint64(300) dataPool := initDataPool(txHash) + accounts := initAccountsMock() + txCoordinatorArgs := ArgTransactionCoordinator{ Hasher: &hashingMocks.HasherMock{}, Marshalizer: &mock.MarshalizerMock{}, ShardCoordinator: mock.NewMultiShardsCoordinatorMock(3), - Accounts: initAccountsMock(), + Accounts: accounts, MiniBlockPool: dataPool.MiniBlocks(), RequestHandler: &testscommon.RequestHandlerStub{}, - PreProcessors: createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock()), + PreProcessors: createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock(), accounts), InterProcessors: createInterimProcessorContainer(), GasHandler: &testscommon.GasHandlerStub{}, FeeHandler: &mock.FeeAccumulatorStub{}, @@ -3476,14 +3545,16 @@ func TestTransactionCoordinator_VerifyFeesShouldErrMissingTransaction(t *testing t.Parallel() dataPool := initDataPool(txHash) + accounts := initAccountsMock() + txCoordinatorArgs := ArgTransactionCoordinator{ Hasher: &hashingMocks.HasherMock{}, Marshalizer: &mock.MarshalizerMock{}, ShardCoordinator: mock.NewMultiShardsCoordinatorMock(3), - Accounts: initAccountsMock(), + Accounts: accounts, MiniBlockPool: dataPool.MiniBlocks(), RequestHandler: &testscommon.RequestHandlerStub{}, - PreProcessors: createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock()), + PreProcessors: createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock(), accounts), InterProcessors: createInterimProcessorContainer(), GasHandler: &testscommon.GasHandlerStub{}, FeeHandler: &mock.FeeAccumulatorStub{}, @@ -3531,14 +3602,16 @@ func TestTransactionCoordinator_VerifyFeesShouldErrMaxAccumulatedFeesExceeded(t tx1GasLimit := uint64(100) dataPool := initDataPool(txHash) + accounts := initAccountsMock() + txCoordinatorArgs := ArgTransactionCoordinator{ Hasher: &hashingMocks.HasherMock{}, Marshalizer: &mock.MarshalizerMock{}, ShardCoordinator: mock.NewMultiShardsCoordinatorMock(3), - Accounts: initAccountsMock(), + Accounts: accounts, MiniBlockPool: dataPool.MiniBlocks(), RequestHandler: &testscommon.RequestHandlerStub{}, - PreProcessors: createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock()), + PreProcessors: createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock(), accounts), InterProcessors: createInterimProcessorContainer(), GasHandler: &testscommon.GasHandlerStub{}, FeeHandler: &mock.FeeAccumulatorStub{}, @@ -3600,14 +3673,16 @@ func TestTransactionCoordinator_VerifyFeesShouldErrMaxDeveloperFeesExceeded(t *t tx1GasLimit := uint64(100) dataPool := initDataPool(txHash) + accounts := initAccountsMock() + txCoordinatorArgs := ArgTransactionCoordinator{ Hasher: &hashingMocks.HasherMock{}, Marshalizer: &mock.MarshalizerMock{}, ShardCoordinator: mock.NewMultiShardsCoordinatorMock(3), - Accounts: initAccountsMock(), + Accounts: accounts, MiniBlockPool: dataPool.MiniBlocks(), RequestHandler: &testscommon.RequestHandlerStub{}, - PreProcessors: createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock()), + PreProcessors: createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock(), accounts), InterProcessors: createInterimProcessorContainer(), GasHandler: &testscommon.GasHandlerStub{}, FeeHandler: &mock.FeeAccumulatorStub{}, @@ -3670,14 +3745,16 @@ func TestTransactionCoordinator_VerifyFeesShouldErrMaxAccumulatedFeesExceededWhe enableEpochsHandlerStub := enableEpochsHandlerMock.NewEnableEpochsHandlerStub() dataPool := initDataPool(txHash) + accounts := initAccountsMock() + txCoordinatorArgs := ArgTransactionCoordinator{ Hasher: &hashingMocks.HasherMock{}, Marshalizer: &mock.MarshalizerMock{}, ShardCoordinator: mock.NewMultiShardsCoordinatorMock(3), - Accounts: initAccountsMock(), + Accounts: accounts, MiniBlockPool: dataPool.MiniBlocks(), RequestHandler: &testscommon.RequestHandlerStub{}, - PreProcessors: createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock()), + PreProcessors: createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock(), accounts), InterProcessors: createInterimProcessorContainer(), GasHandler: &testscommon.GasHandlerStub{}, FeeHandler: &mock.FeeAccumulatorStub{}, @@ -3754,14 +3831,16 @@ func TestTransactionCoordinator_VerifyFeesShouldErrMaxDeveloperFeesExceededWhenS enableEpochsHandlerStub := enableEpochsHandlerMock.NewEnableEpochsHandlerStub() dataPool := initDataPool(txHash) + accounts := initAccountsMock() + txCoordinatorArgs := ArgTransactionCoordinator{ Hasher: &hashingMocks.HasherMock{}, Marshalizer: &mock.MarshalizerMock{}, ShardCoordinator: mock.NewMultiShardsCoordinatorMock(3), - Accounts: initAccountsMock(), + Accounts: accounts, MiniBlockPool: dataPool.MiniBlocks(), RequestHandler: &testscommon.RequestHandlerStub{}, - PreProcessors: createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock()), + PreProcessors: createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock(), accounts), InterProcessors: createInterimProcessorContainer(), GasHandler: &testscommon.GasHandlerStub{}, FeeHandler: &mock.FeeAccumulatorStub{}, @@ -3838,14 +3917,16 @@ func TestTransactionCoordinator_VerifyFeesShouldWork(t *testing.T) { enableEpochsHandlerStub := enableEpochsHandlerMock.NewEnableEpochsHandlerStub() dataPool := initDataPool(txHash) + accounts := initAccountsMock() + txCoordinatorArgs := ArgTransactionCoordinator{ Hasher: &hashingMocks.HasherMock{}, Marshalizer: &mock.MarshalizerMock{}, ShardCoordinator: mock.NewMultiShardsCoordinatorMock(3), - Accounts: initAccountsMock(), + Accounts: accounts, MiniBlockPool: dataPool.MiniBlocks(), RequestHandler: &testscommon.RequestHandlerStub{}, - PreProcessors: createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock()), + PreProcessors: createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock(), accounts), InterProcessors: createInterimProcessorContainer(), GasHandler: &testscommon.GasHandlerStub{}, FeeHandler: &mock.FeeAccumulatorStub{}, @@ -3925,14 +4006,16 @@ func TestTransactionCoordinator_GetMaxAccumulatedAndDeveloperFeesShouldErr(t *te t.Parallel() dataPool := initDataPool(txHash) + accounts := initAccountsMock() + txCoordinatorArgs := ArgTransactionCoordinator{ Hasher: &hashingMocks.HasherMock{}, Marshalizer: &mock.MarshalizerMock{}, ShardCoordinator: mock.NewMultiShardsCoordinatorMock(3), - Accounts: initAccountsMock(), + Accounts: accounts, MiniBlockPool: dataPool.MiniBlocks(), RequestHandler: &testscommon.RequestHandlerStub{}, - PreProcessors: createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock()), + PreProcessors: createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock(), accounts), InterProcessors: createInterimProcessorContainer(), GasHandler: &testscommon.GasHandlerStub{}, FeeHandler: &mock.FeeAccumulatorStub{}, @@ -3977,14 +4060,16 @@ func TestTransactionCoordinator_GetMaxAccumulatedAndDeveloperFeesShouldWork(t *t tx3GasLimit := uint64(300) dataPool := initDataPool(txHash) + accounts := initAccountsMock() + txCoordinatorArgs := ArgTransactionCoordinator{ Hasher: &hashingMocks.HasherMock{}, Marshalizer: &mock.MarshalizerMock{}, ShardCoordinator: mock.NewMultiShardsCoordinatorMock(3), - Accounts: initAccountsMock(), + Accounts: accounts, MiniBlockPool: dataPool.MiniBlocks(), RequestHandler: &testscommon.RequestHandlerStub{}, - PreProcessors: createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock()), + PreProcessors: createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock(), accounts), InterProcessors: createInterimProcessorContainer(), GasHandler: &testscommon.GasHandlerStub{}, FeeHandler: &mock.FeeAccumulatorStub{}, @@ -4043,14 +4128,16 @@ func TestTransactionCoordinator_RevertIfNeededShouldWork(t *testing.T) { numTxsFeesReverted := 0 dataPool := initDataPool(txHash) + accounts := initAccountsMock() + txCoordinatorArgs := ArgTransactionCoordinator{ Hasher: &hashingMocks.HasherMock{}, Marshalizer: &mock.MarshalizerMock{}, ShardCoordinator: mock.NewMultiShardsCoordinatorMock(3), - Accounts: initAccountsMock(), + Accounts: accounts, MiniBlockPool: dataPool.MiniBlocks(), RequestHandler: &testscommon.RequestHandlerStub{}, - PreProcessors: createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock()), + PreProcessors: createPreProcessorContainerWithDataPool(dataPool, FeeHandlerMock(), accounts), InterProcessors: createInterimProcessorContainer(), GasHandler: &mock.GasHandlerMock{ RestoreGasSinceLastResetCalled: func(key []byte) { diff --git a/process/errors.go b/process/errors.go index 83e8095dcb3..4088162fd7e 100644 --- a/process/errors.go +++ b/process/errors.go @@ -1059,9 +1059,6 @@ var ErrNilIsMaxBlockSizeReachedHandler = errors.New("nil handler for max block s // ErrNilTxMaxTotalCostHandler signals a nil transaction max total cost var ErrNilTxMaxTotalCostHandler = errors.New("nil transaction max total cost") -// ErrNilAccountTxsPerShard signals a nil mapping for account transactions to shard -var ErrNilAccountTxsPerShard = errors.New("nil account transactions per shard mapping") - // ErrScheduledRootHashDoesNotMatch signals that scheduled root hash does not match var ErrScheduledRootHashDoesNotMatch = errors.New("scheduled root hash does not match") diff --git a/process/interface.go b/process/interface.go index 747103f26ca..715c02cbf5c 100644 --- a/process/interface.go +++ b/process/interface.go @@ -39,6 +39,7 @@ import ( type TransactionProcessor interface { ProcessTransaction(transaction *transaction.Transaction) (vmcommon.ReturnCode, error) VerifyTransaction(transaction *transaction.Transaction) error + VerifyGuardian(tx *transaction.Transaction, account state.UserAccountHandler) error IsInterfaceNil() bool } diff --git a/process/transaction/baseProcess.go b/process/transaction/baseProcess.go index b1e95a71339..12f29b05bb6 100644 --- a/process/transaction/baseProcess.go +++ b/process/transaction/baseProcess.go @@ -119,7 +119,7 @@ func (txProc *baseTxProcessor) checkTxValues( acntSnd, acntDst state.UserAccountHandler, isUserTxOfRelayed bool, ) error { - err := txProc.verifyGuardian(tx, acntSnd) + err := txProc.VerifyGuardian(tx, acntSnd) if err != nil { return err } @@ -288,7 +288,7 @@ func (txProc *baseTxProcessor) checkGuardedAccountUnguardedTxPermission(tx *tran return nil } -func (txProc *baseTxProcessor) verifyGuardian(tx *transaction.Transaction, account state.UserAccountHandler) error { +func (txProc *baseTxProcessor) VerifyGuardian(tx *transaction.Transaction, account state.UserAccountHandler) error { if check.IfNil(account) { return nil } diff --git a/process/transaction/baseProcess_test.go b/process/transaction/baseProcess_test.go index 7795c1a0f6a..7d4605239a9 100644 --- a/process/transaction/baseProcess_test.go +++ b/process/transaction/baseProcess_test.go @@ -231,7 +231,7 @@ func TestBaseTxProcessor_VerifyGuardian(t *testing.T) { t.Parallel() localBaseProc := baseProc - err := localBaseProc.verifyGuardian(&transaction.Transaction{}, nil) + err := localBaseProc.VerifyGuardian(&transaction.Transaction{}, nil) assert.Nil(t, err) }) t.Run("guarded account with a not guarded transaction should error", func(t *testing.T) { @@ -244,7 +244,7 @@ func TestBaseTxProcessor_VerifyGuardian(t *testing.T) { }, } - err := localBaseProc.verifyGuardian(&transaction.Transaction{}, guardedAccount) + err := localBaseProc.VerifyGuardian(&transaction.Transaction{}, guardedAccount) assert.ErrorIs(t, err, process.ErrTransactionNotExecutable) assert.Contains(t, err.Error(), "not allowed to bypass guardian") }) @@ -258,7 +258,7 @@ func TestBaseTxProcessor_VerifyGuardian(t *testing.T) { }, } - err := localBaseProc.verifyGuardian(&transaction.Transaction{}, notGuardedAccount) + err := localBaseProc.VerifyGuardian(&transaction.Transaction{}, notGuardedAccount) assert.ErrorIs(t, err, process.ErrTransactionNotExecutable) assert.Contains(t, err.Error(), process.ErrGuardedTransactionNotExpected.Error()) }) @@ -272,7 +272,7 @@ func TestBaseTxProcessor_VerifyGuardian(t *testing.T) { }, } - err := localBaseProc.verifyGuardian(&transaction.Transaction{}, notGuardedAccount) + err := localBaseProc.VerifyGuardian(&transaction.Transaction{}, notGuardedAccount) assert.Nil(t, err) }) t.Run("get active guardian fails should error", func(t *testing.T) { @@ -290,7 +290,7 @@ func TestBaseTxProcessor_VerifyGuardian(t *testing.T) { }, } - err := localBaseProc.verifyGuardian(&transaction.Transaction{}, guardedAccount) + err := localBaseProc.VerifyGuardian(&transaction.Transaction{}, guardedAccount) assert.ErrorIs(t, err, process.ErrTransactionNotExecutable) assert.Contains(t, err.Error(), expectedErr.Error()) }) @@ -309,7 +309,7 @@ func TestBaseTxProcessor_VerifyGuardian(t *testing.T) { }, } - err := localBaseProc.verifyGuardian(tx, guardedAccount) + err := localBaseProc.VerifyGuardian(tx, guardedAccount) assert.ErrorIs(t, err, process.ErrTransactionNotExecutable) assert.Contains(t, err.Error(), process.ErrTransactionAndAccountGuardianMismatch.Error()) }) @@ -328,7 +328,7 @@ func TestBaseTxProcessor_VerifyGuardian(t *testing.T) { }, } - err := localBaseProc.verifyGuardian(tx, guardedAccount) + err := localBaseProc.VerifyGuardian(tx, guardedAccount) assert.Nil(t, err) }) } diff --git a/process/transaction/export_test.go b/process/transaction/export_test.go index cd657c3991d..940633ca6b4 100644 --- a/process/transaction/export_test.go +++ b/process/transaction/export_test.go @@ -100,11 +100,6 @@ func (inTx *InterceptedTransaction) CheckMaxGasPrice() error { return inTx.checkMaxGasPrice() } -// VerifyGuardian calls the un-exported method verifyGuardian -func (txProc *txProcessor) VerifyGuardian(tx *transaction.Transaction, account state.UserAccountHandler) error { - return txProc.verifyGuardian(tx, account) -} - // ShouldIncreaseNonce calls the un-exported method shouldIncreaseNonce func (txProc *txProcessor) ShouldIncreaseNonce(executionErr error) bool { return txProc.shouldIncreaseNonce(executionErr) diff --git a/state/accountsDB.go b/state/accountsDB.go index 4274e5138d6..707361ae68d 100644 --- a/state/accountsDB.go +++ b/state/accountsDB.go @@ -482,9 +482,7 @@ func (adb *AccountsDB) saveDataTrie(accountHandler baseAccountHandler) error { } func (adb *AccountsDB) saveAccountToTrie(accountHandler vmcommon.AccountHandler, mainTrie common.Trie) error { - log.Trace("accountsDB.saveAccountToTrie", - "address", hex.EncodeToString(accountHandler.AddressBytes()), - ) + log.Trace("accountsDB.saveAccountToTrie", "address", accountHandler.AddressBytes()) // pass the reference to marshaller, otherwise it will fail marshalling balance buff, err := adb.marshaller.Marshal(accountHandler) @@ -601,9 +599,7 @@ func (adb *AccountsDB) LoadAccount(address []byte) (vmcommon.AccountHandler, err return nil, fmt.Errorf("%w in LoadAccount", ErrNilAddress) } - log.Trace("accountsDB.LoadAccount", - "address", hex.EncodeToString(address), - ) + log.Trace("accountsDB.LoadAccount", "address", address) mainTrie := adb.getMainTrie() acnt, err := adb.getAccount(address, mainTrie) @@ -653,9 +649,7 @@ func (adb *AccountsDB) GetExistingAccount(address []byte) (vmcommon.AccountHandl return nil, fmt.Errorf("%w in GetExistingAccount", ErrNilAddress) } - log.Trace("accountsDB.GetExistingAccount", - "address", hex.EncodeToString(address), - ) + log.Trace("accountsDB.GetExistingAccount", "address", address) mainTrie := adb.getMainTrie() acnt, err := adb.getAccount(address, mainTrie) diff --git a/storage/constants.go b/storage/constants.go index 9cd37571521..4924cbf06dd 100644 --- a/storage/constants.go +++ b/storage/constants.go @@ -19,9 +19,6 @@ const PathEpochPlaceholder = "[E]" // PathIdentifierPlaceholder represents the placeholder for the identifier in paths const PathIdentifierPlaceholder = "[I]" -// TxPoolNumTxsToPreemptivelyEvict instructs tx pool eviction algorithm to remove this many transactions when eviction takes place -const TxPoolNumTxsToPreemptivelyEvict = uint32(1000) - // DefaultDBPath is the default path for nodes databases const DefaultDBPath = "db" @@ -33,3 +30,12 @@ const DefaultStaticDbString = "Static" // DefaultShardString is the default folder root name for per shard databases const DefaultShardString = "Shard" + +// TxPoolSourceMeNumItemsToPreemptivelyEvict is a configuration of the eviction algorithm +const TxPoolSourceMeNumItemsToPreemptivelyEvict = uint32(50000) + +// TxPoolDestinationMeNumItemsToPreemptivelyEvict is a configuration of the eviction algorithm +const TxPoolDestinationMeNumItemsToPreemptivelyEvict = uint32(1000) + +// ShardedDataNumItemsToPreemptivelyEvict is a configuration of the eviction algorithm +const ShardedDataNumItemsToPreemptivelyEvict = uint32(1000) diff --git a/storage/txcache/txcache.go b/storage/txcache/txcache.go index 218a3749805..01e9212ea15 100644 --- a/storage/txcache/txcache.go +++ b/storage/txcache/txcache.go @@ -2,13 +2,20 @@ package txcache import ( "github.com/multiversx/mx-chain-storage-go/txcache" + "github.com/multiversx/mx-chain-storage-go/types" ) // WrappedTransaction contains a transaction, its hash and extra information type WrappedTransaction = txcache.WrappedTransaction -// TxGasHandler handles a transaction gas and gas cost -type TxGasHandler = txcache.TxGasHandler +// AccountState represents the state of an account (as seen by the mempool) +type AccountState = types.AccountState + +// MempoolHost provides blockchain information for mempool operations +type MempoolHost = txcache.MempoolHost + +// SelectionSession provides blockchain information for transaction selection +type SelectionSession = txcache.SelectionSession // ForEachTransaction is an iterator callback type ForEachTransaction = txcache.ForEachTransaction @@ -29,8 +36,8 @@ type DisabledCache = txcache.DisabledCache type CrossTxCache = txcache.CrossTxCache // NewTxCache creates a new transaction cache -func NewTxCache(config ConfigSourceMe, txGasHandler TxGasHandler) (*TxCache, error) { - return txcache.NewTxCache(config, txGasHandler) +func NewTxCache(config ConfigSourceMe, host MempoolHost) (*TxCache, error) { + return txcache.NewTxCache(config, host) } // NewDisabledCache creates a new disabled cache diff --git a/storage/txcache/txcache_test.go b/storage/txcache/txcache_test.go index cd0ded4f133..5b84daba2d5 100644 --- a/storage/txcache/txcache_test.go +++ b/storage/txcache/txcache_test.go @@ -16,37 +16,33 @@ func TestNewTxCache(t *testing.T) { t.Parallel() cfg := ConfigSourceMe{ - Name: "test", - NumChunks: 1, - NumBytesThreshold: 1000, - NumBytesPerSenderThreshold: 100, - CountThreshold: 10, - CountPerSenderThreshold: 100, - NumSendersToPreemptivelyEvict: 1, + Name: "test", + NumChunks: 1, + NumBytesThreshold: 1000, + NumBytesPerSenderThreshold: 100, + CountThreshold: 10, + CountPerSenderThreshold: 100, + NumItemsToPreemptivelyEvict: 1, } cache, err := NewTxCache(cfg, nil) assert.Nil(t, cache) - assert.Equal(t, common.ErrNilTxGasHandler, err) + assert.ErrorContains(t, err, "nil mempool host") }) t.Run("should work", func(t *testing.T) { t.Parallel() cfg := ConfigSourceMe{ - Name: "test", - NumChunks: 1, - NumBytesThreshold: 1000, - NumBytesPerSenderThreshold: 100, - CountThreshold: 10, - CountPerSenderThreshold: 100, - NumSendersToPreemptivelyEvict: 1, + Name: "test", + NumChunks: 1, + NumBytesThreshold: 1000, + NumBytesPerSenderThreshold: 100, + CountThreshold: 10, + CountPerSenderThreshold: 100, + NumItemsToPreemptivelyEvict: 1, } - cache, err := NewTxCache(cfg, &txcachemocks.TxGasHandlerMock{ - GasProcessingDivisor: 1, - MinimumGasPrice: 1, - MinimumGasMove: 1, - }) + cache, err := NewTxCache(cfg, txcachemocks.NewMempoolHostMock()) assert.NotNil(t, cache) assert.Nil(t, err) }) diff --git a/testscommon/components/components.go b/testscommon/components/components.go index 55b09fe99de..01828396e90 100644 --- a/testscommon/components/components.go +++ b/testscommon/components/components.go @@ -439,10 +439,25 @@ func GetProcessArgs( ) bootstrapComponentsFactoryArgs := GetBootStrapFactoryArgs() - bootstrapComponentsFactory, _ := bootstrapComp.NewBootstrapComponentsFactory(bootstrapComponentsFactoryArgs) - bootstrapComponents, _ := bootstrapComp.NewTestManagedBootstrapComponents(bootstrapComponentsFactory) - _ = bootstrapComponents.Create() - _ = bootstrapComponents.SetShardCoordinator(shardCoordinator) + bootstrapComponentsFactory, err := bootstrapComp.NewBootstrapComponentsFactory(bootstrapComponentsFactoryArgs) + if err != nil { + panic(err) + } + + bootstrapComponents, err := bootstrapComp.NewTestManagedBootstrapComponents(bootstrapComponentsFactory) + if err != nil { + panic(err) + } + + err = bootstrapComponents.Create() + if err != nil { + panic(err) + } + + err = bootstrapComponents.SetShardCoordinator(shardCoordinator) + if err != nil { + panic(err) + } return processComp.ProcessComponentsFactoryArgs{ Config: testscommon.GetGeneralConfig(), @@ -720,9 +735,21 @@ func GetNetworkComponents(cryptoComp factory.CryptoComponentsHolder) factory.Net // GetDataComponents - func GetDataComponents(coreComponents factory.CoreComponentsHolder, shardCoordinator sharding.Coordinator) factory.DataComponentsHolder { dataArgs := GetDataArgs(coreComponents, shardCoordinator) - dataComponentsFactory, _ := dataComp.NewDataComponentsFactory(dataArgs) - dataComponents, _ := dataComp.NewManagedDataComponents(dataComponentsFactory) - _ = dataComponents.Create() + dataComponentsFactory, err := dataComp.NewDataComponentsFactory(dataArgs) + if err != nil { + panic(err) + } + + dataComponents, err := dataComp.NewManagedDataComponents(dataComponentsFactory) + if err != nil { + panic(err) + } + + err = dataComponents.Create() + if err != nil { + panic(err) + } + return dataComponents } diff --git a/testscommon/dataRetriever/poolFactory.go b/testscommon/dataRetriever/poolFactory.go index a8f4374e800..71a27a718f2 100644 --- a/testscommon/dataRetriever/poolFactory.go +++ b/testscommon/dataRetriever/poolFactory.go @@ -40,11 +40,8 @@ func CreateTxPool(numShards uint32, selfShard uint32) (dataRetriever.ShardedData }, NumberOfShards: numShards, SelfShardID: selfShard, - TxGasHandler: &txcachemocks.TxGasHandlerMock{ - MinimumGasMove: 50000, - MinimumGasPrice: 200000000000, - GasProcessingDivisor: 100, - }, + TxGasHandler: txcachemocks.NewTxGasHandlerMock(), + Marshalizer: &marshal.GogoProtoMarshalizer{}, }, ) } diff --git a/testscommon/dataRetriever/poolsHolderMock.go b/testscommon/dataRetriever/poolsHolderMock.go index d3d30562954..75321b6854c 100644 --- a/testscommon/dataRetriever/poolsHolderMock.go +++ b/testscommon/dataRetriever/poolsHolderMock.go @@ -4,6 +4,7 @@ import ( "time" "github.com/multiversx/mx-chain-core-go/core/check" + "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/config" "github.com/multiversx/mx-chain-go/dataRetriever" @@ -49,11 +50,8 @@ func NewPoolsHolderMock() *PoolsHolderMock { SizeInBytesPerSender: 10000000, Shards: 16, }, - TxGasHandler: &txcachemocks.TxGasHandlerMock{ - MinimumGasMove: 50000, - MinimumGasPrice: 200000000000, - GasProcessingDivisor: 100, - }, + TxGasHandler: txcachemocks.NewTxGasHandlerMock(), + Marshalizer: &marshal.GogoProtoMarshalizer{}, NumberOfShards: 1, }, ) diff --git a/testscommon/state/accountsAdapterStub.go b/testscommon/state/accountsAdapterStub.go index 034d1f6ae26..70be0b5cc95 100644 --- a/testscommon/state/accountsAdapterStub.go +++ b/testscommon/state/accountsAdapterStub.go @@ -11,6 +11,8 @@ import ( var errNotImplemented = errors.New("not implemented") +var _ state.AccountsAdapter = (*AccountsStub)(nil) + // AccountsStub - type AccountsStub struct { GetExistingAccountCalled func(addressContainer []byte) (vmcommon.AccountHandler, error) diff --git a/testscommon/state/userAccountStub.go b/testscommon/state/userAccountStub.go index ce54f059252..02192ccd8ee 100644 --- a/testscommon/state/userAccountStub.go +++ b/testscommon/state/userAccountStub.go @@ -22,6 +22,7 @@ type UserAccountStub struct { Address []byte CodeMetadata []byte CodeHash []byte + Nonce uint64 AddToBalanceCalled func(value *big.Int) error DataTrieTrackerCalled func() state.DataTrieTracker @@ -106,7 +107,7 @@ func (u *UserAccountStub) IncreaseNonce(_ uint64) { // GetNonce - func (u *UserAccountStub) GetNonce() uint64 { - return 0 + return u.Nonce } // SetCode - diff --git a/testscommon/txProcessorMock.go b/testscommon/txProcessorMock.go index c8c47dbf89e..7b39ef87d13 100644 --- a/testscommon/txProcessorMock.go +++ b/testscommon/txProcessorMock.go @@ -5,6 +5,7 @@ import ( "github.com/multiversx/mx-chain-core-go/data/smartContractResult" "github.com/multiversx/mx-chain-core-go/data/transaction" + "github.com/multiversx/mx-chain-go/state" vmcommon "github.com/multiversx/mx-chain-vm-common-go" ) @@ -12,6 +13,7 @@ import ( type TxProcessorMock struct { ProcessTransactionCalled func(transaction *transaction.Transaction) (vmcommon.ReturnCode, error) VerifyTransactionCalled func(tx *transaction.Transaction) error + VerifyGuardianCalled func(tx *transaction.Transaction, account state.UserAccountHandler) error SetBalancesToTrieCalled func(accBalance map[string]*big.Int) (rootHash []byte, err error) ProcessSmartContractResultCalled func(scr *smartContractResult.SmartContractResult) (vmcommon.ReturnCode, error) } @@ -34,6 +36,15 @@ func (etm *TxProcessorMock) VerifyTransaction(tx *transaction.Transaction) error return nil } +// VerifyGuardian - +func (etm *TxProcessorMock) VerifyGuardian(tx *transaction.Transaction, account state.UserAccountHandler) error { + if etm.VerifyGuardianCalled != nil { + return etm.VerifyGuardianCalled(tx, account) + } + + return nil +} + // SetBalancesToTrie - func (etm *TxProcessorMock) SetBalancesToTrie(accBalance map[string]*big.Int) (rootHash []byte, err error) { if etm.SetBalancesToTrieCalled != nil { diff --git a/testscommon/txProcessorStub.go b/testscommon/txProcessorStub.go index 1070fd21d74..7407800ef6f 100644 --- a/testscommon/txProcessorStub.go +++ b/testscommon/txProcessorStub.go @@ -2,6 +2,7 @@ package testscommon import ( "github.com/multiversx/mx-chain-core-go/data/transaction" + "github.com/multiversx/mx-chain-go/state" vmcommon "github.com/multiversx/mx-chain-vm-common-go" ) @@ -9,6 +10,7 @@ import ( type TxProcessorStub struct { ProcessTransactionCalled func(transaction *transaction.Transaction) (vmcommon.ReturnCode, error) VerifyTransactionCalled func(tx *transaction.Transaction) error + VerifyGuardianCalled func(tx *transaction.Transaction, account state.UserAccountHandler) error } // ProcessTransaction - @@ -29,6 +31,15 @@ func (tps *TxProcessorStub) VerifyTransaction(tx *transaction.Transaction) error return nil } +// VerifyGuardian - +func (tps *TxProcessorStub) VerifyGuardian(tx *transaction.Transaction, account state.UserAccountHandler) error { + if tps.VerifyGuardianCalled != nil { + return tps.VerifyGuardianCalled(tx, account) + } + + return nil +} + // IsInterfaceNil - func (tps *TxProcessorStub) IsInterfaceNil() bool { return tps == nil diff --git a/testscommon/txcachemocks/mempoolHostMock.go b/testscommon/txcachemocks/mempoolHostMock.go new file mode 100644 index 00000000000..3e9ecc995ec --- /dev/null +++ b/testscommon/txcachemocks/mempoolHostMock.go @@ -0,0 +1,41 @@ +package txcachemocks + +import ( + "math/big" + + "github.com/multiversx/mx-chain-core-go/data" +) + +// MempoolHostMock - +type MempoolHostMock struct { + ComputeTxFeeCalled func(tx data.TransactionWithFeeHandler) *big.Int + GetTransferredValueCalled func(tx data.TransactionHandler) *big.Int +} + +// NewMempoolHostMock - +func NewMempoolHostMock() *MempoolHostMock { + return &MempoolHostMock{} +} + +// ComputeTxFee - +func (mock *MempoolHostMock) ComputeTxFee(tx data.TransactionWithFeeHandler) *big.Int { + if mock.ComputeTxFeeCalled != nil { + return mock.ComputeTxFeeCalled(tx) + } + + return big.NewInt(0) +} + +// GetTransferredValue - +func (mock *MempoolHostMock) GetTransferredValue(tx data.TransactionHandler) *big.Int { + if mock.GetTransferredValueCalled != nil { + return mock.GetTransferredValueCalled(tx) + } + + return tx.GetValue() +} + +// IsInterfaceNil - +func (mock *MempoolHostMock) IsInterfaceNil() bool { + return mock == nil +} diff --git a/testscommon/txcachemocks/txCacheMock.go b/testscommon/txcachemocks/txCacheMock.go new file mode 100644 index 00000000000..c34db2d53b0 --- /dev/null +++ b/testscommon/txcachemocks/txCacheMock.go @@ -0,0 +1,220 @@ +package txcachemocks + +import "github.com/multiversx/mx-chain-storage-go/txcache" + +// TxCacheMock - +type TxCacheMock struct { + ClearCalled func() + PutCalled func(key []byte, value interface{}, sizeInBytes int) (evicted bool) + GetCalled func(key []byte) (value interface{}, ok bool) + HasCalled func(key []byte) bool + PeekCalled func(key []byte) (value interface{}, ok bool) + HasOrAddCalled func(key []byte, value interface{}, sizeInBytes int) (has, added bool) + RemoveCalled func(key []byte) + RemoveOldestCalled func() + KeysCalled func() [][]byte + LenCalled func() int + MaxSizeCalled func() int + RegisterHandlerCalled func(func(key []byte, value interface{})) + UnRegisterHandlerCalled func(id string) + CloseCalled func() error + + AddTxCalled func(tx *txcache.WrappedTransaction) (ok bool, added bool) + GetByTxHashCalled func(txHash []byte) (*txcache.WrappedTransaction, bool) + RemoveTxByHashCalled func(txHash []byte) bool + ImmunizeTxsAgainstEvictionCalled func(keys [][]byte) + ForEachTransactionCalled func(txcache.ForEachTransaction) + NumBytesCalled func() int + DiagnoseCalled func(deep bool) + GetTransactionsPoolForSenderCalled func(sender string) []*txcache.WrappedTransaction +} + +// NewTxCacheStub - +func NewTxCacheStub() *TxCacheMock { + return &TxCacheMock{} +} + +// Clear - +func (cache *TxCacheMock) Clear() { + if cache.ClearCalled != nil { + cache.ClearCalled() + } +} + +// Put - +func (cache *TxCacheMock) Put(key []byte, value interface{}, sizeInBytes int) (evicted bool) { + if cache.PutCalled != nil { + return cache.PutCalled(key, value, sizeInBytes) + } + + return false +} + +// Get - +func (cache *TxCacheMock) Get(key []byte) (value interface{}, ok bool) { + if cache.GetCalled != nil { + return cache.GetCalled(key) + } + + return nil, false +} + +// Has - +func (cache *TxCacheMock) Has(key []byte) bool { + if cache.HasCalled != nil { + return cache.HasCalled(key) + } + + return false +} + +// Peek - +func (cache *TxCacheMock) Peek(key []byte) (value interface{}, ok bool) { + if cache.PeekCalled != nil { + return cache.PeekCalled(key) + } + + return nil, false +} + +// HasOrAdd - +func (cache *TxCacheMock) HasOrAdd(key []byte, value interface{}, sizeInBytes int) (has, added bool) { + if cache.HasOrAddCalled != nil { + return cache.HasOrAddCalled(key, value, sizeInBytes) + } + + return false, false +} + +// Remove - +func (cache *TxCacheMock) Remove(key []byte) { + if cache.RemoveCalled != nil { + cache.RemoveCalled(key) + } +} + +// Keys - +func (cache *TxCacheMock) Keys() [][]byte { + if cache.KeysCalled != nil { + return cache.KeysCalled() + } + + return make([][]byte, 0) +} + +// Len - +func (cache *TxCacheMock) Len() int { + if cache.LenCalled != nil { + return cache.LenCalled() + } + + return 0 +} + +// SizeInBytesContained - +func (cache *TxCacheMock) SizeInBytesContained() uint64 { + return 0 +} + +// MaxSize - +func (cache *TxCacheMock) MaxSize() int { + if cache.MaxSizeCalled != nil { + return cache.MaxSizeCalled() + } + + return 0 +} + +// RegisterHandler - +func (cache *TxCacheMock) RegisterHandler(handler func(key []byte, value interface{}), _ string) { + if cache.RegisterHandlerCalled != nil { + cache.RegisterHandlerCalled(handler) + } +} + +// UnRegisterHandler - +func (cache *TxCacheMock) UnRegisterHandler(id string) { + if cache.UnRegisterHandlerCalled != nil { + cache.UnRegisterHandlerCalled(id) + } +} + +// Close - +func (cache *TxCacheMock) Close() error { + if cache.CloseCalled != nil { + return cache.CloseCalled() + } + + return nil +} + +// AddTx - +func (cache *TxCacheMock) AddTx(tx *txcache.WrappedTransaction) (ok bool, added bool) { + if cache.AddTxCalled != nil { + return cache.AddTxCalled(tx) + } + + return false, false +} + +// GetByTxHash - +func (cache *TxCacheMock) GetByTxHash(txHash []byte) (*txcache.WrappedTransaction, bool) { + if cache.GetByTxHashCalled != nil { + return cache.GetByTxHashCalled(txHash) + } + + return nil, false +} + +// RemoveTxByHash - +func (cache *TxCacheMock) RemoveTxByHash(txHash []byte) bool { + if cache.RemoveTxByHashCalled != nil { + return cache.RemoveTxByHashCalled(txHash) + } + + return false +} + +// ImmunizeTxsAgainstEviction - +func (cache *TxCacheMock) ImmunizeTxsAgainstEviction(keys [][]byte) { + if cache.ImmunizeTxsAgainstEvictionCalled != nil { + cache.ImmunizeTxsAgainstEvictionCalled(keys) + } +} + +// ForEachTransaction - +func (cache *TxCacheMock) ForEachTransaction(fn txcache.ForEachTransaction) { + if cache.ForEachTransactionCalled != nil { + cache.ForEachTransactionCalled(fn) + } +} + +// NumBytes - +func (cache *TxCacheMock) NumBytes() int { + if cache.NumBytesCalled != nil { + return cache.NumBytesCalled() + } + + return 0 +} + +// Diagnose - +func (cache *TxCacheMock) Diagnose(deep bool) { + if cache.DiagnoseCalled != nil { + cache.DiagnoseCalled(deep) + } +} + +// GetTransactionsPoolForSender - +func (cache *TxCacheMock) GetTransactionsPoolForSender(sender string) []*txcache.WrappedTransaction { + if cache.GetTransactionsPoolForSenderCalled != nil { + return cache.GetTransactionsPoolForSenderCalled(sender) + } + + return nil +} + +// IsInterfaceNil - +func (cache *TxCacheMock) IsInterfaceNil() bool { + return cache == nil +} diff --git a/testscommon/txcachemocks/txGasHandlerMock.go b/testscommon/txcachemocks/txGasHandlerMock.go index f9da5dfad09..a624e29372a 100644 --- a/testscommon/txcachemocks/txGasHandlerMock.go +++ b/testscommon/txcachemocks/txGasHandlerMock.go @@ -1,56 +1,84 @@ package txcachemocks import ( + "math/big" + + "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/data" ) // TxGasHandler - type TxGasHandler interface { - SplitTxGasInCategories(tx data.TransactionWithFeeHandler) (uint64, uint64) - GasPriceForProcessing(tx data.TransactionWithFeeHandler) uint64 - GasPriceForMove(tx data.TransactionWithFeeHandler) uint64 MinGasPrice() uint64 - MinGasLimit() uint64 - MinGasPriceForProcessing() uint64 + MaxGasLimitPerTx() uint64 + ComputeTxFee(tx data.TransactionWithFeeHandler) *big.Int IsInterfaceNil() bool } // TxGasHandlerMock - type TxGasHandlerMock struct { - MinimumGasMove uint64 - MinimumGasPrice uint64 - GasProcessingDivisor uint64 + minGasLimit uint64 + minGasPrice uint64 + maxGasLimitPerTx uint64 + gasPerDataByte uint64 + gasPriceModifier float64 } -// SplitTxGasInCategories - -func (ghm *TxGasHandlerMock) SplitTxGasInCategories(tx data.TransactionWithFeeHandler) (uint64, uint64) { - moveGas := ghm.MinimumGasMove - return moveGas, tx.GetGasLimit() - moveGas +// NewTxGasHandlerMock - +func NewTxGasHandlerMock() *TxGasHandlerMock { + return &TxGasHandlerMock{ + minGasLimit: 50000, + minGasPrice: 1000000000, + maxGasLimitPerTx: 600000000, + gasPerDataByte: 1500, + gasPriceModifier: 0.01, + } } -// GasPriceForProcessing - -func (ghm *TxGasHandlerMock) GasPriceForProcessing(tx data.TransactionWithFeeHandler) uint64 { - return tx.GetGasPrice() / ghm.GasProcessingDivisor +// WithMinGasLimit - +func (ghm *TxGasHandlerMock) WithMinGasLimit(minGasLimit uint64) *TxGasHandlerMock { + ghm.minGasLimit = minGasLimit + return ghm } -// GasPriceForMove - -func (ghm *TxGasHandlerMock) GasPriceForMove(tx data.TransactionWithFeeHandler) uint64 { - return tx.GetGasPrice() +// WithMinGasPrice - +func (ghm *TxGasHandlerMock) WithMinGasPrice(minGasPrice uint64) *TxGasHandlerMock { + ghm.minGasPrice = minGasPrice + return ghm } // MinGasPrice - func (ghm *TxGasHandlerMock) MinGasPrice() uint64 { - return ghm.MinimumGasPrice + return ghm.minGasPrice +} + +// WithGasPriceModifier - +func (ghm *TxGasHandlerMock) WithGasPriceModifier(gasPriceModifier float64) *TxGasHandlerMock { + ghm.gasPriceModifier = gasPriceModifier + return ghm } -// MinGasLimit - -func (ghm *TxGasHandlerMock) MinGasLimit() uint64 { - return ghm.MinimumGasMove +// MaxGasLimitPerTx - +func (ghm *TxGasHandlerMock) MaxGasLimitPerTx() uint64 { + return ghm.maxGasLimitPerTx } -// MinGasPriceForProcessing - -func (ghm *TxGasHandlerMock) MinGasPriceForProcessing() uint64 { - return ghm.MinimumGasPrice / ghm.GasProcessingDivisor +// ComputeTxFee - +func (ghm *TxGasHandlerMock) ComputeTxFee(tx data.TransactionWithFeeHandler) *big.Int { + dataLength := uint64(len(tx.GetData())) + gasPriceForMovement := tx.GetGasPrice() + gasPriceForProcessing := uint64(float64(gasPriceForMovement) * ghm.gasPriceModifier) + + gasLimitForMovement := ghm.minGasLimit + dataLength*ghm.gasPerDataByte + if tx.GetGasLimit() < gasLimitForMovement { + return big.NewInt(0) + } + + gasLimitForProcessing := tx.GetGasLimit() - gasLimitForMovement + feeForMovement := core.SafeMul(gasPriceForMovement, gasLimitForMovement) + feeForProcessing := core.SafeMul(gasPriceForProcessing, gasLimitForProcessing) + fee := big.NewInt(0).Add(feeForMovement, feeForProcessing) + return fee } // IsInterfaceNil - diff --git a/trie/patriciaMerkleTrie.go b/trie/patriciaMerkleTrie.go index 70df549011f..da9eb87a65f 100644 --- a/trie/patriciaMerkleTrie.go +++ b/trie/patriciaMerkleTrie.go @@ -11,13 +11,14 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" + logger "github.com/multiversx/mx-chain-logger-go" + vmcommon "github.com/multiversx/mx-chain-vm-common-go" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/errors" "github.com/multiversx/mx-chain-go/trie/keyBuilder" "github.com/multiversx/mx-chain-go/trie/statistics" - logger "github.com/multiversx/mx-chain-logger-go" - vmcommon "github.com/multiversx/mx-chain-vm-common-go" ) var log = logger.GetOrCreate("trie") @@ -118,10 +119,7 @@ func (tr *patriciaMerkleTrie) Update(key, value []byte) error { tr.mutOperation.Lock() defer tr.mutOperation.Unlock() - log.Trace("update trie", - "key", hex.EncodeToString(key), - "val", hex.EncodeToString(value), - ) + log.Trace("update trie", "key", key, "val", value) return tr.update(key, value, core.NotSpecified) } @@ -131,11 +129,7 @@ func (tr *patriciaMerkleTrie) UpdateWithVersion(key []byte, value []byte, versio tr.mutOperation.Lock() defer tr.mutOperation.Unlock() - log.Trace("update trie with version", - "key", hex.EncodeToString(key), - "val", hex.EncodeToString(value), - "version", version, - ) + log.Trace("update trie with version", "key", key, "val", value, "version", version) return tr.update(key, value, version) } diff --git a/trie/patriciaMerkleTrie_test.go b/trie/patriciaMerkleTrie_test.go index 4612b8ee331..8a02a8edcd9 100644 --- a/trie/patriciaMerkleTrie_test.go +++ b/trie/patriciaMerkleTrie_test.go @@ -17,6 +17,10 @@ import ( "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/hashing/keccak" "github.com/multiversx/mx-chain-core-go/marshal" + vmcommon "github.com/multiversx/mx-chain-vm-common-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/common/errChan" "github.com/multiversx/mx-chain-go/common/holders" @@ -28,9 +32,6 @@ import ( "github.com/multiversx/mx-chain-go/trie" "github.com/multiversx/mx-chain-go/trie/keyBuilder" "github.com/multiversx/mx-chain-go/trie/mock" - vmcommon "github.com/multiversx/mx-chain-vm-common-go" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) var emptyTrieHash = make([]byte, 32) @@ -1708,3 +1709,28 @@ func BenchmarkPatriciaMerkleTrie_RootHashAfterChanging30000NodesInBatchesOf200(b } } } + +func BenchmarkPatriciaMerkleTrie_Update(b *testing.B) { + tr := emptyTrie() + hsh := keccak.NewKeccak() + + nrValuesInTrie := 2000000 + values := make([][]byte, nrValuesInTrie) + + for i := 0; i < nrValuesInTrie; i++ { + key := hsh.Compute(strconv.Itoa(i)) + value := append(key, []byte(strconv.Itoa(i))...) + + _ = tr.Update(key, value) + values[i] = key + } + _ = tr.Commit() + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + for j := 0; j < nrValuesInTrie; j++ { + _ = tr.Update(values[j], values[j]) + } + } +}