From 610dff1ba07f17b339d789474e89529707d98ca3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Wed, 8 Nov 2023 20:35:02 +0200 Subject: [PATCH 01/75] Bump version. --- version/constants.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version/constants.go b/version/constants.go index 7148031a..67526d4b 100644 --- a/version/constants.go +++ b/version/constants.go @@ -7,5 +7,5 @@ const ( var ( // RosettaMiddlewareVersion is the version of this package (application) - RosettaMiddlewareVersion = "v0.4.4" + RosettaMiddlewareVersion = "v0.5.0" ) From 5db6aa47559eb6f52d272aabaeb19d37c46be4ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Thu, 9 Nov 2023 16:46:56 +0200 Subject: [PATCH 02/75] Handle contracts (initial implementation). --- server/services/constants.go | 1 + server/services/networkProviderExtension.go | 3 +- .../services/transactionsFeaturesDetector.go | 36 +++++++++++++++ server/services/transactionsTransformer.go | 44 ++++++++++++++++++- .../services/transactionsTransformer_test.go | 4 +- 5 files changed, 82 insertions(+), 6 deletions(-) diff --git a/server/services/constants.go b/server/services/constants.go index 16ff804c..3cf34392 100644 --- a/server/services/constants.go +++ b/server/services/constants.go @@ -12,6 +12,7 @@ var ( transactionProcessingTypeRelayed = "RelayedTx" transactionProcessingTypeBuiltInFunctionCall = "BuiltInFunctionCall" transactionProcessingTypeMoveBalance = "MoveBalance" + transactionProcessingTypeContractInvoking = "SCInvoking" amountZero = "0" builtInFunctionClaimDeveloperRewards = core.BuiltInFunctionClaimDeveloperRewards builtInFunctionESDTTransfer = core.BuiltInFunctionESDTTransfer diff --git a/server/services/networkProviderExtension.go b/server/services/networkProviderExtension.go index 842a0652..7722d69b 100644 --- a/server/services/networkProviderExtension.go +++ b/server/services/networkProviderExtension.go @@ -67,8 +67,7 @@ func (extension *networkProviderExtension) isAddressObserved(address string) (bo return false, err } - isUserAddress := extension.isUserAddress(address) - return belongsToObservedShard && isUserAddress, nil + return belongsToObservedShard, nil } func (extension *networkProviderExtension) isUserAddress(address string) bool { diff --git a/server/services/transactionsFeaturesDetector.go b/server/services/transactionsFeaturesDetector.go index f22549d3..867da83e 100644 --- a/server/services/transactionsFeaturesDetector.go +++ b/server/services/transactionsFeaturesDetector.go @@ -74,3 +74,39 @@ func (detector *transactionsFeaturesDetector) isRelayedTransactionCompletelyIntr isWithSignalError := detector.eventsController.hasAnySignalError(tx) return isWithSignalError } + +func (detector *transactionsFeaturesDetector) isIntrashardContractCallWithSignalErrorButWithoutContractResultBearingRefundValue( + txInQuestion *transaction.ApiTransactionResult, + allTransactionsInBlock []*transaction.ApiTransactionResult, +) bool { + if !detector.isContractCallWithSignalError(txInQuestion) { + return false + } + + if !detector.isIntrashard(txInQuestion) { + return false + } + + for _, tx := range allTransactionsInBlock { + matchesTypeOnSource := tx.ProcessingTypeOnSource == transactionProcessingTypeMoveBalance + matchesTypeOnDestination := tx.ProcessingTypeOnDestination == transactionProcessingTypeMoveBalance + matchesOriginalTransactionHash := tx.OriginalTransactionHash == txInQuestion.Hash + matchesRefundValue := tx.Value == txInQuestion.Value + + if matchesTypeOnSource && matchesTypeOnDestination && matchesOriginalTransactionHash && matchesRefundValue { + return false + } + } + + return true +} + +func (detector *transactionsFeaturesDetector) isContractCallWithSignalError(tx *transaction.ApiTransactionResult) bool { + return tx.ProcessingTypeOnSource == transactionProcessingTypeContractInvoking && + tx.ProcessingTypeOnDestination == transactionProcessingTypeContractInvoking && + detector.eventsController.hasAnySignalError(tx) +} + +func (detector *transactionsFeaturesDetector) isIntrashard(tx *transaction.ApiTransactionResult) bool { + return tx.SourceShard == tx.DestinationShard +} diff --git a/server/services/transactionsTransformer.go b/server/services/transactionsTransformer.go index caea5468..ff8483fb 100644 --- a/server/services/transactionsTransformer.go +++ b/server/services/transactionsTransformer.go @@ -88,7 +88,7 @@ func (transformer *transactionsTransformer) txToRosettaTx(tx *transaction.ApiTra switch tx.Type { case string(transaction.TxTypeNormal): - rosettaTx, err = transformer.normalTxToRosetta(tx) + rosettaTx, err = transformer.normalTxToRosetta(tx, txsInBlock) if err != nil { return nil, err } @@ -115,6 +115,15 @@ func (transformer *transactionsTransformer) unsignedTxToRosettaTx( txsInBlock []*transaction.ApiTransactionResult, ) *types.Transaction { if scr.IsRefund { + if scr.Sender == scr.Receiver && !transformer.extension.isUserAddress(scr.Sender) { + log.Info("unsignedTxToRosettaTx: dismissed refund", "hash", scr.Hash, "originalTxHash", scr.OriginalTransactionHash) + + return &types.Transaction{ + TransactionIdentifier: hashToTransactionIdentifier(scr.Hash), + Operations: []*types.Operation{}, + } + } + return &types.Transaction{ TransactionIdentifier: hashToTransactionIdentifier(scr.Hash), Operations: []*types.Operation{ @@ -171,7 +180,10 @@ func (transformer *transactionsTransformer) rewardTxToRosettaTx(tx *transaction. } } -func (transformer *transactionsTransformer) normalTxToRosetta(tx *transaction.ApiTransactionResult) (*types.Transaction, error) { +func (transformer *transactionsTransformer) normalTxToRosetta( + tx *transaction.ApiTransactionResult, + allTransactionsInBlock []*transaction.ApiTransactionResult, +) (*types.Transaction, error) { hasValue := !isZeroAmount(tx.Value) operations := make([]*types.Operation, 0) @@ -200,7 +212,13 @@ func (transformer *transactionsTransformer) normalTxToRosetta(tx *transaction.Ap return nil, err } + valueRefundOperationIfContractCallWithSignalError, err := transformer.createValueReturnOperationsIfIntrashardContractCallWithSignalError(tx, allTransactionsInBlock) + if err != nil { + return nil, err + } + operations = append(operations, innerTxOperationsIfRelayedCompletelyIntrashardWithSignalError...) + operations = append(operations, valueRefundOperationIfContractCallWithSignalError...) return &types.Transaction{ TransactionIdentifier: hashToTransactionIdentifier(tx.Hash), @@ -239,6 +257,28 @@ func (transformer *transactionsTransformer) extractInnerTxOperationsIfRelayedCom }, nil } +func (transformer *transactionsTransformer) createValueReturnOperationsIfIntrashardContractCallWithSignalError( + tx *transaction.ApiTransactionResult, + allTransactionsInBlock []*transaction.ApiTransactionResult, +) ([]*types.Operation, error) { + if !transformer.featuresDetector.isIntrashardContractCallWithSignalErrorButWithoutContractResultBearingRefundValue(tx, allTransactionsInBlock) { + return []*types.Operation{}, nil + } + + return []*types.Operation{ + { + Type: opTransfer, + Account: addressToAccountIdentifier(tx.Sender), + Amount: transformer.extension.valueToNativeAmount(tx.Value), + }, + { + Type: opTransfer, + Account: addressToAccountIdentifier(tx.Receiver), + Amount: transformer.extension.valueToNativeAmount("-" + tx.Value), + }, + }, nil +} + func (transformer *transactionsTransformer) refundReceiptToRosettaTx(receipt *transaction.ApiReceipt) (*types.Transaction, error) { receiptHash, err := transformer.provider.ComputeReceiptHash(receipt) if err != nil { diff --git a/server/services/transactionsTransformer_test.go b/server/services/transactionsTransformer_test.go index 6995b944..f2b3197c 100644 --- a/server/services/transactionsTransformer_test.go +++ b/server/services/transactionsTransformer_test.go @@ -50,7 +50,7 @@ func TestTransactionsTransformer_NormalTxToRosettaTx(t *testing.T) { Metadata: extractTransactionMetadata(tx), } - rosettaTx, err := transformer.normalTxToRosetta(tx) + rosettaTx, err := transformer.normalTxToRosetta(tx, nil) require.NoError(t, err) require.Equal(t, expectedRosettaTx, rosettaTx) }) @@ -94,7 +94,7 @@ func TestTransactionsTransformer_NormalTxToRosettaTx(t *testing.T) { Metadata: extractTransactionMetadata(tx), } - rosettaTx, err := transformer.normalTxToRosetta(tx) + rosettaTx, err := transformer.normalTxToRosetta(tx, nil) require.NoError(t, err) require.Equal(t, expectedRosettaTx, rosettaTx) }) From dc116d29204815fc7d343edef16204fde1bd2478 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Thu, 9 Nov 2023 22:11:14 +0200 Subject: [PATCH 03/75] Handle transfer value only (re-added, changed). Work in progress. --- server/services/constants.go | 1 + server/services/networkProviderExtension.go | 8 +++-- server/services/transactionEvents.go | 8 +++++ .../services/transactionEventsController.go | 31 +++++++++++++++++++ server/services/transactionsTransformer.go | 5 +++ 5 files changed, 51 insertions(+), 2 deletions(-) diff --git a/server/services/constants.go b/server/services/constants.go index 3cf34392..4a47cf9d 100644 --- a/server/services/constants.go +++ b/server/services/constants.go @@ -25,6 +25,7 @@ var ( var ( transactionEventSignalError = core.SignalErrorOperation + transactionEventTransferValueOnly = "transferValueOnly" transactionEventESDTTransfer = "ESDTTransfer" transactionEventESDTNFTTransfer = "ESDTNFTTransfer" transactionEventMultiESDTNFTTransfer = "MultiESDTNFTTransfer" diff --git a/server/services/networkProviderExtension.go b/server/services/networkProviderExtension.go index 7722d69b..5d77f61d 100644 --- a/server/services/networkProviderExtension.go +++ b/server/services/networkProviderExtension.go @@ -71,11 +71,15 @@ func (extension *networkProviderExtension) isAddressObserved(address string) (bo } func (extension *networkProviderExtension) isUserAddress(address string) bool { - pubkey, err := extension.provider.ConvertAddressToPubKey(address) + pubKey, err := extension.provider.ConvertAddressToPubKey(address) if err != nil { // E.g., when address = "metachain" return false } - return !core.IsSmartContractAddress(pubkey) + return extension.isUserPubKey(pubKey) +} + +func (extension *networkProviderExtension) isUserPubKey(pubKey []byte) bool { + return !core.IsSmartContractAddress(pubKey) } diff --git a/server/services/transactionEvents.go b/server/services/transactionEvents.go index f43d076c..d2a0e370 100644 --- a/server/services/transactionEvents.go +++ b/server/services/transactionEvents.go @@ -4,6 +4,14 @@ import ( "fmt" ) +type eventTransferValueOnly struct { + sender string + senderPubKey []byte + receiver string + receiverPubKey []byte + value string +} + type eventESDT struct { senderAddress string receiverAddress string diff --git a/server/services/transactionEventsController.go b/server/services/transactionEventsController.go index 9d344658..8f091ae9 100644 --- a/server/services/transactionEventsController.go +++ b/server/services/transactionEventsController.go @@ -18,6 +18,37 @@ func newTransactionEventsController(provider NetworkProvider) *transactionEvents } } +func (controller *transactionEventsController) extractEventTransferValueOnly(tx *transaction.ApiTransactionResult) ([]*eventTransferValueOnly, error) { + rawEvents := controller.findManyEventsByIdentifier(tx, transactionEventTransferValueOnly) + + typedEvents := make([]*eventTransferValueOnly, 0) + + for _, event := range rawEvents { + numTopics := len(event.Topics) + if numTopics != 3 { + return nil, fmt.Errorf("%w: bad number of topics for 'transferValueOnly' = %d", errCannotRecognizeEvent, numTopics) + } + + senderPubKey := event.Topics[0] + receiverPubKey := event.Topics[1] + valueBytes := event.Topics[2] + + sender := controller.provider.ConvertPubKeyToAddress(senderPubKey) + receiver := controller.provider.ConvertPubKeyToAddress(receiverPubKey) + value := big.NewInt(0).SetBytes(valueBytes) + + typedEvents = append(typedEvents, &eventTransferValueOnly{ + sender: sender, + senderPubKey: senderPubKey, + receiver: receiver, + receiverPubKey: receiverPubKey, + value: value.String(), + }) + } + + return typedEvents, nil +} + func (controller *transactionEventsController) hasAnySignalError(tx *transaction.ApiTransactionResult) bool { if !controller.hasEvents(tx) { return false diff --git a/server/services/transactionsTransformer.go b/server/services/transactionsTransformer.go index ff8483fb..c76709fe 100644 --- a/server/services/transactionsTransformer.go +++ b/server/services/transactionsTransformer.go @@ -361,6 +361,11 @@ func (transformer *transactionsTransformer) mempoolMoveBalanceTxToRosettaTx(tx * } func (transformer *transactionsTransformer) addOperationsGivenTransactionEvents(tx *transaction.ApiTransactionResult, rosettaTx *types.Transaction) error { + eventsTransferValueOnly, err := transformer.eventsController.extractEventTransferValueOnly(tx) + if err != nil { + return err + } + eventsESDTTransfer, err := transformer.eventsController.extractEventsESDTOrESDTNFTTransfers(tx) if err != nil { return err From 25b1f9d1019acd40a120a41398173563cfd081ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Fri, 10 Nov 2023 15:04:20 +0200 Subject: [PATCH 04/75] Handle exception regarding scheduled miniblocks. --- server/provider/scheduledMiniblocks.go | 11 +++ .../scheduledMiniblocksCrossShardException.go | 82 +++++++++++++++++++ 2 files changed, 93 insertions(+) create mode 100644 server/provider/scheduledMiniblocksCrossShardException.go diff --git a/server/provider/scheduledMiniblocks.go b/server/provider/scheduledMiniblocks.go index 3691e1ed..eaa36279 100644 --- a/server/provider/scheduledMiniblocks.go +++ b/server/provider/scheduledMiniblocks.go @@ -17,16 +17,27 @@ func (provider *networkProvider) simplifyBlockWithScheduledTransactions(block *a return err } + reportScheduledMiniblock(block) doSimplifyBlockWithScheduledTransactions(previousBlock, block, nextBlock) deduplicatePreviouslyAppearingContractResultsInReceipts(previousBlock, block) return nil } +func reportScheduledMiniblock(block *api.Block) { + for _, miniblock := range block.MiniBlocks { + if miniblock.ProcessingType == dataBlock.Scheduled.String() { + log.Info("reportScheduledMiniblock()", "block", block.Nonce, "miniblock", miniblock) + } + } +} + func doSimplifyBlockWithScheduledTransactions(previousBlock *api.Block, block *api.Block, nextBlock *api.Block) { // Discard "processed" miniblocks in block N, since they already produced effects in N-1 removeProcessedMiniblocksOfBlock(block) + handleContractResultsHavingOriginalTransactionInCrossShardScheduledMiniblockInPreviousBlock(previousBlock, block, nextBlock) + // Move "processed" miniblocks from N+1 to N processedMiniblocksInNextBlock := findProcessedMiniblocks(nextBlock) appendMiniblocksToBlock(block, processedMiniblocksInNextBlock) diff --git a/server/provider/scheduledMiniblocksCrossShardException.go b/server/provider/scheduledMiniblocksCrossShardException.go new file mode 100644 index 00000000..1b46621c --- /dev/null +++ b/server/provider/scheduledMiniblocksCrossShardException.go @@ -0,0 +1,82 @@ +package provider + +import ( + "github.com/multiversx/mx-chain-core-go/data/api" + dataBlock "github.com/multiversx/mx-chain-core-go/data/block" + "github.com/multiversx/mx-chain-core-go/data/transaction" +) + +func handleContractResultsHavingOriginalTransactionInCrossShardScheduledMiniblockInPreviousBlock(previousBlock *api.Block, block *api.Block, nextBlock *api.Block) { + // Gather transactions in cross-shard scheduled, final miniblocks. + // Afterwards, we will handle their contract results in two steps. + transactionsInCrossShardScheduledMiniblockInPreviousBlock := gatherTransactionsInCrossShardScheduledFinalMiniblock(previousBlock) + transactionsInCrossShardScheduledMiniblockInCurrentBlock := gatherTransactionsInCrossShardScheduledFinalMiniblock(block) + + // First, remove corresponding contract results from the current block (since they already produced their effects in the previous block) + for _, miniblock := range block.MiniBlocks { + if miniblock.Type != dataBlock.SmartContractResultBlock.String() { + continue + } + + contractResultsToKeep := make([]*transaction.ApiTransactionResult, 0, len(miniblock.Transactions)) + + for _, contractResult := range miniblock.Transactions { + _, ok := transactionsInCrossShardScheduledMiniblockInPreviousBlock[contractResult.OriginalTransactionHash] + if ok { + // Discard this contract result + continue + } + + contractResultsToKeep = append(contractResultsToKeep, contractResult) + } + + miniblock.Transactions = contractResultsToKeep + } + + // Secondly, bring here (in the current block) the corresponding contract results of the next block (since they produced their effects in the current block) + contractResultsToMove := make([]*transaction.ApiTransactionResult, 0, len(nextBlock.MiniBlocks)) + + for _, miniblockInNextBlock := range nextBlock.MiniBlocks { + if miniblockInNextBlock.Type != dataBlock.SmartContractResultBlock.String() { + continue + } + + for _, contractResultInNextBlock := range miniblockInNextBlock.Transactions { + _, ok := transactionsInCrossShardScheduledMiniblockInCurrentBlock[contractResultInNextBlock.OriginalTransactionHash] + if ok { + contractResultsToMove = append(contractResultsToMove, contractResultInNextBlock) + } + } + } + + if len(contractResultsToMove) > 0 { + // Extremely rare case. + log.Info("handleContractResultsHavingOriginalTransactionInCrossShardScheduledMiniblockInPreviousBlock", "currentBlock", block.Nonce, "numContractResultsToMove", len(contractResultsToMove)) + + artificialMiniblock := &api.MiniBlock{ + Type: dataBlock.SmartContractResultBlock.String(), + Transactions: contractResultsToMove, + } + + block.MiniBlocks = append(block.MiniBlocks, artificialMiniblock) + } +} + +func gatherTransactionsInCrossShardScheduledFinalMiniblock(block *api.Block) map[string]struct{} { + gathered := make(map[string]struct{}) + + for _, miniblockInPreviousBlock := range block.MiniBlocks { + isScheduled := miniblockInPreviousBlock.ProcessingType == dataBlock.Scheduled.String() + isFinal := miniblockInPreviousBlock.ConstructionState == dataBlock.Final.String() + isCrossShard := miniblockInPreviousBlock.SourceShard != miniblockInPreviousBlock.DestinationShard + if !isScheduled || !isFinal || !isCrossShard { + continue + } + + for _, txInPreviousBlock := range miniblockInPreviousBlock.Transactions { + gathered[txInPreviousBlock.Hash] = struct{}{} + } + } + + return gathered +} From eef16156bed248f70bac81fb5eb39f1ea9ea77fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Sat, 11 Nov 2023 19:26:40 +0200 Subject: [PATCH 05/75] Fix extractInnerTxOperationsIfRelayedCompletelyIntrashardWithSignalError. Handle receiver, as well. --- server/services/transactionsTransformer.go | 6 ++++++ server/services/transactionsTransformer_test.go | 10 ++++++++++ 2 files changed, 16 insertions(+) diff --git a/server/services/transactionsTransformer.go b/server/services/transactionsTransformer.go index c76709fe..ed5e51db 100644 --- a/server/services/transactionsTransformer.go +++ b/server/services/transactionsTransformer.go @@ -247,6 +247,7 @@ func (transformer *transactionsTransformer) extractInnerTxOperationsIfRelayedCom } senderAddress := transformer.provider.ConvertPubKeyToAddress(innerTx.SenderPubKey) + receiverAddress := transformer.provider.ConvertPubKeyToAddress(innerTx.ReceiverPubKey) return []*types.Operation{ { @@ -254,6 +255,11 @@ func (transformer *transactionsTransformer) extractInnerTxOperationsIfRelayedCom Account: addressToAccountIdentifier(senderAddress), Amount: transformer.extension.valueToNativeAmount("-" + innerTx.Value.String()), }, + { + Type: opTransfer, + Account: addressToAccountIdentifier(receiverAddress), + Amount: transformer.extension.valueToNativeAmount(innerTx.Value.String()), + }, }, nil } diff --git a/server/services/transactionsTransformer_test.go b/server/services/transactionsTransformer_test.go index f2b3197c..abd188cd 100644 --- a/server/services/transactionsTransformer_test.go +++ b/server/services/transactionsTransformer_test.go @@ -90,6 +90,11 @@ func TestTransactionsTransformer_NormalTxToRosettaTx(t *testing.T) { Account: addressToAccountIdentifier(testscommon.TestUserCShard0.Address), Amount: extension.valueToNativeAmount("-1000000000000000000"), }, + { + Type: opTransfer, + Account: addressToAccountIdentifier(testscommon.TestAddressOfContract), + Amount: extension.valueToNativeAmount("1000000000000000000"), + }, }, Metadata: extractTransactionMetadata(tx), } @@ -153,6 +158,11 @@ func TestTransactionsTransformer_ExtractInnerTxOperationsIfRelayedCompletelyIntr Account: addressToAccountIdentifier(testscommon.TestUserCShard0.Address), Amount: extension.valueToNativeAmount("-1000000000000000000"), }, + { + Type: opTransfer, + Account: addressToAccountIdentifier(testscommon.TestAddressOfContract), + Amount: extension.valueToNativeAmount("1000000000000000000"), + }, }, operations) }) From 4025639bdef721330d3f1d2b0552f0a3a8c7e31e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Sat, 11 Nov 2023 22:25:52 +0200 Subject: [PATCH 06/75] Create deep copies of blocks, since they are cached & mutated. --- server/provider/networkProvider.go | 43 ++++++++++++++++++---- server/provider/networkProvider_test.go | 48 +++++++++++++++++++++++-- 2 files changed, 83 insertions(+), 8 deletions(-) diff --git a/server/provider/networkProvider.go b/server/provider/networkProvider.go index 78ae136f..583bd9c2 100644 --- a/server/provider/networkProvider.go +++ b/server/provider/networkProvider.go @@ -228,7 +228,7 @@ func (provider *networkProvider) doGetBlockByNonce(nonce uint64) (*api.Block, er block, ok := provider.getBlockByNonceCached(nonce) if ok { - return block, nil + return createBlockCopy(block), nil } response, err := provider.observerFacade.GetBlockByNonce(provider.observedActualShard, nonce, queryOptions) @@ -243,7 +243,40 @@ func (provider *networkProvider) doGetBlockByNonce(nonce uint64) (*api.Block, er provider.cacheBlockByNonce(nonce, block) - return block, nil + return createBlockCopy(block), nil +} + +func createBlockCopy(block *api.Block) *api.Block { + miniblocksCopy := make([]*api.MiniBlock, len(block.MiniBlocks)) + + for i, miniblock := range block.MiniBlocks { + miniblocksCopy[i] = &api.MiniBlock{ + Type: miniblock.Type, + Hash: miniblock.Hash, + ProcessingType: miniblock.ProcessingType, + ConstructionState: miniblock.ConstructionState, + IsFromReceiptsStorage: miniblock.IsFromReceiptsStorage, + SourceShard: miniblock.SourceShard, + DestinationShard: miniblock.DestinationShard, + // This is sufficient for our purposes (we don't mutate the transactions themselves, we only mutate the list of transactions within a miniblock). + Transactions: miniblock.Transactions, + Receipts: miniblock.Receipts, + } + } + + return &api.Block{ + Nonce: block.Nonce, + Round: block.Round, + Epoch: block.Epoch, + Shard: block.Shard, + NumTxs: block.NumTxs, + Hash: block.Hash, + PrevBlockHash: block.PrevBlockHash, + StateRootHash: block.StateRootHash, + Status: block.Status, + Timestamp: block.Timestamp, + MiniBlocks: miniblocksCopy, + } } func (provider *networkProvider) getBlockByNonceCached(nonce uint64) (*api.Block, bool) { @@ -251,8 +284,7 @@ func (provider *networkProvider) getBlockByNonceCached(nonce uint64) (*api.Block if ok { block, ok := blockUntyped.(*api.Block) if ok { - blockCopy := *block - return &blockCopy, true + return block, true } } @@ -260,8 +292,7 @@ func (provider *networkProvider) getBlockByNonceCached(nonce uint64) (*api.Block } func (provider *networkProvider) cacheBlockByNonce(nonce uint64, block *api.Block) { - blockCopy := *block - _ = provider.blocksCache.Put(blockNonceToBytes(nonce), &blockCopy, 1) + _ = provider.blocksCache.Put(blockNonceToBytes(nonce), block, 1) } // GetBlockByHash gets a block by hash diff --git a/server/provider/networkProvider_test.go b/server/provider/networkProvider_test.go index 54cec5ee..c124bd0f 100644 --- a/server/provider/networkProvider_test.go +++ b/server/provider/networkProvider_test.go @@ -93,7 +93,8 @@ func TestNetworkProvider_DoGetBlockByNonce(t *testing.T) { return &data.BlockApiResponse{ Data: data.BlockApiResponsePayload{ Block: api.Block{ - Nonce: 42, + Nonce: 42, + MiniBlocks: []*api.MiniBlock{}, }, }, }, nil @@ -137,7 +138,7 @@ func TestNetworkProvider_DoGetBlockByNonce(t *testing.T) { require.Equal(t, blocksCacheCapacity, provider.blocksCache.Len()) }) - t.Run("the cache holds block copies", func(t *testing.T) { + t.Run("the cache holds block copies (1)", func(t *testing.T) { provider.blocksCache.Clear() observerFacade.GetBlockByNonceCalled = func(shardID uint32, nonce uint64, options common.BlockQueryOptions) (*data.BlockApiResponse, error) { @@ -172,6 +173,49 @@ func TestNetworkProvider_DoGetBlockByNonce(t *testing.T) { require.False(t, &block == &cachedBlock) require.Equal(t, 1, provider.blocksCache.Len()) }) + + t.Run("the cache holds block copies (2)", func(t *testing.T) { + provider.blocksCache.Clear() + + observerFacade.GetBlockByNonceCalled = func(shardID uint32, nonce uint64, options common.BlockQueryOptions) (*data.BlockApiResponse, error) { + return &data.BlockApiResponse{ + Data: data.BlockApiResponsePayload{ + Block: api.Block{ + Nonce: nonce, + MiniBlocks: []*api.MiniBlock{ + { + Hash: "aaaa", + Transactions: []*transaction.ApiTransactionResult{ + {Hash: "cccc"}, + {Hash: "dddd"}, + }, + }, + {Hash: "bbbb"}, + }, + }, + }, + }, nil + } + + block, err := provider.doGetBlockByNonce(7) + require.Nil(t, err) + require.Equal(t, uint64(7), block.Nonce) + require.Len(t, block.MiniBlocks, 2) + require.Equal(t, 1, provider.blocksCache.Len()) + + // Simulate mutations performed by downstream handling of blocks, i.e. "simplifyBlockWithScheduledTransactions": + block.MiniBlocks[0].Transactions = []*transaction.ApiTransactionResult{ + {Hash: "aaaa"}, + } + + cachedBlock, err := provider.doGetBlockByNonce(7) + require.Nil(t, err) + require.Equal(t, uint64(7), cachedBlock.Nonce) + require.Len(t, cachedBlock.MiniBlocks, 2) + require.Len(t, cachedBlock.MiniBlocks[0].Transactions, 2) + require.False(t, &block == &cachedBlock) + require.Equal(t, 1, provider.blocksCache.Len()) + }) } func Test_ComputeShardIdOfPubKey(t *testing.T) { From e7b4eb43043ba12e41b2a442d1e0fa22fa7eaac9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Tue, 14 Nov 2023 00:01:35 +0200 Subject: [PATCH 07/75] Reimplement handling of scheduled transactions. --- server/provider/blocks.go | 21 -- server/provider/blocks_test.go | 42 ---- server/provider/scheduledMiniblocks.go | 228 ++++++++++-------- .../scheduledMiniblocksCrossShardException.go | 82 ------- server/provider/transactions.go | 48 ---- server/provider/transactions_test.go | 54 ----- 6 files changed, 123 insertions(+), 352 deletions(-) delete mode 100644 server/provider/blocks.go delete mode 100644 server/provider/blocks_test.go delete mode 100644 server/provider/scheduledMiniblocksCrossShardException.go delete mode 100644 server/provider/transactions.go delete mode 100644 server/provider/transactions_test.go diff --git a/server/provider/blocks.go b/server/provider/blocks.go deleted file mode 100644 index d0b8ff09..00000000 --- a/server/provider/blocks.go +++ /dev/null @@ -1,21 +0,0 @@ -package provider - -import "github.com/multiversx/mx-chain-core-go/data/api" - -func removeMiniblocksFromBlock(block *api.Block, shouldRemove func(miniblock *api.MiniBlock) bool) { - miniblocksToKeep := make([]*api.MiniBlock, 0, len(block.MiniBlocks)) - - for _, miniblock := range block.MiniBlocks { - if shouldRemove(miniblock) { - continue - } - - miniblocksToKeep = append(miniblocksToKeep, miniblock) - } - - block.MiniBlocks = miniblocksToKeep -} - -func appendMiniblocksToBlock(block *api.Block, miniblocks []*api.MiniBlock) { - block.MiniBlocks = append(block.MiniBlocks, miniblocks...) -} diff --git a/server/provider/blocks_test.go b/server/provider/blocks_test.go deleted file mode 100644 index f30652c0..00000000 --- a/server/provider/blocks_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package provider - -import ( - "testing" - - "github.com/multiversx/mx-chain-core-go/data/api" - "github.com/stretchr/testify/require" -) - -func TestRemoveMiniblocksFromBlock(t *testing.T) { - block := &api.Block{ - MiniBlocks: []*api.MiniBlock{ - {Hash: "aaaa", SourceShard: 7}, - {Hash: "bbbb", SourceShard: 8}, - {Hash: "aabb", SourceShard: 7}, - {Hash: "abba", SourceShard: 8}, - }, - } - - shouldRemove := func(miniblock *api.MiniBlock) bool { - return miniblock.SourceShard == 8 - } - - removeMiniblocksFromBlock(block, shouldRemove) - require.Len(t, block.MiniBlocks, 2) - require.Equal(t, "aaaa", block.MiniBlocks[0].Hash) - require.Equal(t, "aabb", block.MiniBlocks[1].Hash) -} - -func TestAppendMiniblocksToBlock(t *testing.T) { - block := &api.Block{ - MiniBlocks: []*api.MiniBlock{ - {Hash: "aaaa"}, - {Hash: "bbbb"}, - }, - } - - appendMiniblocksToBlock(block, []*api.MiniBlock{{Hash: "abcd"}, {Hash: "dcba"}}) - require.Len(t, block.MiniBlocks, 4) - require.Equal(t, "abcd", block.MiniBlocks[2].Hash) - require.Equal(t, "dcba", block.MiniBlocks[3].Hash) -} diff --git a/server/provider/scheduledMiniblocks.go b/server/provider/scheduledMiniblocks.go index eaa36279..a8348c07 100644 --- a/server/provider/scheduledMiniblocks.go +++ b/server/provider/scheduledMiniblocks.go @@ -17,168 +17,186 @@ func (provider *networkProvider) simplifyBlockWithScheduledTransactions(block *a return err } - reportScheduledMiniblock(block) + reportScheduledTransactions(block) doSimplifyBlockWithScheduledTransactions(previousBlock, block, nextBlock) - deduplicatePreviouslyAppearingContractResultsInReceipts(previousBlock, block) return nil } -func reportScheduledMiniblock(block *api.Block) { +func reportScheduledTransactions(block *api.Block) { + numScheduled := 0 + numProcessed := 0 + numInvalid := 0 + for _, miniblock := range block.MiniBlocks { if miniblock.ProcessingType == dataBlock.Scheduled.String() { - log.Info("reportScheduledMiniblock()", "block", block.Nonce, "miniblock", miniblock) + numScheduled += len(miniblock.Transactions) + } else if miniblock.ProcessingType == dataBlock.Processed.String() { + numProcessed += len(miniblock.Transactions) + } else if miniblock.Type == dataBlock.InvalidBlock.String() { + numInvalid += len(miniblock.Transactions) } } + + if numScheduled > 0 || numProcessed > 0 { + log.Info("reportScheduledTransactions()", "scheduled", numScheduled, "processed", numProcessed, "invalid", numInvalid, "block", block.Nonce) + } } func doSimplifyBlockWithScheduledTransactions(previousBlock *api.Block, block *api.Block, nextBlock *api.Block) { - // Discard "processed" miniblocks in block N, since they already produced effects in N-1 - removeProcessedMiniblocksOfBlock(block) + txs := gatherEffectiveTransactions(block.Shard, previousBlock, block, nextBlock) + receipts := gatherAllReceipts(block) + + block.MiniBlocks = []*api.MiniBlock{ + { + Type: "Artificial", + Transactions: txs, + }, + { + Type: "Artificial", + Receipts: receipts, + }, + } +} - handleContractResultsHavingOriginalTransactionInCrossShardScheduledMiniblockInPreviousBlock(previousBlock, block, nextBlock) +func gatherEffectiveTransactions(selfShard uint32, previousBlock *api.Block, currentBlock *api.Block, nextBlock *api.Block) []*transaction.ApiTransactionResult { + txsInCurrentBlock := gatherAllTransactions(currentBlock) - // Move "processed" miniblocks from N+1 to N - processedMiniblocksInNextBlock := findProcessedMiniblocks(nextBlock) - appendMiniblocksToBlock(block, processedMiniblocksInNextBlock) + scheduledTxsInPreviousBlock := gatherScheduledTransactions(previousBlock) + scheduledTxsInCurrentBlock := gatherScheduledTransactions(currentBlock) - // Build an artificial miniblock holding the "invalid" transactions that produced their effects in block N, - // and replace the existing (one or two "invalid" miniblocks). - invalidTxs := gatherInvalidTransactions(previousBlock, block, nextBlock) - invalidMiniblock := &api.MiniBlock{ - Type: dataBlock.InvalidBlock.String(), - Transactions: invalidTxs, + if len(scheduledTxsInPreviousBlock) == 0 && len(scheduledTxsInCurrentBlock) == 0 { + return txsInCurrentBlock } - removeInvalidMiniblocks(block) - if len(invalidMiniblock.Transactions) > 0 { - appendMiniblocksToBlock(block, []*api.MiniBlock{invalidMiniblock}) + var previouslyExecutedResults []*transaction.ApiTransactionResult + var currentlyExecutedResults []*transaction.ApiTransactionResult + + if len(scheduledTxsInPreviousBlock) > 0 { + previouslyExecutedResults = findImmediatelyExecutingContractResults(selfShard, scheduledTxsInPreviousBlock, txsInCurrentBlock) + } + if len(scheduledTxsInCurrentBlock) > 0 { + txsInNextBlock := gatherAllTransactions(nextBlock) + currentlyExecutedResults = findImmediatelyExecutingContractResults(selfShard, scheduledTxsInCurrentBlock, txsInNextBlock) } - // Discard "scheduled" miniblocks of N, since we've already brought the "processed" ones from N+1, - // and also handled the "invalid" ones. - removeScheduledMiniblocks(block) -} + // effectiveTxs + // = txsInCurrentBlock + // - txsInPreviousBlock (excludes transactions in "processed" miniblocks, for example) + // - previouslyExecutedResults + // + currentlyExecutedResults -func removeProcessedMiniblocksOfBlock(block *api.Block) { - removeMiniblocksFromBlock(block, func(miniblock *api.MiniBlock) bool { - return miniblock.ProcessingType == dataBlock.Processed.String() - }) -} + effectiveTxs := make([]*transaction.ApiTransactionResult, 0) + effectiveTxsByHash := make(map[string]*transaction.ApiTransactionResult) -func removeScheduledMiniblocks(block *api.Block) { - removeMiniblocksFromBlock(block, func(miniblock *api.MiniBlock) bool { - hasProcessingTypeScheduled := miniblock.ProcessingType == dataBlock.Scheduled.String() - hasConstructionStateNotFinal := miniblock.ConstructionState != dataBlock.Final.String() - shouldRemove := hasProcessingTypeScheduled && hasConstructionStateNotFinal - return shouldRemove - }) -} + for _, tx := range txsInCurrentBlock { + effectiveTxsByHash[tx.Hash] = tx + } -func removeInvalidMiniblocks(block *api.Block) { - removeMiniblocksFromBlock(block, func(miniblock *api.MiniBlock) bool { - return miniblock.Type == dataBlock.InvalidBlock.String() - }) -} + if len(scheduledTxsInPreviousBlock) > 0 { + txsInPreviousBlock := gatherAllTransactions(previousBlock) -func gatherInvalidTransactions(previousBlock *api.Block, block *api.Block, nextBlock *api.Block) []*transaction.ApiTransactionResult { - // Find "invalid" transactions that are "final" in N - invalidTxsInBlock := findInvalidTransactions(block) - // If also present in N-1, discard them - scheduledTxsHashesPreviousBlock := findScheduledTransactionsHashes(previousBlock) - invalidTxsInBlock = discardTransactions(invalidTxsInBlock, scheduledTxsHashesPreviousBlock) + for _, tx := range txsInPreviousBlock { + delete(effectiveTxsByHash, tx.Hash) + } + + for _, tx := range previouslyExecutedResults { + delete(effectiveTxsByHash, tx.Hash) + } + } - // Find "invalid" transactions in N+1 that are "scheduled" in N - invalidTxsInNextBlock := findInvalidTransactions(nextBlock) - scheduledTxsHashesInBlock := findScheduledTransactionsHashes(block) - invalidTxsScheduledInBlock := filterTransactions(invalidTxsInNextBlock, scheduledTxsHashesInBlock) + if len(scheduledTxsInCurrentBlock) > 0 { + for _, tx := range currentlyExecutedResults { + effectiveTxsByHash[tx.Hash] = tx + } + } - // Duplication might occur, since a block can contain two "invalid" miniblocks, - // one added to block body, one saved in the receipts unit (at times, they have different content, different hashes). - invalidTxs := append(invalidTxsInBlock, invalidTxsScheduledInBlock...) - invalidTxs = deduplicateTransactions(invalidTxs) + for _, tx := range effectiveTxsByHash { + effectiveTxs = append(effectiveTxs, tx) + } - return invalidTxs + return effectiveTxs } -func findScheduledTransactionsHashes(block *api.Block) map[string]struct{} { - txs := make(map[string]struct{}) +func findImmediatelyExecutingContractResults( + selfShard uint32, + transactions []*transaction.ApiTransactionResult, + maybeContractResults []*transaction.ApiTransactionResult, +) []*transaction.ApiTransactionResult { + immediateleyExecutingContractResults := make([]*transaction.ApiTransactionResult, 0) + nextContractResultsByHash := make(map[string][]*transaction.ApiTransactionResult) - for _, miniblock := range block.MiniBlocks { - hasProcessingTypeScheduled := miniblock.ProcessingType == dataBlock.Scheduled.String() - hasConstructionStateNotFinal := miniblock.ConstructionState != dataBlock.Final.String() - shouldAccumulateTxs := hasProcessingTypeScheduled && hasConstructionStateNotFinal - - if shouldAccumulateTxs { - for _, tx := range miniblock.Transactions { - txs[tx.Hash] = struct{}{} - } - } + for _, item := range maybeContractResults { + nextContractResultsByHash[item.PreviousTransactionHash] = append(nextContractResultsByHash[item.PreviousTransactionHash], item) } - return txs + for _, tx := range transactions { + immediateleyExecutingContractResultsPart := findImmediatelyExecutingContractResultsOfTransaction(selfShard, tx, nextContractResultsByHash) + immediateleyExecutingContractResults = append(immediateleyExecutingContractResults, immediateleyExecutingContractResultsPart...) + } + + return immediateleyExecutingContractResults } -func findProcessedMiniblocks(block *api.Block) []*api.MiniBlock { - foundMiniblocks := make([]*api.MiniBlock, 0, len(block.MiniBlocks)) +func findImmediatelyExecutingContractResultsOfTransaction( + selfShard uint32, + tx *transaction.ApiTransactionResult, + nextContractResultsByHash map[string][]*transaction.ApiTransactionResult, +) []*transaction.ApiTransactionResult { + immediatelyExecutingContractResults := make([]*transaction.ApiTransactionResult, 0) - for _, miniblock := range block.MiniBlocks { - if miniblock.ProcessingType == dataBlock.Processed.String() { - foundMiniblocks = append(foundMiniblocks, miniblock) + for _, nextContractResult := range nextContractResultsByHash[tx.Hash] { + // Not immediately executing. + if nextContractResult.SourceShard != selfShard { + continue } + + immediatelyExecutingContractResults = append(immediatelyExecutingContractResults, nextContractResult) + // Recursive call: + immediatelyExecutingContractResultsPart := findImmediatelyExecutingContractResultsOfTransaction(selfShard, nextContractResult, nextContractResultsByHash) + immediatelyExecutingContractResults = append(immediatelyExecutingContractResults, immediatelyExecutingContractResultsPart...) } - return foundMiniblocks + return immediatelyExecutingContractResults } -func findInvalidTransactions(block *api.Block) []*transaction.ApiTransactionResult { - invalidTxs := make([]*transaction.ApiTransactionResult, 0) +func gatherScheduledTransactions(block *api.Block) []*transaction.ApiTransactionResult { + scheduledTxs := make([]*transaction.ApiTransactionResult, 0) for _, miniblock := range block.MiniBlocks { - if miniblock.Type == dataBlock.InvalidBlock.String() { - for _, tx := range miniblock.Transactions { - invalidTxs = append(invalidTxs, tx) - } + isScheduled := miniblock.ProcessingType == dataBlock.Scheduled.String() + if !isScheduled { + continue } - } - return invalidTxs -} + for _, tx := range miniblock.Transactions { + scheduledTxs = append(scheduledTxs, tx) + } + } -// Sometimes, an invalid transaction processed in a scheduled miniblock -// might have its smart contract result (if any) saved in the receipts unit of both blocks N and N+1. -// This function ignores the duplicate entries in block N. -func deduplicatePreviouslyAppearingContractResultsInReceipts(previousBlock *api.Block, block *api.Block) { - previouslyAppearing := findContractResultsInReceipts(previousBlock) - removeContractResultsInReceipts(block, previouslyAppearing) + return scheduledTxs } -func findContractResultsInReceipts(block *api.Block) map[string]struct{} { - txs := make(map[string]struct{}) +func gatherAllTransactions(block *api.Block) []*transaction.ApiTransactionResult { + txs := make([]*transaction.ApiTransactionResult, 0) for _, miniblock := range block.MiniBlocks { - if !isContractResultsMiniblockInReceipts(miniblock) { - continue - } - for _, tx := range miniblock.Transactions { - txs[tx.Hash] = struct{}{} + txs = append(txs, tx) } } return txs } -func removeContractResultsInReceipts(block *api.Block, txsHashes map[string]struct{}) { +func gatherAllReceipts(block *api.Block) []*transaction.ApiReceipt { + receipts := make([]*transaction.ApiReceipt, 0) + for _, miniblock := range block.MiniBlocks { - if !isContractResultsMiniblockInReceipts(miniblock) { - continue + for _, receipt := range miniblock.Receipts { + receipts = append(receipts, receipt) } - - miniblock.Transactions = discardTransactions(miniblock.Transactions, txsHashes) } -} -func isContractResultsMiniblockInReceipts(miniblock *api.MiniBlock) bool { - return miniblock.Type == dataBlock.SmartContractResultBlock.String() && miniblock.IsFromReceiptsStorage + return receipts } diff --git a/server/provider/scheduledMiniblocksCrossShardException.go b/server/provider/scheduledMiniblocksCrossShardException.go deleted file mode 100644 index 1b46621c..00000000 --- a/server/provider/scheduledMiniblocksCrossShardException.go +++ /dev/null @@ -1,82 +0,0 @@ -package provider - -import ( - "github.com/multiversx/mx-chain-core-go/data/api" - dataBlock "github.com/multiversx/mx-chain-core-go/data/block" - "github.com/multiversx/mx-chain-core-go/data/transaction" -) - -func handleContractResultsHavingOriginalTransactionInCrossShardScheduledMiniblockInPreviousBlock(previousBlock *api.Block, block *api.Block, nextBlock *api.Block) { - // Gather transactions in cross-shard scheduled, final miniblocks. - // Afterwards, we will handle their contract results in two steps. - transactionsInCrossShardScheduledMiniblockInPreviousBlock := gatherTransactionsInCrossShardScheduledFinalMiniblock(previousBlock) - transactionsInCrossShardScheduledMiniblockInCurrentBlock := gatherTransactionsInCrossShardScheduledFinalMiniblock(block) - - // First, remove corresponding contract results from the current block (since they already produced their effects in the previous block) - for _, miniblock := range block.MiniBlocks { - if miniblock.Type != dataBlock.SmartContractResultBlock.String() { - continue - } - - contractResultsToKeep := make([]*transaction.ApiTransactionResult, 0, len(miniblock.Transactions)) - - for _, contractResult := range miniblock.Transactions { - _, ok := transactionsInCrossShardScheduledMiniblockInPreviousBlock[contractResult.OriginalTransactionHash] - if ok { - // Discard this contract result - continue - } - - contractResultsToKeep = append(contractResultsToKeep, contractResult) - } - - miniblock.Transactions = contractResultsToKeep - } - - // Secondly, bring here (in the current block) the corresponding contract results of the next block (since they produced their effects in the current block) - contractResultsToMove := make([]*transaction.ApiTransactionResult, 0, len(nextBlock.MiniBlocks)) - - for _, miniblockInNextBlock := range nextBlock.MiniBlocks { - if miniblockInNextBlock.Type != dataBlock.SmartContractResultBlock.String() { - continue - } - - for _, contractResultInNextBlock := range miniblockInNextBlock.Transactions { - _, ok := transactionsInCrossShardScheduledMiniblockInCurrentBlock[contractResultInNextBlock.OriginalTransactionHash] - if ok { - contractResultsToMove = append(contractResultsToMove, contractResultInNextBlock) - } - } - } - - if len(contractResultsToMove) > 0 { - // Extremely rare case. - log.Info("handleContractResultsHavingOriginalTransactionInCrossShardScheduledMiniblockInPreviousBlock", "currentBlock", block.Nonce, "numContractResultsToMove", len(contractResultsToMove)) - - artificialMiniblock := &api.MiniBlock{ - Type: dataBlock.SmartContractResultBlock.String(), - Transactions: contractResultsToMove, - } - - block.MiniBlocks = append(block.MiniBlocks, artificialMiniblock) - } -} - -func gatherTransactionsInCrossShardScheduledFinalMiniblock(block *api.Block) map[string]struct{} { - gathered := make(map[string]struct{}) - - for _, miniblockInPreviousBlock := range block.MiniBlocks { - isScheduled := miniblockInPreviousBlock.ProcessingType == dataBlock.Scheduled.String() - isFinal := miniblockInPreviousBlock.ConstructionState == dataBlock.Final.String() - isCrossShard := miniblockInPreviousBlock.SourceShard != miniblockInPreviousBlock.DestinationShard - if !isScheduled || !isFinal || !isCrossShard { - continue - } - - for _, txInPreviousBlock := range miniblockInPreviousBlock.Transactions { - gathered[txInPreviousBlock.Hash] = struct{}{} - } - } - - return gathered -} diff --git a/server/provider/transactions.go b/server/provider/transactions.go deleted file mode 100644 index 80850bc2..00000000 --- a/server/provider/transactions.go +++ /dev/null @@ -1,48 +0,0 @@ -package provider - -import "github.com/multiversx/mx-chain-core-go/data/transaction" - -func discardTransactions(txs []*transaction.ApiTransactionResult, txsHashesToDiscard map[string]struct{}) []*transaction.ApiTransactionResult { - txsToKeep := make([]*transaction.ApiTransactionResult, 0, len(txs)) - - for _, tx := range txs { - _, shouldDiscard := txsHashesToDiscard[tx.Hash] - if shouldDiscard { - continue - } - - txsToKeep = append(txsToKeep, tx) - } - - return txsToKeep -} - -func filterTransactions(txs []*transaction.ApiTransactionResult, txsHashesToKeep map[string]struct{}) []*transaction.ApiTransactionResult { - txsToKeep := make([]*transaction.ApiTransactionResult, 0, len(txs)) - - for _, tx := range txs { - _, shouldKeep := txsHashesToKeep[tx.Hash] - if shouldKeep { - txsToKeep = append(txsToKeep, tx) - } - } - - return txsToKeep -} - -func deduplicateTransactions(txs []*transaction.ApiTransactionResult) []*transaction.ApiTransactionResult { - deduplicatedTxs := make([]*transaction.ApiTransactionResult, 0, len(txs)) - seenTxsHashes := make(map[string]struct{}) - - for _, tx := range txs { - _, alreadySeen := seenTxsHashes[tx.Hash] - if alreadySeen { - continue - } - - deduplicatedTxs = append(deduplicatedTxs, tx) - seenTxsHashes[tx.Hash] = struct{}{} - } - - return deduplicatedTxs -} diff --git a/server/provider/transactions_test.go b/server/provider/transactions_test.go deleted file mode 100644 index f7ac41ed..00000000 --- a/server/provider/transactions_test.go +++ /dev/null @@ -1,54 +0,0 @@ -package provider - -import ( - "testing" - - "github.com/multiversx/mx-chain-core-go/data/transaction" - "github.com/stretchr/testify/require" -) - -func TestDiscardTransactions(t *testing.T) { - transactions := []*transaction.ApiTransactionResult{ - {Hash: "aaaa"}, - {Hash: "bbbb"}, - {Hash: "aabb"}, - {Hash: "abba"}, - } - - txsHashesToDiscard := map[string]struct{}{"aabb": {}, "abba": {}} - result := discardTransactions(transactions, txsHashesToDiscard) - require.Len(t, result, 2) - require.Equal(t, "aaaa", result[0].Hash) - require.Equal(t, "bbbb", result[1].Hash) -} - -func TestFilterTransactions(t *testing.T) { - transactions := []*transaction.ApiTransactionResult{ - {Hash: "aaaa"}, - {Hash: "bbbb"}, - {Hash: "aabb"}, - {Hash: "abba"}, - } - - txsHashesToKeep := map[string]struct{}{"aabb": {}, "abba": {}} - result := filterTransactions(transactions, txsHashesToKeep) - require.Len(t, result, 2) - require.Equal(t, "aabb", result[0].Hash) - require.Equal(t, "abba", result[1].Hash) -} - -func TestDeduplicateTransactions(t *testing.T) { - transactions := []*transaction.ApiTransactionResult{ - {Hash: "aaaa"}, - {Hash: "bbbb"}, - {Hash: "aabb"}, - {Hash: "bbbb"}, - {Hash: "aaaa"}, - } - - result := deduplicateTransactions(transactions) - require.Len(t, result, 3) - require.Equal(t, "aaaa", result[0].Hash) - require.Equal(t, "bbbb", result[1].Hash) - require.Equal(t, "aabb", result[2].Hash) -} From b67f06c3e6f1622144ad74452f6525420571b616 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Tue, 14 Nov 2023 00:04:45 +0200 Subject: [PATCH 08/75] Deduplication of transferValueOnly <> smart contract results. --- .../services/transactionEventsController.go | 38 ++++++++++++++++++- server/services/transactionsTransformer.go | 29 +++++++++++++- 2 files changed, 63 insertions(+), 4 deletions(-) diff --git a/server/services/transactionEventsController.go b/server/services/transactionEventsController.go index 8f091ae9..c13e5c2f 100644 --- a/server/services/transactionEventsController.go +++ b/server/services/transactionEventsController.go @@ -49,6 +49,40 @@ func (controller *transactionEventsController) extractEventTransferValueOnly(tx return typedEvents, nil } +func filterOutTransferValueOnlyEventsThatAreAlreadyCapturedAsContractResults( + events []*eventTransferValueOnly, + txsInBlock []*transaction.ApiTransactionResult, +) []*eventTransferValueOnly { + // First, we find all contract results in this block, and we "summarize" them (in a map). + contractResultsSummaries := make(map[string]struct{}) + + for _, tx := range txsInBlock { + isContractResult := tx.Type == string(transaction.TxTypeUnsigned) + if !isContractResult { + continue + } + + summary := fmt.Sprintf("%s-%s-%s", tx.Sender, tx.Receiver, tx.Value) + contractResultsSummaries[summary] = struct{}{} + } + + eventsToKeep := make([]*eventTransferValueOnly, 0, len(events)) + + for _, event := range events { + summary := fmt.Sprintf("%s-%s-%s", event.sender, event.receiver, event.value) + + _, isAlreadyCaptured := contractResultsSummaries[summary] + if isAlreadyCaptured { + continue + } + + // Event not captured as contract result, so we should keep it. + eventsToKeep = append(eventsToKeep, event) + } + + return eventsToKeep +} + func (controller *transactionEventsController) hasAnySignalError(tx *transaction.ApiTransactionResult) bool { if !controller.hasEvents(tx) { return false @@ -96,7 +130,7 @@ func (controller *transactionEventsController) extractEventsESDTOrESDTNFTTransfe return nil, fmt.Errorf("%w: bad number of topics for (ESDT|ESDTNFT)Transfer event = %d", errCannotRecognizeEvent, numTopics) } - identifider := event.Topics[0] + identifier := event.Topics[0] nonceAsBytes := event.Topics[1] valueBytes := event.Topics[2] receiverPubkey := event.Topics[3] @@ -107,7 +141,7 @@ func (controller *transactionEventsController) extractEventsESDTOrESDTNFTTransfe typedEvents = append(typedEvents, &eventESDT{ senderAddress: event.Address, receiverAddress: receiver, - identifier: string(identifider), + identifier: string(identifier), nonceAsBytes: nonceAsBytes, value: value.String(), }) diff --git a/server/services/transactionsTransformer.go b/server/services/transactionsTransformer.go index ed5e51db..a0876bc3 100644 --- a/server/services/transactionsTransformer.go +++ b/server/services/transactionsTransformer.go @@ -102,7 +102,7 @@ func (transformer *transactionsTransformer) txToRosettaTx(tx *transaction.ApiTra return nil, fmt.Errorf("unknown transaction type: %s", tx.Type) } - err = transformer.addOperationsGivenTransactionEvents(tx, rosettaTx) + err = transformer.addOperationsGivenTransactionEvents(tx, txsInBlock, rosettaTx) if err != nil { return nil, err } @@ -366,7 +366,11 @@ func (transformer *transactionsTransformer) mempoolMoveBalanceTxToRosettaTx(tx * } } -func (transformer *transactionsTransformer) addOperationsGivenTransactionEvents(tx *transaction.ApiTransactionResult, rosettaTx *types.Transaction) error { +func (transformer *transactionsTransformer) addOperationsGivenTransactionEvents( + tx *transaction.ApiTransactionResult, + txsInBlock []*transaction.ApiTransactionResult, + rosettaTx *types.Transaction, +) error { eventsTransferValueOnly, err := transformer.eventsController.extractEventTransferValueOnly(tx) if err != nil { return err @@ -392,6 +396,27 @@ func (transformer *transactionsTransformer) addOperationsGivenTransactionEvents( return err } + eventsTransferValueOnly = filterOutTransferValueOnlyEventsThatAreAlreadyCapturedAsContractResults(eventsTransferValueOnly, txsInBlock) + + for _, event := range eventsTransferValueOnly { + log.Info("eventTransferValueOnly (effective)", "tx", tx.Hash) + + operations := []*types.Operation{ + { + Type: opTransfer, + Account: addressToAccountIdentifier(event.sender), + Amount: transformer.extension.valueToNativeAmount("-" + event.value), + }, + { + Type: opTransfer, + Account: addressToAccountIdentifier(event.receiver), + Amount: transformer.extension.valueToNativeAmount(event.value), + }, + } + + rosettaTx.Operations = append(rosettaTx.Operations, operations...) + } + for _, event := range eventsESDTTransfer { if !transformer.provider.HasCustomCurrency(event.identifier) { // We are only emitting balance-changing operations for supported currencies. From df1aa65156b4e447ddd30a9534d8eef7f02e47fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Tue, 14 Nov 2023 17:35:06 +0200 Subject: [PATCH 09/75] Fetch NFT balances when appropriate. --- server/provider/accounts.go | 53 +++++++++++++++++++++++++++++++++++-- server/provider/errors.go | 5 ++++ server/provider/urls.go | 5 ++++ 3 files changed, 61 insertions(+), 2 deletions(-) diff --git a/server/provider/accounts.go b/server/provider/accounts.go index 4ef605b8..8b5915b1 100644 --- a/server/provider/accounts.go +++ b/server/provider/accounts.go @@ -1,6 +1,12 @@ package provider -import "github.com/multiversx/mx-chain-rosetta/server/resources" +import ( + "fmt" + "strconv" + "strings" + + "github.com/multiversx/mx-chain-rosetta/server/resources" +) // TODO: Merge the methods in this file into a single method, e.g. GetAccountWithBalance(address, tokenIdentifier, options), where tokenIdentifier can be the native token or an ESDT. @@ -54,10 +60,19 @@ func (provider *networkProvider) GetAccountNativeBalance(address string, options // GetAccountESDTBalance gets the ESDT balance by address and tokenIdentifier // TODO: Return nonce for ESDT, as well (an additional request might be needed). func (provider *networkProvider) GetAccountESDTBalance(address string, tokenIdentifier string, options resources.AccountQueryOptions) (*resources.AccountESDTBalance, error) { + tokenIdentifierParts, err := parseExtendedIdentifierParts(tokenIdentifier) + if err != nil { + return nil, err + } + url := buildUrlGetAccountESDTBalance(address, tokenIdentifier, options) + if tokenIdentifierParts.nonce > 0 { + url = buildUrlGetAccountNFTBalance(address, fmt.Sprintf("%s-%s", tokenIdentifierParts.ticker, tokenIdentifierParts.randomSequence), tokenIdentifierParts.nonce, options) + } + response := &resources.AccountESDTBalanceApiResponse{} - err := provider.getResource(url, response) + err = provider.getResource(url, response) if err != nil { return nil, newErrCannotGetAccount(address, err) } @@ -78,3 +93,37 @@ func (provider *networkProvider) GetAccountESDTBalance(address string, tokenIden BlockCoordinates: data.BlockCoordinates, }, nil } + +type tokenIdentifierParts struct { + ticker string + randomSequence string + nonce uint64 +} + +func parseExtendedIdentifierParts(tokenIdentifier string) (*tokenIdentifierParts, error) { + parts := strings.Split(tokenIdentifier, "-") + + if len(parts) == 2 { + return &tokenIdentifierParts{ + ticker: parts[0], + randomSequence: parts[1], + nonce: 0, + }, nil + } + + if len(parts) == 3 { + nonceHex := parts[2] + nonce, err := strconv.ParseUint(nonceHex, 16, 64) + if err != nil { + return nil, newErrCannotParseTokenIdentifier(tokenIdentifier, err) + } + + return &tokenIdentifierParts{ + ticker: parts[0], + randomSequence: parts[1], + nonce: nonce, + }, nil + } + + return nil, newErrCannotParseTokenIdentifier(tokenIdentifier, nil) +} diff --git a/server/provider/errors.go b/server/provider/errors.go index 933f4641..174d7f4c 100644 --- a/server/provider/errors.go +++ b/server/provider/errors.go @@ -11,6 +11,7 @@ var errCannotGetBlock = errors.New("cannot get block") var errCannotGetAccount = errors.New("cannot get account") var errCannotGetTransaction = errors.New("cannot get transaction") var errCannotGetLatestBlockNonce = errors.New("cannot get latest block nonce, maybe the node didn't start syncing") +var errCannotParseTokenIdentifier = errors.New("cannot parse token identifier") func newErrCannotGetBlockByNonce(nonce uint64, innerError error) error { return fmt.Errorf("%w: %v, nonce = %d", errCannotGetBlock, innerError, nonce) @@ -28,6 +29,10 @@ func newErrCannotGetTransaction(hash string, innerError error) error { return fmt.Errorf("%w: %v, address = %s", errCannotGetTransaction, innerError, hash) } +func newErrCannotParseTokenIdentifier(tokenIdentifier string, innerError error) error { + return fmt.Errorf("%w: %v, tokenIdentifier = %s", errCannotParseTokenIdentifier, innerError, tokenIdentifier) +} + // In proxy-go, the function CallGetRestEndPoint() returns an error message as the JSON content of the erroneous HTTP response. // Here, we attempt to decode that JSON and create an error with a "flat" error message. func convertStructuredApiErrToFlatErr(apiErr error) error { diff --git a/server/provider/urls.go b/server/provider/urls.go index 41d6b5ed..23cabdd8 100644 --- a/server/provider/urls.go +++ b/server/provider/urls.go @@ -16,6 +16,7 @@ var ( urlPathGetAccount = "/address/%s" urlPathGetAccountNativeBalance = "/address/%s" urlPathGetAccountESDTBalance = "/address/%s/esdt/%s" + urlPathGetAccountNFTBalance = "/address/%s/nft/%s/nonce/%d" urlParameterAccountQueryOptionsOnFinalBlock = "onFinalBlock" urlParameterAccountQueryOptionsBlockNonce = "blockNonce" urlParameterAccountQueryOptionsBlockHash = "blockHash" @@ -38,6 +39,10 @@ func buildUrlGetAccountESDTBalance(address string, tokenIdentifier string, optio return buildUrlWithAccountQueryOptions(fmt.Sprintf(urlPathGetAccountESDTBalance, address, tokenIdentifier), options) } +func buildUrlGetAccountNFTBalance(address string, tokenIdentifier string, nonce uint64, options resources.AccountQueryOptions) string { + return buildUrlWithAccountQueryOptions(fmt.Sprintf(urlPathGetAccountNFTBalance, address, tokenIdentifier, nonce), options) +} + func buildUrlWithAccountQueryOptions(path string, options resources.AccountQueryOptions) string { if options.OnFinalBlock { return buildUrlWithQueryParameter(path, urlParameterAccountQueryOptionsOnFinalBlock, "true") From 6935e2672ee29f45db157eb3a77da6fabe03de6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Tue, 14 Nov 2023 17:37:11 +0200 Subject: [PATCH 10/75] Handle additional ESDT events (for NFT / SFT creation, burning). --- server/services/constants.go | 3 + server/services/transactionEvents.go | 4 +- .../services/transactionEventsController.go | 82 +++++++++++++++++++ server/services/transactionEvents_test.go | 6 +- server/services/transactionsTransformer.go | 76 +++++++++++++++-- 5 files changed, 161 insertions(+), 10 deletions(-) diff --git a/server/services/constants.go b/server/services/constants.go index 4a47cf9d..e4274c58 100644 --- a/server/services/constants.go +++ b/server/services/constants.go @@ -28,6 +28,9 @@ var ( transactionEventTransferValueOnly = "transferValueOnly" transactionEventESDTTransfer = "ESDTTransfer" transactionEventESDTNFTTransfer = "ESDTNFTTransfer" + transactionEventESDTNFTCreate = "ESDTNFTCreate" + transactionEventESDTNFTBurn = "ESDTNFTBurn" + transactionEventESDTNFTAddQuantity = "ESDTNFTAddQuantity" transactionEventMultiESDTNFTTransfer = "MultiESDTNFTTransfer" transactionEventESDTLocalBurn = "ESDTLocalBurn" transactionEventESDTLocalMint = "ESDTLocalMint" diff --git a/server/services/transactionEvents.go b/server/services/transactionEvents.go index d2a0e370..39cfda21 100644 --- a/server/services/transactionEvents.go +++ b/server/services/transactionEvents.go @@ -26,8 +26,8 @@ func (event *eventESDT) getBaseIdentifier() string { return event.identifier } -// getComposedIdentifier returns the "full" token identifier for all types of ESDTs -func (event *eventESDT) getComposedIdentifier() string { +// getExtendedIdentifier returns the "full" token identifier for all types of ESDTs +func (event *eventESDT) getExtendedIdentifier() string { if len(event.nonceAsBytes) > 0 { return fmt.Sprintf("%s-%x", event.identifier, event.nonceAsBytes) } diff --git a/server/services/transactionEventsController.go b/server/services/transactionEventsController.go index c13e5c2f..b22ad76e 100644 --- a/server/services/transactionEventsController.go +++ b/server/services/transactionEventsController.go @@ -261,6 +261,88 @@ func (controller *transactionEventsController) extractEventsESDTWipe(tx *transac return typedEvents, nil } +func (controller *transactionEventsController) extractEventsESDTNFTCreate(tx *transaction.ApiTransactionResult) ([]*eventESDT, error) { + rawEvents := controller.findManyEventsByIdentifier(tx, transactionEventESDTNFTCreate) + typedEvents := make([]*eventESDT, 0, len(rawEvents)) + + for _, event := range rawEvents { + numTopics := len(event.Topics) + if numTopics != 4 { + return nil, fmt.Errorf("%w: bad number of topics for %s event = %d", errCannotRecognizeEvent, transactionEventESDTNFTCreate, numTopics) + } + + identifider := event.Topics[0] + nonceAsBytes := event.Topics[1] + valueBytes := event.Topics[2] + // We ignore the 4th topic. + + value := big.NewInt(0).SetBytes(valueBytes) + + typedEvents = append(typedEvents, &eventESDT{ + otherAddress: event.Address, + identifier: string(identifider), + nonceAsBytes: nonceAsBytes, + value: value.String(), + }) + } + + return typedEvents, nil +} + +func (controller *transactionEventsController) extractEventsESDTNFTBurn(tx *transaction.ApiTransactionResult) ([]*eventESDT, error) { + rawEvents := controller.findManyEventsByIdentifier(tx, transactionEventESDTNFTBurn) + typedEvents := make([]*eventESDT, 0, len(rawEvents)) + + for _, event := range rawEvents { + numTopics := len(event.Topics) + if numTopics != 3 { + return nil, fmt.Errorf("%w: bad number of topics for %s event = %d", errCannotRecognizeEvent, transactionEventESDTNFTBurn, numTopics) + } + + identifider := event.Topics[0] + nonceAsBytes := event.Topics[1] + valueBytes := event.Topics[2] + + value := big.NewInt(0).SetBytes(valueBytes) + + typedEvents = append(typedEvents, &eventESDT{ + otherAddress: event.Address, + identifier: string(identifider), + nonceAsBytes: nonceAsBytes, + value: value.String(), + }) + } + + return typedEvents, nil +} + +func (controller *transactionEventsController) extractEventsESDTNFTAddQuantity(tx *transaction.ApiTransactionResult) ([]*eventESDT, error) { + rawEvents := controller.findManyEventsByIdentifier(tx, transactionEventESDTNFTAddQuantity) + typedEvents := make([]*eventESDT, 0, len(rawEvents)) + + for _, event := range rawEvents { + numTopics := len(event.Topics) + if numTopics != 3 { + return nil, fmt.Errorf("%w: bad number of topics for %s event = %d", errCannotRecognizeEvent, transactionEventESDTNFTAddQuantity, numTopics) + } + + identifider := event.Topics[0] + nonceAsBytes := event.Topics[1] + valueBytes := event.Topics[2] + + value := big.NewInt(0).SetBytes(valueBytes) + + typedEvents = append(typedEvents, &eventESDT{ + otherAddress: event.Address, + identifier: string(identifider), + nonceAsBytes: nonceAsBytes, + value: value.String(), + }) + } + + return typedEvents, nil +} + func (controller *transactionEventsController) findManyEventsByIdentifier(tx *transaction.ApiTransactionResult, identifier string) []*transaction.Events { events := make([]*transaction.Events, 0) diff --git a/server/services/transactionEvents_test.go b/server/services/transactionEvents_test.go index 79d96b95..0bb2749b 100644 --- a/server/services/transactionEvents_test.go +++ b/server/services/transactionEvents_test.go @@ -16,7 +16,7 @@ func TestEventESDT(t *testing.T) { } require.Equal(t, "FOO-abcdef", event.getBaseIdentifier()) - require.Equal(t, "FOO-abcdef", event.getComposedIdentifier()) + require.Equal(t, "FOO-abcdef", event.getExtendedIdentifier()) event = eventESDT{ identifier: "FOO-abcdef", @@ -24,7 +24,7 @@ func TestEventESDT(t *testing.T) { } require.Equal(t, "FOO-abcdef", event.getBaseIdentifier()) - require.Equal(t, "FOO-abcdef", event.getComposedIdentifier()) + require.Equal(t, "FOO-abcdef", event.getExtendedIdentifier()) }) t.Run("with nonce (SFT, MetaESDT, NFT)", func(t *testing.T) { @@ -34,7 +34,7 @@ func TestEventESDT(t *testing.T) { } require.Equal(t, "FOO-abcdef", event.getBaseIdentifier()) - require.Equal(t, "FOO-abcdef-2a", event.getComposedIdentifier()) + require.Equal(t, "FOO-abcdef-2a", event.getExtendedIdentifier()) }) } diff --git a/server/services/transactionsTransformer.go b/server/services/transactionsTransformer.go index a0876bc3..5c55b9de 100644 --- a/server/services/transactionsTransformer.go +++ b/server/services/transactionsTransformer.go @@ -396,6 +396,21 @@ func (transformer *transactionsTransformer) addOperationsGivenTransactionEvents( return err } + eventsESDTNFTCreate, err := transformer.eventsController.extractEventsESDTNFTCreate(tx) + if err != nil { + return err + } + + eventsESDTNFTBurn, err := transformer.eventsController.extractEventsESDTNFTBurn(tx) + if err != nil { + return err + } + + eventsESDTNFTAddQuantity, err := transformer.eventsController.extractEventsESDTNFTAddQuantity(tx) + if err != nil { + return err + } + eventsTransferValueOnly = filterOutTransferValueOnlyEventsThatAreAlreadyCapturedAsContractResults(eventsTransferValueOnly, txsInBlock) for _, event := range eventsTransferValueOnly { @@ -427,12 +442,12 @@ func (transformer *transactionsTransformer) addOperationsGivenTransactionEvents( { Type: opCustomTransfer, Account: addressToAccountIdentifier(event.senderAddress), - Amount: transformer.extension.valueToCustomAmount("-"+event.value, event.getComposedIdentifier()), + Amount: transformer.extension.valueToCustomAmount("-"+event.value, event.getExtendedIdentifier()), }, { Type: opCustomTransfer, Account: addressToAccountIdentifier(event.receiverAddress), - Amount: transformer.extension.valueToCustomAmount(event.value, event.getComposedIdentifier()), + Amount: transformer.extension.valueToCustomAmount(event.value, event.getExtendedIdentifier()), }, } @@ -449,7 +464,7 @@ func (transformer *transactionsTransformer) addOperationsGivenTransactionEvents( { Type: opCustomTransfer, Account: addressToAccountIdentifier(event.otherAddress), - Amount: transformer.extension.valueToCustomAmount("-"+event.value, event.getComposedIdentifier()), + Amount: transformer.extension.valueToCustomAmount("-"+event.value, event.getExtendedIdentifier()), }, } @@ -466,7 +481,7 @@ func (transformer *transactionsTransformer) addOperationsGivenTransactionEvents( { Type: opCustomTransfer, Account: addressToAccountIdentifier(event.otherAddress), - Amount: transformer.extension.valueToCustomAmount(event.value, event.getComposedIdentifier()), + Amount: transformer.extension.valueToCustomAmount(event.value, event.getExtendedIdentifier()), }, } @@ -483,7 +498,58 @@ func (transformer *transactionsTransformer) addOperationsGivenTransactionEvents( { Type: opCustomTransfer, Account: addressToAccountIdentifier(event.otherAddress), - Amount: transformer.extension.valueToCustomAmount("-"+event.value, event.getComposedIdentifier()), + Amount: transformer.extension.valueToCustomAmount("-"+event.value, event.getExtendedIdentifier()), + }, + } + + rosettaTx.Operations = append(rosettaTx.Operations, operations...) + } + + for _, event := range eventsESDTNFTCreate { + if !transformer.provider.HasCustomCurrency(event.identifier) { + // We are only emitting balance-changing operations for supported currencies. + continue + } + + operations := []*types.Operation{ + { + Type: opCustomTransfer, + Account: addressToAccountIdentifier(event.otherAddress), + Amount: transformer.extension.valueToCustomAmount(event.value, event.getExtendedIdentifier()), + }, + } + + rosettaTx.Operations = append(rosettaTx.Operations, operations...) + } + + for _, event := range eventsESDTNFTBurn { + if !transformer.provider.HasCustomCurrency(event.identifier) { + // We are only emitting balance-changing operations for supported currencies. + continue + } + + operations := []*types.Operation{ + { + Type: opCustomTransfer, + Account: addressToAccountIdentifier(event.otherAddress), + Amount: transformer.extension.valueToCustomAmount("-"+event.value, event.getExtendedIdentifier()), + }, + } + + rosettaTx.Operations = append(rosettaTx.Operations, operations...) + } + + for _, event := range eventsESDTNFTAddQuantity { + if !transformer.provider.HasCustomCurrency(event.identifier) { + // We are only emitting balance-changing operations for supported currencies. + continue + } + + operations := []*types.Operation{ + { + Type: opCustomTransfer, + Account: addressToAccountIdentifier(event.otherAddress), + Amount: transformer.extension.valueToCustomAmount(event.value, event.getExtendedIdentifier()), }, } From 256731734facc154cc863b1196d4204a8b41f0ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Wed, 15 Nov 2023 14:36:52 +0200 Subject: [PATCH 11/75] Handle relayed V2 (in progress). --- server/services/errors.go | 1 + server/services/relayedTransactions.go | 32 +++++++++++++++++-- server/services/relayedTransactions_test.go | 4 +-- .../transactionsFeaturesDetector_test.go | 8 ++--- .../services/transactionsTransformer_test.go | 16 +++++----- 5 files changed, 44 insertions(+), 17 deletions(-) diff --git a/server/services/errors.go b/server/services/errors.go index 76f3bab8..3f9c76b8 100644 --- a/server/services/errors.go +++ b/server/services/errors.go @@ -202,3 +202,4 @@ func (factory *errFactory) getPrototypeByCode(code errCode) errPrototype { var errEventNotFound = errors.New("transaction event not found") var errCannotRecognizeEvent = errors.New("cannot recognize transaction event") var errCannotParseRelayedV1 = errors.New("cannot parse relayed V1 transaction") +var errCannotParseRelayedV2 = errors.New("cannot parse relayed V2 transaction") diff --git a/server/services/relayedTransactions.go b/server/services/relayedTransactions.go index af83ca0f..62917eb1 100644 --- a/server/services/relayedTransactions.go +++ b/server/services/relayedTransactions.go @@ -11,16 +11,26 @@ import ( // innerTransactionOfRelayedV1 is used to parse the inner transaction of a relayed V1 transaction, and holds only the fields handled by Rosetta. type innerTransactionOfRelayedV1 struct { - Nonce uint64 `json:"nonce"` Value big.Int `json:"value"` ReceiverPubKey []byte `json:"receiver"` SenderPubKey []byte `json:"sender"` } +// innerTransactionOfRelayedV2 is used to parse the inner transaction of a relayed V2 transaction, and holds only the fields handled by Rosetta. +type innerTransactionOfRelayedV2 struct { + ReceiverPubKey []byte `json:"receiver"` +} + func isRelayedV1Transaction(tx *transaction.ApiTransactionResult) bool { return (tx.Type == string(transaction.TxTypeNormal)) && - (tx.ProcessingTypeOnSource == transactionProcessingTypeRelayed) && - (tx.ProcessingTypeOnDestination == transactionProcessingTypeRelayed) + (tx.ProcessingTypeOnSource == transactionProcessingTypeRelayedV1) && + (tx.ProcessingTypeOnDestination == transactionProcessingTypeRelayedV1) +} + +func isRelayedV2Transaction(tx *transaction.ApiTransactionResult) bool { + return (tx.Type == string(transaction.TxTypeNormal)) && + (tx.ProcessingTypeOnSource == transactionProcessingTypeRelayedV2) && + (tx.ProcessingTypeOnDestination == transactionProcessingTypeRelayedV2) } func parseInnerTxOfRelayedV1(tx *transaction.ApiTransactionResult) (*innerTransactionOfRelayedV1, error) { @@ -43,3 +53,19 @@ func parseInnerTxOfRelayedV1(tx *transaction.ApiTransactionResult) (*innerTransa return &innerTx, nil } + +func parseInnerTxOfRelayedV2(tx *transaction.ApiTransactionResult) (*innerTransactionOfRelayedV2, error) { + subparts := strings.Split(string(tx.Data), argumentsSeparator) + if len(subparts) != 5 { + return nil, errCannotParseRelayedV2 + } + + receiverPubKey, err := hex.DecodeString(subparts[1]) + if err != nil { + return nil, err + } + + return &innerTransactionOfRelayedV2{ + ReceiverPubKey: receiverPubKey, + }, nil +} diff --git a/server/services/relayedTransactions_test.go b/server/services/relayedTransactions_test.go index 029db0be..aca493bd 100644 --- a/server/services/relayedTransactions_test.go +++ b/server/services/relayedTransactions_test.go @@ -17,8 +17,8 @@ func Test_IsRelayedV1Transaction(t *testing.T) { t.Run("relayed v1 tx", func(t *testing.T) { tx := &transaction.ApiTransactionResult{ Type: string(transaction.TxTypeNormal), - ProcessingTypeOnSource: transactionProcessingTypeRelayed, - ProcessingTypeOnDestination: transactionProcessingTypeRelayed, + ProcessingTypeOnSource: transactionProcessingTypeRelayedV1, + ProcessingTypeOnDestination: transactionProcessingTypeRelayedV1, } require.True(t, isRelayedV1Transaction(tx)) diff --git a/server/services/transactionsFeaturesDetector_test.go b/server/services/transactionsFeaturesDetector_test.go index 440fc8e6..c99d71a0 100644 --- a/server/services/transactionsFeaturesDetector_test.go +++ b/server/services/transactionsFeaturesDetector_test.go @@ -75,7 +75,7 @@ func TestTransactionsFeaturesDetector_IsInvalidTransactionOfTypeMoveBalanceThatO }) } -func TestTransactionsFeaturesDetector_IsRelayedTransactionCompletelyIntrashardWithSignalError(t *testing.T) { +func TestTransactionsFeaturesDetector_isRelayedV1TransactionCompletelyIntrashardWithSignalError(t *testing.T) { networkProvider := testscommon.NewNetworkProviderMock() detector := newTransactionsFeaturesDetector(networkProvider) @@ -90,7 +90,7 @@ func TestTransactionsFeaturesDetector_IsRelayedTransactionCompletelyIntrashardWi ReceiverPubKey: testscommon.TestUserShard2.PubKey, } - featureDetected := detector.isRelayedTransactionCompletelyIntrashardWithSignalError(tx, innerTx) + featureDetected := detector.isRelayedV1TransactionCompletelyIntrashardWithSignalError(tx, innerTx) require.False(t, featureDetected) }) @@ -105,7 +105,7 @@ func TestTransactionsFeaturesDetector_IsRelayedTransactionCompletelyIntrashardWi ReceiverPubKey: testscommon.TestUserBShard0.PubKey, } - featureDetected := detector.isRelayedTransactionCompletelyIntrashardWithSignalError(tx, innerTx) + featureDetected := detector.isRelayedV1TransactionCompletelyIntrashardWithSignalError(tx, innerTx) require.False(t, featureDetected) }) @@ -127,7 +127,7 @@ func TestTransactionsFeaturesDetector_IsRelayedTransactionCompletelyIntrashardWi ReceiverPubKey: testscommon.TestUserBShard0.PubKey, } - featureDetected := detector.isRelayedTransactionCompletelyIntrashardWithSignalError(tx, innerTx) + featureDetected := detector.isRelayedV1TransactionCompletelyIntrashardWithSignalError(tx, innerTx) require.True(t, featureDetected) }) } diff --git a/server/services/transactionsTransformer_test.go b/server/services/transactionsTransformer_test.go index abd188cd..efed7081 100644 --- a/server/services/transactionsTransformer_test.go +++ b/server/services/transactionsTransformer_test.go @@ -58,8 +58,8 @@ func TestTransactionsTransformer_NormalTxToRosettaTx(t *testing.T) { t.Run("relayed tx, completely intrashard, with signal error", func(t *testing.T) { tx := &transaction.ApiTransactionResult{ Type: string(transaction.TxTypeNormal), - ProcessingTypeOnSource: transactionProcessingTypeRelayed, - ProcessingTypeOnDestination: transactionProcessingTypeRelayed, + ProcessingTypeOnSource: transactionProcessingTypeRelayedV1, + ProcessingTypeOnDestination: transactionProcessingTypeRelayedV1, Hash: "aaaa", Sender: testscommon.TestUserAShard0.Address, Receiver: testscommon.TestUserBShard0.Address, @@ -123,8 +123,8 @@ func TestTransactionsTransformer_ExtractInnerTxOperationsIfRelayedCompletelyIntr t.Run("relayed tx (badly formatted)", func(t *testing.T) { tx := &transaction.ApiTransactionResult{ Type: string(transaction.TxTypeNormal), - ProcessingTypeOnSource: transactionProcessingTypeRelayed, - ProcessingTypeOnDestination: transactionProcessingTypeRelayed, + ProcessingTypeOnSource: transactionProcessingTypeRelayedV1, + ProcessingTypeOnDestination: transactionProcessingTypeRelayedV1, Data: []byte("bad"), } @@ -136,8 +136,8 @@ func TestTransactionsTransformer_ExtractInnerTxOperationsIfRelayedCompletelyIntr t.Run("relayed tx, completely intrashard, with signal error, inner tx has non-zero value", func(t *testing.T) { tx := &transaction.ApiTransactionResult{ Type: string(transaction.TxTypeNormal), - ProcessingTypeOnSource: transactionProcessingTypeRelayed, - ProcessingTypeOnDestination: transactionProcessingTypeRelayed, + ProcessingTypeOnSource: transactionProcessingTypeRelayedV1, + ProcessingTypeOnDestination: transactionProcessingTypeRelayedV1, SourceShard: 0, DestinationShard: 0, Data: []byte("relayedTx@7b226e6f6e6365223a372c2273656e646572223a226e69424758747949504349644a78793373796c6c455a474c78506a503148734a45646e43732b6d726577413d222c227265636569766572223a224141414141414141414141464145356c3079623173734a3933504433672f4b396f48384579366d576958513d222c2276616c7565223a313030303030303030303030303030303030302c226761735072696365223a313030303030303030302c226761734c696d6974223a35303030302c2264617461223a22222c227369676e6174757265223a226e6830743338585a77614b6a725878446969716f6d364d6a5671724d612f6b70767474696a33692b5a6d43492f3778626830596762363548424151445744396f7036575567674541755430756e5253595736455341413d3d222c22636861696e4944223a224d513d3d222c2276657273696f6e223a327d"), @@ -169,8 +169,8 @@ func TestTransactionsTransformer_ExtractInnerTxOperationsIfRelayedCompletelyIntr t.Run("relayed tx, completely intrashard, with signal error, inner tx has zero value", func(t *testing.T) { tx := &transaction.ApiTransactionResult{ Type: string(transaction.TxTypeNormal), - ProcessingTypeOnSource: transactionProcessingTypeRelayed, - ProcessingTypeOnDestination: transactionProcessingTypeRelayed, + ProcessingTypeOnSource: transactionProcessingTypeRelayedV1, + ProcessingTypeOnDestination: transactionProcessingTypeRelayedV1, SourceShard: 0, DestinationShard: 0, Data: []byte("relayedTx@7b226e6f6e6365223a372c2273656e646572223a226e69424758747949504349644a78793373796c6c455a474c78506a503148734a45646e43732b6d726577413d222c227265636569766572223a224141414141414141414141464145356c3079623173734a3933504433672f4b396f48384579366d576958513d222c2276616c7565223a302c226761735072696365223a313030303030303030302c226761734c696d6974223a35303030302c2264617461223a22222c227369676e6174757265223a22336c644e7a32435734416143675069495863636c466b4654324149586a4a4757316a526a306c542b4f3161736b6241394a744e365a764173396e394f58716d343130574a49665332332b4168666e48793267446c41773d3d222c22636861696e4944223a224d513d3d222c2276657273696f6e223a327d"), From ff44b5812141121bc92c40f98e9446a54df502e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Thu, 16 Nov 2023 13:12:16 +0200 Subject: [PATCH 12/75] Delayed commits. Handle txs with signal error (ignore all other events). Added some TODOs. --- server/provider/constants.go | 2 +- server/provider/currenciesProvider.go | 7 +++-- server/provider/networkProvider.go | 2 ++ server/services/constants.go | 3 +- .../services/transactionsFeaturesDetector.go | 2 +- server/services/transactionsTransformer.go | 31 +++++++++++++++++-- 6 files changed, 39 insertions(+), 8 deletions(-) diff --git a/server/provider/constants.go b/server/provider/constants.go index d4d572bf..74cd0c97 100644 --- a/server/provider/constants.go +++ b/server/provider/constants.go @@ -3,5 +3,5 @@ package provider var ( nativeCurrencyNumDecimals = 18 genesisBlockNonce = 0 - blocksCacheCapacity = 256 + blocksCacheCapacity = 1024 ) diff --git a/server/provider/currenciesProvider.go b/server/provider/currenciesProvider.go index 2e5df112..14b8eff3 100644 --- a/server/provider/currenciesProvider.go +++ b/server/provider/currenciesProvider.go @@ -1,6 +1,8 @@ package provider -import "github.com/multiversx/mx-chain-rosetta/server/resources" +import ( + "github.com/multiversx/mx-chain-rosetta/server/resources" +) type currenciesProvider struct { nativeCurrency resources.Currency @@ -61,6 +63,5 @@ func (provider *currenciesProvider) GetCustomCurrencyBySymbol(symbol string) (re // HasCustomCurrency checks whether a custom currency (ESDT) is enabled (supported) func (provider *currenciesProvider) HasCustomCurrency(symbol string) bool { - _, ok := provider.customCurrenciesBySymbol[symbol] - return ok + return true } diff --git a/server/provider/networkProvider.go b/server/provider/networkProvider.go index 583bd9c2..da5302cc 100644 --- a/server/provider/networkProvider.go +++ b/server/provider/networkProvider.go @@ -212,6 +212,7 @@ func (provider *networkProvider) GetBlockByNonce(nonce uint64) (*api.Block, erro return nil, err } + // TODO: returns a mutate copy instead. err = provider.simplifyBlockWithScheduledTransactions(block) if err != nil { return nil, err @@ -246,6 +247,7 @@ func (provider *networkProvider) doGetBlockByNonce(nonce uint64) (*api.Block, er return createBlockCopy(block), nil } +// TOOD: Remove this. Cache the simplified block, instead. func createBlockCopy(block *api.Block) *api.Block { miniblocksCopy := make([]*api.MiniBlock, len(block.MiniBlocks)) diff --git a/server/services/constants.go b/server/services/constants.go index e4274c58..1e072383 100644 --- a/server/services/constants.go +++ b/server/services/constants.go @@ -9,7 +9,8 @@ import ( var ( transactionVersion = 1 - transactionProcessingTypeRelayed = "RelayedTx" + transactionProcessingTypeRelayedV1 = "RelayedTx" + transactionProcessingTypeRelayedV2 = "RelayedTxV2" transactionProcessingTypeBuiltInFunctionCall = "BuiltInFunctionCall" transactionProcessingTypeMoveBalance = "MoveBalance" transactionProcessingTypeContractInvoking = "SCInvoking" diff --git a/server/services/transactionsFeaturesDetector.go b/server/services/transactionsFeaturesDetector.go index 867da83e..0d36e0a5 100644 --- a/server/services/transactionsFeaturesDetector.go +++ b/server/services/transactionsFeaturesDetector.go @@ -60,7 +60,7 @@ func (detector *transactionsFeaturesDetector) isInvalidTransactionOfTypeMoveBala return withSendingValueToNonPayableContract || withMetaTransactionIsInvalid } -func (detector *transactionsFeaturesDetector) isRelayedTransactionCompletelyIntrashardWithSignalError(tx *transaction.ApiTransactionResult, innerTx *innerTransactionOfRelayedV1) bool { +func (detector *transactionsFeaturesDetector) isRelayedV1TransactionCompletelyIntrashardWithSignalError(tx *transaction.ApiTransactionResult, innerTx *innerTransactionOfRelayedV1) bool { innerTxSenderShard := detector.networkProvider.ComputeShardIdOfPubKey(innerTx.SenderPubKey) innerTxReceiverShard := detector.networkProvider.ComputeShardIdOfPubKey(innerTx.ReceiverPubKey) diff --git a/server/services/transactionsTransformer.go b/server/services/transactionsTransformer.go index 5c55b9de..abdb50f8 100644 --- a/server/services/transactionsTransformer.go +++ b/server/services/transactionsTransformer.go @@ -29,6 +29,9 @@ func newTransactionsTransformer(provider NetworkProvider) *transactionsTransform } func (transformer *transactionsTransformer) transformBlockTxs(block *api.Block) ([]*types.Transaction, error) { + // TODO: Remove + log.Info("transformBlockTxs", "block", block.Nonce, "numTxs", block.NumTxs) + txs := make([]*transaction.ApiTransactionResult, 0) receipts := make([]*transaction.ApiReceipt, 0) @@ -83,6 +86,13 @@ func (transformer *transactionsTransformer) transformBlockTxs(block *api.Block) } func (transformer *transactionsTransformer) txToRosettaTx(tx *transaction.ApiTransactionResult, txsInBlock []*transaction.ApiTransactionResult) (*types.Transaction, error) { + // TODO: Remove + if isRelayedV1Transaction(tx) { + log.Info("txToRosettaTx: relayed V1", "hash", tx.Hash) + } else if isRelayedV2Transaction(tx) { + log.Info("txToRosettaTx: relayed V2", "hash", tx.Hash) + } + var rosettaTx *types.Transaction var err error @@ -217,6 +227,14 @@ func (transformer *transactionsTransformer) normalTxToRosetta( return nil, err } + // TODO: Remove (maybe) + if len(innerTxOperationsIfRelayedCompletelyIntrashardWithSignalError) > 0 { + log.Info("innerTxOperationsIfRelayedCompletelyIntrashardWithSignalError", "tx", tx.Hash, "block", tx.BlockNonce) + } + if len(valueRefundOperationIfContractCallWithSignalError) > 0 { + log.Info("valueRefundOperationIfContractCallWithSignalError", "tx", tx.Hash, "block", tx.BlockNonce) + } + operations = append(operations, innerTxOperationsIfRelayedCompletelyIntrashardWithSignalError...) operations = append(operations, valueRefundOperationIfContractCallWithSignalError...) @@ -227,7 +245,9 @@ func (transformer *transactionsTransformer) normalTxToRosetta( }, nil } +// This only handles operations for the native balance. func (transformer *transactionsTransformer) extractInnerTxOperationsIfRelayedCompletelyIntrashardWithSignalError(tx *transaction.ApiTransactionResult) ([]*types.Operation, error) { + // Only relayed V1 is handled. Relayed V2 cannot bear native value in the inner transaction. isRelayedTransaction := isRelayedV1Transaction(tx) if !isRelayedTransaction { return []*types.Operation{}, nil @@ -242,7 +262,7 @@ func (transformer *transactionsTransformer) extractInnerTxOperationsIfRelayedCom return []*types.Operation{}, nil } - if !transformer.featuresDetector.isRelayedTransactionCompletelyIntrashardWithSignalError(tx, innerTx) { + if !transformer.featuresDetector.isRelayedV1TransactionCompletelyIntrashardWithSignalError(tx, innerTx) { return []*types.Operation{}, nil } @@ -371,6 +391,13 @@ func (transformer *transactionsTransformer) addOperationsGivenTransactionEvents( txsInBlock []*transaction.ApiTransactionResult, rosettaTx *types.Transaction, ) error { + hasSignalError := transformer.featuresDetector.eventsController.hasAnySignalError(tx) + if hasSignalError { + // TODO: Remove + log.Info("hasSignalError, will ignore events", "tx", tx.Hash, "block", tx.BlockNonce) + return nil + } + eventsTransferValueOnly, err := transformer.eventsController.extractEventTransferValueOnly(tx) if err != nil { return err @@ -414,7 +441,7 @@ func (transformer *transactionsTransformer) addOperationsGivenTransactionEvents( eventsTransferValueOnly = filterOutTransferValueOnlyEventsThatAreAlreadyCapturedAsContractResults(eventsTransferValueOnly, txsInBlock) for _, event := range eventsTransferValueOnly { - log.Info("eventTransferValueOnly (effective)", "tx", tx.Hash) + log.Info("eventTransferValueOnly (effective)", "tx", tx.Hash, "block", tx.BlockNonce) operations := []*types.Operation{ { From 4ed3ba23ee1710ae11f62585b750cda0fcb377cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Thu, 16 Nov 2023 13:12:51 +0200 Subject: [PATCH 13/75] Handle transfer value only - before vs. after Sirius. --- .../services/transactionEventsController.go | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/server/services/transactionEventsController.go b/server/services/transactionEventsController.go index b22ad76e..f1ced03b 100644 --- a/server/services/transactionEventsController.go +++ b/server/services/transactionEventsController.go @@ -25,14 +25,26 @@ func (controller *transactionEventsController) extractEventTransferValueOnly(tx for _, event := range rawEvents { numTopics := len(event.Topics) - if numTopics != 3 { + isBeforeSirius := numTopics == 3 + isAfterSirius := numTopics == 2 + + var senderPubKey []byte + var receiverPubKey []byte + var valueBytes []byte + + if isBeforeSirius { + senderPubKey = event.Topics[0] + receiverPubKey = event.Topics[1] + valueBytes = event.Topics[2] + } else if isAfterSirius { + // https://github.com/multiversx/mx-specs/blob/main/releases/protocol/release-specs-v1.6.0-Sirius.md#17-logs--events-changes-5490 + senderPubKey = []byte(event.Address) + receiverPubKey = event.Topics[0] + valueBytes = event.Topics[1] + } else { return nil, fmt.Errorf("%w: bad number of topics for 'transferValueOnly' = %d", errCannotRecognizeEvent, numTopics) } - senderPubKey := event.Topics[0] - receiverPubKey := event.Topics[1] - valueBytes := event.Topics[2] - sender := controller.provider.ConvertPubKeyToAddress(senderPubKey) receiver := controller.provider.ConvertPubKeyToAddress(receiverPubKey) value := big.NewInt(0).SetBytes(valueBytes) From e94592b93015104734f2f378397330b4b6ea8ead Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Thu, 16 Nov 2023 13:46:08 +0200 Subject: [PATCH 14/75] Handle SC deployments with value. --- server/services/constants.go | 8 +++-- server/services/transactionEvents.go | 5 ++++ .../services/transactionEventsController.go | 24 +++++++++++++++ server/services/transactionsTransformer.go | 30 ++++++++++++++++++- 4 files changed, 63 insertions(+), 4 deletions(-) diff --git a/server/services/constants.go b/server/services/constants.go index 1e072383..cb929fbc 100644 --- a/server/services/constants.go +++ b/server/services/constants.go @@ -22,10 +22,12 @@ var ( sendingValueToNonPayableContractDataPrefix = argumentsSeparator + hex.EncodeToString([]byte("sending value to non payable contract")) emptyHash = strings.Repeat("0", 64) nodeVersionForOfflineRosetta = "N / A" + systemContractDeployAddress = "erd1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq6gq4hu" ) var ( transactionEventSignalError = core.SignalErrorOperation + transactionEventSCDeploy = core.SCDeployIdentifier transactionEventTransferValueOnly = "transferValueOnly" transactionEventESDTTransfer = "ESDTTransfer" transactionEventESDTNFTTransfer = "ESDTNFTTransfer" @@ -33,9 +35,9 @@ var ( transactionEventESDTNFTBurn = "ESDTNFTBurn" transactionEventESDTNFTAddQuantity = "ESDTNFTAddQuantity" transactionEventMultiESDTNFTTransfer = "MultiESDTNFTTransfer" - transactionEventESDTLocalBurn = "ESDTLocalBurn" - transactionEventESDTLocalMint = "ESDTLocalMint" - transactionEventESDTWipe = "ESDTWipe" + transactionEventESDTLocalBurn = core.BuiltInFunctionESDTLocalBurn + transactionEventESDTLocalMint = core.BuiltInFunctionESDTLocalMint + transactionEventESDTWipe = core.BuiltInFunctionESDTWipe transactionEventTopicInvalidMetaTransaction = "meta transaction is invalid" transactionEventTopicInvalidMetaTransactionNotEnoughGas = "meta transaction is invalid: not enough gas" ) diff --git a/server/services/transactionEvents.go b/server/services/transactionEvents.go index 39cfda21..43d6e986 100644 --- a/server/services/transactionEvents.go +++ b/server/services/transactionEvents.go @@ -34,3 +34,8 @@ func (event *eventESDT) getExtendedIdentifier() string { return event.identifier } + +type eventSCDeploy struct { + contractAddress string + deployerAddress string +} diff --git a/server/services/transactionEventsController.go b/server/services/transactionEventsController.go index f1ced03b..2865f912 100644 --- a/server/services/transactionEventsController.go +++ b/server/services/transactionEventsController.go @@ -18,6 +18,30 @@ func newTransactionEventsController(provider NetworkProvider) *transactionEvents } } +func (controller *transactionEventsController) extractEventSCDeploy(tx *transaction.ApiTransactionResult) ([]*eventSCDeploy, error) { + rawEvents := controller.findManyEventsByIdentifier(tx, transactionEventSCDeploy) + + typedEvents := make([]*eventSCDeploy, 0, len(rawEvents)) + + for _, event := range rawEvents { + numTopics := len(event.Topics) + if numTopics != 2 { + return nil, fmt.Errorf("%w: bad number of topics for %s event = %d", errCannotRecognizeEvent, transactionEventSCDeploy, numTopics) + } + + contractAddress := event.Address + deployerPubKey := event.Topics[1] + deployerAddress := controller.provider.ConvertPubKeyToAddress(deployerPubKey) + + typedEvents = append(typedEvents, &eventSCDeploy{ + contractAddress: contractAddress, + deployerAddress: deployerAddress, + }) + } + + return typedEvents, nil +} + func (controller *transactionEventsController) extractEventTransferValueOnly(tx *transaction.ApiTransactionResult) ([]*eventTransferValueOnly, error) { rawEvents := controller.findManyEventsByIdentifier(tx, transactionEventTransferValueOnly) diff --git a/server/services/transactionsTransformer.go b/server/services/transactionsTransformer.go index abdb50f8..79ac170a 100644 --- a/server/services/transactionsTransformer.go +++ b/server/services/transactionsTransformer.go @@ -398,11 +398,18 @@ func (transformer *transactionsTransformer) addOperationsGivenTransactionEvents( return nil } + eventsSCDeploy, err := transformer.eventsController.extractEventSCDeploy(tx) + if err != nil { + return err + } + eventsTransferValueOnly, err := transformer.eventsController.extractEventTransferValueOnly(tx) if err != nil { return err } + eventsTransferValueOnly = filterOutTransferValueOnlyEventsThatAreAlreadyCapturedAsContractResults(eventsTransferValueOnly, txsInBlock) + eventsESDTTransfer, err := transformer.eventsController.extractEventsESDTOrESDTNFTTransfers(tx) if err != nil { return err @@ -438,7 +445,28 @@ func (transformer *transactionsTransformer) addOperationsGivenTransactionEvents( return err } - eventsTransferValueOnly = filterOutTransferValueOnlyEventsThatAreAlreadyCapturedAsContractResults(eventsTransferValueOnly, txsInBlock) + for _, event := range eventsSCDeploy { + log.Info("eventSCDeploy", "tx", tx.Hash, "block", tx.BlockNonce, "contract", event.contractAddress, "deployer", event.deployerAddress) + + // Handle deployments with transfer of value + + operations := []*types.Operation{ + // Deployer's balance change is already captured in non-events-based operations. + // Let's simulate the transfer from the System deployment address to the contract address. + { + Type: opTransfer, + Account: addressToAccountIdentifier(systemContractDeployAddress), + Amount: transformer.extension.valueToNativeAmount("-" + tx.Value), + }, + { + Type: opTransfer, + Account: addressToAccountIdentifier(event.contractAddress), + Amount: transformer.extension.valueToNativeAmount(tx.Value), + }, + } + + rosettaTx.Operations = append(rosettaTx.Operations, operations...) + } for _, event := range eventsTransferValueOnly { log.Info("eventTransferValueOnly (effective)", "tx", tx.Hash, "block", tx.BlockNonce) From 0f07181a184952bcf2450974735cf793b59edc12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Thu, 16 Nov 2023 17:37:43 +0200 Subject: [PATCH 15/75] Fix transfer value only for Sirius. --- server/services/transactionEventsController.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/services/transactionEventsController.go b/server/services/transactionEventsController.go index 2865f912..c2153599 100644 --- a/server/services/transactionEventsController.go +++ b/server/services/transactionEventsController.go @@ -63,8 +63,8 @@ func (controller *transactionEventsController) extractEventTransferValueOnly(tx } else if isAfterSirius { // https://github.com/multiversx/mx-specs/blob/main/releases/protocol/release-specs-v1.6.0-Sirius.md#17-logs--events-changes-5490 senderPubKey = []byte(event.Address) - receiverPubKey = event.Topics[0] - valueBytes = event.Topics[1] + receiverPubKey = event.Topics[1] + valueBytes = event.Topics[0] } else { return nil, fmt.Errorf("%w: bad number of topics for 'transferValueOnly' = %d", errCannotRecognizeEvent, numTopics) } From 24a2b4384e21df4bf49bfba50414976c8460566f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Thu, 16 Nov 2023 17:49:29 +0200 Subject: [PATCH 16/75] Fix events for Sirius. --- server/services/transactionEvents.go | 8 +++--- .../services/transactionEventsController.go | 27 ++++++++++--------- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/server/services/transactionEvents.go b/server/services/transactionEvents.go index 43d6e986..ea0a030d 100644 --- a/server/services/transactionEvents.go +++ b/server/services/transactionEvents.go @@ -5,11 +5,9 @@ import ( ) type eventTransferValueOnly struct { - sender string - senderPubKey []byte - receiver string - receiverPubKey []byte - value string + sender string + receiver string + value string } type eventESDT struct { diff --git a/server/services/transactionEventsController.go b/server/services/transactionEventsController.go index c2153599..536c5355 100644 --- a/server/services/transactionEventsController.go +++ b/server/services/transactionEventsController.go @@ -52,33 +52,34 @@ func (controller *transactionEventsController) extractEventTransferValueOnly(tx isBeforeSirius := numTopics == 3 isAfterSirius := numTopics == 2 - var senderPubKey []byte - var receiverPubKey []byte + var sender string + var receiver string var valueBytes []byte if isBeforeSirius { - senderPubKey = event.Topics[0] - receiverPubKey = event.Topics[1] + senderPubKey := event.Topics[0] + receiverPubKey := event.Topics[1] valueBytes = event.Topics[2] + + sender = controller.provider.ConvertPubKeyToAddress(senderPubKey) + receiver = controller.provider.ConvertPubKeyToAddress(receiverPubKey) } else if isAfterSirius { // https://github.com/multiversx/mx-specs/blob/main/releases/protocol/release-specs-v1.6.0-Sirius.md#17-logs--events-changes-5490 - senderPubKey = []byte(event.Address) - receiverPubKey = event.Topics[1] + sender = event.Address + receiverPubKey := event.Topics[1] valueBytes = event.Topics[0] + + receiver = controller.provider.ConvertPubKeyToAddress(receiverPubKey) } else { return nil, fmt.Errorf("%w: bad number of topics for 'transferValueOnly' = %d", errCannotRecognizeEvent, numTopics) } - sender := controller.provider.ConvertPubKeyToAddress(senderPubKey) - receiver := controller.provider.ConvertPubKeyToAddress(receiverPubKey) value := big.NewInt(0).SetBytes(valueBytes) typedEvents = append(typedEvents, &eventTransferValueOnly{ - sender: sender, - senderPubKey: senderPubKey, - receiver: receiver, - receiverPubKey: receiverPubKey, - value: value.String(), + sender: sender, + receiver: receiver, + value: value.String(), }) } From 1d457e78dbce9312899af4b6cb5b4fc92a864b52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Thu, 16 Nov 2023 18:06:39 +0200 Subject: [PATCH 17/75] Fix handling of SCDeploy. --- server/services/transactionEventsController.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/server/services/transactionEventsController.go b/server/services/transactionEventsController.go index 536c5355..3700972f 100644 --- a/server/services/transactionEventsController.go +++ b/server/services/transactionEventsController.go @@ -25,8 +25,10 @@ func (controller *transactionEventsController) extractEventSCDeploy(tx *transact for _, event := range rawEvents { numTopics := len(event.Topics) - if numTopics != 2 { - return nil, fmt.Errorf("%w: bad number of topics for %s event = %d", errCannotRecognizeEvent, transactionEventSCDeploy, numTopics) + if numTopics < 2 { + // Before Sirius, there are 2 topics: contract address, deployer address. + // For Sirius, there are 3 topics: contract address, deployer address, codehash. + return nil, fmt.Errorf("%w: bad number of topics for %s event = '%d', tx = %s", errCannotRecognizeEvent, transactionEventSCDeploy, numTopics, tx.Hash) } contractAddress := event.Address @@ -71,7 +73,7 @@ func (controller *transactionEventsController) extractEventTransferValueOnly(tx receiver = controller.provider.ConvertPubKeyToAddress(receiverPubKey) } else { - return nil, fmt.Errorf("%w: bad number of topics for 'transferValueOnly' = %d", errCannotRecognizeEvent, numTopics) + return nil, fmt.Errorf("%w: bad number of topics for '%s' = %d, tx = %s", errCannotRecognizeEvent, transactionEventTransferValueOnly, numTopics, tx.Hash) } value := big.NewInt(0).SetBytes(valueBytes) From b9a14b50c1de01c2654e931c64d58e7240b403b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Fri, 17 Nov 2023 11:20:38 +0200 Subject: [PATCH 18/75] Workaround for sc deploy. --- server/services/transactionsTransformer.go | 31 +++++++++++----------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/server/services/transactionsTransformer.go b/server/services/transactionsTransformer.go index 79ac170a..f5b3ae43 100644 --- a/server/services/transactionsTransformer.go +++ b/server/services/transactionsTransformer.go @@ -449,23 +449,24 @@ func (transformer *transactionsTransformer) addOperationsGivenTransactionEvents( log.Info("eventSCDeploy", "tx", tx.Hash, "block", tx.BlockNonce, "contract", event.contractAddress, "deployer", event.deployerAddress) // Handle deployments with transfer of value + if tx.Receiver == systemContractDeployAddress { + operations := []*types.Operation{ + // Deployer's balance change is already captured in non-events-based operations. + // Let's simulate the transfer from the System deployment address to the contract address. + { + Type: opTransfer, + Account: addressToAccountIdentifier(tx.Receiver), + Amount: transformer.extension.valueToNativeAmount("-" + tx.Value), + }, + { + Type: opTransfer, + Account: addressToAccountIdentifier(event.contractAddress), + Amount: transformer.extension.valueToNativeAmount(tx.Value), + }, + } - operations := []*types.Operation{ - // Deployer's balance change is already captured in non-events-based operations. - // Let's simulate the transfer from the System deployment address to the contract address. - { - Type: opTransfer, - Account: addressToAccountIdentifier(systemContractDeployAddress), - Amount: transformer.extension.valueToNativeAmount("-" + tx.Value), - }, - { - Type: opTransfer, - Account: addressToAccountIdentifier(event.contractAddress), - Amount: transformer.extension.valueToNativeAmount(tx.Value), - }, + rosettaTx.Operations = append(rosettaTx.Operations, operations...) } - - rosettaTx.Operations = append(rosettaTx.Operations, operations...) } for _, event := range eventsTransferValueOnly { From 79669becb8c263b7de55628a8764ea55c4204f26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Fri, 17 Nov 2023 20:32:00 +0200 Subject: [PATCH 19/75] Temporary workaround (slight optimization). --- server/services/transactionEventsController.go | 13 ++++++++++--- server/services/transactionsTransformer.go | 2 +- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/server/services/transactionEventsController.go b/server/services/transactionEventsController.go index 3700972f..c7c7d227 100644 --- a/server/services/transactionEventsController.go +++ b/server/services/transactionEventsController.go @@ -88,20 +88,27 @@ func (controller *transactionEventsController) extractEventTransferValueOnly(tx return typedEvents, nil } +// TODO: Use a cached "contractResultsSummaries" (this a must)! func filterOutTransferValueOnlyEventsThatAreAlreadyCapturedAsContractResults( + txOfInterest *transaction.ApiTransactionResult, events []*eventTransferValueOnly, txsInBlock []*transaction.ApiTransactionResult, ) []*eventTransferValueOnly { // First, we find all contract results in this block, and we "summarize" them (in a map). contractResultsSummaries := make(map[string]struct{}) - for _, tx := range txsInBlock { - isContractResult := tx.Type == string(transaction.TxTypeUnsigned) + for _, item := range txsInBlock { + isContractResult := item.Type == string(transaction.TxTypeUnsigned) if !isContractResult { continue } - summary := fmt.Sprintf("%s-%s-%s", tx.Sender, tx.Receiver, tx.Value) + // If not part of the same transaction graph, then skip it. + if !(item.OriginalTransactionHash == txOfInterest.Hash) && !(item.OriginalTransactionHash == txOfInterest.OriginalTransactionHash) { + continue + } + + summary := fmt.Sprintf("%s-%s-%s", item.Sender, item.Receiver, item.Value) contractResultsSummaries[summary] = struct{}{} } diff --git a/server/services/transactionsTransformer.go b/server/services/transactionsTransformer.go index f5b3ae43..3472b263 100644 --- a/server/services/transactionsTransformer.go +++ b/server/services/transactionsTransformer.go @@ -408,7 +408,7 @@ func (transformer *transactionsTransformer) addOperationsGivenTransactionEvents( return err } - eventsTransferValueOnly = filterOutTransferValueOnlyEventsThatAreAlreadyCapturedAsContractResults(eventsTransferValueOnly, txsInBlock) + eventsTransferValueOnly = filterOutTransferValueOnlyEventsThatAreAlreadyCapturedAsContractResults(tx, eventsTransferValueOnly, txsInBlock) eventsESDTTransfer, err := transformer.eventsController.extractEventsESDTOrESDTNFTTransfers(tx) if err != nil { From d4065751befa3dfba582b003bb681cc510cf6351 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Fri, 17 Nov 2023 21:55:29 +0200 Subject: [PATCH 20/75] Handle exception for claim developer rewards. --- server/services/exceptions.go | 14 ++++++++++++++ server/services/transactionsTransformer.go | 15 +++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 server/services/exceptions.go diff --git a/server/services/exceptions.go b/server/services/exceptions.go new file mode 100644 index 00000000..8965ab26 --- /dev/null +++ b/server/services/exceptions.go @@ -0,0 +1,14 @@ +package services + +import ( + "github.com/multiversx/mx-chain-core-go/data/transaction" +) + +func isContractResultOfOpaquelyClaimingDeveloperRewards(tx *transaction.ApiTransactionResult) bool { + if tx.BlockNonce == 15693863 && tx.SourceShard == 2 && tx.Hash == "8ddecc831c70ecf0ca312a46e04bbdc7508fabada494714ec41fd49c8ec13915" { + log.Warn("Exception: 8ddecc831c70ecf0ca312a46e04bbdc7508fabada494714ec41fd49c8ec13915") + return true + } + + return false +} diff --git a/server/services/transactionsTransformer.go b/server/services/transactionsTransformer.go index 3472b263..25b3409a 100644 --- a/server/services/transactionsTransformer.go +++ b/server/services/transactionsTransformer.go @@ -37,6 +37,8 @@ func (transformer *transactionsTransformer) transformBlockTxs(block *api.Block) for _, miniblock := range block.MiniBlocks { for _, tx := range miniblock.Transactions { + // Make sure SCRs also have the block nonce set. + tx.BlockNonce = block.Nonce txs = append(txs, tx) } for _, receipt := range miniblock.Receipts { @@ -159,6 +161,19 @@ func (transformer *transactionsTransformer) unsignedTxToRosettaTx( } } + if isContractResultOfOpaquelyClaimingDeveloperRewards(scr) { + return &types.Transaction{ + TransactionIdentifier: hashToTransactionIdentifier(scr.Hash), + Operations: []*types.Operation{ + { + Type: opScResult, + Account: addressToAccountIdentifier(scr.Receiver), + Amount: transformer.extension.valueToNativeAmount(scr.Value), + }, + }, + } + } + return &types.Transaction{ TransactionIdentifier: hashToTransactionIdentifier(scr.Hash), Operations: []*types.Operation{ From ecbce7f1635c4fac2521f8752096321cff4ef0d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Sun, 19 Nov 2023 00:44:49 +0200 Subject: [PATCH 21/75] Handle internal VM errors (without signal error, if error happens on callbacks). --- server/services/constants.go | 1 + server/services/transactionEventsController.go | 15 +++++++++++++++ server/services/transactionsTransformer.go | 7 +++++++ 3 files changed, 23 insertions(+) diff --git a/server/services/constants.go b/server/services/constants.go index cb929fbc..ca85520f 100644 --- a/server/services/constants.go +++ b/server/services/constants.go @@ -27,6 +27,7 @@ var ( var ( transactionEventSignalError = core.SignalErrorOperation + transactionEventInternalVMErrors = "internalVMErrors" transactionEventSCDeploy = core.SCDeployIdentifier transactionEventTransferValueOnly = "transferValueOnly" transactionEventESDTTransfer = "ESDTTransfer" diff --git a/server/services/transactionEventsController.go b/server/services/transactionEventsController.go index c7c7d227..88591733 100644 --- a/server/services/transactionEventsController.go +++ b/server/services/transactionEventsController.go @@ -144,6 +144,21 @@ func (controller *transactionEventsController) hasAnySignalError(tx *transaction return false } +func (controller *transactionEventsController) hasAnyInternalVMError(tx *transaction.ApiTransactionResult) bool { + if !controller.hasEvents(tx) { + return false + } + + for _, event := range tx.Logs.Events { + hasError := event.Identifier == transactionEventInternalVMErrors + if hasError { + return true + } + } + + return false +} + func (controller *transactionEventsController) hasSignalErrorOfSendingValueToNonPayableContract(tx *transaction.ApiTransactionResult) bool { if !controller.hasEvents(tx) { return false diff --git a/server/services/transactionsTransformer.go b/server/services/transactionsTransformer.go index 25b3409a..ab13e662 100644 --- a/server/services/transactionsTransformer.go +++ b/server/services/transactionsTransformer.go @@ -413,6 +413,13 @@ func (transformer *transactionsTransformer) addOperationsGivenTransactionEvents( return nil } + hasInternalVMError := transformer.featuresDetector.eventsController.hasAnyInternalVMError(tx) + if hasInternalVMError { + // TODO: Remove + log.Info("hasInternalVMError, will ignore events", "tx", tx.Hash, "block", tx.BlockNonce) + return nil + } + eventsSCDeploy, err := transformer.eventsController.extractEventSCDeploy(tx) if err != nil { return err From 2c24305cc301d361121ac5eb434890c8b058f26f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Wed, 22 Nov 2023 20:43:50 +0200 Subject: [PATCH 22/75] Handle internalVMErrors that happen on legacy callback - keep existing (previous) events, handle them. --- server/services/transactionEventsController.go | 13 ++++++++++--- server/services/transactionsTransformer.go | 2 +- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/server/services/transactionEventsController.go b/server/services/transactionEventsController.go index 88591733..dbbb0550 100644 --- a/server/services/transactionEventsController.go +++ b/server/services/transactionEventsController.go @@ -144,16 +144,23 @@ func (controller *transactionEventsController) hasAnySignalError(tx *transaction return false } -func (controller *transactionEventsController) hasAnyInternalVMError(tx *transaction.ApiTransactionResult) bool { +func (controller *transactionEventsController) hasAnyInternalVMErrorNotOnLegacyCallback(tx *transaction.ApiTransactionResult) bool { if !controller.hasEvents(tx) { return false } for _, event := range tx.Logs.Events { hasError := event.Identifier == transactionEventInternalVMErrors - if hasError { - return true + if !hasError { + continue + } + + isAboutLegacyCallback := strings.Contains(string(event.Data), "[execution failed] [callBack]") + if isAboutLegacyCallback { + continue } + + return true } return false diff --git a/server/services/transactionsTransformer.go b/server/services/transactionsTransformer.go index ab13e662..d91a724e 100644 --- a/server/services/transactionsTransformer.go +++ b/server/services/transactionsTransformer.go @@ -413,7 +413,7 @@ func (transformer *transactionsTransformer) addOperationsGivenTransactionEvents( return nil } - hasInternalVMError := transformer.featuresDetector.eventsController.hasAnyInternalVMError(tx) + hasInternalVMError := transformer.featuresDetector.eventsController.hasAnyInternalVMErrorNotOnLegacyCallback(tx) if hasInternalVMError { // TODO: Remove log.Info("hasInternalVMError, will ignore events", "tx", tx.Hash, "block", tx.BlockNonce) From 3789d1d222e03765cb4ed6e7edb15a7befb72bf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Wed, 13 Dec 2023 16:19:30 +0200 Subject: [PATCH 23/75] Handle deployments with signal error - create refund operations. --- server/services/constants.go | 1 + server/services/transactionsFeaturesDetector.go | 6 ++++++ server/services/transactionsTransformer.go | 13 ++++++++----- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/server/services/constants.go b/server/services/constants.go index ca85520f..449eaaa0 100644 --- a/server/services/constants.go +++ b/server/services/constants.go @@ -14,6 +14,7 @@ var ( transactionProcessingTypeBuiltInFunctionCall = "BuiltInFunctionCall" transactionProcessingTypeMoveBalance = "MoveBalance" transactionProcessingTypeContractInvoking = "SCInvoking" + transactionProcessingTypeContractDeployment = "SCDeployment" amountZero = "0" builtInFunctionClaimDeveloperRewards = core.BuiltInFunctionClaimDeveloperRewards builtInFunctionESDTTransfer = core.BuiltInFunctionESDTTransfer diff --git a/server/services/transactionsFeaturesDetector.go b/server/services/transactionsFeaturesDetector.go index 0d36e0a5..7b0da59d 100644 --- a/server/services/transactionsFeaturesDetector.go +++ b/server/services/transactionsFeaturesDetector.go @@ -107,6 +107,12 @@ func (detector *transactionsFeaturesDetector) isContractCallWithSignalError(tx * detector.eventsController.hasAnySignalError(tx) } +func (detector *transactionsFeaturesDetector) isContractDeploymentWithSignalError(tx *transaction.ApiTransactionResult) bool { + return tx.ProcessingTypeOnSource == transactionProcessingTypeContractDeployment && + tx.ProcessingTypeOnDestination == transactionProcessingTypeContractDeployment && + detector.eventsController.hasAnySignalError(tx) +} + func (detector *transactionsFeaturesDetector) isIntrashard(tx *transaction.ApiTransactionResult) bool { return tx.SourceShard == tx.DestinationShard } diff --git a/server/services/transactionsTransformer.go b/server/services/transactionsTransformer.go index d91a724e..7a13be68 100644 --- a/server/services/transactionsTransformer.go +++ b/server/services/transactionsTransformer.go @@ -237,7 +237,7 @@ func (transformer *transactionsTransformer) normalTxToRosetta( return nil, err } - valueRefundOperationIfContractCallWithSignalError, err := transformer.createValueReturnOperationsIfIntrashardContractCallWithSignalError(tx, allTransactionsInBlock) + valueRefundOperationIfContractCallOrDeploymentWithSignalError, err := transformer.createValueReturnOperationsIfIntrashardContractCallOrContractDeploymentWithSignalError(tx, allTransactionsInBlock) if err != nil { return nil, err } @@ -246,12 +246,12 @@ func (transformer *transactionsTransformer) normalTxToRosetta( if len(innerTxOperationsIfRelayedCompletelyIntrashardWithSignalError) > 0 { log.Info("innerTxOperationsIfRelayedCompletelyIntrashardWithSignalError", "tx", tx.Hash, "block", tx.BlockNonce) } - if len(valueRefundOperationIfContractCallWithSignalError) > 0 { - log.Info("valueRefundOperationIfContractCallWithSignalError", "tx", tx.Hash, "block", tx.BlockNonce) + if len(valueRefundOperationIfContractCallOrDeploymentWithSignalError) > 0 { + log.Info("valueRefundOperationIfContractCallOrDeploymentWithSignalError", "tx", tx.Hash, "block", tx.BlockNonce) } operations = append(operations, innerTxOperationsIfRelayedCompletelyIntrashardWithSignalError...) - operations = append(operations, valueRefundOperationIfContractCallWithSignalError...) + operations = append(operations, valueRefundOperationIfContractCallOrDeploymentWithSignalError...) return &types.Transaction{ TransactionIdentifier: hashToTransactionIdentifier(tx.Hash), @@ -298,13 +298,16 @@ func (transformer *transactionsTransformer) extractInnerTxOperationsIfRelayedCom }, nil } -func (transformer *transactionsTransformer) createValueReturnOperationsIfIntrashardContractCallWithSignalError( +func (transformer *transactionsTransformer) createValueReturnOperationsIfIntrashardContractCallOrContractDeploymentWithSignalError( tx *transaction.ApiTransactionResult, allTransactionsInBlock []*transaction.ApiTransactionResult, ) ([]*types.Operation, error) { if !transformer.featuresDetector.isIntrashardContractCallWithSignalErrorButWithoutContractResultBearingRefundValue(tx, allTransactionsInBlock) { return []*types.Operation{}, nil } + if !transformer.featuresDetector.isContractDeploymentWithSignalError(tx) { + return []*types.Operation{}, nil + } return []*types.Operation{ { From c3c0b7e8655f79b0a9d9497d3a27bded534e1066 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Wed, 13 Dec 2023 17:02:44 +0200 Subject: [PATCH 24/75] Fix check. --- server/services/transactionsTransformer.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/server/services/transactionsTransformer.go b/server/services/transactionsTransformer.go index 7a13be68..1ec639e9 100644 --- a/server/services/transactionsTransformer.go +++ b/server/services/transactionsTransformer.go @@ -302,10 +302,9 @@ func (transformer *transactionsTransformer) createValueReturnOperationsIfIntrash tx *transaction.ApiTransactionResult, allTransactionsInBlock []*transaction.ApiTransactionResult, ) ([]*types.Operation, error) { - if !transformer.featuresDetector.isIntrashardContractCallWithSignalErrorButWithoutContractResultBearingRefundValue(tx, allTransactionsInBlock) { - return []*types.Operation{}, nil - } - if !transformer.featuresDetector.isContractDeploymentWithSignalError(tx) { + isContractCallWithError := transformer.featuresDetector.isIntrashardContractCallWithSignalErrorButWithoutContractResultBearingRefundValue(tx, allTransactionsInBlock) + isContractDeploymentWithError := transformer.featuresDetector.isContractDeploymentWithSignalError(tx) + if !isContractCallWithError && !isContractDeploymentWithError { return []*types.Operation{}, nil } From cbc50dc61d512b8dc01c98973efb68099ca6ff59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Mon, 8 Apr 2024 17:36:26 +0300 Subject: [PATCH 25/75] Handle exceptions around "claim developer rewards". --- server/services/exceptions.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/server/services/exceptions.go b/server/services/exceptions.go index 8965ab26..979ec12d 100644 --- a/server/services/exceptions.go +++ b/server/services/exceptions.go @@ -4,11 +4,19 @@ import ( "github.com/multiversx/mx-chain-core-go/data/transaction" ) +// See: +// - https://explorer.multiversx.com/transactions?function=claimRewardsAllContracts +// - https://github.com/multiversx/mx-chain-vm-common-go/pull/247 func isContractResultOfOpaquelyClaimingDeveloperRewards(tx *transaction.ApiTransactionResult) bool { if tx.BlockNonce == 15693863 && tx.SourceShard == 2 && tx.Hash == "8ddecc831c70ecf0ca312a46e04bbdc7508fabada494714ec41fd49c8ec13915" { log.Warn("Exception: 8ddecc831c70ecf0ca312a46e04bbdc7508fabada494714ec41fd49c8ec13915") return true } + if tx.BlockNonce == 18098197 && tx.SourceShard == 2 && tx.Hash == "90357d8f72bb1c749b46e394a4be349d4b2700a8f734470f815a524059f7031b" { + log.Warn("Exception: 90357d8f72bb1c749b46e394a4be349d4b2700a8f734470f815a524059f7031b") + return true + } + return false } From 790f1815a522f4af87653c5462e4dd739007882a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Mon, 8 Apr 2024 20:27:00 +0300 Subject: [PATCH 26/75] Fix detection of ClaimDeveloperRewards. --- server/services/transactionsFeaturesDetector.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/server/services/transactionsFeaturesDetector.go b/server/services/transactionsFeaturesDetector.go index 7b0da59d..1ecddb8c 100644 --- a/server/services/transactionsFeaturesDetector.go +++ b/server/services/transactionsFeaturesDetector.go @@ -1,8 +1,6 @@ package services import ( - "bytes" - "github.com/multiversx/mx-chain-core-go/data/transaction" ) @@ -28,9 +26,9 @@ func (detector *transactionsFeaturesDetector) doesContractResultHoldRewardsOfCla for _, tx := range allTransactionsInBlock { matchesTypeOnSource := tx.ProcessingTypeOnSource == transactionProcessingTypeBuiltInFunctionCall matchesTypeOnDestination := tx.ProcessingTypeOnDestination == transactionProcessingTypeBuiltInFunctionCall - matchesData := bytes.Equal(tx.Data, []byte(builtInFunctionClaimDeveloperRewards)) + matchesOperation := tx.Operation == builtInFunctionClaimDeveloperRewards - if matchesTypeOnSource && matchesTypeOnDestination && matchesData { + if matchesTypeOnSource && matchesTypeOnDestination && matchesOperation { claimDeveloperRewardsTxs[tx.Hash] = struct{}{} } } From 7d19982959f030624236faae2e2486eb4898b867 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Thu, 11 Apr 2024 16:54:56 +0300 Subject: [PATCH 27/75] extractEventsClaimDeveloperRewards. --- server/services/constants.go | 1 + server/services/transactionEvents.go | 5 ++ .../services/transactionEventsController.go | 49 ++++++++++++++----- server/services/transactionsTransformer.go | 17 +++++++ 4 files changed, 60 insertions(+), 12 deletions(-) diff --git a/server/services/constants.go b/server/services/constants.go index 449eaaa0..cb4fac1e 100644 --- a/server/services/constants.go +++ b/server/services/constants.go @@ -40,6 +40,7 @@ var ( transactionEventESDTLocalBurn = core.BuiltInFunctionESDTLocalBurn transactionEventESDTLocalMint = core.BuiltInFunctionESDTLocalMint transactionEventESDTWipe = core.BuiltInFunctionESDTWipe + transactionEventClaimDeveloperRewards = core.BuiltInFunctionClaimDeveloperRewards transactionEventTopicInvalidMetaTransaction = "meta transaction is invalid" transactionEventTopicInvalidMetaTransactionNotEnoughGas = "meta transaction is invalid: not enough gas" ) diff --git a/server/services/transactionEvents.go b/server/services/transactionEvents.go index ea0a030d..4deaf200 100644 --- a/server/services/transactionEvents.go +++ b/server/services/transactionEvents.go @@ -37,3 +37,8 @@ type eventSCDeploy struct { contractAddress string deployerAddress string } + +type eventClaimDeveloperRewards struct { + value string + receiverAddress string +} diff --git a/server/services/transactionEventsController.go b/server/services/transactionEventsController.go index dbbb0550..0b3b8bf2 100644 --- a/server/services/transactionEventsController.go +++ b/server/services/transactionEventsController.go @@ -258,14 +258,14 @@ func (controller *transactionEventsController) extractEventsESDTLocalBurn(tx *tr return nil, fmt.Errorf("%w: bad number of topics for ESDTLocalBurn event = %d", errCannotRecognizeEvent, numTopics) } - identifider := event.Topics[0] + identifier := event.Topics[0] nonceAsBytes := event.Topics[1] valueBytes := event.Topics[2] value := big.NewInt(0).SetBytes(valueBytes) typedEvents = append(typedEvents, &eventESDT{ otherAddress: event.Address, - identifier: string(identifider), + identifier: string(identifier), nonceAsBytes: nonceAsBytes, value: value.String(), }) @@ -284,14 +284,14 @@ func (controller *transactionEventsController) extractEventsESDTLocalMint(tx *tr return nil, fmt.Errorf("%w: bad number of topics for ESDTLocalMint event = %d", errCannotRecognizeEvent, numTopics) } - identifider := event.Topics[0] + identifier := event.Topics[0] nonceAsBytes := event.Topics[1] valueBytes := event.Topics[2] value := big.NewInt(0).SetBytes(valueBytes) typedEvents = append(typedEvents, &eventESDT{ otherAddress: event.Address, - identifier: string(identifider), + identifier: string(identifier), nonceAsBytes: nonceAsBytes, value: value.String(), }) @@ -310,7 +310,7 @@ func (controller *transactionEventsController) extractEventsESDTWipe(tx *transac return nil, fmt.Errorf("%w: bad number of topics for ESDTWipe event = %d", errCannotRecognizeEvent, numTopics) } - identifider := event.Topics[0] + identifier := event.Topics[0] nonceAsBytes := event.Topics[1] valueBytes := event.Topics[2] accountPubkey := event.Topics[3] @@ -320,7 +320,7 @@ func (controller *transactionEventsController) extractEventsESDTWipe(tx *transac typedEvents = append(typedEvents, &eventESDT{ otherAddress: accountAddress, - identifier: string(identifider), + identifier: string(identifier), nonceAsBytes: nonceAsBytes, value: value.String(), }) @@ -339,7 +339,7 @@ func (controller *transactionEventsController) extractEventsESDTNFTCreate(tx *tr return nil, fmt.Errorf("%w: bad number of topics for %s event = %d", errCannotRecognizeEvent, transactionEventESDTNFTCreate, numTopics) } - identifider := event.Topics[0] + identifier := event.Topics[0] nonceAsBytes := event.Topics[1] valueBytes := event.Topics[2] // We ignore the 4th topic. @@ -348,7 +348,7 @@ func (controller *transactionEventsController) extractEventsESDTNFTCreate(tx *tr typedEvents = append(typedEvents, &eventESDT{ otherAddress: event.Address, - identifier: string(identifider), + identifier: string(identifier), nonceAsBytes: nonceAsBytes, value: value.String(), }) @@ -367,7 +367,7 @@ func (controller *transactionEventsController) extractEventsESDTNFTBurn(tx *tran return nil, fmt.Errorf("%w: bad number of topics for %s event = %d", errCannotRecognizeEvent, transactionEventESDTNFTBurn, numTopics) } - identifider := event.Topics[0] + identifier := event.Topics[0] nonceAsBytes := event.Topics[1] valueBytes := event.Topics[2] @@ -375,7 +375,7 @@ func (controller *transactionEventsController) extractEventsESDTNFTBurn(tx *tran typedEvents = append(typedEvents, &eventESDT{ otherAddress: event.Address, - identifier: string(identifider), + identifier: string(identifier), nonceAsBytes: nonceAsBytes, value: value.String(), }) @@ -394,7 +394,7 @@ func (controller *transactionEventsController) extractEventsESDTNFTAddQuantity(t return nil, fmt.Errorf("%w: bad number of topics for %s event = %d", errCannotRecognizeEvent, transactionEventESDTNFTAddQuantity, numTopics) } - identifider := event.Topics[0] + identifier := event.Topics[0] nonceAsBytes := event.Topics[1] valueBytes := event.Topics[2] @@ -402,7 +402,7 @@ func (controller *transactionEventsController) extractEventsESDTNFTAddQuantity(t typedEvents = append(typedEvents, &eventESDT{ otherAddress: event.Address, - identifier: string(identifider), + identifier: string(identifier), nonceAsBytes: nonceAsBytes, value: value.String(), }) @@ -411,6 +411,31 @@ func (controller *transactionEventsController) extractEventsESDTNFTAddQuantity(t return typedEvents, nil } +func (controller *transactionEventsController) extractEventsClaimDeveloperRewards(tx *transaction.ApiTransactionResult) ([]*eventClaimDeveloperRewards, error) { + rawEvents := controller.findManyEventsByIdentifier(tx, transactionEventClaimDeveloperRewards) + typedEvents := make([]*eventClaimDeveloperRewards, 0, len(rawEvents)) + + for _, event := range rawEvents { + numTopics := len(event.Topics) + if numTopics != 2 { + return nil, fmt.Errorf("%w: bad number of topics for %s event = %d", errCannotRecognizeEvent, transactionEventClaimDeveloperRewards, numTopics) + } + + valueBytes := event.Topics[0] + receiverPubkey := event.Topics[1] + + value := big.NewInt(0).SetBytes(valueBytes) + receiver := controller.provider.ConvertPubKeyToAddress(receiverPubkey) + + typedEvents = append(typedEvents, &eventClaimDeveloperRewards{ + value: value.String(), + receiverAddress: receiver, + }) + } + + return typedEvents, nil +} + func (controller *transactionEventsController) findManyEventsByIdentifier(tx *transaction.ApiTransactionResult, identifier string) []*transaction.Events { events := make([]*transaction.Events, 0) diff --git a/server/services/transactionsTransformer.go b/server/services/transactionsTransformer.go index 1ec639e9..096f6cc3 100644 --- a/server/services/transactionsTransformer.go +++ b/server/services/transactionsTransformer.go @@ -469,6 +469,11 @@ func (transformer *transactionsTransformer) addOperationsGivenTransactionEvents( return err } + eventsClaimDeveloperRewards, err := transformer.eventsController.extractEventsClaimDeveloperRewards(tx) + if err != nil { + return err + } + for _, event := range eventsSCDeploy { log.Info("eventSCDeploy", "tx", tx.Hash, "block", tx.BlockNonce, "contract", event.contractAddress, "deployer", event.deployerAddress) @@ -636,5 +641,17 @@ func (transformer *transactionsTransformer) addOperationsGivenTransactionEvents( rosettaTx.Operations = append(rosettaTx.Operations, operations...) } + for _, event := range eventsClaimDeveloperRewards { + operations := []*types.Operation{ + { + Type: opDeveloperRewardsAsScResult, + Account: addressToAccountIdentifier(event.receiverAddress), + Amount: transformer.extension.valueToNativeAmount(event.value), + }, + } + + rosettaTx.Operations = append(rosettaTx.Operations, operations...) + } + return nil } From 515815f52623a49ca4b0344d0bad614a3b6cd853 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Thu, 11 Apr 2024 17:16:39 +0300 Subject: [PATCH 28/75] Remove logic around hasAnyInternalVMErrorNotOnLegacyCallback. --- .../services/transactionEventsController.go | 22 ------------------- server/services/transactionsTransformer.go | 7 ------ 2 files changed, 29 deletions(-) diff --git a/server/services/transactionEventsController.go b/server/services/transactionEventsController.go index 0b3b8bf2..4ce959a3 100644 --- a/server/services/transactionEventsController.go +++ b/server/services/transactionEventsController.go @@ -144,28 +144,6 @@ func (controller *transactionEventsController) hasAnySignalError(tx *transaction return false } -func (controller *transactionEventsController) hasAnyInternalVMErrorNotOnLegacyCallback(tx *transaction.ApiTransactionResult) bool { - if !controller.hasEvents(tx) { - return false - } - - for _, event := range tx.Logs.Events { - hasError := event.Identifier == transactionEventInternalVMErrors - if !hasError { - continue - } - - isAboutLegacyCallback := strings.Contains(string(event.Data), "[execution failed] [callBack]") - if isAboutLegacyCallback { - continue - } - - return true - } - - return false -} - func (controller *transactionEventsController) hasSignalErrorOfSendingValueToNonPayableContract(tx *transaction.ApiTransactionResult) bool { if !controller.hasEvents(tx) { return false diff --git a/server/services/transactionsTransformer.go b/server/services/transactionsTransformer.go index 096f6cc3..e0a0b68d 100644 --- a/server/services/transactionsTransformer.go +++ b/server/services/transactionsTransformer.go @@ -415,13 +415,6 @@ func (transformer *transactionsTransformer) addOperationsGivenTransactionEvents( return nil } - hasInternalVMError := transformer.featuresDetector.eventsController.hasAnyInternalVMErrorNotOnLegacyCallback(tx) - if hasInternalVMError { - // TODO: Remove - log.Info("hasInternalVMError, will ignore events", "tx", tx.Hash, "block", tx.BlockNonce) - return nil - } - eventsSCDeploy, err := transformer.eventsController.extractEventSCDeploy(tx) if err != nil { return err From ff34b3e8d2b9b6a72459a7793f0710d8fe604ecb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Thu, 11 Apr 2024 17:43:51 +0300 Subject: [PATCH 29/75] Handle claim developer rewards - in two ways: old and new. --- server/services/features.go | 21 ++++++++ server/services/transactionsTransformer.go | 60 ++++++++++++---------- 2 files changed, 53 insertions(+), 28 deletions(-) create mode 100644 server/services/features.go diff --git a/server/services/features.go b/server/services/features.go new file mode 100644 index 00000000..c68ebf73 --- /dev/null +++ b/server/services/features.go @@ -0,0 +1,21 @@ +package services + +import ( + "os" + "strconv" +) + +func areClaimDeveloperRewardsEventsEnabled(blockNonce uint64) bool { + nonceAsString := os.Getenv("CLAIM_DEVELOPER_REWARDS_EVENTS_ENABLED_NONCE") + if nonceAsString == "" { + return false + } + + nonce, err := strconv.Atoi(nonceAsString) + if err != nil { + return false + } + + enabled := blockNonce >= uint64(nonce) + return enabled +} diff --git a/server/services/transactionsTransformer.go b/server/services/transactionsTransformer.go index e0a0b68d..7087e837 100644 --- a/server/services/transactionsTransformer.go +++ b/server/services/transactionsTransformer.go @@ -148,29 +148,31 @@ func (transformer *transactionsTransformer) unsignedTxToRosettaTx( } } - if transformer.featuresDetector.doesContractResultHoldRewardsOfClaimDeveloperRewards(scr, txsInBlock) { - return &types.Transaction{ - TransactionIdentifier: hashToTransactionIdentifier(scr.Hash), - Operations: []*types.Operation{ - { - Type: opDeveloperRewardsAsScResult, - Account: addressToAccountIdentifier(scr.Receiver), - Amount: transformer.extension.valueToNativeAmount(scr.Value), + if !areClaimDeveloperRewardsEventsEnabled(scr.BlockNonce) { + if transformer.featuresDetector.doesContractResultHoldRewardsOfClaimDeveloperRewards(scr, txsInBlock) { + return &types.Transaction{ + TransactionIdentifier: hashToTransactionIdentifier(scr.Hash), + Operations: []*types.Operation{ + { + Type: opDeveloperRewardsAsScResult, + Account: addressToAccountIdentifier(scr.Receiver), + Amount: transformer.extension.valueToNativeAmount(scr.Value), + }, }, - }, + } } - } - if isContractResultOfOpaquelyClaimingDeveloperRewards(scr) { - return &types.Transaction{ - TransactionIdentifier: hashToTransactionIdentifier(scr.Hash), - Operations: []*types.Operation{ - { - Type: opScResult, - Account: addressToAccountIdentifier(scr.Receiver), - Amount: transformer.extension.valueToNativeAmount(scr.Value), + if isContractResultOfOpaquelyClaimingDeveloperRewards(scr) { + return &types.Transaction{ + TransactionIdentifier: hashToTransactionIdentifier(scr.Hash), + Operations: []*types.Operation{ + { + Type: opScResult, + Account: addressToAccountIdentifier(scr.Receiver), + Amount: transformer.extension.valueToNativeAmount(scr.Value), + }, }, - }, + } } } @@ -634,16 +636,18 @@ func (transformer *transactionsTransformer) addOperationsGivenTransactionEvents( rosettaTx.Operations = append(rosettaTx.Operations, operations...) } - for _, event := range eventsClaimDeveloperRewards { - operations := []*types.Operation{ - { - Type: opDeveloperRewardsAsScResult, - Account: addressToAccountIdentifier(event.receiverAddress), - Amount: transformer.extension.valueToNativeAmount(event.value), - }, - } + if areClaimDeveloperRewardsEventsEnabled(tx.BlockNonce) { + for _, event := range eventsClaimDeveloperRewards { + operations := []*types.Operation{ + { + Type: opDeveloperRewardsAsScResult, + Account: addressToAccountIdentifier(event.receiverAddress), + Amount: transformer.extension.valueToNativeAmount(event.value), + }, + } - rosettaTx.Operations = append(rosettaTx.Operations, operations...) + rosettaTx.Operations = append(rosettaTx.Operations, operations...) + } } return nil From 2604bd4d109e92b7d5abc8e49c9d97915fdc8ade Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Thu, 11 Apr 2024 17:44:19 +0300 Subject: [PATCH 30/75] Bump version. --- version/constants.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version/constants.go b/version/constants.go index 67526d4b..dd45828b 100644 --- a/version/constants.go +++ b/version/constants.go @@ -7,5 +7,5 @@ const ( var ( // RosettaMiddlewareVersion is the version of this package (application) - RosettaMiddlewareVersion = "v0.5.0" + RosettaMiddlewareVersion = "v0.5.1" ) From 794b0ed7d30e1976d01775400b9859e3aa11b63b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Wed, 7 Aug 2024 12:36:08 +0300 Subject: [PATCH 31/75] Reference newest mx-chain-* packages. --- go.mod | 51 ++++++------ go.sum | 120 ++++++++++++++--------------- server/factory/provider.go | 3 +- server/provider/networkProvider.go | 8 +- testscommon/networkProviderMock.go | 10 ++- testscommon/realworld.go | 3 +- 6 files changed, 101 insertions(+), 94 deletions(-) diff --git a/go.mod b/go.mod index d26bed1c..1bcae42e 100644 --- a/go.mod +++ b/go.mod @@ -5,60 +5,57 @@ go 1.18 require ( github.com/coinbase/rosetta-sdk-go v0.8.3 github.com/coinbase/rosetta-sdk-go/types v1.0.0 - github.com/multiversx/mx-chain-core-go v1.1.37 - github.com/multiversx/mx-chain-go v1.5.13 - github.com/multiversx/mx-chain-logger-go v1.0.11 - github.com/multiversx/mx-chain-proxy-go v1.1.39 - github.com/multiversx/mx-chain-storage-go v1.0.7 + github.com/gin-gonic/gin v1.9.1 + github.com/multiversx/mx-chain-core-go v1.2.21 + github.com/multiversx/mx-chain-go v1.7.17 + github.com/multiversx/mx-chain-logger-go v1.0.15 + github.com/multiversx/mx-chain-proxy-go v1.1.50 + github.com/multiversx/mx-chain-storage-go v1.0.16 github.com/stretchr/testify v1.8.4 github.com/urfave/cli v1.22.10 ) require ( github.com/btcsuite/btcd/btcutil v1.1.3 // indirect - github.com/bytedance/sonic v1.8.0 // indirect + github.com/bytedance/sonic v1.9.1 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/denisbrodbeck/machineid v1.0.1 // indirect + github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/gin-contrib/sse v0.1.0 // indirect - github.com/gin-gonic/gin v1.9.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.11.2 // indirect - github.com/goccy/go-json v0.10.0 // indirect + github.com/go-playground/validator/v10 v10.14.0 // indirect + github.com/goccy/go-json v0.10.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.3 // indirect - github.com/golang/snappy v0.0.4 // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/hashicorp/golang-lru v0.6.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.2.5 // indirect - github.com/leodido/go-urn v1.2.1 // indirect + github.com/leodido/go-urn v1.2.4 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mr-tron/base58 v1.2.0 // indirect - github.com/multiversx/concurrent-map v0.1.4 // indirect - github.com/multiversx/mx-chain-crypto-go v1.2.6 // indirect - github.com/multiversx/mx-chain-es-indexer-go v1.3.20 // indirect - github.com/multiversx/mx-chain-p2p-go v1.0.18 // indirect - github.com/multiversx/mx-chain-vm-common-go v1.3.42 // indirect + github.com/multiversx/mx-chain-communication-go v1.1.0 // indirect + github.com/multiversx/mx-chain-crypto-go v1.2.12 // indirect + github.com/multiversx/mx-chain-es-indexer-go v1.7.4 // indirect + github.com/multiversx/mx-chain-vm-common-go v1.5.13 // indirect github.com/pelletier/go-toml v1.9.3 // indirect - github.com/pelletier/go-toml/v2 v2.0.6 // indirect + github.com/pelletier/go-toml/v2 v2.0.8 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/russross/blackfriday/v2 v2.0.1 // indirect - github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect - github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect - github.com/ugorji/go/codec v1.2.9 // indirect - golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect - golang.org/x/crypto v0.9.0 // indirect - golang.org/x/net v0.10.0 // indirect - golang.org/x/sys v0.8.0 // indirect - golang.org/x/text v0.9.0 // indirect + github.com/ugorji/go/codec v1.2.11 // indirect + golang.org/x/arch v0.3.0 // indirect + golang.org/x/crypto v0.10.0 // indirect + golang.org/x/net v0.11.0 // indirect + golang.org/x/sys v0.10.0 // indirect + golang.org/x/text v0.10.0 // indirect google.golang.org/protobuf v1.30.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 79c3ff4f..a4674b67 100644 --- a/go.sum +++ b/go.sum @@ -21,8 +21,8 @@ github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= -github.com/bytedance/sonic v1.8.0 h1:ea0Xadu+sHlu7x5O3gKhRpQ1IKiMrSiHttPF0ybECuA= -github.com/bytedance/sonic v1.8.0/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= +github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= +github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= @@ -31,8 +31,8 @@ github.com/coinbase/rosetta-sdk-go v0.8.3/go.mod h1:ChOHc+BNq7zqJDDkui0DA124GOvl github.com/coinbase/rosetta-sdk-go/types v1.0.0 h1:jpVIwLcPoOeCR6o1tU+Xv7r5bMONNbHU7MuEHboiFuA= github.com/coinbase/rosetta-sdk-go/types v1.0.0/go.mod h1:eq7W2TMRH22GTW0N0beDnN931DW0/WOI1R2sdHNHG4c= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -44,22 +44,24 @@ github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMS github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI= github.com/elastic/go-elasticsearch/v7 v7.12.0 h1:j4tvcMrZJLp39L2NYvBb7f+lHKPqPHSL3nvB8+/DV+s= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= +github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= github.com/gin-contrib/cors v1.4.0 h1:oJ6gwtUl3lqV0WEIwM/LxPF1QZ5qe2lGWdY2+bz7y0g= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= -github.com/gin-gonic/gin v1.9.0 h1:OjyFBKICoexlu99ctXNR2gg+c5pKrKMuyjgARg9qeY8= -github.com/gin-gonic/gin v1.9.0/go.mod h1:W1Me9+hsUSyj3CePGrd1/QrKJMSJ1Tu/0hFEH89961k= +github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= +github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.11.2 h1:q3SHpufmypg+erIExEKUmsgmhDTyhcJ38oeKGACXohU= -github.com/go-playground/validator/v10 v10.11.2/go.mod h1:NieE624vt4SCTJtD87arVLvdmjPAeV8BQlHtMnw9D7s= -github.com/goccy/go-json v0.10.0 h1:mXKd9Qw4NuzShiRlOXKews24ufknHO7gx30lsDyokKA= -github.com/goccy/go-json v0.10.0/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js= +github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -77,14 +79,14 @@ github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/hashicorp/golang-lru v0.6.0 h1:uL2shRDx7RTrOrTCUZEGP/wJUFiUI8QT6E7z5o8jga4= github.com/hashicorp/golang-lru v0.6.0/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/herumi/bls-go-binary v1.0.0 h1:PRPF6vPd35zyDy+tp86HwNnGdufCH2lZL0wZGxYvkRs= +github.com/herumi/bls-go-binary v1.28.2 h1:F0AezsC0M1a9aZjk7g0l2hMb1F56Xtpfku97pDndNZE= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -97,10 +99,8 @@ github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= -github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= +github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= +github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= @@ -113,79 +113,80 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiversx/concurrent-map v0.1.4 h1:hdnbM8VE4b0KYJaGY5yJS2aNIW9TFFsUYwbO0993uPI= -github.com/multiversx/concurrent-map v0.1.4/go.mod h1:8cWFRJDOrWHOTNSqgYCUvwT7c7eFQ4U2vKMOp4A/9+o= -github.com/multiversx/mx-chain-core-go v1.1.37 h1:2EYoUWjr+8zUYEt3TBMnQ+0UUZwDb71HA+KBwqDUpVQ= -github.com/multiversx/mx-chain-core-go v1.1.37/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= -github.com/multiversx/mx-chain-crypto-go v1.2.6 h1:yxsjAQGh62los+iYmORMfh3w9qen0xbYlmwU0juNSeg= -github.com/multiversx/mx-chain-crypto-go v1.2.6/go.mod h1:rOj0Rr19HTOYt9YTeym7RKxlHt91NXln3LVKjHKVmA0= -github.com/multiversx/mx-chain-es-indexer-go v1.3.20 h1:/82eDBsjlEm6dNgW44WQOBqKByRAWr3n6OWjYpi43vE= -github.com/multiversx/mx-chain-es-indexer-go v1.3.20/go.mod h1:B8ahv6M93qHallA9fN+yknQFzdrDhHDgVlCy/0PVcG0= -github.com/multiversx/mx-chain-go v1.5.13 h1:30Eipz1R8Mqe/xjc3bKUYtOnrbhddtsi3pBkaiDrLb4= -github.com/multiversx/mx-chain-go v1.5.13/go.mod h1:c22Et9ftMVQk+wx65n+d8am8SFsOSgQD6WR47vpP0Tg= -github.com/multiversx/mx-chain-logger-go v1.0.11 h1:DFsHa+sc5fKwhDR50I8uBM99RTDTEW68ESyr5ALRDwE= -github.com/multiversx/mx-chain-logger-go v1.0.11/go.mod h1:1srDkP0DQucWQ+rYfaq0BX2qLnULsUdRPADpYUTM6dA= -github.com/multiversx/mx-chain-p2p-go v1.0.18 h1:Dc+ngk8AV1UdBeZMzZP2MkxDyfoNrD/vY77umvo3SqQ= -github.com/multiversx/mx-chain-p2p-go v1.0.18/go.mod h1:Xy1B8X+AaGafDNcI6QZsTxO6bBf4akEyq+WdGRWSCpI= -github.com/multiversx/mx-chain-proxy-go v1.1.39 h1:hrZQeio1qPU8rDapTVzi3E7cl7Br2KJbqUi39QRy0b0= -github.com/multiversx/mx-chain-proxy-go v1.1.39/go.mod h1:b1gukLqxvi976gWZ4bHLvIaaAsyuLHi2JxNRLBAkcR8= -github.com/multiversx/mx-chain-storage-go v1.0.7 h1:UqLo/OLTD3IHiE/TB/SEdNRV1GG2f1R6vIP5ehHwCNw= -github.com/multiversx/mx-chain-storage-go v1.0.7/go.mod h1:gtKoV32Cg2Uy8deHzF8Ud0qAl0zv92FvWgPSYIP0Zmg= -github.com/multiversx/mx-chain-vm-common-go v1.3.42 h1:avhgUwi6f+wpHqaBk76j6islLzUlSRBXwisKoZnUXpk= -github.com/multiversx/mx-chain-vm-common-go v1.3.42/go.mod h1:r+aILrY07ue89PH+D+B+Pp0viO1U3kN98t1pXneSgkE= -github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= +github.com/multiversx/mx-chain-communication-go v1.1.0 h1:J7bX6HoN3HiHY7cUeEjG8AJWgQDDPcY+OPDOsSUOkRE= +github.com/multiversx/mx-chain-communication-go v1.1.0/go.mod h1:WK6bP4pGEHGDDna/AYRIMtl6G9OA0NByI1Lw8PmOnRM= +github.com/multiversx/mx-chain-core-go v1.2.21 h1:+XVKznPTlUU5EFS1A8chtS8fStW60upRIyF4Pgml19I= +github.com/multiversx/mx-chain-core-go v1.2.21/go.mod h1:B5zU4MFyJezmEzCsAHE9YNULmGCm2zbPHvl9hazNxmE= +github.com/multiversx/mx-chain-crypto-go v1.2.12 h1:zWip7rpUS4CGthJxfKn5MZfMfYPjVjIiCID6uX5BSOk= +github.com/multiversx/mx-chain-crypto-go v1.2.12/go.mod h1:HzcPpCm1zanNct/6h2rIh+MFrlXbjA5C8+uMyXj3LI4= +github.com/multiversx/mx-chain-es-indexer-go v1.7.4 h1:SjJk9G9SN8baz0sFIU2jymYCfx3XiikGEB2wW0jwvfw= +github.com/multiversx/mx-chain-es-indexer-go v1.7.4/go.mod h1:oGcRK2E3Syv6vRTszWrrb/TqD8akq0yeoMr1wPPiTO4= +github.com/multiversx/mx-chain-go v1.7.17 h1:8E19lc0d7Skj6WgWgiybUMCNCXaCvzhSINGla3ebfVM= +github.com/multiversx/mx-chain-go v1.7.17/go.mod h1:MPlFDdtae1jMI8E6QJiPGwGWEacZ+5jNgaNkLYmDZ5A= +github.com/multiversx/mx-chain-logger-go v1.0.15 h1:HlNdK8etyJyL9NQ+6mIXyKPEBo+wRqOwi3n+m2QIHXc= +github.com/multiversx/mx-chain-logger-go v1.0.15/go.mod h1:t3PRKaWB1M+i6gUfD27KXgzLJJC+mAQiN+FLlL1yoGQ= +github.com/multiversx/mx-chain-proxy-go v1.1.50 h1:XFx+YotnSoPa34Ia2qIPeb3Sh02WsjHXplb5aYeCYmc= +github.com/multiversx/mx-chain-proxy-go v1.1.50/go.mod h1:bg8/xUsn+7PcIPNC81Y4vfL5uwgwcsoLKzAwAiiOa5g= +github.com/multiversx/mx-chain-storage-go v1.0.16 h1:l2lJq+EAN3YwLbjJrnoKfFd1/1Xmo9DcAUECND2obLs= +github.com/multiversx/mx-chain-storage-go v1.0.16/go.mod h1:uM/z7YyqTOD3wgyH8TfapyEl5sb+7x/Jaxne4cfG4HI= +github.com/multiversx/mx-chain-vm-common-go v1.5.13 h1:ymnIHJW4Z4mFa0hZzla4fozkF30vjH5O1q+Y7Ftc+pQ= +github.com/multiversx/mx-chain-vm-common-go v1.5.13/go.mod h1:OSvFbzdWThfRbLZbUsEr7bikBSaLrPJQ2iUm9jw9nXQ= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= -github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= +github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= +github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= -github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= +github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d h1:vfofYNRScrDdvS342BElfbETmL1Aiz3i2t0zfRj16Hs= +github.com/tklauser/go-sysconf v0.3.4 h1:HT8SVixZd3IzLdfs/xlpq0jeSfTX57g1v6wB1EuzV7M= +github.com/tklauser/numcpus v0.2.1 h1:ct88eFm+Q7m2ZfXJdan1xYoXKlmwsfP+k88q05KvlZc= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= -github.com/ugorji/go/codec v1.2.9 h1:rmenucSohSTiyL09Y+l2OCk+FrMxGMzho2+tjr5ticU= -github.com/ugorji/go/codec v1.2.9/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= +github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/urfave/cli v1.22.10 h1:p8Fspmz3iTctJstry1PYS3HVdllxnEzTEsgIgtxTrCk= github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -golang.org/x/arch v0.0.0-20210923205945-b76863e36670 h1:18EFjUmQOcUvxNYSkA6jO9VAiXCnxFY6NyDX0bHDmkU= +github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= +golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= -golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= +golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= +golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -196,8 +197,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= +golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -214,13 +215,13 @@ golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= +golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -228,7 +229,6 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -240,16 +240,14 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/server/factory/provider.go b/server/factory/provider.go index 6f15a689..a9ab7ab3 100644 --- a/server/factory/provider.go +++ b/server/factory/provider.go @@ -18,6 +18,7 @@ const ( hasherType = "blake2b" marshalizerForHashingType = "gogo protobuf" pubKeyLength = 32 + bech32Prefix = "erd" notApplicableConfigurationFilePath = "not applicable" notApplicableFullHistoryNodesMessage = "not applicable" @@ -56,7 +57,7 @@ func CreateNetworkProvider(args ArgsCreateNetworkProvider) (NetworkProvider, err return nil, err } - pubKeyConverter, err := pubkeyConverter.NewBech32PubkeyConverter(pubKeyLength, log) + pubKeyConverter, err := pubkeyConverter.NewBech32PubkeyConverter(pubKeyLength, bech32Prefix) if err != nil { return nil, err } diff --git a/server/provider/networkProvider.go b/server/provider/networkProvider.go index 41ba8660..2f96e5c6 100644 --- a/server/provider/networkProvider.go +++ b/server/provider/networkProvider.go @@ -363,7 +363,13 @@ func (provider *networkProvider) ComputeShardIdOfPubKey(pubKey []byte) uint32 { // ConvertPubKeyToAddress converts a public key to an address func (provider *networkProvider) ConvertPubKeyToAddress(pubkey []byte) string { - return provider.pubKeyConverter.Encode(pubkey) + address, err := provider.pubKeyConverter.Encode(pubkey) + if err != nil { + log.Warn("networkProvider.ConvertPubKeyToAddress()", "pubkey", pubkey, "err", err) + return "" + } + + return address } // ConvertAddressToPubKey converts an address to a pubkey diff --git a/testscommon/networkProviderMock.go b/testscommon/networkProviderMock.go index 91843473..496745f5 100644 --- a/testscommon/networkProviderMock.go +++ b/testscommon/networkProviderMock.go @@ -50,7 +50,7 @@ type networkProviderMock struct { // NewNetworkProviderMock - func NewNetworkProviderMock() *networkProviderMock { - pubKeyConverter, _ := pubkeyConverter.NewBech32PubkeyConverter(32, log) + pubKeyConverter, _ := pubkeyConverter.NewBech32PubkeyConverter(32, "erd") return &networkProviderMock{ pubKeyConverter: pubKeyConverter, @@ -299,7 +299,13 @@ func (mock *networkProviderMock) ComputeShardIdOfPubKey(pubKey []byte) uint32 { // ConvertPubKeyToAddress - func (mock *networkProviderMock) ConvertPubKeyToAddress(pubkey []byte) string { - return mock.pubKeyConverter.Encode(pubkey) + address, err := mock.pubKeyConverter.Encode(pubkey) + if err != nil { + log.Error("networkProviderMock.ConvertPubKeyToAddress() failed", "pubkey", pubkey, "error", err) + return "" + } + + return address } // ConvertAddressToPubKey - diff --git a/testscommon/realworld.go b/testscommon/realworld.go index 200613d5..a61f5884 100644 --- a/testscommon/realworld.go +++ b/testscommon/realworld.go @@ -4,11 +4,10 @@ import ( "github.com/multiversx/mx-chain-core-go/core/pubkeyConverter" hasherFactory "github.com/multiversx/mx-chain-core-go/hashing/factory" marshalFactory "github.com/multiversx/mx-chain-core-go/marshal/factory" - logger "github.com/multiversx/mx-chain-logger-go" ) // RealWorldBech32PubkeyConverter is a bech32 converter, to be used in tests -var RealWorldBech32PubkeyConverter, _ = pubkeyConverter.NewBech32PubkeyConverter(32, logger.GetOrCreate("testscommon")) +var RealWorldBech32PubkeyConverter, _ = pubkeyConverter.NewBech32PubkeyConverter(32, "erd") // RealWorldBlake2bHasher is a blake2b hasher, to be used in tests var RealWorldBlake2bHasher, _ = hasherFactory.NewHasher("blake2b") From 60337db892b6580c550a9b9b82b6910d806092ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Wed, 7 Aug 2024 12:37:29 +0300 Subject: [PATCH 32/75] Bit of cleanup. --- server/services/transactionsTransformer.go | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/server/services/transactionsTransformer.go b/server/services/transactionsTransformer.go index 7087e837..2caf9f5f 100644 --- a/server/services/transactionsTransformer.go +++ b/server/services/transactionsTransformer.go @@ -29,9 +29,6 @@ func newTransactionsTransformer(provider NetworkProvider) *transactionsTransform } func (transformer *transactionsTransformer) transformBlockTxs(block *api.Block) ([]*types.Transaction, error) { - // TODO: Remove - log.Info("transformBlockTxs", "block", block.Nonce, "numTxs", block.NumTxs) - txs := make([]*transaction.ApiTransactionResult, 0) receipts := make([]*transaction.ApiReceipt, 0) @@ -88,13 +85,6 @@ func (transformer *transactionsTransformer) transformBlockTxs(block *api.Block) } func (transformer *transactionsTransformer) txToRosettaTx(tx *transaction.ApiTransactionResult, txsInBlock []*transaction.ApiTransactionResult) (*types.Transaction, error) { - // TODO: Remove - if isRelayedV1Transaction(tx) { - log.Info("txToRosettaTx: relayed V1", "hash", tx.Hash) - } else if isRelayedV2Transaction(tx) { - log.Info("txToRosettaTx: relayed V2", "hash", tx.Hash) - } - var rosettaTx *types.Transaction var err error @@ -149,6 +139,7 @@ func (transformer *transactionsTransformer) unsignedTxToRosettaTx( } if !areClaimDeveloperRewardsEventsEnabled(scr.BlockNonce) { + // Handle developer rewards in a legacy manner (without looking at events / logs) if transformer.featuresDetector.doesContractResultHoldRewardsOfClaimDeveloperRewards(scr, txsInBlock) { return &types.Transaction{ TransactionIdentifier: hashToTransactionIdentifier(scr.Hash), @@ -244,12 +235,11 @@ func (transformer *transactionsTransformer) normalTxToRosetta( return nil, err } - // TODO: Remove (maybe) if len(innerTxOperationsIfRelayedCompletelyIntrashardWithSignalError) > 0 { - log.Info("innerTxOperationsIfRelayedCompletelyIntrashardWithSignalError", "tx", tx.Hash, "block", tx.BlockNonce) + log.Info("normalTxToRosetta(): innerTxOperationsIfRelayedCompletelyIntrashardWithSignalError", "tx", tx.Hash, "block", tx.BlockNonce) } if len(valueRefundOperationIfContractCallOrDeploymentWithSignalError) > 0 { - log.Info("valueRefundOperationIfContractCallOrDeploymentWithSignalError", "tx", tx.Hash, "block", tx.BlockNonce) + log.Info("normalTxToRosetta(): valueRefundOperationIfContractCallOrDeploymentWithSignalError", "tx", tx.Hash, "block", tx.BlockNonce) } operations = append(operations, innerTxOperationsIfRelayedCompletelyIntrashardWithSignalError...) @@ -412,8 +402,6 @@ func (transformer *transactionsTransformer) addOperationsGivenTransactionEvents( ) error { hasSignalError := transformer.featuresDetector.eventsController.hasAnySignalError(tx) if hasSignalError { - // TODO: Remove - log.Info("hasSignalError, will ignore events", "tx", tx.Hash, "block", tx.BlockNonce) return nil } From f96e64ca4ddff446410f7a62f3a40935359a8bb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Wed, 7 Aug 2024 13:37:41 +0300 Subject: [PATCH 33/75] Handle native transfers within multi esdt transfers. --- server/provider/currenciesProvider.go | 3 +- server/services/constants.go | 1 + server/services/transactionsTransformer.go | 47 +++++++++++++--------- 3 files changed, 32 insertions(+), 19 deletions(-) diff --git a/server/provider/currenciesProvider.go b/server/provider/currenciesProvider.go index 394568f8..96f483a4 100644 --- a/server/provider/currenciesProvider.go +++ b/server/provider/currenciesProvider.go @@ -62,5 +62,6 @@ func (provider *currenciesProvider) GetCustomCurrencyBySymbol(symbol string) (re // HasCustomCurrency checks whether a custom currency (ESDT) is enabled (supported) func (provider *currenciesProvider) HasCustomCurrency(symbol string) bool { - return true + _, ok := provider.customCurrenciesBySymbol[symbol] + return ok } diff --git a/server/services/constants.go b/server/services/constants.go index cb4fac1e..579f9341 100644 --- a/server/services/constants.go +++ b/server/services/constants.go @@ -24,6 +24,7 @@ var ( emptyHash = strings.Repeat("0", 64) nodeVersionForOfflineRosetta = "N / A" systemContractDeployAddress = "erd1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq6gq4hu" + nativeAsESDTIdentifier = "EGLD-000000" ) var ( diff --git a/server/services/transactionsTransformer.go b/server/services/transactionsTransformer.go index 2caf9f5f..09f75959 100644 --- a/server/services/transactionsTransformer.go +++ b/server/services/transactionsTransformer.go @@ -458,8 +458,6 @@ func (transformer *transactionsTransformer) addOperationsGivenTransactionEvents( } for _, event := range eventsSCDeploy { - log.Info("eventSCDeploy", "tx", tx.Hash, "block", tx.BlockNonce, "contract", event.contractAddress, "deployer", event.deployerAddress) - // Handle deployments with transfer of value if tx.Receiver == systemContractDeployAddress { operations := []*types.Operation{ @@ -501,25 +499,38 @@ func (transformer *transactionsTransformer) addOperationsGivenTransactionEvents( } for _, event := range eventsESDTTransfer { - if !transformer.provider.HasCustomCurrency(event.identifier) { + if event.identifier == nativeAsESDTIdentifier { + operations := []*types.Operation{ + { + Type: opTransfer, + Account: addressToAccountIdentifier(event.senderAddress), + Amount: transformer.extension.valueToNativeAmount("-" + event.value), + }, + { + Type: opTransfer, + Account: addressToAccountIdentifier(event.receiverAddress), + Amount: transformer.extension.valueToNativeAmount(event.value), + }, + } + + rosettaTx.Operations = append(rosettaTx.Operations, operations...) + } else if transformer.provider.HasCustomCurrency(event.identifier) { // We are only emitting balance-changing operations for supported currencies. - continue - } + operations := []*types.Operation{ + { + Type: opCustomTransfer, + Account: addressToAccountIdentifier(event.senderAddress), + Amount: transformer.extension.valueToCustomAmount("-"+event.value, event.getExtendedIdentifier()), + }, + { + Type: opCustomTransfer, + Account: addressToAccountIdentifier(event.receiverAddress), + Amount: transformer.extension.valueToCustomAmount(event.value, event.getExtendedIdentifier()), + }, + } - operations := []*types.Operation{ - { - Type: opCustomTransfer, - Account: addressToAccountIdentifier(event.senderAddress), - Amount: transformer.extension.valueToCustomAmount("-"+event.value, event.getExtendedIdentifier()), - }, - { - Type: opCustomTransfer, - Account: addressToAccountIdentifier(event.receiverAddress), - Amount: transformer.extension.valueToCustomAmount(event.value, event.getExtendedIdentifier()), - }, + rosettaTx.Operations = append(rosettaTx.Operations, operations...) } - - rosettaTx.Operations = append(rosettaTx.Operations, operations...) } for _, event := range eventsESDTLocalBurn { From b648deccaf64659f3512db2fc74c6170fccb6650 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Wed, 7 Aug 2024 13:57:36 +0300 Subject: [PATCH 34/75] Additional tests. --- systemtests/adder.wasm | Bin 0 -> 697 bytes systemtests/broadcast_samples.py | 290 ++++++++++++++++++++++++++ systemtests/config.py | 17 ++ systemtests/proxyToObserverAdapter.go | 3 +- 4 files changed, 309 insertions(+), 1 deletion(-) create mode 100644 systemtests/adder.wasm create mode 100644 systemtests/broadcast_samples.py diff --git a/systemtests/adder.wasm b/systemtests/adder.wasm new file mode 100644 index 0000000000000000000000000000000000000000..67af5e5cf2fa468f4fbbb1476d191b4ad48eae8b GIT binary patch literal 697 zcmZuvO^?$s5S_8JZ8l~b@3sPU#UbPbCvJ!%sYDSHTCKEjPTa;#t2W8XNkBcNKY=R} z2YwHKs*IBs35i2I_Iq#Md?=u*9034dxd}i5*e0xP`x6puA>nQJw!s3}vhPfAU3{G` zHrdCb{-EnGMYYKaokk{Wa$VHGE$?T#zH9O`SOHp8SgG2*H>NbbZko@U)k>SWDf3xp zdG-TlGsY+Wp)QTev< zGk^sr{fzTXv8mR7+j1;**;7%P!hj18(cwe8NpeHr1D~}J{W`ef6o&WiAlG?m_6!`d ztKoV;3wmgBxZ#RS#_m$>b~Cly!erPyW=5qNuDv);Rjp!Oq#Q&XueExN7pl@6z2E4K z5ZegklMo~XAg1>D^WfU=C4&gJA-rLhnSjIq=FU4`L None: + self.configuration = configuration + self.mnemonic = Mnemonic(configuration.users_mnemonic) + self.sponsor = self._create_sponsor() + self.users: List[Account] = [] + self.users_by_shard: List[List[Account]] = [[], [], []] + self.users_by_bech32: Dict[str, Account] = {} + self.contracts: List[Address] = [] + self.contracts_by_shard: List[List[Address]] = [[], [], []] + + address_computer = AddressComputer() + + for i in range(32): + user = self._create_user(i) + shard = address_computer.get_shard_of_address(user.address) + self.users.append(user) + self.users_by_shard[shard].append(user) + self.users_by_bech32[user.address.to_bech32()] = user + + for item in configuration.known_contracts: + contract_address = Address.from_bech32(item) + shard = address_computer.get_shard_of_address(contract_address) + self.contracts.append(contract_address) + self.contracts_by_shard[shard].append(contract_address) + + def _create_sponsor(self) -> "Account": + sponsor_secret_key = UserSecretKey(self.configuration.sponsor_secret_key) + sponsor_signer = UserSigner(sponsor_secret_key) + return Account(sponsor_signer) + + def _create_user(self, index: int) -> "Account": + user_secret_key = self.mnemonic.derive_key(index) + user_signer = UserSigner(user_secret_key) + return Account(user_signer) + + def get_user(self, shard: int, index: int) -> "Account": + return self.users_by_shard[shard][index] + + def get_account_by_bech32(self, address: str) -> "Account": + if self.sponsor.address.to_bech32() == address: + return self.sponsor + + return self.users_by_bech32[address] + + +class Controller: + def __init__(self, configuration: Configuration, accounts: BunchOfAccounts) -> None: + self.configuration = configuration + self.accounts = accounts + self.custom_currencies = CustomCurrencies(configuration) + self.transactions_factory_config = TransactionsFactoryConfig(chain_id=configuration.network_id) + self.network_provider = ProxyNetworkProvider(configuration.proxy_url) + self.nonces_tracker = NoncesTracker(configuration.proxy_url) + self.transfer_transactions_factory = TransferTransactionsFactory(self.transactions_factory_config) + self.relayed_transactions_factory = RelayedTransactionsFactory(self.transactions_factory_config) + self.contracts_transactions_factory = SmartContractTransactionsFactory(self.transactions_factory_config) + + def create_airdrops(self) -> List[Transaction]: + transactions: List[Transaction] = [] + + for user in self.accounts.users: + transaction = self.transfer_transactions_factory.create_transaction_for_native_token_transfer( + sender=self.accounts.sponsor.address, + receiver=user.address, + native_amount=10000000000000000 + ) + + transaction = self.transfer_transactions_factory.create_transaction_for_esdt_token_transfer( + sender=self.accounts.sponsor.address, + receiver=user.address, + token_transfers=[TokenTransfer(Token(self.custom_currencies.currency), 1000000)] + ) + + transactions.append(transaction) + + return transactions + + def create_contract_deployments(self) -> List[Transaction]: + transactions: List[Transaction] = [] + + # Non-payable contracts, in each shard + transactions.append(self.contracts_transactions_factory.create_transaction_for_deploy( + sender=self.accounts.get_user(shard=0, index=0).address, + bytecode=CONTRACT_PATH_ADDER, + gas_limit=5000000, + arguments=[0] + )) + + transactions.append(self.contracts_transactions_factory.create_transaction_for_deploy( + sender=self.accounts.get_user(shard=1, index=0).address, + bytecode=CONTRACT_PATH_ADDER, + gas_limit=5000000, + arguments=[0] + )) + + transactions.append(self.contracts_transactions_factory.create_transaction_for_deploy( + sender=self.accounts.get_user(shard=2, index=0).address, + bytecode=CONTRACT_PATH_ADDER, + gas_limit=5000000, + arguments=[0] + )) + + return transactions + + def create_simple_move_balance_with_refund(self, sender: "Account", receiver: Address) -> Transaction: + transaction = self.transfer_transactions_factory.create_transaction_for_native_token_transfer( + sender=sender.address, + receiver=receiver, + native_amount=42 + ) + + transaction.gas_limit += 42000 + return transaction + + def create_invalid_move_balance_with_refund(self, sender: "Account", receiver: Address) -> Transaction: + transaction = self.transfer_transactions_factory.create_transaction_for_native_token_transfer( + sender=sender.address, + receiver=receiver, + native_amount=1000000000000000000000000 + ) + + transaction.gas_limit += 42000 + return transaction + + def create_native_transfer_within_multiesdt(self, sender: "Account", receiver: Address) -> Transaction: + transaction = self.transfer_transactions_factory.create_transaction_for_transfer( + sender=sender.address, + receiver=receiver, + native_amount=42, + token_transfers=[TokenTransfer(Token(self.custom_currencies.currency), 7)] + ) + + return transaction + + def send_multiple(self, transactions: List[Transaction]): + for transaction in transactions: + sender = self.accounts.get_account_by_bech32(transaction.sender) + self._apply_nonce(transaction, sender) + self._sign(transaction, sender) + + self.network_provider.send_transactions(transactions) + + def send(self, transaction: Transaction): + sender = self.accounts.get_account_by_bech32(transaction.sender) + self._apply_nonce(transaction, sender) + self._sign(transaction, sender) + transaction_hash = self.network_provider.send_transaction(transaction) + print(f"{self.configuration.explorer_url}/transactions/{transaction_hash}") + + def _apply_nonce(self, transaction: Transaction, user: "Account"): + transaction.nonce = self.nonces_tracker.get_then_increment_nonce(user.address) + + def _sign(self, transaction: Transaction, user: "Account"): + computer = TransactionComputer() + bytes_for_signing = computer.compute_bytes_for_signing(transaction) + transaction.signature = user.signer.sign(bytes_for_signing) + + +class Account: + def __init__(self, signer: UserSigner) -> None: + self.signer = signer + self.address: Address = signer.get_pubkey().to_address("erd") + + +class NoncesTracker: + def __init__(self, proxy_url: str) -> None: + self.nonces_by_address: Dict[str, int] = {} + self.network_provider = ProxyNetworkProvider(proxy_url) + + def get_then_increment_nonce(self, address: Address): + nonce = self.get_nonce(address) + self.increment_nonce(address) + return nonce + + def get_nonce(self, address: Address) -> int: + if address.to_bech32() not in self.nonces_by_address: + self.recall_nonce(address) + + return self.nonces_by_address[address.to_bech32()] + + def recall_nonce(self, address: Address): + account = self.network_provider.get_account(address) + self.nonces_by_address[address.to_bech32()] = account.nonce + + def increment_nonce(self, address: Address): + self.nonces_by_address[address.to_bech32()] += 1 + + +class CustomCurrencies: + def __init__(self, configuration: Configuration) -> None: + self.currency = "ROSETTA-7783de" + + +if __name__ == '__main__': + main() diff --git a/systemtests/config.py b/systemtests/config.py index 2a4b2a34..644b9385 100644 --- a/systemtests/config.py +++ b/systemtests/config.py @@ -1,4 +1,6 @@ +import os from dataclasses import dataclass +from typing import List @dataclass @@ -14,6 +16,11 @@ class Configuration: check_construction_custom_configuration_file: str check_data_configuration_file: str check_data_directory: str + legacy_delegation_contract: str + known_contracts: List[str] + explorer_url: str + sponsor_secret_key: bytes = bytes.fromhex(os.environ.get("SPONSOR_SECRET_KEY", "")) + users_mnemonic: str = os.environ.get("USERS_MNEMONIC", "") CONFIGURATIONS = { @@ -29,6 +36,9 @@ class Configuration: check_construction_custom_configuration_file="systemtests/mesh_cli_config/devnet-construction-custom.json", check_data_configuration_file="systemtests/mesh_cli_config/check-data.json", check_data_directory="systemtests/devnet-data", + legacy_delegation_contract="erd1qqqqqqqqqqqqqpgq97wezxw6l7lgg7k9rxvycrz66vn92ksh2tssxwf7ep", + known_contracts=[], + explorer_url="https://devnet-explorer.multiversx.com", ), "testnet": Configuration( network_shard=0, @@ -42,5 +52,12 @@ class Configuration: check_construction_custom_configuration_file="systemtests/mesh_cli_config/testnet-construction-custom.json", check_data_configuration_file="systemtests/mesh_cli_config/check-data.json", check_data_directory="systemtests/testnet-data", + known_contracts=[ + "erd1qqqqqqqqqqqqqpgqagjekf5mxv86hy5c62vvtug5vc6jmgcsq6uq8reras", + "erd1qqqqqqqqqqqqqpgq89t5xm4x04tnt9lv747wdrsaycf3rcwcggzsa7crse", + "erd1qqqqqqqqqqqqqpgqeesfamasje5zru7ku79m8p4xqfqnywvqxj0qhtyzdr" + ], + legacy_delegation_contract="erd1qqqqqqqqqqqqqpgq97wezxw6l7lgg7k9rxvycrz66vn92ksh2tssxwf7ep", + explorer_url="https://testnet-explorer.multiversx.com", ), } diff --git a/systemtests/proxyToObserverAdapter.go b/systemtests/proxyToObserverAdapter.go index a099add3..f31997b7 100644 --- a/systemtests/proxyToObserverAdapter.go +++ b/systemtests/proxyToObserverAdapter.go @@ -57,7 +57,8 @@ func startAdapter(ctx *cli.Context) error { sleepDuration: ctx.GlobalUint(cliFlagSleep.Name), } - router := gin.Default() + router := gin.New() + router.Use(gin.Recovery()) router.GET("/node/status", adapter.getNodeStatus) router.GET("/node/epoch-start/:epoch", adapter.getEpochStart) router.GET("/block/by-nonce/:nonce", adapter.getBlockByNonce) From 1c0813b76ee9d9e6b76242fecbf0074293c2e02d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Fri, 9 Aug 2024 13:25:59 +0300 Subject: [PATCH 35/75] Claim developer rewards: remove exceptions (isContractResultOfOpaquelyClaimingDeveloperRewards). --- server/services/exceptions.go | 22 ---------------------- server/services/transactionsTransformer.go | 13 ------------- systemtests/check_with_mesh_cli.py | 2 +- systemtests/config.py | 5 ++++- systemtests/constants.py | 2 +- 5 files changed, 6 insertions(+), 38 deletions(-) delete mode 100644 server/services/exceptions.go diff --git a/server/services/exceptions.go b/server/services/exceptions.go deleted file mode 100644 index 979ec12d..00000000 --- a/server/services/exceptions.go +++ /dev/null @@ -1,22 +0,0 @@ -package services - -import ( - "github.com/multiversx/mx-chain-core-go/data/transaction" -) - -// See: -// - https://explorer.multiversx.com/transactions?function=claimRewardsAllContracts -// - https://github.com/multiversx/mx-chain-vm-common-go/pull/247 -func isContractResultOfOpaquelyClaimingDeveloperRewards(tx *transaction.ApiTransactionResult) bool { - if tx.BlockNonce == 15693863 && tx.SourceShard == 2 && tx.Hash == "8ddecc831c70ecf0ca312a46e04bbdc7508fabada494714ec41fd49c8ec13915" { - log.Warn("Exception: 8ddecc831c70ecf0ca312a46e04bbdc7508fabada494714ec41fd49c8ec13915") - return true - } - - if tx.BlockNonce == 18098197 && tx.SourceShard == 2 && tx.Hash == "90357d8f72bb1c749b46e394a4be349d4b2700a8f734470f815a524059f7031b" { - log.Warn("Exception: 90357d8f72bb1c749b46e394a4be349d4b2700a8f734470f815a524059f7031b") - return true - } - - return false -} diff --git a/server/services/transactionsTransformer.go b/server/services/transactionsTransformer.go index 09f75959..fbcfa89e 100644 --- a/server/services/transactionsTransformer.go +++ b/server/services/transactionsTransformer.go @@ -152,19 +152,6 @@ func (transformer *transactionsTransformer) unsignedTxToRosettaTx( }, } } - - if isContractResultOfOpaquelyClaimingDeveloperRewards(scr) { - return &types.Transaction{ - TransactionIdentifier: hashToTransactionIdentifier(scr.Hash), - Operations: []*types.Operation{ - { - Type: opScResult, - Account: addressToAccountIdentifier(scr.Receiver), - Amount: transformer.extension.valueToNativeAmount(scr.Value), - }, - }, - } - } } return &types.Transaction{ diff --git a/systemtests/check_with_mesh_cli.py b/systemtests/check_with_mesh_cli.py index 36d040cf..270aa431 100644 --- a/systemtests/check_with_mesh_cli.py +++ b/systemtests/check_with_mesh_cli.py @@ -61,7 +61,7 @@ def run_rosetta(configuration: Configuration): command = [ str(constants.PATH_ROSETTA), f"--port={constants.PORT_ROSETTA}", - f"--observer-http-url=http://localhost:{constants.PORT_OBSERVER_SURROGATE}", + f"--observer-http-url={constants.URL_OBSERVER_SURROGATE}", f"--observer-actual-shard={configuration.network_shard}", f"--network-id={configuration.network_id}", f"--network-name={configuration.network_name}", diff --git a/systemtests/config.py b/systemtests/config.py index 644b9385..89a09bf2 100644 --- a/systemtests/config.py +++ b/systemtests/config.py @@ -11,6 +11,7 @@ class Configuration: native_currency: str config_file_custom_currencies: str num_historical_epochs: int + observer_url: str proxy_url: str check_construction_native_configuration_file: str check_construction_custom_configuration_file: str @@ -31,6 +32,7 @@ class Configuration: native_currency="EGLD", config_file_custom_currencies="systemtests/rosetta_config/devnet-custom-currencies.json", num_historical_epochs=2, + observer_url="", proxy_url="https://devnet-gateway.multiversx.com", check_construction_native_configuration_file="systemtests/mesh_cli_config/devnet-construction-native.json", check_construction_custom_configuration_file="systemtests/mesh_cli_config/devnet-construction-custom.json", @@ -46,7 +48,8 @@ class Configuration: network_name="untitled", native_currency="EGLD", config_file_custom_currencies="systemtests/rosetta_config/testnet-custom-currencies.json", - num_historical_epochs=2, + num_historical_epochs=1, + observer_url="", proxy_url="https://testnet-gateway.multiversx.com", check_construction_native_configuration_file="systemtests/mesh_cli_config/testnet-construction-native.json", check_construction_custom_configuration_file="systemtests/mesh_cli_config/testnet-construction-custom.json", diff --git a/systemtests/constants.py b/systemtests/constants.py index 777c19f3..dbdc1881 100644 --- a/systemtests/constants.py +++ b/systemtests/constants.py @@ -4,5 +4,5 @@ PATH_ROSETTA = PATH_REPOSITORY / "cmd" / "rosetta" / "rosetta" PATH_PROXY_TO_OBSERVER_ADAPTER = PATH_REPOSITORY / "systemtests" / "proxyToObserverAdapter" PORT_ROSETTA = 7091 -PORT_OBSERVER_SURROGATE = 8080 +URL_OBSERVER_SURROGATE = "http://localhost:8080" ADAPTER_DELAY_IN_MILLISECONDS = 100 From 2ff19465b5ed5de0e7f27217f423ad70ffaac4f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Fri, 9 Aug 2024 16:49:11 +0300 Subject: [PATCH 36/75] Config for localnet etc. --- server/services/constants.go | 1 + systemtests/config.py | 29 ++++++++++++++++--- .../localnet-custom-currencies.json | 1 + 3 files changed, 27 insertions(+), 4 deletions(-) create mode 100644 systemtests/rosetta_config/localnet-custom-currencies.json diff --git a/server/services/constants.go b/server/services/constants.go index 579f9341..2eb5bef3 100644 --- a/server/services/constants.go +++ b/server/services/constants.go @@ -11,6 +11,7 @@ var ( transactionVersion = 1 transactionProcessingTypeRelayedV1 = "RelayedTx" transactionProcessingTypeRelayedV2 = "RelayedTxV2" + transactionProcessingTypeRelayedV3 = "RelayedTxV3" transactionProcessingTypeBuiltInFunctionCall = "BuiltInFunctionCall" transactionProcessingTypeMoveBalance = "MoveBalance" transactionProcessingTypeContractInvoking = "SCInvoking" diff --git a/systemtests/config.py b/systemtests/config.py index 89a09bf2..addabc2f 100644 --- a/systemtests/config.py +++ b/systemtests/config.py @@ -17,7 +17,6 @@ class Configuration: check_construction_custom_configuration_file: str check_data_configuration_file: str check_data_directory: str - legacy_delegation_contract: str known_contracts: List[str] explorer_url: str sponsor_secret_key: bytes = bytes.fromhex(os.environ.get("SPONSOR_SECRET_KEY", "")) @@ -38,8 +37,11 @@ class Configuration: check_construction_custom_configuration_file="systemtests/mesh_cli_config/devnet-construction-custom.json", check_data_configuration_file="systemtests/mesh_cli_config/check-data.json", check_data_directory="systemtests/devnet-data", - legacy_delegation_contract="erd1qqqqqqqqqqqqqpgq97wezxw6l7lgg7k9rxvycrz66vn92ksh2tssxwf7ep", - known_contracts=[], + known_contracts=[ + "erd1qqqqqqqqqqqqqpgqagjekf5mxv86hy5c62vvtug5vc6jmgcsq6uq8reras", + "erd1qqqqqqqqqqqqqpgq89t5xm4x04tnt9lv747wdrsaycf3rcwcggzsa7crse", + "erd1qqqqqqqqqqqqqpgqeesfamasje5zru7ku79m8p4xqfqnywvqxj0qhtyzdr" + ], explorer_url="https://devnet-explorer.multiversx.com", ), "testnet": Configuration( @@ -60,7 +62,26 @@ class Configuration: "erd1qqqqqqqqqqqqqpgq89t5xm4x04tnt9lv747wdrsaycf3rcwcggzsa7crse", "erd1qqqqqqqqqqqqqpgqeesfamasje5zru7ku79m8p4xqfqnywvqxj0qhtyzdr" ], - legacy_delegation_contract="erd1qqqqqqqqqqqqqpgq97wezxw6l7lgg7k9rxvycrz66vn92ksh2tssxwf7ep", explorer_url="https://testnet-explorer.multiversx.com", ), + "localnet": Configuration( + network_shard=0, + network_id="localnet", + network_name="untitled", + native_currency="EGLD", + config_file_custom_currencies="systemtests/rosetta_config/localnet-custom-currencies.json", + num_historical_epochs=2, + observer_url="", + proxy_url="http://localhost:7950", + check_construction_native_configuration_file="systemtests/mesh_cli_config/localnet-construction-native.json", + check_construction_custom_configuration_file="systemtests/mesh_cli_config/localnet-construction-custom.json", + check_data_configuration_file="systemtests/mesh_cli_config/check-data.json", + check_data_directory="systemtests/localnet-data", + known_contracts=[ + "erd1qqqqqqqqqqqqqpgqagjekf5mxv86hy5c62vvtug5vc6jmgcsq6uq8reras", + "erd1qqqqqqqqqqqqqpgq89t5xm4x04tnt9lv747wdrsaycf3rcwcggzsa7crse", + "erd1qqqqqqqqqqqqqpgqeesfamasje5zru7ku79m8p4xqfqnywvqxj0qhtyzdr" + ], + explorer_url="https://localnet-explorer.multiversx.com", + ), } diff --git a/systemtests/rosetta_config/localnet-custom-currencies.json b/systemtests/rosetta_config/localnet-custom-currencies.json new file mode 100644 index 00000000..fe51488c --- /dev/null +++ b/systemtests/rosetta_config/localnet-custom-currencies.json @@ -0,0 +1 @@ +[] From 77cc55859b997efc7415fb0e0b1a872c6970a4ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Mon, 12 Aug 2024 12:13:25 +0300 Subject: [PATCH 37/75] Checker should also work directly against observers (no adapter). --- systemtests/check_with_mesh_cli.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/systemtests/check_with_mesh_cli.py b/systemtests/check_with_mesh_cli.py index 270aa431..de3354be 100644 --- a/systemtests/check_with_mesh_cli.py +++ b/systemtests/check_with_mesh_cli.py @@ -22,13 +22,13 @@ def main() -> int: configuration = CONFIGURATIONS[args.network] process_rosetta = run_rosetta(configuration) - process_adapter = run_proxy_to_observer_adapter(configuration) + process_adapter = optionally_run_proxy_to_observer_adapter(configuration) process_checker = run_mesh_cli(mode, configuration) # Handle termination signals def signal_handler(sig: Any, frame: Any): process_rosetta.kill() - process_adapter.kill() + process_adapter.kill() if process_adapter else None process_checker.kill() sys.exit(1) @@ -39,7 +39,7 @@ def signal_handler(sig: Any, frame: Any): exit_code = process_checker.wait() process_rosetta.kill() - process_adapter.kill() + process_adapter.kill() if process_adapter else None time.sleep(1) @@ -74,7 +74,11 @@ def run_rosetta(configuration: Configuration): return subprocess.Popen(command) -def run_proxy_to_observer_adapter(configuration: Configuration): +def optionally_run_proxy_to_observer_adapter(configuration: Configuration) -> Any: + if configuration.observer_url: + # If observer URL is provided, we don't need the adapter. + return None + command = [ str(constants.PATH_PROXY_TO_OBSERVER_ADAPTER), f"--proxy={configuration.proxy_url}", From b0d9018ec3e30daede1a0f8f7acf5c3947516481 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Mon, 12 Aug 2024 13:31:02 +0300 Subject: [PATCH 38/75] New CLI flag: --handle-contracts. --- cmd/rosetta/cli.go | 17 ++++ cmd/rosetta/main.go | 1 + server/factory/provider.go | 2 + server/provider/networkProvider.go | 16 ++-- server/provider/networkProvider_test.go | 94 +++++++++++++++++++ server/services/blockService.go | 2 +- server/services/networkProviderExtension.go | 9 -- .../services/networkProviderExtension_test.go | 19 ---- server/services/transactionsTransformer.go | 2 +- 9 files changed, 126 insertions(+), 36 deletions(-) diff --git a/cmd/rosetta/cli.go b/cmd/rosetta/cli.go index 20b2d7c2..514f22f6 100644 --- a/cmd/rosetta/cli.go +++ b/cmd/rosetta/cli.go @@ -156,11 +156,22 @@ VERSION: Required: true, } + cliFlagShouldHandleContracts = cli.BoolFlag{ + Name: "handle-contracts", + Usage: "Whether to handle balance changes of smart contracts or not.", + } + cliFlagConfigFileCustomCurrencies = cli.StringFlag{ Name: "config-custom-currencies", Usage: "Specifies the configuration file for custom currencies.", Required: false, } + + cliFlagConfigFileNodeFeatures = cli.StringFlag{ + Name: "config-node-features", + Usage: "Specifies the configuration file for features activation on Node's side.", + Required: false, + } ) func getAllCliFlags() []cli.Flag { @@ -187,7 +198,9 @@ func getAllCliFlags() []cli.Flag { cliFlagNativeCurrencySymbol, cliFlagFirstHistoricalEpoch, cliFlagNumHistoricalEpochs, + cliFlagShouldHandleContracts, cliFlagConfigFileCustomCurrencies, + cliFlagConfigFileNodeFeatures, } } @@ -215,7 +228,9 @@ type parsedCliFlags struct { nativeCurrencySymbol string firstHistoricalEpoch uint32 numHistoricalEpochs uint32 + shouldHandleContracts bool configFileCustomCurrencies string + configFileNodeFeatures string } func getParsedCliFlags(ctx *cli.Context) parsedCliFlags { @@ -243,6 +258,8 @@ func getParsedCliFlags(ctx *cli.Context) parsedCliFlags { nativeCurrencySymbol: ctx.GlobalString(cliFlagNativeCurrencySymbol.Name), firstHistoricalEpoch: uint32(ctx.GlobalUint(cliFlagFirstHistoricalEpoch.Name)), numHistoricalEpochs: uint32(ctx.GlobalUint(cliFlagNumHistoricalEpochs.Name)), + shouldHandleContracts: ctx.GlobalBool(cliFlagShouldHandleContracts.Name), configFileCustomCurrencies: ctx.GlobalString(cliFlagConfigFileCustomCurrencies.Name), + configFileNodeFeatures: ctx.GlobalString(cliFlagConfigFileNodeFeatures.Name), } } diff --git a/cmd/rosetta/main.go b/cmd/rosetta/main.go index fc81360c..904a6658 100644 --- a/cmd/rosetta/main.go +++ b/cmd/rosetta/main.go @@ -81,6 +81,7 @@ func startRosetta(ctx *cli.Context) error { GenesisBlockHash: cliFlags.genesisBlock, FirstHistoricalEpoch: cliFlags.firstHistoricalEpoch, NumHistoricalEpochs: cliFlags.numHistoricalEpochs, + ShouldHandleContracts: cliFlags.shouldHandleContracts, }) if err != nil { return err diff --git a/server/factory/provider.go b/server/factory/provider.go index a9ab7ab3..b945ed46 100644 --- a/server/factory/provider.go +++ b/server/factory/provider.go @@ -48,6 +48,7 @@ type ArgsCreateNetworkProvider struct { GenesisTimestamp int64 FirstHistoricalEpoch uint32 NumHistoricalEpochs uint32 + ShouldHandleContracts bool } // CreateNetworkProvider creates a network provider @@ -138,6 +139,7 @@ func CreateNetworkProvider(args ArgsCreateNetworkProvider) (NetworkProvider, err GenesisTimestamp: args.GenesisTimestamp, FirstHistoricalEpoch: args.FirstHistoricalEpoch, NumHistoricalEpochs: args.NumHistoricalEpochs, + ShouldHandleContracts: args.ShouldHandleContracts, ObserverFacade: &components.ObserverFacade{ Processor: baseProcessor, diff --git a/server/provider/networkProvider.go b/server/provider/networkProvider.go index 2f96e5c6..0549f460 100644 --- a/server/provider/networkProvider.go +++ b/server/provider/networkProvider.go @@ -41,6 +41,7 @@ type ArgsNewNetworkProvider struct { GenesisTimestamp int64 FirstHistoricalEpoch uint32 NumHistoricalEpochs uint32 + ShouldHandleContracts bool ObserverFacade observerFacade @@ -62,6 +63,7 @@ type networkProvider struct { genesisTimestamp int64 firstHistoricalEpoch uint32 numHistoricalEpochs uint32 + shouldHandleContracts bool observerFacade observerFacade @@ -101,6 +103,7 @@ func NewNetworkProvider(args ArgsNewNetworkProvider) (*networkProvider, error) { genesisTimestamp: args.GenesisTimestamp, firstHistoricalEpoch: args.FirstHistoricalEpoch, numHistoricalEpochs: args.NumHistoricalEpochs, + shouldHandleContracts: args.ShouldHandleContracts, observerFacade: args.ObserverFacade, @@ -345,14 +348,15 @@ func (provider *networkProvider) IsAddressObserved(address string) (bool, error) } shard := provider.observerFacade.ComputeShardId(pubKey) - isObservedActualShard := shard == provider.observedActualShard - isObservedProjectedShard := pubKey[len(pubKey)-1] == byte(provider.observedProjectedShard) - if provider.observedProjectedShardIsSet { - return isObservedProjectedShard, nil - } + noConstraintAboutProjectedShard := !provider.observedProjectedShardIsSet + noConstraintAboutHandlingContracts := provider.shouldHandleContracts + + passesConstraintAboutObservedActualShard := shard == provider.observedActualShard + passesConstraintAboutObservedProjectedShard := noConstraintAboutProjectedShard || pubKey[len(pubKey)-1] == byte(provider.observedProjectedShard) + passesConstraintAboutHandlingContracts := noConstraintAboutHandlingContracts || !core.IsSmartContractAddress(pubKey) - return isObservedActualShard, nil + return passesConstraintAboutObservedActualShard && passesConstraintAboutObservedProjectedShard && passesConstraintAboutHandlingContracts, nil } // ComputeShardIdOfPubKey computes the shard ID of a public key diff --git a/server/provider/networkProvider_test.go b/server/provider/networkProvider_test.go index 6b26403a..9f38e4bc 100644 --- a/server/provider/networkProvider_test.go +++ b/server/provider/networkProvider_test.go @@ -39,6 +39,7 @@ func TestNewNetworkProvider(t *testing.T) { GenesisTimestamp: 123456789, FirstHistoricalEpoch: 1000, NumHistoricalEpochs: 1024, + ShouldHandleContracts: true, ObserverFacade: testscommon.NewObserverFacadeMock(), Hasher: testscommon.RealWorldBlake2bHasher, MarshalizerForHashing: testscommon.MarshalizerForHashing, @@ -71,6 +72,7 @@ func TestNewNetworkProvider(t *testing.T) { assert.Equal(t, int64(123456789), provider.GetGenesisTimestamp()) assert.Equal(t, uint32(1000), provider.firstHistoricalEpoch) assert.Equal(t, uint32(1024), provider.numHistoricalEpochs) + assert.Equal(t, true, provider.shouldHandleContracts) } func TestNetworkProvider_DoGetBlockByNonce(t *testing.T) { @@ -280,6 +282,98 @@ func Test_ComputeTransactionFeeForMoveBalance(t *testing.T) { }) } +func TestNetworkProvider_IsAddressObserved(t *testing.T) { + t.Run("no projected shard, do not handle contracts", func(t *testing.T) { + args := createDefaultArgsNewNetworkProvider() + args.ObservedActualShard = 0 + args.ShouldHandleContracts = false + + provider, err := NewNetworkProvider(args) + require.Nil(t, err) + require.NotNil(t, provider) + + isObserved, err := provider.IsAddressObserved("erd1qyu5wthldzr8wx5c9ucg8kjagg0jfs53s8nr3zpz3hypefsdd8ssycr6th") + require.NoError(t, err) + require.False(t, isObserved) + + isObserved, err = provider.IsAddressObserved("erd1spyavw0956vq68xj8y4tenjpq2wd5a9p2c6j8gsz7ztyrnpxrruqzu66jx") + require.NoError(t, err) + require.True(t, isObserved) + + isObserved, err = provider.IsAddressObserved("erd1k2s324ww2g0yj38qn2ch2jwctdy8mnfxep94q9arncc6xecg3xaq6mjse8") + require.NoError(t, err) + require.False(t, isObserved) + + isObserved, err = provider.IsAddressObserved("erd1qqqqqqqqqqqqqpgqws44xjx2t056nn79fn29q0rjwfrd3m43396ql35kxy") + require.NoError(t, err) + require.False(t, isObserved) + }) + + t.Run("with projected shard, do not handle contracts", func(t *testing.T) { + args := createDefaultArgsNewNetworkProvider() + args.ObservedActualShard = 0 + args.ObservedProjectedShard = 0 + args.ObservedProjectedShardIsSet = true + args.ShouldHandleContracts = false + + provider, err := NewNetworkProvider(args) + require.Nil(t, err) + require.NotNil(t, provider) + + isObserved, err := provider.IsAddressObserved("erd1ldjsdetjvegjdnda0qw2h62kq6rpvrklkc5pw9zxm0nwulfhtyqqtyc4vq") + require.NoError(t, err) + require.True(t, isObserved) + + isObserved, err = provider.IsAddressObserved("erd1spyavw0956vq68xj8y4tenjpq2wd5a9p2c6j8gsz7ztyrnpxrruqzu66jx") + require.NoError(t, err) + require.False(t, isObserved) + + isObserved, err = provider.IsAddressObserved("erd1qqqqqqqqqqqqqpgqws44xjx2t056nn79fn29q0rjwfrd3m43396ql35kxy") + require.NoError(t, err) + require.False(t, isObserved) + }) + + t.Run("no projected shard, handle contracts", func(t *testing.T) { + args := createDefaultArgsNewNetworkProvider() + args.ObservedActualShard = 0 + args.ShouldHandleContracts = true + + provider, err := NewNetworkProvider(args) + require.Nil(t, err) + require.NotNil(t, provider) + + isObserved, err := provider.IsAddressObserved("erd1qyu5wthldzr8wx5c9ucg8kjagg0jfs53s8nr3zpz3hypefsdd8ssycr6th") + require.NoError(t, err) + require.False(t, isObserved) + + isObserved, err = provider.IsAddressObserved("erd1spyavw0956vq68xj8y4tenjpq2wd5a9p2c6j8gsz7ztyrnpxrruqzu66jx") + require.NoError(t, err) + require.True(t, isObserved) + + isObserved, err = provider.IsAddressObserved("erd1k2s324ww2g0yj38qn2ch2jwctdy8mnfxep94q9arncc6xecg3xaq6mjse8") + require.NoError(t, err) + require.False(t, isObserved) + + isObserved, err = provider.IsAddressObserved("erd1qqqqqqqqqqqqqpgqws44xjx2t056nn79fn29q0rjwfrd3m43396ql35kxy") + require.NoError(t, err) + require.True(t, isObserved) + }) + + t.Run("with error", func(t *testing.T) { + args := createDefaultArgsNewNetworkProvider() + args.ObservedActualShard = 0 + args.ShouldHandleContracts = true + + provider, err := NewNetworkProvider(args) + require.Nil(t, err) + require.NotNil(t, provider) + + isObserved, err := provider.IsAddressObserved("erd1test") + require.Error(t, err) + require.False(t, isObserved) + }) +} + func createDefaultArgsNewNetworkProvider() ArgsNewNetworkProvider { return ArgsNewNetworkProvider{ IsOffline: false, diff --git a/server/services/blockService.go b/server/services/blockService.go index 2de30597..c96b9201 100644 --- a/server/services/blockService.go +++ b/server/services/blockService.go @@ -145,7 +145,7 @@ func (service *blockService) createGenesisOperations(balances []*resources.Genes operations = append(operations, operation) } - operations, err := filterOperationsByAddress(operations, service.extension.isAddressObserved) + operations, err := filterOperationsByAddress(operations, service.provider.IsAddressObserved) if err != nil { return nil, err } diff --git a/server/services/networkProviderExtension.go b/server/services/networkProviderExtension.go index 35bf0a15..cfae1930 100644 --- a/server/services/networkProviderExtension.go +++ b/server/services/networkProviderExtension.go @@ -70,15 +70,6 @@ func (extension *networkProviderExtension) getGenesisBlockIdentifier() *types.Bl return blockSummaryToIdentifier(summary) } -func (extension *networkProviderExtension) isAddressObserved(address string) (bool, error) { - belongsToObservedShard, err := extension.provider.IsAddressObserved(address) - if err != nil { - return false, err - } - - return belongsToObservedShard, nil -} - func (extension *networkProviderExtension) isUserAddress(address string) bool { pubKey, err := extension.provider.ConvertAddressToPubKey(address) if err != nil { diff --git a/server/services/networkProviderExtension_test.go b/server/services/networkProviderExtension_test.go index f1df647e..5440ddd0 100644 --- a/server/services/networkProviderExtension_test.go +++ b/server/services/networkProviderExtension_test.go @@ -40,22 +40,3 @@ func TestNetworkProviderExtension_ValueToCustomAmount(t *testing.T) { require.Equal(t, expectedAmount, amount) } - -func TestNetworkProviderExtension_IsAddressObserved(t *testing.T) { - networkProvider := testscommon.NewNetworkProviderMock() - extension := newNetworkProviderExtension(networkProvider) - - networkProvider.MockObservedActualShard = 0 - - isObserved, err := extension.isAddressObserved("erd1qyu5wthldzr8wx5c9ucg8kjagg0jfs53s8nr3zpz3hypefsdd8ssycr6th") - require.NoError(t, err) - require.False(t, isObserved) - - isObserved, err = extension.isAddressObserved("erd1spyavw0956vq68xj8y4tenjpq2wd5a9p2c6j8gsz7ztyrnpxrruqzu66jx") - require.NoError(t, err) - require.True(t, isObserved) - - isObserved, err = extension.isAddressObserved("erd1k2s324ww2g0yj38qn2ch2jwctdy8mnfxep94q9arncc6xecg3xaq6mjse8") - require.NoError(t, err) - require.False(t, isObserved) -} diff --git a/server/services/transactionsTransformer.go b/server/services/transactionsTransformer.go index fbcfa89e..c4519f53 100644 --- a/server/services/transactionsTransformer.go +++ b/server/services/transactionsTransformer.go @@ -68,7 +68,7 @@ func (transformer *transactionsTransformer) transformBlockTxs(block *api.Block) } for _, rosettaTx := range rosettaTxs { - filteredOperations, err := filterOperationsByAddress(rosettaTx.Operations, transformer.extension.isAddressObserved) + filteredOperations, err := filterOperationsByAddress(rosettaTx.Operations, transformer.provider.IsAddressObserved) if err != nil { return nil, err } From 7c49f40f06ab0bc405a12af365774efc909b6b3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Mon, 12 Aug 2024 13:31:45 +0300 Subject: [PATCH 39/75] Fix tests. --- server/services/relayedTransactions_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/server/services/relayedTransactions_test.go b/server/services/relayedTransactions_test.go index aca493bd..51267a68 100644 --- a/server/services/relayedTransactions_test.go +++ b/server/services/relayedTransactions_test.go @@ -42,7 +42,6 @@ func Test_ParseInnerTxOfRelayedV1(t *testing.T) { require.NoError(t, err) require.NotNil(t, innerTx) - require.Equal(t, uint64(7), innerTx.Nonce) require.Equal(t, "1000000000000000000", innerTx.Value.String()) require.Equal(t, testscommon.TestPubKeyAlice, innerTx.SenderPubKey) require.Equal(t, testscommon.TestPubKeyBob, innerTx.ReceiverPubKey) From 846df8bbca1bbb521718a885fed572269cb733f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Mon, 12 Aug 2024 14:23:29 +0300 Subject: [PATCH 40/75] Additional transactions samples. --- systemtests/broadcast_samples.py | 157 ++++++++++++++++++++++++++++--- 1 file changed, 146 insertions(+), 11 deletions(-) diff --git a/systemtests/broadcast_samples.py b/systemtests/broadcast_samples.py index 2376c877..5d85e1ce 100644 --- a/systemtests/broadcast_samples.py +++ b/systemtests/broadcast_samples.py @@ -91,6 +91,62 @@ def main(): receiver=accounts.contracts_by_shard[1][0], )) + # Intra-shard, relayed v1 transaction with MoveBalance + controller.send(controller.create_relayed_v1_with_move_balance( + relayer=accounts.get_user(shard=0, index=0), + sender=accounts.get_user(shard=0, index=1), + receiver=accounts.get_user(shard=0, index=2).address, + amount=42 + )) + + # Relayed v3, senders and receivers in same shard + controller.send(controller.create_relayed_v3_with_a_few_inner_move_balances( + relayer=accounts.get_user(shard=0, index=0), + senders=accounts.users_by_shard[0][1:3], + receivers=[account.address for account in accounts.users_by_shard[0][3:5]], + amount=42 + )) + + # Relayed v3, senders and receivers in different shards + controller.send(controller.create_relayed_v3_with_a_few_inner_move_balances( + relayer=accounts.get_user(shard=0, index=0), + senders=accounts.users_by_shard[0][1:3], + receivers=[account.address for account in accounts.users_by_shard[1][3:5]], + amount=42 + )) + + # Relayed v3, senders and receivers in same shard (insufficient balance) + controller.send(controller.create_relayed_v3_with_a_few_inner_move_balances( + relayer=accounts.get_user(shard=0, index=0), + senders=accounts.users_by_shard[0][1:3], + receivers=[account.address for account in accounts.users_by_shard[0][3:5]], + amount=1000000000000000000000 + )) + + # Relayed v3, senders and receivers in different shards (insufficient balance) + controller.send(controller.create_relayed_v3_with_a_few_inner_move_balances( + relayer=accounts.get_user(shard=0, index=0), + senders=accounts.users_by_shard[0][1:3], + receivers=[account.address for account in accounts.users_by_shard[1][3:5]], + amount=1000000000000000000000 + )) + + # Relayed v3, senders and receivers in same shard, sending to non-payable contract + controller.send(controller.create_relayed_v3_with_a_few_inner_move_balances( + relayer=accounts.get_user(shard=0, index=0), + senders=[accounts.get_user(shard=0, index=5)], + receivers=[accounts.contracts_by_shard[0][0]], + amount=1000000000000000000 + )) + + # Intra-shard, relayed v1 transaction with MoveBalance + controller.send(controller.create_relayed_v1_with_move_balance( + relayer=accounts.get_user(shard=1, index=0), + sender=accounts.get_user(shard=1, index=9), + receiver=Address.from_bech32("erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqzllls8a5w6u"), + amount=1000000000000000000 + )) + class BunchOfAccounts: def __init__(self, configuration: Configuration) -> None: @@ -157,7 +213,7 @@ def create_airdrops(self) -> List[Transaction]: transaction = self.transfer_transactions_factory.create_transaction_for_native_token_transfer( sender=self.accounts.sponsor.address, receiver=user.address, - native_amount=10000000000000000 + native_amount=1000000000000000000 ) transaction = self.transfer_transactions_factory.create_transaction_for_esdt_token_transfer( @@ -227,28 +283,107 @@ def create_native_transfer_within_multiesdt(self, sender: "Account", receiver: A return transaction + def create_relayed_v1_with_move_balance(self, relayer: "Account", sender: "Account", receiver: Address, amount: int) -> Transaction: + inner_transaction = self.transfer_transactions_factory.create_transaction_for_native_token_transfer( + sender=sender.address, + receiver=receiver, + native_amount=amount + ) + + self._apply_nonce(inner_transaction) + self._sign(inner_transaction) + + transaction = self.relayed_transactions_factory.create_relayed_v1_transaction( + inner_transaction=inner_transaction, + relayer_address=relayer.address, + ) + + return transaction + + def create_relayed_v1_with_esdt_transfer(self, relayer: "Account", sender: "Account", receiver: Address, amount: int) -> Transaction: + inner_transaction = self.transfer_transactions_factory.create_transaction_for_esdt_token_transfer( + sender=sender.address, + receiver=receiver, + token_transfers=[TokenTransfer(Token(self.custom_currencies.currency), amount)] + ) + + self._apply_nonce(inner_transaction) + self._sign(inner_transaction) + + transaction = self.relayed_transactions_factory.create_relayed_v1_transaction( + inner_transaction=inner_transaction, + relayer_address=relayer.address, + ) + + return transaction + + def create_relayed_v2_with_move_balance(self, relayer: "Account", sender: "Account", receiver: Address, amount: int) -> Transaction: + inner_transaction = self.transfer_transactions_factory.create_transaction_for_native_token_transfer( + sender=sender.address, + receiver=receiver, + native_amount=amount + ) + + inner_transaction.gas_limit = 0 + + self._apply_nonce(inner_transaction) + self._sign(inner_transaction) + + transaction = self.relayed_transactions_factory.create_relayed_v2_transaction( + inner_transaction=inner_transaction, + inner_transaction_gas_limit=100000, + relayer_address=relayer.address, + ) + + return transaction + + def create_relayed_v3_with_a_few_inner_move_balances(self, relayer: "Account", senders: List["Account"], receivers: List[Address], amount: int) -> Transaction: + if len(senders) != len(receivers): + raise ValueError("senders and receivers must have the same length", len(senders), len(receivers)) + + inner_transactions: List[Transaction] = [] + + for sender, receiver in zip(senders, receivers): + inner_transaction = self.transfer_transactions_factory.create_transaction_for_native_token_transfer( + sender=sender.address, + receiver=receiver, + native_amount=amount, + ) + + inner_transaction.relayer = relayer.address.to_bech32() + self._apply_nonce(inner_transaction) + self._sign(inner_transaction) + inner_transactions.append(inner_transaction) + + transaction = self.relayed_transactions_factory.create_relayed_v3_transaction( + relayer_address=relayer.address, + inner_transactions=inner_transactions, + ) + + return transaction + def send_multiple(self, transactions: List[Transaction]): for transaction in transactions: - sender = self.accounts.get_account_by_bech32(transaction.sender) - self._apply_nonce(transaction, sender) - self._sign(transaction, sender) + self._apply_nonce(transaction) + self._sign(transaction) self.network_provider.send_transactions(transactions) def send(self, transaction: Transaction): - sender = self.accounts.get_account_by_bech32(transaction.sender) - self._apply_nonce(transaction, sender) - self._sign(transaction, sender) + self._apply_nonce(transaction) + self._sign(transaction) transaction_hash = self.network_provider.send_transaction(transaction) print(f"{self.configuration.explorer_url}/transactions/{transaction_hash}") - def _apply_nonce(self, transaction: Transaction, user: "Account"): - transaction.nonce = self.nonces_tracker.get_then_increment_nonce(user.address) + def _apply_nonce(self, transaction: Transaction): + sender = self.accounts.get_account_by_bech32(transaction.sender) + transaction.nonce = self.nonces_tracker.get_then_increment_nonce(sender.address) - def _sign(self, transaction: Transaction, user: "Account"): + def _sign(self, transaction: Transaction): + sender = self.accounts.get_account_by_bech32(transaction.sender) computer = TransactionComputer() bytes_for_signing = computer.compute_bytes_for_signing(transaction) - transaction.signature = user.signer.sign(bytes_for_signing) + transaction.signature = sender.signer.sign(bytes_for_signing) class Account: From 8c51425419f0e548e8012cf7c2107ab2304cf269 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Mon, 12 Aug 2024 14:30:38 +0300 Subject: [PATCH 41/75] Optimization (attempt) for claim developer rewards. --- .../services/transactionsFeaturesDetector.go | 30 ++++++++++++++----- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/server/services/transactionsFeaturesDetector.go b/server/services/transactionsFeaturesDetector.go index 1ecddb8c..98dceb8a 100644 --- a/server/services/transactionsFeaturesDetector.go +++ b/server/services/transactionsFeaturesDetector.go @@ -21,23 +21,37 @@ func (detector *transactionsFeaturesDetector) doesContractResultHoldRewardsOfCla contractResult *transaction.ApiTransactionResult, allTransactionsInBlock []*transaction.ApiTransactionResult, ) bool { - claimDeveloperRewardsTxs := make(map[string]struct{}) + hasData := len(contractResult.Data) > 0 + hasNonZeroNonce := contractResult.Nonce > 0 + if hasData || hasNonZeroNonce { + return false + } for _, tx := range allTransactionsInBlock { matchesTypeOnSource := tx.ProcessingTypeOnSource == transactionProcessingTypeBuiltInFunctionCall + if !matchesTypeOnSource { + continue + } + matchesTypeOnDestination := tx.ProcessingTypeOnDestination == transactionProcessingTypeBuiltInFunctionCall + if !matchesTypeOnDestination { + continue + } + matchesOperation := tx.Operation == builtInFunctionClaimDeveloperRewards + if !matchesOperation { + continue + } - if matchesTypeOnSource && matchesTypeOnDestination && matchesOperation { - claimDeveloperRewardsTxs[tx.Hash] = struct{}{} + matchesOriginalTransactionHash := tx.Hash == contractResult.OriginalTransactionHash + if !matchesOriginalTransactionHash { + continue } - } - _, isResultOfClaimDeveloperRewards := claimDeveloperRewardsTxs[contractResult.OriginalTransactionHash] - hasNoData := len(contractResult.Data) == 0 - hasZeroNonce := contractResult.Nonce == 0 + return true + } - return isResultOfClaimDeveloperRewards && hasNoData && hasZeroNonce + return false } // isInvalidTransactionOfTypeMoveBalanceThatOnlyConsumesDataMovementGas detects (intra-shard) invalid transactions From bd46ed347b8401dcaa129aca69ea0435c9773924 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Mon, 19 Aug 2024 11:40:56 +0300 Subject: [PATCH 42/75] Some tests & refactoring. --- server/services/constants.go | 1 + .../services/transactionEventsController.go | 10 +++---- .../transactionEventsController_test.go | 29 +++++++++++++++++++ 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/server/services/constants.go b/server/services/constants.go index 435ce316..349a671d 100644 --- a/server/services/constants.go +++ b/server/services/constants.go @@ -56,4 +56,5 @@ var ( numTopicsOfEventESDTNFTCreate = 4 numTopicsOfEventESDTNFTBurn = 3 numTopicsOfEventESDTNFTAddQuantity = 3 + numTopicsOfEventSCDeployBeforeSirius = 2 ) diff --git a/server/services/transactionEventsController.go b/server/services/transactionEventsController.go index c902c7c4..502f668a 100644 --- a/server/services/transactionEventsController.go +++ b/server/services/transactionEventsController.go @@ -20,17 +20,17 @@ func newTransactionEventsController(provider NetworkProvider) *transactionEvents func (controller *transactionEventsController) extractEventSCDeploy(tx *transaction.ApiTransactionResult) ([]*eventSCDeploy, error) { rawEvents := controller.findManyEventsByIdentifier(tx, transactionEventSCDeploy) - typedEvents := make([]*eventSCDeploy, 0, len(rawEvents)) for _, event := range rawEvents { numTopics := len(event.Topics) - if numTopics < 2 { + if numTopics < numTopicsOfEventSCDeployBeforeSirius { // Before Sirius, there are 2 topics: contract address, deployer address. - // For Sirius, there are 3 topics: contract address, deployer address, codehash. - return nil, fmt.Errorf("%w: bad number of topics for %s event = '%d', tx = %s", errCannotRecognizeEvent, transactionEventSCDeploy, numTopics, tx.Hash) + // For Sirius, there are 3 topics: contract address, deployer address, codehash (not used). + return nil, fmt.Errorf("%w: bad number of topics for SCdeploy event = %d", errCannotRecognizeEvent, numTopics) } + // "event.Address" is same as "event.Topics[0]"" (the address of the deployed contract). contractAddress := event.Address deployerPubKey := event.Topics[1] deployerAddress := controller.provider.ConvertPubKeyToAddress(deployerPubKey) @@ -73,7 +73,7 @@ func (controller *transactionEventsController) extractEventTransferValueOnly(tx receiver = controller.provider.ConvertPubKeyToAddress(receiverPubKey) } else { - return nil, fmt.Errorf("%w: bad number of topics for '%s' = %d, tx = %s", errCannotRecognizeEvent, transactionEventTransferValueOnly, numTopics, tx.Hash) + return nil, fmt.Errorf("%w: bad number of topics for '%s' = %d", errCannotRecognizeEvent, transactionEventTransferValueOnly, numTopics) } value := big.NewInt(0).SetBytes(valueBytes) diff --git a/server/services/transactionEventsController_test.go b/server/services/transactionEventsController_test.go index ae96c734..b977ad98 100644 --- a/server/services/transactionEventsController_test.go +++ b/server/services/transactionEventsController_test.go @@ -1,6 +1,7 @@ package services import ( + "encoding/hex" "testing" "github.com/multiversx/mx-chain-core-go/data/transaction" @@ -174,6 +175,34 @@ func TestTransactionEventsController_ExtractEvents(t *testing.T) { networkProvider := testscommon.NewNetworkProviderMock() controller := newTransactionEventsController(networkProvider) + t.Run("SCDeploy", func(t *testing.T) { + topic0, _ := hex.DecodeString("00000000000000000500def8dad1161f8f0c38f3e6e73515ed81058f0b5606b8") + topic1, _ := hex.DecodeString("5cf4abc83e50c5309d807fc3f676988759a1e301001bc9a0265804f42af806b8") + topic2, _ := hex.DecodeString("be5560e0d7d3857d438a3678269039f8f80ded90dcbc2cda268a0847ba9cb379") + + tx := &transaction.ApiTransactionResult{ + Logs: &transaction.ApiLogs{ + Events: []*transaction.Events{ + { + Identifier: "SCDeploy", + Address: "erd1qqqqqqqqqqqqqpgqmmud45gkr78scw8numnn290dsyzc7z6kq6uqw2jcza", + Topics: [][]byte{ + topic0, + topic1, + topic2, + }, + }, + }, + }, + } + + events, err := controller.extractEventSCDeploy(tx) + require.NoError(t, err) + require.Len(t, events, 1) + require.Equal(t, "erd1qqqqqqqqqqqqqpgqmmud45gkr78scw8numnn290dsyzc7z6kq6uqw2jcza", events[0].contractAddress) + require.Equal(t, "erd1tn62hjp72rznp8vq0lplva5csav6rccpqqdungpxtqz0g2hcq6uq9k4cc6", events[0].deployerAddress) + }) + t.Run("ESDTNFTCreate", func(t *testing.T) { tx := &transaction.ApiTransactionResult{ Logs: &transaction.ApiLogs{ From f64dd9e44d4e98b4a6272929f7ed26a3ec0cb444 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Mon, 19 Aug 2024 15:10:28 +0300 Subject: [PATCH 43/75] Add unit tests. --- ...ocks_with_direct_sc_deploy_with_value.json | 51 +++++++++++++++ server/services/transactionsTransformer.go | 6 +- .../services/transactionsTransformer_test.go | 60 ++++++++++++++++++ systemtests/broadcast_samples.py | 18 ++++++ systemtests/dummy.wasm | Bin 0 -> 115 bytes 5 files changed, 132 insertions(+), 3 deletions(-) create mode 100644 server/services/testdata/blocks_with_direct_sc_deploy_with_value.json create mode 100755 systemtests/dummy.wasm diff --git a/server/services/testdata/blocks_with_direct_sc_deploy_with_value.json b/server/services/testdata/blocks_with_direct_sc_deploy_with_value.json new file mode 100644 index 00000000..070f6d84 --- /dev/null +++ b/server/services/testdata/blocks_with_direct_sc_deploy_with_value.json @@ -0,0 +1,51 @@ +[ + { + "miniBlocks": [ + { + "transactions": [ + { + "type": "normal", + "processingTypeOnSource": "SCDeployment", + "processingTypeOnDestination": "SCDeployment", + "hash": "2459bb2b9a64c1c920777ecbdaf0fa33d7fe8bcd24d7164562f341b2e4f702da", + "value": "10000000000000000", + "receiver": "erd1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq6gq4hu", + "sender": "erd1tn62hjp72rznp8vq0lplva5csav6rccpqqdungpxtqz0g2hcq6uq9k4cc6", + "logs": { + "address": "erd1tn62hjp72rznp8vq0lplva5csav6rccpqqdungpxtqz0g2hcq6uq9k4cc6", + "events": [ + { + "address": "erd1qqqqqqqqqqqqqpgqc4vdnqgc48ww26ljxqe2flgl86jewg0nq6uqna2ymj", + "identifier": "SCDeploy", + "topics": [ + "AAAAAAAAAAAFAMVY2YEYqdzla/IwMqT9Hz6llyHzBrg=", + "XPSryD5QxTCdgH/D9naYh1mh4wEAG8mgJlgE9Cr4Brg=", + "v5gJ+IK1KdPGJfGYibrKij1SO0bzAAhfUVsODJ8midg=" + ], + "data": null, + "additionalData": null + }, + { + "address": "erd1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq6gq4hu", + "identifier": "writeLog", + "topics": [ + "XPSryD5QxTCdgH/D9naYh1mh4wEAG8mgJlgE9Cr4Brg=", + "QHRvbyBtdWNoIGdhcyBwcm92aWRlZCBmb3IgcHJvY2Vzc2luZzogZ2FzIHByb3ZpZGVkID0gNDU4ODUwMCwgZ2FzIHVzZWQgPSAzMzQ1MDU=" + ], + "data": "QDZmNmI=", + "additionalData": ["QDZmNmI="] + } + ] + }, + "status": "success", + "operation": "scDeploy", + "initiallyPaidFee": "457385000000000", + "chainID": "T", + "version": 2, + "options": 0 + } + ] + } + ] + } +] diff --git a/server/services/transactionsTransformer.go b/server/services/transactionsTransformer.go index 782616e9..045afa77 100644 --- a/server/services/transactionsTransformer.go +++ b/server/services/transactionsTransformer.go @@ -445,11 +445,11 @@ func (transformer *transactionsTransformer) addOperationsGivenTransactionEvents( } for _, event := range eventsSCDeploy { - // Handle deployments with transfer of value + // Handle direct deployments with transfer of value (indirect deployments are currently excluded to prevent any potential misinterpretations).. if tx.Receiver == systemContractDeployAddress { operations := []*types.Operation{ - // Deployer's balance change is already captured in non-events-based operations. - // Let's simulate the transfer from the System deployment address to the contract address. + // Deployer's balance change is already captured in operations recovered not from logs / events, but from the transaction itself. + // It remains to "simulate" the transfer from the System deployment address to the contract address. { Type: opTransfer, Account: addressToAccountIdentifier(tx.Receiver), diff --git a/server/services/transactionsTransformer_test.go b/server/services/transactionsTransformer_test.go index e6b7fbe8..4b1cd7a8 100644 --- a/server/services/transactionsTransformer_test.go +++ b/server/services/transactionsTransformer_test.go @@ -287,6 +287,66 @@ func TestTransactionsTransformer_InvalidTxToRosettaTx(t *testing.T) { require.Equal(t, expectedTx, rosettaTx) } +func TestTransactionsTransformer_TransformBlockTxsHavingDirectSCDeployWithValue(t *testing.T) { + networkProvider := testscommon.NewNetworkProviderMock() + networkProvider.MockObservedActualShard = 0 + + extension := newNetworkProviderExtension(networkProvider) + transformer := newTransactionsTransformer(networkProvider) + + blocks, err := readTestBlocks("testdata/blocks_with_direct_sc_deploy_with_value.json") + require.Nil(t, err) + + txs, err := transformer.transformBlockTxs(blocks[0]) + require.Nil(t, err) + require.Len(t, txs, 1) + require.Len(t, txs[0].Operations, 5) + + expectedTx := &types.Transaction{ + TransactionIdentifier: hashToTransactionIdentifier("2459bb2b9a64c1c920777ecbdaf0fa33d7fe8bcd24d7164562f341b2e4f702da"), + Operations: []*types.Operation{ + { + Type: opTransfer, + OperationIdentifier: indexToOperationIdentifier(0), + Account: addressToAccountIdentifier("erd1tn62hjp72rznp8vq0lplva5csav6rccpqqdungpxtqz0g2hcq6uq9k4cc6"), + Amount: extension.valueToNativeAmount("-10000000000000000"), + Status: &opStatusSuccess, + }, + { + Type: opTransfer, + OperationIdentifier: indexToOperationIdentifier(1), + Account: addressToAccountIdentifier("erd1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq6gq4hu"), + Amount: extension.valueToNativeAmount("10000000000000000"), + Status: &opStatusSuccess, + }, + { + Type: opFee, + OperationIdentifier: indexToOperationIdentifier(2), + Account: addressToAccountIdentifier("erd1tn62hjp72rznp8vq0lplva5csav6rccpqqdungpxtqz0g2hcq6uq9k4cc6"), + Amount: extension.valueToNativeAmount("-457385000000000"), + Status: &opStatusSuccess, + }, + { + Type: opTransfer, + OperationIdentifier: indexToOperationIdentifier(3), + Account: addressToAccountIdentifier("erd1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq6gq4hu"), + Amount: extension.valueToNativeAmount("-10000000000000000"), + Status: &opStatusSuccess, + }, + { + Type: opTransfer, + OperationIdentifier: indexToOperationIdentifier(4), + Account: addressToAccountIdentifier("erd1qqqqqqqqqqqqqpgqc4vdnqgc48ww26ljxqe2flgl86jewg0nq6uqna2ymj"), + Amount: extension.valueToNativeAmount("10000000000000000"), + Status: &opStatusSuccess, + }, + }, + Metadata: extractTransactionMetadata(blocks[0].MiniBlocks[0].Transactions[0]), + } + + require.Equal(t, expectedTx, txs[0]) +} + func TestTransactionsTransformer_TransformBlockTxsHavingESDTIssue(t *testing.T) { networkProvider := testscommon.NewNetworkProviderMock() networkProvider.MockCustomCurrencies = []resources.Currency{{Symbol: "FOO-6d28db"}} diff --git a/systemtests/broadcast_samples.py b/systemtests/broadcast_samples.py index 5d85e1ce..62d3dd19 100644 --- a/systemtests/broadcast_samples.py +++ b/systemtests/broadcast_samples.py @@ -13,6 +13,7 @@ from systemtests.config import CONFIGURATIONS, Configuration CONTRACT_PATH_ADDER = Path(__file__).parent / "adder.wasm" +CONTRACT_PATH_DUMMY = Path(__file__).parent / "dummy.wasm" def main(): @@ -147,6 +148,12 @@ def main(): amount=1000000000000000000 )) + # Direct contract deployment with MoveBalance + controller.send(controller.create_contract_deployment_with_move_balance( + sender=accounts.get_user(shard=0, index=0), + amount=10000000000000000 + )) + class BunchOfAccounts: def __init__(self, configuration: Configuration) -> None: @@ -362,6 +369,17 @@ def create_relayed_v3_with_a_few_inner_move_balances(self, relayer: "Account", s return transaction + def create_contract_deployment_with_move_balance(self, sender: "Account", amount: int) -> Transaction: + transaction = self.contracts_transactions_factory.create_transaction_for_deploy( + sender=sender.address, + bytecode=CONTRACT_PATH_DUMMY, + gas_limit=5000000, + arguments=[0], + native_transfer_amount=amount + ) + + return transaction + def send_multiple(self, transactions: List[Transaction]): for transaction in transactions: self._apply_nonce(transaction) diff --git a/systemtests/dummy.wasm b/systemtests/dummy.wasm new file mode 100755 index 0000000000000000000000000000000000000000..3c36df962a52b73fa104b51e38b89b635f769a43 GIT binary patch literal 115 zcmWN{u?~VT6ouh)?yW%RLi7n-oejG_2_zKl(3Sw2uqdOCZU21ZkA4XRz)CuP&;fFy zy7;y}uI+X7(JxOxjgmYlAw|V6Z~M>K@@$IWLYg;$$>R!^ujw)OgM;Xp60|<5z-Be> JmuAnw{sGKc6}kWb literal 0 HcmV?d00001 From ec1082c540bc16ae24a1cad9c456fd0acb63be4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Tue, 20 Aug 2024 11:56:19 +0300 Subject: [PATCH 44/75] New test ESDT. Formatting. --- cmd/rosetta/testdata/custom-currencies.json | 16 ++-- .../testdata/blocks_with_esdt_burn.json | 6 +- .../testdata/blocks_with_esdt_mint.json | 6 +- systemtests/mesh_cli_config/check-data.json | 86 +++++++++---------- .../devnet-construction-custom.json | 84 +++++++++--------- .../devnet-construction-native.json | 62 ++++++------- .../testnet-construction-custom.json | 84 +++++++++--------- .../testnet-construction-custom.ros | 6 +- .../testnet-construction-native.json | 62 ++++++------- .../devnet-custom-currencies.json | 8 +- .../mainnet-custom-currencies.json | 24 +++--- .../testnet-custom-currencies.json | 8 +- 12 files changed, 222 insertions(+), 230 deletions(-) diff --git a/cmd/rosetta/testdata/custom-currencies.json b/cmd/rosetta/testdata/custom-currencies.json index 8a90dd3c..9fcd369b 100644 --- a/cmd/rosetta/testdata/custom-currencies.json +++ b/cmd/rosetta/testdata/custom-currencies.json @@ -1,10 +1,10 @@ [ - { - "symbol": "WEGLD-bd4d79", - "decimals": 18 - }, - { - "symbol": "USDC-c76f1f", - "decimals": 6 - } + { + "symbol": "WEGLD-bd4d79", + "decimals": 18 + }, + { + "symbol": "USDC-c76f1f", + "decimals": 6 + } ] diff --git a/server/services/testdata/blocks_with_esdt_burn.json b/server/services/testdata/blocks_with_esdt_burn.json index fe31dd29..5d9fa744 100644 --- a/server/services/testdata/blocks_with_esdt_burn.json +++ b/server/services/testdata/blocks_with_esdt_burn.json @@ -16,11 +16,7 @@ { "address": "erd1r69gk66fmedhhcg24g2c5kn2f2a5k4kvpr6jfw67dn2lyydd8cfswy6ede", "identifier": "ESDTLocalBurn", - "topics": [ - "VEVTVC00ODRmYTE=", - "", - "Mg==" - ], + "topics": ["VEVTVC00ODRmYTE=", "", "Mg=="], "data": null }, { diff --git a/server/services/testdata/blocks_with_esdt_mint.json b/server/services/testdata/blocks_with_esdt_mint.json index e83231af..b10d5ea7 100644 --- a/server/services/testdata/blocks_with_esdt_mint.json +++ b/server/services/testdata/blocks_with_esdt_mint.json @@ -16,11 +16,7 @@ { "address": "erd1r69gk66fmedhhcg24g2c5kn2f2a5k4kvpr6jfw67dn2lyydd8cfswy6ede", "identifier": "ESDTLocalMint", - "topics": [ - "VEVTVC00ODRmYTE=", - "", - "yA==" - ], + "topics": ["VEVTVC00ODRmYTE=", "", "yA=="], "data": null }, { diff --git a/systemtests/mesh_cli_config/check-data.json b/systemtests/mesh_cli_config/check-data.json index 85e0c701..a50018e1 100644 --- a/systemtests/mesh_cli_config/check-data.json +++ b/systemtests/mesh_cli_config/check-data.json @@ -1,46 +1,46 @@ { - "network": { - "blockchain": "MultiversX", - "network": "untitled" - }, - "http_timeout": 30, - "max_retries": 10, - "retry_elapsed_time": 20, - "max_online_connections": 8, - "max_sync_concurrency": 4, - "tip_delay": 10, - "max_reorg_depth": 100, - "log_configuration": false, - "compression_disabled": false, - "l0_in_memory_enabled": false, - "error_stack_trace_disabled": false, - "coin_supported": false, - "construction": null, - "data": { - "active_reconciliation_concurrency": 8, - "inactive_reconciliation_concurrency": 1, - "inactive_reconciliation_frequency": 8, - "log_blocks": false, - "log_transactions": true, - "log_balance_changes": true, - "log_reconciliations": true, - "ignore_reconciliation_error": false, - "reconciliation_disabled": false, - "reconciliation_drain_disabled": false, - "inactive_discrepancy_search_disabled": false, - "balance_tracking_disabled": false, - "coin_tracking_disabled": false, - "status_port": 9091, - "results_output_file": "", - "pruning_block_disabled": false, - "pruning_balance_disabled": false, - "initial_balance_fetch_disabled": false, - "historical_balance_disabled": false, - "end_conditions": { - "reconciliation_coverage": { - "coverage": 1, - "from_tip": true - } + "network": { + "blockchain": "MultiversX", + "network": "untitled" + }, + "http_timeout": 30, + "max_retries": 10, + "retry_elapsed_time": 20, + "max_online_connections": 8, + "max_sync_concurrency": 4, + "tip_delay": 10, + "max_reorg_depth": 100, + "log_configuration": false, + "compression_disabled": false, + "l0_in_memory_enabled": false, + "error_stack_trace_disabled": false, + "coin_supported": false, + "construction": null, + "data": { + "active_reconciliation_concurrency": 8, + "inactive_reconciliation_concurrency": 1, + "inactive_reconciliation_frequency": 8, + "log_blocks": false, + "log_transactions": true, + "log_balance_changes": true, + "log_reconciliations": true, + "ignore_reconciliation_error": false, + "reconciliation_disabled": false, + "reconciliation_drain_disabled": false, + "inactive_discrepancy_search_disabled": false, + "balance_tracking_disabled": false, + "coin_tracking_disabled": false, + "status_port": 9091, + "results_output_file": "", + "pruning_block_disabled": false, + "pruning_balance_disabled": false, + "initial_balance_fetch_disabled": false, + "historical_balance_disabled": false, + "end_conditions": { + "reconciliation_coverage": { + "coverage": 1, + "from_tip": true + } + } } - } } diff --git a/systemtests/mesh_cli_config/devnet-construction-custom.json b/systemtests/mesh_cli_config/devnet-construction-custom.json index e7400b75..1b8b88da 100644 --- a/systemtests/mesh_cli_config/devnet-construction-custom.json +++ b/systemtests/mesh_cli_config/devnet-construction-custom.json @@ -1,46 +1,46 @@ { - "network": { - "blockchain": "MultiversX", - "network": "untitled" - }, - "data_directory": "", - "http_timeout": 200, - "max_retries": 1, - "max_online_connections": 120, - "max_sync_concurrency": 1, - "construction": { - "end_conditions": { - "transfer": 1 + "network": { + "blockchain": "MultiversX", + "network": "untitled" }, - "stale_depth": 10, - "broadcast_limit": 5, - "constructor_dsl_file": "devnet-construction-custom.ros", - "prefunded_accounts": [ - { - "account_identifier": { - "address": "erd1ldjsdetjvegjdnda0qw2h62kq6rpvrklkc5pw9zxm0nwulfhtyqqtyc4vq" + "data_directory": "", + "http_timeout": 200, + "max_retries": 1, + "max_online_connections": 120, + "max_sync_concurrency": 1, + "construction": { + "end_conditions": { + "transfer": 1 }, - "privkey": "3e4e89e501eb542c12403fb15c52479e8721f2f4dedc3b3ef0f3b47b37de006c", - "curve_type": "edwards25519", - "currency": { - "symbol": "ROSETTA-2c0a37", - "decimals": 2 - } - }, - { - "account_identifier": { - "address": "erd1ldjsdetjvegjdnda0qw2h62kq6rpvrklkc5pw9zxm0nwulfhtyqqtyc4vq" - }, - "privkey": "3e4e89e501eb542c12403fb15c52479e8721f2f4dedc3b3ef0f3b47b37de006c", - "curve_type": "edwards25519", - "currency": { - "symbol": "EGLD", - "decimals": 18 - } - } - ] - }, - "data": { - "inactive_discrepancy_search_disabled": true - } + "stale_depth": 10, + "broadcast_limit": 5, + "constructor_dsl_file": "devnet-construction-custom.ros", + "prefunded_accounts": [ + { + "account_identifier": { + "address": "erd1ldjsdetjvegjdnda0qw2h62kq6rpvrklkc5pw9zxm0nwulfhtyqqtyc4vq" + }, + "privkey": "3e4e89e501eb542c12403fb15c52479e8721f2f4dedc3b3ef0f3b47b37de006c", + "curve_type": "edwards25519", + "currency": { + "symbol": "ROSETTA-2c0a37", + "decimals": 2 + } + }, + { + "account_identifier": { + "address": "erd1ldjsdetjvegjdnda0qw2h62kq6rpvrklkc5pw9zxm0nwulfhtyqqtyc4vq" + }, + "privkey": "3e4e89e501eb542c12403fb15c52479e8721f2f4dedc3b3ef0f3b47b37de006c", + "curve_type": "edwards25519", + "currency": { + "symbol": "EGLD", + "decimals": 18 + } + } + ] + }, + "data": { + "inactive_discrepancy_search_disabled": true + } } diff --git a/systemtests/mesh_cli_config/devnet-construction-native.json b/systemtests/mesh_cli_config/devnet-construction-native.json index 711b1c6a..2568c1ae 100644 --- a/systemtests/mesh_cli_config/devnet-construction-native.json +++ b/systemtests/mesh_cli_config/devnet-construction-native.json @@ -1,35 +1,35 @@ { - "network": { - "blockchain": "MultiversX", - "network": "untitled" - }, - "data_directory": "", - "http_timeout": 200, - "max_retries": 1, - "max_online_connections": 120, - "max_sync_concurrency": 1, - "construction": { - "end_conditions": { - "transfer": 1 + "network": { + "blockchain": "MultiversX", + "network": "untitled" }, - "stale_depth": 10, - "broadcast_limit": 5, - "constructor_dsl_file": "devnet-construction-native.ros", - "prefunded_accounts": [ - { - "account_identifier": { - "address": "erd1ldjsdetjvegjdnda0qw2h62kq6rpvrklkc5pw9zxm0nwulfhtyqqtyc4vq" + "data_directory": "", + "http_timeout": 200, + "max_retries": 1, + "max_online_connections": 120, + "max_sync_concurrency": 1, + "construction": { + "end_conditions": { + "transfer": 1 }, - "privkey": "3e4e89e501eb542c12403fb15c52479e8721f2f4dedc3b3ef0f3b47b37de006c", - "curve_type": "edwards25519", - "currency": { - "symbol": "EGLD", - "decimals": 18 - } - } - ] - }, - "data": { - "inactive_discrepancy_search_disabled": true - } + "stale_depth": 10, + "broadcast_limit": 5, + "constructor_dsl_file": "devnet-construction-native.ros", + "prefunded_accounts": [ + { + "account_identifier": { + "address": "erd1ldjsdetjvegjdnda0qw2h62kq6rpvrklkc5pw9zxm0nwulfhtyqqtyc4vq" + }, + "privkey": "3e4e89e501eb542c12403fb15c52479e8721f2f4dedc3b3ef0f3b47b37de006c", + "curve_type": "edwards25519", + "currency": { + "symbol": "EGLD", + "decimals": 18 + } + } + ] + }, + "data": { + "inactive_discrepancy_search_disabled": true + } } diff --git a/systemtests/mesh_cli_config/testnet-construction-custom.json b/systemtests/mesh_cli_config/testnet-construction-custom.json index 1c93e0d5..9b3441d9 100644 --- a/systemtests/mesh_cli_config/testnet-construction-custom.json +++ b/systemtests/mesh_cli_config/testnet-construction-custom.json @@ -1,46 +1,46 @@ { - "network": { - "blockchain": "MultiversX", - "network": "untitled" - }, - "data_directory": "", - "http_timeout": 200, - "max_retries": 1, - "max_online_connections": 120, - "max_sync_concurrency": 1, - "construction": { - "end_conditions": { - "transfer": 1 + "network": { + "blockchain": "MultiversX", + "network": "untitled" }, - "stale_depth": 10, - "broadcast_limit": 5, - "constructor_dsl_file": "testnet-construction-custom.ros", - "prefunded_accounts": [ - { - "account_identifier": { - "address": "erd1ldjsdetjvegjdnda0qw2h62kq6rpvrklkc5pw9zxm0nwulfhtyqqtyc4vq" + "data_directory": "", + "http_timeout": 200, + "max_retries": 1, + "max_online_connections": 120, + "max_sync_concurrency": 1, + "construction": { + "end_conditions": { + "transfer": 1 }, - "privkey": "3e4e89e501eb542c12403fb15c52479e8721f2f4dedc3b3ef0f3b47b37de006c", - "curve_type": "edwards25519", - "currency": { - "symbol": "ROSETTA-7783de", - "decimals": 4 - } - }, - { - "account_identifier": { - "address": "erd1ldjsdetjvegjdnda0qw2h62kq6rpvrklkc5pw9zxm0nwulfhtyqqtyc4vq" - }, - "privkey": "3e4e89e501eb542c12403fb15c52479e8721f2f4dedc3b3ef0f3b47b37de006c", - "curve_type": "edwards25519", - "currency": { - "symbol": "EGLD", - "decimals": 18 - } - } - ] - }, - "data": { - "inactive_discrepancy_search_disabled": true - } + "stale_depth": 10, + "broadcast_limit": 5, + "constructor_dsl_file": "testnet-construction-custom.ros", + "prefunded_accounts": [ + { + "account_identifier": { + "address": "erd1ldjsdetjvegjdnda0qw2h62kq6rpvrklkc5pw9zxm0nwulfhtyqqtyc4vq" + }, + "privkey": "3e4e89e501eb542c12403fb15c52479e8721f2f4dedc3b3ef0f3b47b37de006c", + "curve_type": "edwards25519", + "currency": { + "symbol": "ROSETTA-de9b08", + "decimals": 4 + } + }, + { + "account_identifier": { + "address": "erd1ldjsdetjvegjdnda0qw2h62kq6rpvrklkc5pw9zxm0nwulfhtyqqtyc4vq" + }, + "privkey": "3e4e89e501eb542c12403fb15c52479e8721f2f4dedc3b3ef0f3b47b37de006c", + "curve_type": "edwards25519", + "currency": { + "symbol": "EGLD", + "decimals": 18 + } + } + ] + }, + "data": { + "inactive_discrepancy_search_disabled": true + } } diff --git a/systemtests/mesh_cli_config/testnet-construction-custom.ros b/systemtests/mesh_cli_config/testnet-construction-custom.ros index fa70a322..3c6631b9 100644 --- a/systemtests/mesh_cli_config/testnet-construction-custom.ros +++ b/systemtests/mesh_cli_config/testnet-construction-custom.ros @@ -1,13 +1,13 @@ transfer(1){ transfer{ transfer.network = {"network":"untitled", "blockchain":"MultiversX"}; - custom_currency = {"symbol":"ROSETTA-7783de", "decimals":4}; + custom_currency = {"symbol":"ROSETTA-de9b08", "decimals":4}; sender = { "account_identifier": { "address": "erd1ldjsdetjvegjdnda0qw2h62kq6rpvrklkc5pw9zxm0nwulfhtyqqtyc4vq" }, "currency": { - "symbol": "ROSETTA-7783de", + "symbol": "ROSETTA-de9b08", "decimals": 4 } }; @@ -24,7 +24,7 @@ transfer(1){ "address": "erd1xtslmt67utuewwv8jsx729mxjxaa8dvyyzp7492hy99dl7hvcuqq30l98v" }, "currency": { - "symbol": "ROSETTA-7783de", + "symbol": "ROSETTA-de9b08", "decimals": 4 } }; diff --git a/systemtests/mesh_cli_config/testnet-construction-native.json b/systemtests/mesh_cli_config/testnet-construction-native.json index 2113da29..75c66677 100644 --- a/systemtests/mesh_cli_config/testnet-construction-native.json +++ b/systemtests/mesh_cli_config/testnet-construction-native.json @@ -1,35 +1,35 @@ { - "network": { - "blockchain": "MultiversX", - "network": "untitled" - }, - "data_directory": "", - "http_timeout": 200, - "max_retries": 1, - "max_online_connections": 120, - "max_sync_concurrency": 1, - "construction": { - "end_conditions": { - "transfer": 1 + "network": { + "blockchain": "MultiversX", + "network": "untitled" }, - "stale_depth": 10, - "broadcast_limit": 5, - "constructor_dsl_file": "testnet-construction-native.ros", - "prefunded_accounts": [ - { - "account_identifier": { - "address": "erd1ldjsdetjvegjdnda0qw2h62kq6rpvrklkc5pw9zxm0nwulfhtyqqtyc4vq" + "data_directory": "", + "http_timeout": 200, + "max_retries": 1, + "max_online_connections": 120, + "max_sync_concurrency": 1, + "construction": { + "end_conditions": { + "transfer": 1 }, - "privkey": "3e4e89e501eb542c12403fb15c52479e8721f2f4dedc3b3ef0f3b47b37de006c", - "curve_type": "edwards25519", - "currency": { - "symbol": "EGLD", - "decimals": 18 - } - } - ] - }, - "data": { - "inactive_discrepancy_search_disabled": true - } + "stale_depth": 10, + "broadcast_limit": 5, + "constructor_dsl_file": "testnet-construction-native.ros", + "prefunded_accounts": [ + { + "account_identifier": { + "address": "erd1ldjsdetjvegjdnda0qw2h62kq6rpvrklkc5pw9zxm0nwulfhtyqqtyc4vq" + }, + "privkey": "3e4e89e501eb542c12403fb15c52479e8721f2f4dedc3b3ef0f3b47b37de006c", + "curve_type": "edwards25519", + "currency": { + "symbol": "EGLD", + "decimals": 18 + } + } + ] + }, + "data": { + "inactive_discrepancy_search_disabled": true + } } diff --git a/systemtests/rosetta_config/devnet-custom-currencies.json b/systemtests/rosetta_config/devnet-custom-currencies.json index be7ce62e..b6161d1b 100644 --- a/systemtests/rosetta_config/devnet-custom-currencies.json +++ b/systemtests/rosetta_config/devnet-custom-currencies.json @@ -1,6 +1,6 @@ [ - { - "symbol": "ROSETTA-2c0a37", - "decimals": 2 - } + { + "symbol": "ROSETTA-2c0a37", + "decimals": 2 + } ] diff --git a/systemtests/rosetta_config/mainnet-custom-currencies.json b/systemtests/rosetta_config/mainnet-custom-currencies.json index 18cffb72..b2fb7bb6 100644 --- a/systemtests/rosetta_config/mainnet-custom-currencies.json +++ b/systemtests/rosetta_config/mainnet-custom-currencies.json @@ -1,14 +1,14 @@ [ - { - "symbol": "WEGLD-bd4d79", - "decimals": 18 - }, - { - "symbol": "MEX-455c57", - "decimals": 18 - }, - { - "symbol": "USDC-c76f1f", - "decimals": 6 - } + { + "symbol": "WEGLD-bd4d79", + "decimals": 18 + }, + { + "symbol": "MEX-455c57", + "decimals": 18 + }, + { + "symbol": "USDC-c76f1f", + "decimals": 6 + } ] diff --git a/systemtests/rosetta_config/testnet-custom-currencies.json b/systemtests/rosetta_config/testnet-custom-currencies.json index 6939ea7f..dd7aac14 100644 --- a/systemtests/rosetta_config/testnet-custom-currencies.json +++ b/systemtests/rosetta_config/testnet-custom-currencies.json @@ -1,6 +1,6 @@ [ - { - "symbol": "ROSETTA-7783de", - "decimals": 4 - } + { + "symbol": "ROSETTA-de9b08", + "decimals": 4 + } ] From b3abb24a2370f9cadeca14bc9fa5e5d7ee92f2d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Tue, 20 Aug 2024 11:57:08 +0300 Subject: [PATCH 45/75] Adjust tests. --- systemtests/broadcast_samples.py | 43 ++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/systemtests/broadcast_samples.py b/systemtests/broadcast_samples.py index 62d3dd19..8ee8c1d1 100644 --- a/systemtests/broadcast_samples.py +++ b/systemtests/broadcast_samples.py @@ -1,3 +1,4 @@ +import json from argparse import ArgumentParser from pathlib import Path from typing import Dict, List @@ -29,40 +30,40 @@ def main(): controller = Controller(configuration, accounts) if mode == "setup": - controller.send_multiple(controller.create_airdrops()) + # controller.send_multiple(controller.create_airdrops()) controller.send_multiple(controller.create_contract_deployments()) elif mode == "run": - # Intra-shard + print("Intra-shard, simple MoveBalance with refund") controller.send(controller.create_simple_move_balance_with_refund( sender=accounts.get_user(shard=0, index=0), receiver=accounts.get_user(shard=0, index=1).address, )) - # Cross-shard + print("Cross-shard, simple MoveBalance with refund") controller.send(controller.create_simple_move_balance_with_refund( sender=accounts.get_user(shard=0, index=1), receiver=accounts.get_user(shard=1, index=0).address, )) - # Intrashard + print("Intra-shard, invalid MoveBalance with refund") controller.send(controller.create_invalid_move_balance_with_refund( sender=accounts.get_user(shard=0, index=2), receiver=accounts.get_user(shard=0, index=3).address, )) - # Cross-shard + print("Cross-shard, invalid MoveBalance with refund") controller.send(controller.create_invalid_move_balance_with_refund( sender=accounts.get_user(shard=0, index=4), receiver=accounts.get_user(shard=1, index=1).address, )) - # Intra-shard, sending value to non-payable contract + print("Intra-shard, sending value to non-payable contract") controller.send(controller.create_simple_move_balance_with_refund( sender=accounts.get_user(shard=0, index=0), receiver=accounts.contracts_by_shard[0][0], )) - # Cross-shard, sending value to non-payable contract + print("Cross-shard, sending value to non-payable contract") controller.send(controller.create_simple_move_balance_with_refund( sender=accounts.get_user(shard=0, index=1), receiver=accounts.contracts_by_shard[1][0], @@ -74,25 +75,25 @@ def main(): receiver=accounts.get_user(shard=0, index=1).address, )) - # Cross-shard, native transfer within MultiESDTTransfer + print("Cross-shard, native transfer within MultiESDTTransfer") controller.send(controller.create_native_transfer_within_multiesdt( sender=accounts.get_user(shard=0, index=1), receiver=accounts.get_user(shard=1, index=0).address, )) - # Intra-shard, native transfer within MultiESDTTransfer, towards non-payable contract + print("Intra-shard, native transfer within MultiESDTTransfer, towards non-payable contract") controller.send(controller.create_native_transfer_within_multiesdt( sender=accounts.get_user(shard=0, index=0), receiver=accounts.contracts_by_shard[0][0], )) - # Cross-shard, native transfer within MultiESDTTransfer, towards non-payable contract + print("Cross-shard, native transfer within MultiESDTTransfer, towards non-payable contract") controller.send(controller.create_native_transfer_within_multiesdt( sender=accounts.get_user(shard=0, index=1), receiver=accounts.contracts_by_shard[1][0], )) - # Intra-shard, relayed v1 transaction with MoveBalance + print("Intra-shard, relayed v1 transaction with MoveBalance") controller.send(controller.create_relayed_v1_with_move_balance( relayer=accounts.get_user(shard=0, index=0), sender=accounts.get_user(shard=0, index=1), @@ -100,7 +101,7 @@ def main(): amount=42 )) - # Relayed v3, senders and receivers in same shard + print("Relayed v3, senders and receivers in same shard") controller.send(controller.create_relayed_v3_with_a_few_inner_move_balances( relayer=accounts.get_user(shard=0, index=0), senders=accounts.users_by_shard[0][1:3], @@ -108,7 +109,7 @@ def main(): amount=42 )) - # Relayed v3, senders and receivers in different shards + print("Relayed v3, senders and receivers in different shards") controller.send(controller.create_relayed_v3_with_a_few_inner_move_balances( relayer=accounts.get_user(shard=0, index=0), senders=accounts.users_by_shard[0][1:3], @@ -116,7 +117,7 @@ def main(): amount=42 )) - # Relayed v3, senders and receivers in same shard (insufficient balance) + print("Relayed v3, senders and receivers in same shard (insufficient balance)") controller.send(controller.create_relayed_v3_with_a_few_inner_move_balances( relayer=accounts.get_user(shard=0, index=0), senders=accounts.users_by_shard[0][1:3], @@ -124,7 +125,7 @@ def main(): amount=1000000000000000000000 )) - # Relayed v3, senders and receivers in different shards (insufficient balance) + print("Relayed v3, senders and receivers in different shards (insufficient balance)") controller.send(controller.create_relayed_v3_with_a_few_inner_move_balances( relayer=accounts.get_user(shard=0, index=0), senders=accounts.users_by_shard[0][1:3], @@ -132,7 +133,7 @@ def main(): amount=1000000000000000000000 )) - # Relayed v3, senders and receivers in same shard, sending to non-payable contract + print("Relayed v3, senders and receivers in same shard, sending to non-payable contract") controller.send(controller.create_relayed_v3_with_a_few_inner_move_balances( relayer=accounts.get_user(shard=0, index=0), senders=[accounts.get_user(shard=0, index=5)], @@ -140,7 +141,7 @@ def main(): amount=1000000000000000000 )) - # Intra-shard, relayed v1 transaction with MoveBalance + print("Intra-shard, relayed v1 transaction with MoveBalance") controller.send(controller.create_relayed_v1_with_move_balance( relayer=accounts.get_user(shard=1, index=0), sender=accounts.get_user(shard=1, index=9), @@ -148,12 +149,14 @@ def main(): amount=1000000000000000000 )) - # Direct contract deployment with MoveBalance + print("Direct contract deployment with MoveBalance") controller.send(controller.create_contract_deployment_with_move_balance( sender=accounts.get_user(shard=0, index=0), amount=10000000000000000 )) + # TODO isIntrashardContractCallWithSignalErrorButWithoutContractResultBearingRefundValue + class BunchOfAccounts: def __init__(self, configuration: Configuration) -> None: @@ -436,7 +439,9 @@ def increment_nonce(self, address: Address): class CustomCurrencies: def __init__(self, configuration: Configuration) -> None: - self.currency = "ROSETTA-7783de" + file_content = Path(configuration.config_file_custom_currencies).read_text() + data = json.loads(file_content) + self.currency = data[0].get("symbol") if __name__ == '__main__': From b90d9bce96402ea88319e28d0c0d2a074357737d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Tue, 20 Aug 2024 11:57:45 +0300 Subject: [PATCH 46/75] Lower ADAPTER_DELAY_IN_MILLISECONDS. --- systemtests/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/systemtests/constants.py b/systemtests/constants.py index dbdc1881..68e743ed 100644 --- a/systemtests/constants.py +++ b/systemtests/constants.py @@ -5,4 +5,4 @@ PATH_PROXY_TO_OBSERVER_ADAPTER = PATH_REPOSITORY / "systemtests" / "proxyToObserverAdapter" PORT_ROSETTA = 7091 URL_OBSERVER_SURROGATE = "http://localhost:8080" -ADAPTER_DELAY_IN_MILLISECONDS = 100 +ADAPTER_DELAY_IN_MILLISECONDS = 80 From b5b0d5b0fa3aafad607b82135294688eb591b77f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Tue, 20 Aug 2024 15:55:47 +0300 Subject: [PATCH 47/75] Simplification post-Sirius: createValueReturnOperationsIfIntrashardContractCallOrContractDeploymentWithSignalError. --- .../services/transactionsFeaturesDetector.go | 26 ------------------- server/services/transactionsTransformer.go | 23 ++++++++-------- .../services/transactionsTransformer_test.go | 4 +-- 3 files changed, 14 insertions(+), 39 deletions(-) diff --git a/server/services/transactionsFeaturesDetector.go b/server/services/transactionsFeaturesDetector.go index 98dceb8a..0a2ceb96 100644 --- a/server/services/transactionsFeaturesDetector.go +++ b/server/services/transactionsFeaturesDetector.go @@ -87,32 +87,6 @@ func (detector *transactionsFeaturesDetector) isRelayedV1TransactionCompletelyIn return isWithSignalError } -func (detector *transactionsFeaturesDetector) isIntrashardContractCallWithSignalErrorButWithoutContractResultBearingRefundValue( - txInQuestion *transaction.ApiTransactionResult, - allTransactionsInBlock []*transaction.ApiTransactionResult, -) bool { - if !detector.isContractCallWithSignalError(txInQuestion) { - return false - } - - if !detector.isIntrashard(txInQuestion) { - return false - } - - for _, tx := range allTransactionsInBlock { - matchesTypeOnSource := tx.ProcessingTypeOnSource == transactionProcessingTypeMoveBalance - matchesTypeOnDestination := tx.ProcessingTypeOnDestination == transactionProcessingTypeMoveBalance - matchesOriginalTransactionHash := tx.OriginalTransactionHash == txInQuestion.Hash - matchesRefundValue := tx.Value == txInQuestion.Value - - if matchesTypeOnSource && matchesTypeOnDestination && matchesOriginalTransactionHash && matchesRefundValue { - return false - } - } - - return true -} - func (detector *transactionsFeaturesDetector) isContractCallWithSignalError(tx *transaction.ApiTransactionResult) bool { return tx.ProcessingTypeOnSource == transactionProcessingTypeContractInvoking && tx.ProcessingTypeOnDestination == transactionProcessingTypeContractInvoking && diff --git a/server/services/transactionsTransformer.go b/server/services/transactionsTransformer.go index 84379478..54a49bb7 100644 --- a/server/services/transactionsTransformer.go +++ b/server/services/transactionsTransformer.go @@ -90,7 +90,7 @@ func (transformer *transactionsTransformer) txToRosettaTx(tx *transaction.ApiTra switch tx.Type { case string(transaction.TxTypeNormal): - rosettaTx, err = transformer.normalTxToRosetta(tx, txsInBlock) + rosettaTx, err = transformer.normalTxToRosetta(tx) if err != nil { return nil, err } @@ -185,10 +185,7 @@ func (transformer *transactionsTransformer) rewardTxToRosettaTx(tx *transaction. } } -func (transformer *transactionsTransformer) normalTxToRosetta( - tx *transaction.ApiTransactionResult, - allTransactionsInBlock []*transaction.ApiTransactionResult, -) (*types.Transaction, error) { +func (transformer *transactionsTransformer) normalTxToRosetta(tx *transaction.ApiTransactionResult) (*types.Transaction, error) { hasValue := !isZeroAmount(tx.Value) operations := make([]*types.Operation, 0) @@ -217,7 +214,7 @@ func (transformer *transactionsTransformer) normalTxToRosetta( return nil, err } - valueRefundOperationIfContractCallOrDeploymentWithSignalError, err := transformer.createValueReturnOperationsIfIntrashardContractCallOrContractDeploymentWithSignalError(tx, allTransactionsInBlock) + valueRefundOperationIfContractCallOrDeploymentWithSignalError, err := transformer.createValueReturnOperationsIfIntrashardContractCallOrContractDeploymentWithSignalError(tx) if err != nil { return nil, err } @@ -277,14 +274,18 @@ func (transformer *transactionsTransformer) extractInnerTxOperationsIfRelayedCom }, nil } +// createValueReturnOperationsIfIntrashardContractCallOrContractDeploymentWithSignalError applies a workaround for: +// - intra-shard contract calls, bearing value, which fail with signal error +// - direct contract deployments, bearing value, which fail with signal error +// For these cases, the protocol does not generate an explicit SCR with the value refund (before Sirius, in some circumstances, it did). +// However, in reality, the value remains at the sender. Thus, in Rosetta, we apply some "correcting" operations. func (transformer *transactionsTransformer) createValueReturnOperationsIfIntrashardContractCallOrContractDeploymentWithSignalError( tx *transaction.ApiTransactionResult, - allTransactionsInBlock []*transaction.ApiTransactionResult, ) ([]*types.Operation, error) { - isContractCallWithError := transformer.featuresDetector.isIntrashardContractCallWithSignalErrorButWithoutContractResultBearingRefundValue(tx, allTransactionsInBlock) - isContractDeploymentWithError := transformer.featuresDetector.isContractDeploymentWithSignalError(tx) - if !isContractCallWithError && !isContractDeploymentWithError { - return []*types.Operation{}, nil + detector := transformer.featuresDetector + shouldApplyCorrection := detector.isContractDeploymentWithSignalError(tx) || (detector.isIntrashard(tx) && detector.isContractCallWithSignalError(tx)) + if !shouldApplyCorrection { + return make([]*types.Operation, 0), nil } return []*types.Operation{ diff --git a/server/services/transactionsTransformer_test.go b/server/services/transactionsTransformer_test.go index 4b1cd7a8..f09b09b9 100644 --- a/server/services/transactionsTransformer_test.go +++ b/server/services/transactionsTransformer_test.go @@ -50,7 +50,7 @@ func TestTransactionsTransformer_NormalTxToRosettaTx(t *testing.T) { Metadata: extractTransactionMetadata(tx), } - rosettaTx, err := transformer.normalTxToRosetta(tx, nil) + rosettaTx, err := transformer.normalTxToRosetta(tx) require.NoError(t, err) require.Equal(t, expectedRosettaTx, rosettaTx) }) @@ -99,7 +99,7 @@ func TestTransactionsTransformer_NormalTxToRosettaTx(t *testing.T) { Metadata: extractTransactionMetadata(tx), } - rosettaTx, err := transformer.normalTxToRosetta(tx, nil) + rosettaTx, err := transformer.normalTxToRosetta(tx) require.NoError(t, err) require.Equal(t, expectedRosettaTx, rosettaTx) }) From aa4eb55346ef630b7b49cd4db1842776b73a8613 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Tue, 20 Aug 2024 16:11:54 +0300 Subject: [PATCH 48/75] Simplification post-Sirius: refactoring. --- .../services/transactionsFeaturesDetector.go | 4 ++ server/services/transactionsTransformer.go | 46 +++---------------- 2 files changed, 11 insertions(+), 39 deletions(-) diff --git a/server/services/transactionsFeaturesDetector.go b/server/services/transactionsFeaturesDetector.go index 0a2ceb96..590e199d 100644 --- a/server/services/transactionsFeaturesDetector.go +++ b/server/services/transactionsFeaturesDetector.go @@ -87,6 +87,10 @@ func (detector *transactionsFeaturesDetector) isRelayedV1TransactionCompletelyIn return isWithSignalError } +func (detector *transactionsFeaturesDetector) isContractDeploymentWithSignalErrorOrIntrashardContractCallWithSignalError(tx *transaction.ApiTransactionResult) bool { + return detector.isContractDeploymentWithSignalError(tx) || (detector.isIntrashard(tx) && detector.isContractCallWithSignalError(tx)) +} + func (detector *transactionsFeaturesDetector) isContractCallWithSignalError(tx *transaction.ApiTransactionResult) bool { return tx.ProcessingTypeOnSource == transactionProcessingTypeContractInvoking && tx.ProcessingTypeOnDestination == transactionProcessingTypeContractInvoking && diff --git a/server/services/transactionsTransformer.go b/server/services/transactionsTransformer.go index 54a49bb7..47ba3aa7 100644 --- a/server/services/transactionsTransformer.go +++ b/server/services/transactionsTransformer.go @@ -186,10 +186,15 @@ func (transformer *transactionsTransformer) rewardTxToRosettaTx(tx *transaction. } func (transformer *transactionsTransformer) normalTxToRosetta(tx *transaction.ApiTransactionResult) (*types.Transaction, error) { - hasValue := !isZeroAmount(tx.Value) operations := make([]*types.Operation, 0) - if hasValue { + // Special handling of: + // - intra-shard contract calls, bearing value, which fail with signal error + // - direct contract deployments, bearing value, which fail with signal error + // For these, the protocol does not generate an explicit SCR with the value refund (before Sirius, in some circumstances, it did). + // However, since the value remains at the sender, we don't emit any operations in these circumstances. + transfersValue := isNonZeroAmount(tx.Value) && !transformer.featuresDetector.isContractDeploymentWithSignalErrorOrIntrashardContractCallWithSignalError(tx) + if transfersValue { operations = append(operations, &types.Operation{ Type: opTransfer, Account: addressToAccountIdentifier(tx.Sender), @@ -214,20 +219,11 @@ func (transformer *transactionsTransformer) normalTxToRosetta(tx *transaction.Ap return nil, err } - valueRefundOperationIfContractCallOrDeploymentWithSignalError, err := transformer.createValueReturnOperationsIfIntrashardContractCallOrContractDeploymentWithSignalError(tx) - if err != nil { - return nil, err - } - if len(innerTxOperationsIfRelayedCompletelyIntrashardWithSignalError) > 0 { log.Info("normalTxToRosetta(): innerTxOperationsIfRelayedCompletelyIntrashardWithSignalError", "tx", tx.Hash, "block", tx.BlockNonce) } - if len(valueRefundOperationIfContractCallOrDeploymentWithSignalError) > 0 { - log.Info("normalTxToRosetta(): valueRefundOperationIfContractCallOrDeploymentWithSignalError", "tx", tx.Hash, "block", tx.BlockNonce) - } operations = append(operations, innerTxOperationsIfRelayedCompletelyIntrashardWithSignalError...) - operations = append(operations, valueRefundOperationIfContractCallOrDeploymentWithSignalError...) return &types.Transaction{ TransactionIdentifier: hashToTransactionIdentifier(tx.Hash), @@ -274,34 +270,6 @@ func (transformer *transactionsTransformer) extractInnerTxOperationsIfRelayedCom }, nil } -// createValueReturnOperationsIfIntrashardContractCallOrContractDeploymentWithSignalError applies a workaround for: -// - intra-shard contract calls, bearing value, which fail with signal error -// - direct contract deployments, bearing value, which fail with signal error -// For these cases, the protocol does not generate an explicit SCR with the value refund (before Sirius, in some circumstances, it did). -// However, in reality, the value remains at the sender. Thus, in Rosetta, we apply some "correcting" operations. -func (transformer *transactionsTransformer) createValueReturnOperationsIfIntrashardContractCallOrContractDeploymentWithSignalError( - tx *transaction.ApiTransactionResult, -) ([]*types.Operation, error) { - detector := transformer.featuresDetector - shouldApplyCorrection := detector.isContractDeploymentWithSignalError(tx) || (detector.isIntrashard(tx) && detector.isContractCallWithSignalError(tx)) - if !shouldApplyCorrection { - return make([]*types.Operation, 0), nil - } - - return []*types.Operation{ - { - Type: opTransfer, - Account: addressToAccountIdentifier(tx.Sender), - Amount: transformer.extension.valueToNativeAmount(tx.Value), - }, - { - Type: opTransfer, - Account: addressToAccountIdentifier(tx.Receiver), - Amount: transformer.extension.valueToNativeAmount("-" + tx.Value), - }, - }, nil -} - func (transformer *transactionsTransformer) refundReceiptToRosettaTx(receipt *transaction.ApiReceipt) (*types.Transaction, error) { receiptHash, err := transformer.provider.ComputeReceiptHash(receipt) if err != nil { From c0978839443d783280946606820f69a4d31f318a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Tue, 20 Aug 2024 16:24:27 +0300 Subject: [PATCH 49/75] Additional examples etc. --- systemtests/broadcast_samples.py | 43 +++++++++++++++++++++++++++++- systemtests/check_with_mesh_cli.py | 3 ++- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/systemtests/broadcast_samples.py b/systemtests/broadcast_samples.py index 8ee8c1d1..738a7ec0 100644 --- a/systemtests/broadcast_samples.py +++ b/systemtests/broadcast_samples.py @@ -155,7 +155,25 @@ def main(): amount=10000000000000000 )) - # TODO isIntrashardContractCallWithSignalErrorButWithoutContractResultBearingRefundValue + print("Intra-shard, contract call with MoveBalance, with signal error") + controller.send(controller.create_contract_call_with_move_balance_with_signal_error( + sender=accounts.get_user(shard=0, index=0), + contract=accounts.contracts_by_shard[0][0], + amount=10000000000000000 + )) + + print("Cross-shard, contract call with MoveBalance, with signal error") + controller.send(controller.create_contract_call_with_move_balance_with_signal_error( + sender=accounts.get_user(shard=0, index=0), + contract=accounts.contracts_by_shard[1][0], + amount=10000000000000000 + )) + + print("Direct contract deployment with MoveBalance, with signal error") + controller.send(controller.create_contract_deployment_with_move_balance_with_signal_error( + sender=accounts.get_user(shard=0, index=0), + amount=77 + )) class BunchOfAccounts: @@ -383,6 +401,29 @@ def create_contract_deployment_with_move_balance(self, sender: "Account", amount return transaction + def create_contract_deployment_with_move_balance_with_signal_error(self, sender: "Account", amount: int) -> Transaction: + transaction = self.contracts_transactions_factory.create_transaction_for_deploy( + sender=sender.address, + bytecode=CONTRACT_PATH_ADDER, + gas_limit=5000000, + arguments=[1, 2, 3, 4, 5], + native_transfer_amount=amount + ) + + return transaction + + def create_contract_call_with_move_balance_with_signal_error(self, sender: "Account", contract: Address, amount: int) -> Transaction: + transaction = self.contracts_transactions_factory.create_transaction_for_execute( + sender=sender.address, + contract=contract, + function="missingFunction", + gas_limit=5000000, + arguments=[1, 2, 3, 4, 5], + native_transfer_amount=amount + ) + + return transaction + def send_multiple(self, transactions: List[Transaction]): for transaction in transactions: self._apply_nonce(transaction) diff --git a/systemtests/check_with_mesh_cli.py b/systemtests/check_with_mesh_cli.py index de3354be..d4272c79 100644 --- a/systemtests/check_with_mesh_cli.py +++ b/systemtests/check_with_mesh_cli.py @@ -57,11 +57,12 @@ def run_rosetta(configuration: Configuration): """ current_epoch = get_current_epoch(configuration) + observer_url = configuration.observer_url or constants.URL_OBSERVER_SURROGATE command = [ str(constants.PATH_ROSETTA), f"--port={constants.PORT_ROSETTA}", - f"--observer-http-url={constants.URL_OBSERVER_SURROGATE}", + f"--observer-http-url={observer_url}", f"--observer-actual-shard={configuration.network_shard}", f"--network-id={configuration.network_id}", f"--network-name={configuration.network_name}", From b519141fcd7fae32fbaf7a1492e4fafd3eb4fb24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Tue, 20 Aug 2024 16:52:38 +0300 Subject: [PATCH 50/75] Additional unit tests. --- .../services/transactionsFeaturesDetector.go | 12 +- .../transactionsFeaturesDetector_test.go | 104 ++++++++++++++++++ 2 files changed, 110 insertions(+), 6 deletions(-) diff --git a/server/services/transactionsFeaturesDetector.go b/server/services/transactionsFeaturesDetector.go index 590e199d..3c33d5bc 100644 --- a/server/services/transactionsFeaturesDetector.go +++ b/server/services/transactionsFeaturesDetector.go @@ -91,18 +91,18 @@ func (detector *transactionsFeaturesDetector) isContractDeploymentWithSignalErro return detector.isContractDeploymentWithSignalError(tx) || (detector.isIntrashard(tx) && detector.isContractCallWithSignalError(tx)) } -func (detector *transactionsFeaturesDetector) isContractCallWithSignalError(tx *transaction.ApiTransactionResult) bool { - return tx.ProcessingTypeOnSource == transactionProcessingTypeContractInvoking && - tx.ProcessingTypeOnDestination == transactionProcessingTypeContractInvoking && - detector.eventsController.hasAnySignalError(tx) -} - func (detector *transactionsFeaturesDetector) isContractDeploymentWithSignalError(tx *transaction.ApiTransactionResult) bool { return tx.ProcessingTypeOnSource == transactionProcessingTypeContractDeployment && tx.ProcessingTypeOnDestination == transactionProcessingTypeContractDeployment && detector.eventsController.hasAnySignalError(tx) } +func (detector *transactionsFeaturesDetector) isContractCallWithSignalError(tx *transaction.ApiTransactionResult) bool { + return tx.ProcessingTypeOnSource == transactionProcessingTypeContractInvoking && + tx.ProcessingTypeOnDestination == transactionProcessingTypeContractInvoking && + detector.eventsController.hasAnySignalError(tx) +} + func (detector *transactionsFeaturesDetector) isIntrashard(tx *transaction.ApiTransactionResult) bool { return tx.SourceShard == tx.DestinationShard } diff --git a/server/services/transactionsFeaturesDetector_test.go b/server/services/transactionsFeaturesDetector_test.go index c99d71a0..70af7207 100644 --- a/server/services/transactionsFeaturesDetector_test.go +++ b/server/services/transactionsFeaturesDetector_test.go @@ -131,3 +131,107 @@ func TestTransactionsFeaturesDetector_isRelayedV1TransactionCompletelyIntrashard require.True(t, featureDetected) }) } + +func TestTransactionsFeatureDetector_isContractDeploymentWithSignalErrorOrIntrashardContractCallWithSignalError(t *testing.T) { + networkProvider := testscommon.NewNetworkProviderMock() + detector := newTransactionsFeaturesDetector(networkProvider) + + t.Run("contract deployment, with signal error", func(t *testing.T) { + tx := &transaction.ApiTransactionResult{ + ProcessingTypeOnSource: transactionProcessingTypeContractDeployment, + ProcessingTypeOnDestination: transactionProcessingTypeContractDeployment, + Logs: &transaction.ApiLogs{ + Events: []*transaction.Events{ + { + Identifier: transactionEventSignalError, + }, + }, + }, + } + + require.True(t, detector.isContractDeploymentWithSignalErrorOrIntrashardContractCallWithSignalError(tx)) + }) + + t.Run("contract deployment, without signal error", func(t *testing.T) { + tx := &transaction.ApiTransactionResult{ + ProcessingTypeOnSource: transactionProcessingTypeContractDeployment, + ProcessingTypeOnDestination: transactionProcessingTypeContractDeployment, + } + + require.False(t, detector.isContractDeploymentWithSignalErrorOrIntrashardContractCallWithSignalError(tx)) + }) + + t.Run("contract call, with signal error, intra-shard", func(t *testing.T) { + tx := &transaction.ApiTransactionResult{ + ProcessingTypeOnSource: transactionProcessingTypeContractInvoking, + ProcessingTypeOnDestination: transactionProcessingTypeContractInvoking, + SourceShard: 2, + DestinationShard: 2, + Logs: &transaction.ApiLogs{ + Events: []*transaction.Events{ + { + Identifier: transactionEventSignalError, + }, + }, + }, + } + + require.True(t, detector.isContractDeploymentWithSignalErrorOrIntrashardContractCallWithSignalError(tx)) + }) + + t.Run("contract call, with signal error, cross-shard", func(t *testing.T) { + tx := &transaction.ApiTransactionResult{ + ProcessingTypeOnSource: transactionProcessingTypeContractInvoking, + ProcessingTypeOnDestination: transactionProcessingTypeContractInvoking, + SourceShard: 0, + DestinationShard: 1, + Logs: &transaction.ApiLogs{ + Events: []*transaction.Events{ + { + Identifier: transactionEventSignalError, + }, + }, + }, + } + + require.False(t, detector.isContractDeploymentWithSignalErrorOrIntrashardContractCallWithSignalError(tx)) + }) + + t.Run("contract call, without signal error", func(t *testing.T) { + tx := &transaction.ApiTransactionResult{ + ProcessingTypeOnSource: transactionProcessingTypeContractInvoking, + ProcessingTypeOnDestination: transactionProcessingTypeContractInvoking, + } + + require.False(t, detector.isContractDeploymentWithSignalErrorOrIntrashardContractCallWithSignalError(tx)) + }) + + t.Run("arbitrary transaction", func(t *testing.T) { + tx := &transaction.ApiTransactionResult{ + ProcessingTypeOnSource: transactionProcessingTypeMoveBalance, + ProcessingTypeOnDestination: transactionProcessingTypeMoveBalance, + } + + require.False(t, detector.isContractDeploymentWithSignalErrorOrIntrashardContractCallWithSignalError(tx)) + }) +} + +func TestTransactionsFeaturesDetector_isIntrashard(t *testing.T) { + networkProvider := testscommon.NewNetworkProviderMock() + detector := newTransactionsFeaturesDetector(networkProvider) + + require.True(t, detector.isIntrashard(&transaction.ApiTransactionResult{ + SourceShard: 0, + DestinationShard: 0, + })) + + require.True(t, detector.isIntrashard(&transaction.ApiTransactionResult{ + SourceShard: 1, + DestinationShard: 1, + })) + + require.False(t, detector.isIntrashard(&transaction.ApiTransactionResult{ + SourceShard: 0, + DestinationShard: 1, + })) +} From cac7baf7446fad5e10854e52656ebc228fa97c28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Tue, 20 Aug 2024 17:09:44 +0300 Subject: [PATCH 51/75] Unit tests (contract deployments with value, with error). --- .../blocks_with_contract_deployments.json | 98 +++++++++++++++ ...ocks_with_direct_sc_deploy_with_value.json | 51 -------- .../services/transactionsTransformer_test.go | 118 +++++++++++------- 3 files changed, 170 insertions(+), 97 deletions(-) create mode 100644 server/services/testdata/blocks_with_contract_deployments.json delete mode 100644 server/services/testdata/blocks_with_direct_sc_deploy_with_value.json diff --git a/server/services/testdata/blocks_with_contract_deployments.json b/server/services/testdata/blocks_with_contract_deployments.json new file mode 100644 index 00000000..ce7acb7a --- /dev/null +++ b/server/services/testdata/blocks_with_contract_deployments.json @@ -0,0 +1,98 @@ +[ + { + "comment": "block with contract deployment, with move balance", + "miniBlocks": [ + { + "transactions": [ + { + "type": "normal", + "processingTypeOnSource": "SCDeployment", + "processingTypeOnDestination": "SCDeployment", + "hash": "2459bb2b9a64c1c920777ecbdaf0fa33d7fe8bcd24d7164562f341b2e4f702da", + "value": "10000000000000000", + "receiver": "erd1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq6gq4hu", + "sender": "erd1tn62hjp72rznp8vq0lplva5csav6rccpqqdungpxtqz0g2hcq6uq9k4cc6", + "logs": { + "address": "erd1tn62hjp72rznp8vq0lplva5csav6rccpqqdungpxtqz0g2hcq6uq9k4cc6", + "events": [ + { + "address": "erd1qqqqqqqqqqqqqpgqc4vdnqgc48ww26ljxqe2flgl86jewg0nq6uqna2ymj", + "identifier": "SCDeploy", + "topics": [ + "AAAAAAAAAAAFAMVY2YEYqdzla/IwMqT9Hz6llyHzBrg=", + "XPSryD5QxTCdgH/D9naYh1mh4wEAG8mgJlgE9Cr4Brg=", + "v5gJ+IK1KdPGJfGYibrKij1SO0bzAAhfUVsODJ8midg=" + ], + "data": null, + "additionalData": null + }, + { + "address": "erd1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq6gq4hu", + "identifier": "writeLog", + "topics": [ + "XPSryD5QxTCdgH/D9naYh1mh4wEAG8mgJlgE9Cr4Brg=", + "QHRvbyBtdWNoIGdhcyBwcm92aWRlZCBmb3IgcHJvY2Vzc2luZzogZ2FzIHByb3ZpZGVkID0gNDU4ODUwMCwgZ2FzIHVzZWQgPSAzMzQ1MDU=" + ], + "data": "QDZmNmI=", + "additionalData": ["QDZmNmI="] + } + ] + }, + "operation": "scDeploy", + "initiallyPaidFee": "457385000000000" + } + ] + } + ] + }, + { + "comment": "block with contract deployment, with move balance, with signal error", + "miniBlocks": [ + { + "transactions": [ + { + "type": "normal", + "processingTypeOnSource": "SCDeployment", + "processingTypeOnDestination": "SCDeployment", + "hash": "52b74f025158f18512353da5f23c2e31c78c49d7b73da7d24f048f86da48b5c5", + "value": "77", + "receiver": "erd1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq6gq4hu", + "sender": "erd1tn62hjp72rznp8vq0lplva5csav6rccpqqdungpxtqz0g2hcq6uq9k4cc6", + "sourceShard": 0, + "destinationShard": 0, + "logs": { + "address": "erd1tn62hjp72rznp8vq0lplva5csav6rccpqqdungpxtqz0g2hcq6uq9k4cc6", + "events": [ + { + "address": "erd1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq6gq4hu", + "identifier": "signalError", + "topics": [ + "XPSryD5QxTCdgH/D9naYh1mh4wEAG8mgJlgE9Cr4Brg=", + "ZnVuY3Rpb24gZG9lcyBub3QgYWNjZXB0IEVHTEQgcGF5bWVudA==" + ], + "data": "QDY1Nzg2NTYzNzU3NDY5NmY2ZTIwNjY2MTY5NmM2NTY0", + "additionalData": [ + "QDY1Nzg2NTYzNzU3NDY5NmY2ZTIwNjY2MTY5NmM2NTY0" + ] + }, + { + "address": "erd1tn62hjp72rznp8vq0lplva5csav6rccpqqdungpxtqz0g2hcq6uq9k4cc6", + "identifier": "internalVMErrors", + "topics": [ + "XPSryD5QxTCdgH/D9naYh1mh4wEAG8mgJlgE9Cr4Brg=", + "X2luaXQ=" + ], + "data": "CglydW50aW1lLmdvOjgzNCBbZXhlY3V0aW9uIGZhaWxlZF0gW10KCXJ1bnRpbWUuZ286ODM0IFtleGVjdXRpb24gZmFpbGVkXSBbXQoJcnVudGltZS5nbzo4MzEgW2Z1bmN0aW9uIGRvZXMgbm90IGFjY2VwdCBFR0xEIHBheW1lbnRd", + "additionalData": [ + "CglydW50aW1lLmdvOjgzNCBbZXhlY3V0aW9uIGZhaWxlZF0gW10KCXJ1bnRpbWUuZ286ODM0IFtleGVjdXRpb24gZmFpbGVkXSBbXQoJcnVudGltZS5nbzo4MzEgW2Z1bmN0aW9uIGRvZXMgbm90IGFjY2VwdCBFR0xEIHBheW1lbnRd" + ] + } + ] + }, + "initiallyPaidFee": "2206715000000000" + } + ] + } + ] + } +] diff --git a/server/services/testdata/blocks_with_direct_sc_deploy_with_value.json b/server/services/testdata/blocks_with_direct_sc_deploy_with_value.json deleted file mode 100644 index 070f6d84..00000000 --- a/server/services/testdata/blocks_with_direct_sc_deploy_with_value.json +++ /dev/null @@ -1,51 +0,0 @@ -[ - { - "miniBlocks": [ - { - "transactions": [ - { - "type": "normal", - "processingTypeOnSource": "SCDeployment", - "processingTypeOnDestination": "SCDeployment", - "hash": "2459bb2b9a64c1c920777ecbdaf0fa33d7fe8bcd24d7164562f341b2e4f702da", - "value": "10000000000000000", - "receiver": "erd1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq6gq4hu", - "sender": "erd1tn62hjp72rznp8vq0lplva5csav6rccpqqdungpxtqz0g2hcq6uq9k4cc6", - "logs": { - "address": "erd1tn62hjp72rznp8vq0lplva5csav6rccpqqdungpxtqz0g2hcq6uq9k4cc6", - "events": [ - { - "address": "erd1qqqqqqqqqqqqqpgqc4vdnqgc48ww26ljxqe2flgl86jewg0nq6uqna2ymj", - "identifier": "SCDeploy", - "topics": [ - "AAAAAAAAAAAFAMVY2YEYqdzla/IwMqT9Hz6llyHzBrg=", - "XPSryD5QxTCdgH/D9naYh1mh4wEAG8mgJlgE9Cr4Brg=", - "v5gJ+IK1KdPGJfGYibrKij1SO0bzAAhfUVsODJ8midg=" - ], - "data": null, - "additionalData": null - }, - { - "address": "erd1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq6gq4hu", - "identifier": "writeLog", - "topics": [ - "XPSryD5QxTCdgH/D9naYh1mh4wEAG8mgJlgE9Cr4Brg=", - "QHRvbyBtdWNoIGdhcyBwcm92aWRlZCBmb3IgcHJvY2Vzc2luZzogZ2FzIHByb3ZpZGVkID0gNDU4ODUwMCwgZ2FzIHVzZWQgPSAzMzQ1MDU=" - ], - "data": "QDZmNmI=", - "additionalData": ["QDZmNmI="] - } - ] - }, - "status": "success", - "operation": "scDeploy", - "initiallyPaidFee": "457385000000000", - "chainID": "T", - "version": 2, - "options": 0 - } - ] - } - ] - } -] diff --git a/server/services/transactionsTransformer_test.go b/server/services/transactionsTransformer_test.go index f09b09b9..5bf89081 100644 --- a/server/services/transactionsTransformer_test.go +++ b/server/services/transactionsTransformer_test.go @@ -287,64 +287,90 @@ func TestTransactionsTransformer_InvalidTxToRosettaTx(t *testing.T) { require.Equal(t, expectedTx, rosettaTx) } -func TestTransactionsTransformer_TransformBlockTxsHavingDirectSCDeployWithValue(t *testing.T) { +func TestTransactionsTransformer_TransformBlockTxsHavingContractDeployments(t *testing.T) { networkProvider := testscommon.NewNetworkProviderMock() networkProvider.MockObservedActualShard = 0 extension := newNetworkProviderExtension(networkProvider) transformer := newTransactionsTransformer(networkProvider) - blocks, err := readTestBlocks("testdata/blocks_with_direct_sc_deploy_with_value.json") + blocks, err := readTestBlocks("testdata/blocks_with_contract_deployments.json") require.Nil(t, err) - txs, err := transformer.transformBlockTxs(blocks[0]) - require.Nil(t, err) - require.Len(t, txs, 1) - require.Len(t, txs[0].Operations, 5) + t.Run("direct contract deployment, with value", func(t *testing.T) { - expectedTx := &types.Transaction{ - TransactionIdentifier: hashToTransactionIdentifier("2459bb2b9a64c1c920777ecbdaf0fa33d7fe8bcd24d7164562f341b2e4f702da"), - Operations: []*types.Operation{ - { - Type: opTransfer, - OperationIdentifier: indexToOperationIdentifier(0), - Account: addressToAccountIdentifier("erd1tn62hjp72rznp8vq0lplva5csav6rccpqqdungpxtqz0g2hcq6uq9k4cc6"), - Amount: extension.valueToNativeAmount("-10000000000000000"), - Status: &opStatusSuccess, - }, - { - Type: opTransfer, - OperationIdentifier: indexToOperationIdentifier(1), - Account: addressToAccountIdentifier("erd1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq6gq4hu"), - Amount: extension.valueToNativeAmount("10000000000000000"), - Status: &opStatusSuccess, - }, - { - Type: opFee, - OperationIdentifier: indexToOperationIdentifier(2), - Account: addressToAccountIdentifier("erd1tn62hjp72rznp8vq0lplva5csav6rccpqqdungpxtqz0g2hcq6uq9k4cc6"), - Amount: extension.valueToNativeAmount("-457385000000000"), - Status: &opStatusSuccess, - }, - { - Type: opTransfer, - OperationIdentifier: indexToOperationIdentifier(3), - Account: addressToAccountIdentifier("erd1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq6gq4hu"), - Amount: extension.valueToNativeAmount("-10000000000000000"), - Status: &opStatusSuccess, + txs, err := transformer.transformBlockTxs(blocks[0]) + require.Nil(t, err) + require.Len(t, txs, 1) + require.Len(t, txs[0].Operations, 5) + + expectedTx := &types.Transaction{ + TransactionIdentifier: hashToTransactionIdentifier("2459bb2b9a64c1c920777ecbdaf0fa33d7fe8bcd24d7164562f341b2e4f702da"), + Operations: []*types.Operation{ + { + Type: opTransfer, + OperationIdentifier: indexToOperationIdentifier(0), + Account: addressToAccountIdentifier("erd1tn62hjp72rznp8vq0lplva5csav6rccpqqdungpxtqz0g2hcq6uq9k4cc6"), + Amount: extension.valueToNativeAmount("-10000000000000000"), + Status: &opStatusSuccess, + }, + { + Type: opTransfer, + OperationIdentifier: indexToOperationIdentifier(1), + Account: addressToAccountIdentifier("erd1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq6gq4hu"), + Amount: extension.valueToNativeAmount("10000000000000000"), + Status: &opStatusSuccess, + }, + { + Type: opFee, + OperationIdentifier: indexToOperationIdentifier(2), + Account: addressToAccountIdentifier("erd1tn62hjp72rznp8vq0lplva5csav6rccpqqdungpxtqz0g2hcq6uq9k4cc6"), + Amount: extension.valueToNativeAmount("-457385000000000"), + Status: &opStatusSuccess, + }, + { + Type: opTransfer, + OperationIdentifier: indexToOperationIdentifier(3), + Account: addressToAccountIdentifier("erd1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq6gq4hu"), + Amount: extension.valueToNativeAmount("-10000000000000000"), + Status: &opStatusSuccess, + }, + { + Type: opTransfer, + OperationIdentifier: indexToOperationIdentifier(4), + Account: addressToAccountIdentifier("erd1qqqqqqqqqqqqqpgqc4vdnqgc48ww26ljxqe2flgl86jewg0nq6uqna2ymj"), + Amount: extension.valueToNativeAmount("10000000000000000"), + Status: &opStatusSuccess, + }, }, - { - Type: opTransfer, - OperationIdentifier: indexToOperationIdentifier(4), - Account: addressToAccountIdentifier("erd1qqqqqqqqqqqqqpgqc4vdnqgc48ww26ljxqe2flgl86jewg0nq6uqna2ymj"), - Amount: extension.valueToNativeAmount("10000000000000000"), - Status: &opStatusSuccess, + Metadata: extractTransactionMetadata(blocks[0].MiniBlocks[0].Transactions[0]), + } + + require.Equal(t, expectedTx, txs[0]) + }) + + t.Run("direct contract deployment, with value, with signal error", func(t *testing.T) { + txs, err := transformer.transformBlockTxs(blocks[1]) + require.Nil(t, err) + require.Len(t, txs, 1) + require.Len(t, txs[0].Operations, 1) + + expectedTx := &types.Transaction{ + TransactionIdentifier: hashToTransactionIdentifier("52b74f025158f18512353da5f23c2e31c78c49d7b73da7d24f048f86da48b5c5"), + Operations: []*types.Operation{ + { + Type: opFee, + OperationIdentifier: indexToOperationIdentifier(0), + Account: addressToAccountIdentifier("erd1tn62hjp72rznp8vq0lplva5csav6rccpqqdungpxtqz0g2hcq6uq9k4cc6"), + Amount: extension.valueToNativeAmount("-2206715000000000"), + Status: &opStatusSuccess, + }, }, - }, - Metadata: extractTransactionMetadata(blocks[0].MiniBlocks[0].Transactions[0]), - } + Metadata: extractTransactionMetadata(blocks[1].MiniBlocks[0].Transactions[0]), + } - require.Equal(t, expectedTx, txs[0]) + require.Equal(t, expectedTx, txs[0]) + }) } func TestTransactionsTransformer_TransformBlockTxsHavingESDTIssue(t *testing.T) { From d0a640443f6aaa12bf203469c153ad869f4dc366 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Tue, 20 Aug 2024 17:14:33 +0300 Subject: [PATCH 52/75] Additional tests: intra-shard contract calls, with value, with signal error. --- .../testdata/blocks_with_contract_calls.json | 52 +++++++++++++++++++ .../services/transactionsTransformer_test.go | 35 ++++++++++++- 2 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 server/services/testdata/blocks_with_contract_calls.json diff --git a/server/services/testdata/blocks_with_contract_calls.json b/server/services/testdata/blocks_with_contract_calls.json new file mode 100644 index 00000000..43f6a7d0 --- /dev/null +++ b/server/services/testdata/blocks_with_contract_calls.json @@ -0,0 +1,52 @@ +[ + { + "comment": "block with intra-shard contract call, with move balance, with signal error", + "miniBlocks": [ + { + "transactions": [ + { + "type": "normal", + "processingTypeOnSource": "SCInvoking", + "processingTypeOnDestination": "SCInvoking", + "hash": "9851ab6cb221bce5800030f9db2a5fbd8ed4a9db4c6f9d190c16113f2b080e57", + "value": "10000000000000000", + "receiver": "erd1qqqqqqqqqqqqqpgqagjekf5mxv86hy5c62vvtug5vc6jmgcsq6uq8reras", + "sender": "erd1tn62hjp72rznp8vq0lplva5csav6rccpqqdungpxtqz0g2hcq6uq9k4cc6", + "sourceShard": 0, + "destinationShard": 0, + "logs": { + "address": "erd1qqqqqqqqqqqqqpgqagjekf5mxv86hy5c62vvtug5vc6jmgcsq6uq8reras", + "events": [ + { + "address": "erd1qqqqqqqqqqqqqpgqagjekf5mxv86hy5c62vvtug5vc6jmgcsq6uq8reras", + "identifier": "signalError", + "topics": [ + "XPSryD5QxTCdgH/D9naYh1mh4wEAG8mgJlgE9Cr4Brg=", + "aW52YWxpZCBmdW5jdGlvbiAobm90IGZvdW5kKQ==" + ], + "data": "QDY2NzU2ZTYzNzQ2OTZmNmUyMDZlNmY3NDIwNjY2Zjc1NmU2NA==", + "additionalData": [ + "QDY2NzU2ZTYzNzQ2OTZmNmUyMDZlNmY3NDIwNjY2Zjc1NmU2NA==" + ] + }, + { + "address": "erd1tn62hjp72rznp8vq0lplva5csav6rccpqqdungpxtqz0g2hcq6uq9k4cc6", + "identifier": "internalVMErrors", + "topics": [ + "AAAAAAAAAAAFAOolmyabMw+rkpjSmMXxFGY1LaMQBrg=", + "bWlzc2luZ0Z1bmN0aW9u" + ], + "data": "CglydW50aW1lLmdvOjgzMSBbaW52YWxpZCBmdW5jdGlvbiAobm90IGZvdW5kKV0gW21pc3NpbmdGdW5jdGlvbl0=", + "additionalData": [ + "CglydW50aW1lLmdvOjgzMSBbaW52YWxpZCBmdW5jdGlvbiAobm90IGZvdW5kKV0gW21pc3NpbmdGdW5jdGlvbl0=" + ] + } + ] + }, + "initiallyPaidFee": "144050000000000" + } + ] + } + ] + } +] diff --git a/server/services/transactionsTransformer_test.go b/server/services/transactionsTransformer_test.go index 5bf89081..eb478050 100644 --- a/server/services/transactionsTransformer_test.go +++ b/server/services/transactionsTransformer_test.go @@ -298,7 +298,6 @@ func TestTransactionsTransformer_TransformBlockTxsHavingContractDeployments(t *t require.Nil(t, err) t.Run("direct contract deployment, with value", func(t *testing.T) { - txs, err := transformer.transformBlockTxs(blocks[0]) require.Nil(t, err) require.Len(t, txs, 1) @@ -373,6 +372,40 @@ func TestTransactionsTransformer_TransformBlockTxsHavingContractDeployments(t *t }) } +func TestTransactionsTransformer_TransformBlockTxsHavingContractCalls(t *testing.T) { + networkProvider := testscommon.NewNetworkProviderMock() + networkProvider.MockObservedActualShard = 0 + + extension := newNetworkProviderExtension(networkProvider) + transformer := newTransactionsTransformer(networkProvider) + + blocks, err := readTestBlocks("testdata/blocks_with_contract_calls.json") + require.Nil(t, err) + + t.Run("direct contract call, with value, with signal error", func(t *testing.T) { + txs, err := transformer.transformBlockTxs(blocks[0]) + require.Nil(t, err) + require.Len(t, txs, 1) + require.Len(t, txs[0].Operations, 1) + + expectedTx := &types.Transaction{ + TransactionIdentifier: hashToTransactionIdentifier("9851ab6cb221bce5800030f9db2a5fbd8ed4a9db4c6f9d190c16113f2b080e57"), + Operations: []*types.Operation{ + { + Type: opFee, + OperationIdentifier: indexToOperationIdentifier(0), + Account: addressToAccountIdentifier("erd1tn62hjp72rznp8vq0lplva5csav6rccpqqdungpxtqz0g2hcq6uq9k4cc6"), + Amount: extension.valueToNativeAmount("-144050000000000"), + Status: &opStatusSuccess, + }, + }, + Metadata: extractTransactionMetadata(blocks[0].MiniBlocks[0].Transactions[0]), + } + + require.Equal(t, expectedTx, txs[0]) + }) +} + func TestTransactionsTransformer_TransformBlockTxsHavingESDTIssue(t *testing.T) { networkProvider := testscommon.NewNetworkProviderMock() networkProvider.MockCustomCurrencies = []resources.Currency{{Symbol: "FOO-6d28db"}} From 5a1d82947301fbe9d06a03b229809bf8fe37dc24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Tue, 20 Aug 2024 17:15:16 +0300 Subject: [PATCH 53/75] Sketch mainnet config for system tests. --- systemtests/config.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/systemtests/config.py b/systemtests/config.py index 6f40fa4c..95bd11b0 100644 --- a/systemtests/config.py +++ b/systemtests/config.py @@ -24,6 +24,23 @@ class Configuration: CONFIGURATIONS = { + "mainnet": Configuration( + network_shard=0, + network_id="1", + network_name="untitled", + native_currency="EGLD", + config_file_custom_currencies="systemtests/rosetta_config/mainnet-custom-currencies.json", + num_historical_epochs=2, + observer_url="", + proxy_url="https://gateway.multiversx.com", + check_construction_native_configuration_file="", + check_construction_custom_configuration_file="", + check_data_configuration_file="systemtests/mesh_cli_config/check-data.json", + check_data_directory="systemtests/mainnet-data", + known_contracts=[ + ], + explorer_url="https://explorer.multiversx.com", + ), "devnet": Configuration( network_shard=0, network_id="D", From 264fd5c8b50a212bbb59fdac4ff4cfb771fd9321 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Wed, 21 Aug 2024 15:21:44 +0300 Subject: [PATCH 54/75] Claim developer rewards - unit tests. Rename operation. --- cmd/rosetta/cli.go | 15 ++-- cmd/rosetta/main.go | 1 + server/factory/interface.go | 1 + server/factory/provider.go | 2 + server/provider/networkProvider.go | 10 ++- server/services/features.go | 21 ----- server/services/interface.go | 1 + server/services/operations.go | 2 + .../blocks_with_claim_developer_rewards.json | 77 +++++++++++++++++++ server/services/transactionsTransformer.go | 10 ++- .../services/transactionsTransformer_test.go | 68 ++++++++++++++++ systemtests/broadcast_samples.py | 15 ++++ testscommon/networkProviderMock.go | 6 ++ 13 files changed, 198 insertions(+), 31 deletions(-) delete mode 100644 server/services/features.go create mode 100644 server/services/testdata/blocks_with_claim_developer_rewards.json diff --git a/cmd/rosetta/cli.go b/cmd/rosetta/cli.go index 514f22f6..9c4cbcb9 100644 --- a/cmd/rosetta/cli.go +++ b/cmd/rosetta/cli.go @@ -1,6 +1,8 @@ package main import ( + "math" + logger "github.com/multiversx/mx-chain-logger-go" "github.com/urfave/cli" ) @@ -167,10 +169,11 @@ VERSION: Required: false, } - cliFlagConfigFileNodeFeatures = cli.StringFlag{ - Name: "config-node-features", - Usage: "Specifies the configuration file for features activation on Node's side.", + cliFlagActivationEpochSpica = cli.UintFlag{ + Name: "activation-epoch-spica", + Usage: "Specifies the activation epoch for Spica release.", Required: false, + Value: math.MaxUint32, } ) @@ -200,7 +203,7 @@ func getAllCliFlags() []cli.Flag { cliFlagNumHistoricalEpochs, cliFlagShouldHandleContracts, cliFlagConfigFileCustomCurrencies, - cliFlagConfigFileNodeFeatures, + cliFlagActivationEpochSpica, } } @@ -230,7 +233,7 @@ type parsedCliFlags struct { numHistoricalEpochs uint32 shouldHandleContracts bool configFileCustomCurrencies string - configFileNodeFeatures string + activationEpochSpica uint32 } func getParsedCliFlags(ctx *cli.Context) parsedCliFlags { @@ -260,6 +263,6 @@ func getParsedCliFlags(ctx *cli.Context) parsedCliFlags { numHistoricalEpochs: uint32(ctx.GlobalUint(cliFlagNumHistoricalEpochs.Name)), shouldHandleContracts: ctx.GlobalBool(cliFlagShouldHandleContracts.Name), configFileCustomCurrencies: ctx.GlobalString(cliFlagConfigFileCustomCurrencies.Name), - configFileNodeFeatures: ctx.GlobalString(cliFlagConfigFileNodeFeatures.Name), + activationEpochSpica: uint32(ctx.GlobalUint(cliFlagActivationEpochSpica.Name)), } } diff --git a/cmd/rosetta/main.go b/cmd/rosetta/main.go index fc51576c..dbb2d423 100644 --- a/cmd/rosetta/main.go +++ b/cmd/rosetta/main.go @@ -82,6 +82,7 @@ func startRosetta(ctx *cli.Context) error { FirstHistoricalEpoch: cliFlags.firstHistoricalEpoch, NumHistoricalEpochs: cliFlags.numHistoricalEpochs, ShouldHandleContracts: cliFlags.shouldHandleContracts, + ActivationEpochSpica: cliFlags.activationEpochSpica, }) if err != nil { return err diff --git a/server/factory/interface.go b/server/factory/interface.go index 916da7fd..861af975 100644 --- a/server/factory/interface.go +++ b/server/factory/interface.go @@ -35,5 +35,6 @@ type NetworkProvider interface { ComputeReceiptHash(apiReceipt *transaction.ApiReceipt) (string, error) ComputeTransactionFeeForMoveBalance(tx *transaction.ApiTransactionResult) *big.Int GetMempoolTransactionByHash(hash string) (*transaction.ApiTransactionResult, error) + IsReleaseSpicaActive(epoch uint32) bool LogDescription() } diff --git a/server/factory/provider.go b/server/factory/provider.go index b945ed46..b3acc26d 100644 --- a/server/factory/provider.go +++ b/server/factory/provider.go @@ -49,6 +49,7 @@ type ArgsCreateNetworkProvider struct { FirstHistoricalEpoch uint32 NumHistoricalEpochs uint32 ShouldHandleContracts bool + ActivationEpochSpica uint32 } // CreateNetworkProvider creates a network provider @@ -140,6 +141,7 @@ func CreateNetworkProvider(args ArgsCreateNetworkProvider) (NetworkProvider, err FirstHistoricalEpoch: args.FirstHistoricalEpoch, NumHistoricalEpochs: args.NumHistoricalEpochs, ShouldHandleContracts: args.ShouldHandleContracts, + ActivationEpochSpica: args.ActivationEpochSpica, ObserverFacade: &components.ObserverFacade{ Processor: baseProcessor, diff --git a/server/provider/networkProvider.go b/server/provider/networkProvider.go index 0549f460..67ac8b69 100644 --- a/server/provider/networkProvider.go +++ b/server/provider/networkProvider.go @@ -42,6 +42,7 @@ type ArgsNewNetworkProvider struct { FirstHistoricalEpoch uint32 NumHistoricalEpochs uint32 ShouldHandleContracts bool + ActivationEpochSpica uint32 ObserverFacade observerFacade @@ -64,6 +65,7 @@ type networkProvider struct { firstHistoricalEpoch uint32 numHistoricalEpochs uint32 shouldHandleContracts bool + activationEpochSpica uint32 observerFacade observerFacade @@ -104,6 +106,7 @@ func NewNetworkProvider(args ArgsNewNetworkProvider) (*networkProvider, error) { firstHistoricalEpoch: args.FirstHistoricalEpoch, numHistoricalEpochs: args.NumHistoricalEpochs, shouldHandleContracts: args.ShouldHandleContracts, + activationEpochSpica: args.ActivationEpochSpica, observerFacade: args.ObserverFacade, @@ -446,7 +449,7 @@ func (provider *networkProvider) GetMempoolTransactionByHash(hash string) (*tran return nil, nil } -// ComputeTransactionFeeForMoveBalance computes the fee for a move-balance transaction. +// ComputeTransactionFeeForMoveBalance computes the fee for a move-balance transaction func (provider *networkProvider) ComputeTransactionFeeForMoveBalance(tx *transaction.ApiTransactionResult) *big.Int { minGasLimit := provider.networkConfig.MinGasLimit extraGasLimitGuardedTx := provider.networkConfig.ExtraGasLimitGuardedTx @@ -462,6 +465,11 @@ func (provider *networkProvider) ComputeTransactionFeeForMoveBalance(tx *transac return fee } +// IsReleaseSpicaActive returns whether the Spica release is active in the provided epoch +func (provider *networkProvider) IsReleaseSpicaActive(epoch uint32) bool { + return epoch >= provider.activationEpochSpica +} + // LogDescription writes a description of the network provider in the log output func (provider *networkProvider) LogDescription() { log.Info("Description of network provider", diff --git a/server/services/features.go b/server/services/features.go deleted file mode 100644 index c68ebf73..00000000 --- a/server/services/features.go +++ /dev/null @@ -1,21 +0,0 @@ -package services - -import ( - "os" - "strconv" -) - -func areClaimDeveloperRewardsEventsEnabled(blockNonce uint64) bool { - nonceAsString := os.Getenv("CLAIM_DEVELOPER_REWARDS_EVENTS_ENABLED_NONCE") - if nonceAsString == "" { - return false - } - - nonce, err := strconv.Atoi(nonceAsString) - if err != nil { - return false - } - - enabled := blockNonce >= uint64(nonce) - return enabled -} diff --git a/server/services/interface.go b/server/services/interface.go index 271c46ad..ca9630dc 100644 --- a/server/services/interface.go +++ b/server/services/interface.go @@ -34,4 +34,5 @@ type NetworkProvider interface { ComputeReceiptHash(apiReceipt *transaction.ApiReceipt) (string, error) ComputeTransactionFeeForMoveBalance(tx *transaction.ApiTransactionResult) *big.Int GetMempoolTransactionByHash(hash string) (*transaction.ApiTransactionResult, error) + IsReleaseSpicaActive(epoch uint32) bool } diff --git a/server/services/operations.go b/server/services/operations.go index 91a39674..11c4fe44 100644 --- a/server/services/operations.go +++ b/server/services/operations.go @@ -12,6 +12,7 @@ const ( opScResult = "SmartContractResult" opFeeRefundAsScResult = "FeeRefundAsSmartContractResult" opDeveloperRewardsAsScResult = "DeveloperRewardsAsSmartContractResult" + opDeveloperRewards = "DeveloperRewards" opFeeOfInvalidTx = "FeeOfInvalidTransaction" opFeeRefund = "FeeRefund" opCustomTransfer = "CustomTransfer" @@ -27,6 +28,7 @@ var ( opScResult, opFeeRefundAsScResult, opDeveloperRewardsAsScResult, + opDeveloperRewards, opFeeOfInvalidTx, opFeeRefund, opCustomTransfer, diff --git a/server/services/testdata/blocks_with_claim_developer_rewards.json b/server/services/testdata/blocks_with_claim_developer_rewards.json new file mode 100644 index 00000000..1ed7360c --- /dev/null +++ b/server/services/testdata/blocks_with_claim_developer_rewards.json @@ -0,0 +1,77 @@ +[ + { + "comment": "block with claim developer rewards", + "miniBlocks": [ + { + "transactions": [ + { + "type": "normal", + "processingTypeOnSource": "BuiltInFunctionCall", + "processingTypeOnDestination": "BuiltInFunctionCall", + "hash": "96b1a0533ff17df3d1777889117023d0b178cc80fa72535d1b8ec1a13bcf3a75", + "epoch": 20, + "value": "0", + "receiver": "erd1qqqqqqqqqqqqqpgqagjekf5mxv86hy5c62vvtug5vc6jmgcsq6uq8reras", + "sender": "erd1tn62hjp72rznp8vq0lplva5csav6rccpqqdungpxtqz0g2hcq6uq9k4cc6", + "data": "Q2xhaW1EZXZlbG9wZXJSZXdhcmRz", + "signature": "132aa54ec093c21e2d0d447bf9a9b1da5c33d4120da66fb53dbc78d1d3d007e2314fa06348302f503e96a91b697dd943189df0ad4c3750416a704de2ac007e06", + "sourceShard": 0, + "destinationShard": 0, + "logs": { + "address": "erd1qqqqqqqqqqqqqpgqagjekf5mxv86hy5c62vvtug5vc6jmgcsq6uq8reras", + "events": [ + { + "address": "erd1qqqqqqqqqqqqqpgqagjekf5mxv86hy5c62vvtug5vc6jmgcsq6uq8reras", + "identifier": "completedTxEvent", + "topics": [ + "lrGgUz/xffPRd3iJEXAj0LF4zID6clNdG47BoTvPOnU=" + ], + "data": null, + "additionalData": null + } + ] + }, + "operation": "ClaimDeveloperRewards", + "initiallyPaidFee": "160685000000000" + } + ] + }, + { + "transactions": [ + { + "type": "unsigned", + "processingTypeOnSource": "MoveBalance", + "processingTypeOnDestination": "MoveBalance", + "hash": "d05c5f65f564d740aa1e81f7a96581d739783a43c232a6c86112afa1e6c318c4", + "epoch": 20, + "value": "1774725000000", + "receiver": "erd1tn62hjp72rznp8vq0lplva5csav6rccpqqdungpxtqz0g2hcq6uq9k4cc6", + "sender": "erd1tn62hjp72rznp8vq0lplva5csav6rccpqqdungpxtqz0g2hcq6uq9k4cc6", + "previousTransactionHash": "96b1a0533ff17df3d1777889117023d0b178cc80fa72535d1b8ec1a13bcf3a75", + "originalTransactionHash": "96b1a0533ff17df3d1777889117023d0b178cc80fa72535d1b8ec1a13bcf3a75", + "originalSender": "erd1tn62hjp72rznp8vq0lplva5csav6rccpqqdungpxtqz0g2hcq6uq9k4cc6", + "sourceShard": 0, + "destinationShard": 0 + }, + { + "type": "unsigned", + "processingTypeOnSource": "MoveBalance", + "processingTypeOnDestination": "MoveBalance", + "hash": "bc89442d5e77113f0b4e7383e5d078776fc6724690ba9f98d1704202c324e090", + "epoch": 20, + "value": "29185000000000", + "receiver": "erd1tn62hjp72rznp8vq0lplva5csav6rccpqqdungpxtqz0g2hcq6uq9k4cc6", + "sender": "erd1qqqqqqqqqqqqqpgqagjekf5mxv86hy5c62vvtug5vc6jmgcsq6uq8reras", + "previousTransactionHash": "96b1a0533ff17df3d1777889117023d0b178cc80fa72535d1b8ec1a13bcf3a75", + "originalTransactionHash": "96b1a0533ff17df3d1777889117023d0b178cc80fa72535d1b8ec1a13bcf3a75", + "sourceShard": 0, + "destinationShard": 0, + "nonce": 29, + "data": "QDZmNmI=", + "isRefund": true + } + ] + } + ] + } +] diff --git a/server/services/transactionsTransformer.go b/server/services/transactionsTransformer.go index 9e5ed04d..73683caf 100644 --- a/server/services/transactionsTransformer.go +++ b/server/services/transactionsTransformer.go @@ -138,7 +138,7 @@ func (transformer *transactionsTransformer) unsignedTxToRosettaTx( } } - if !areClaimDeveloperRewardsEventsEnabled(scr.BlockNonce) { + if !transformer.areClaimDeveloperRewardsEventsEnabled(scr.Epoch) { // Handle developer rewards in a legacy manner (without looking at events / logs) if transformer.featuresDetector.doesContractResultHoldRewardsOfClaimDeveloperRewards(scr, txsInBlock) { return &types.Transaction{ @@ -561,11 +561,11 @@ func (transformer *transactionsTransformer) addOperationsGivenTransactionEvents( rosettaTx.Operations = append(rosettaTx.Operations, operations...) } - if areClaimDeveloperRewardsEventsEnabled(tx.BlockNonce) { + if transformer.areClaimDeveloperRewardsEventsEnabled(tx.Epoch) { for _, event := range eventsClaimDeveloperRewards { operations := []*types.Operation{ { - Type: opDeveloperRewardsAsScResult, + Type: opDeveloperRewards, Account: addressToAccountIdentifier(event.receiverAddress), Amount: transformer.extension.valueToNativeAmount(event.value), }, @@ -612,3 +612,7 @@ func (transformer *transactionsTransformer) extractOperationsFromEventESDT(event return make([]*types.Operation, 0) } + +func (transformer *transactionsTransformer) areClaimDeveloperRewardsEventsEnabled(epoch uint32) bool { + return transformer.provider.IsReleaseSpicaActive(epoch) +} diff --git a/server/services/transactionsTransformer_test.go b/server/services/transactionsTransformer_test.go index eb478050..e9056190 100644 --- a/server/services/transactionsTransformer_test.go +++ b/server/services/transactionsTransformer_test.go @@ -998,6 +998,74 @@ func TestTransactionsTransformer_TransformBlockTxsHavingNFTBurn(t *testing.T) { require.Equal(t, expectedNftBurnTx, txs[0]) } +func TestTransactionsTransformer_TransformBlockTxsHavingClaimDeveloperRewards(t *testing.T) { + networkProvider := testscommon.NewNetworkProviderMock() + networkProvider.MockObservedActualShard = 0 + networkProvider.MockActivationEpochSpica = 42 + + extension := newNetworkProviderExtension(networkProvider) + transformer := newTransactionsTransformer(networkProvider) + + blocks, err := readTestBlocks("testdata/blocks_with_claim_developer_rewards.json") + require.Nil(t, err) + + t.Run("recover operations in legacy manner (without using events)", func(t *testing.T) { + txs, err := transformer.transformBlockTxs(blocks[0]) + require.Nil(t, err) + require.Len(t, txs, 3) + require.Len(t, txs[0].Operations, 1) + require.Len(t, txs[1].Operations, 1) + require.Len(t, txs[2].Operations, 1) + + expectedTx0 := &types.Transaction{ + TransactionIdentifier: hashToTransactionIdentifier("96b1a0533ff17df3d1777889117023d0b178cc80fa72535d1b8ec1a13bcf3a75"), + Operations: []*types.Operation{ + { + Type: opFee, + OperationIdentifier: indexToOperationIdentifier(0), + Account: addressToAccountIdentifier("erd1tn62hjp72rznp8vq0lplva5csav6rccpqqdungpxtqz0g2hcq6uq9k4cc6"), + Amount: extension.valueToNativeAmount("-160685000000000"), + Status: &opStatusSuccess, + }, + }, + Metadata: extractTransactionMetadata(blocks[0].MiniBlocks[0].Transactions[0]), + } + + require.Equal(t, expectedTx0, txs[0]) + + expectedTx1 := &types.Transaction{ + TransactionIdentifier: hashToTransactionIdentifier("d05c5f65f564d740aa1e81f7a96581d739783a43c232a6c86112afa1e6c318c4"), + Operations: []*types.Operation{ + { + Type: opDeveloperRewardsAsScResult, + OperationIdentifier: indexToOperationIdentifier(0), + Account: addressToAccountIdentifier("erd1tn62hjp72rznp8vq0lplva5csav6rccpqqdungpxtqz0g2hcq6uq9k4cc6"), + Amount: extension.valueToNativeAmount("1774725000000"), + Status: &opStatusSuccess, + }, + }, + } + + require.Equal(t, expectedTx1, txs[1]) + + // Fee refund + expectedTx2 := &types.Transaction{ + TransactionIdentifier: hashToTransactionIdentifier("bc89442d5e77113f0b4e7383e5d078776fc6724690ba9f98d1704202c324e090"), + Operations: []*types.Operation{ + { + Type: opFeeRefundAsScResult, + OperationIdentifier: indexToOperationIdentifier(0), + Account: addressToAccountIdentifier("erd1tn62hjp72rznp8vq0lplva5csav6rccpqqdungpxtqz0g2hcq6uq9k4cc6"), + Amount: extension.valueToNativeAmount("29185000000000"), + Status: &opStatusSuccess, + }, + }, + } + + require.Equal(t, expectedTx2, txs[2]) + }) +} + func readTestBlocks(filePath string) ([]*api.Block, error) { var blocks []*api.Block diff --git a/systemtests/broadcast_samples.py b/systemtests/broadcast_samples.py index 738a7ec0..d4334bc3 100644 --- a/systemtests/broadcast_samples.py +++ b/systemtests/broadcast_samples.py @@ -175,6 +175,11 @@ def main(): amount=77 )) + controller.send(controller.create_claim_developer_rewards_on_directly_owned_contract( + sender=accounts.get_user(shard=0, index=0), + contract=accounts.contracts_by_shard[0][0], + )) + class BunchOfAccounts: def __init__(self, configuration: Configuration) -> None: @@ -424,6 +429,16 @@ def create_contract_call_with_move_balance_with_signal_error(self, sender: "Acco return transaction + def create_claim_developer_rewards_on_directly_owned_contract(self, sender: "Account", contract: Address) -> Transaction: + transaction = self.contracts_transactions_factory.create_transaction_for_execute( + sender=sender.address, + contract=contract, + function="ClaimDeveloperRewards", + gas_limit=8000000, + ) + + return transaction + def send_multiple(self, transactions: List[Transaction]): for transaction in transactions: self._apply_nonce(transaction) diff --git a/testscommon/networkProviderMock.go b/testscommon/networkProviderMock.go index 9d185f9a..38c5606e 100644 --- a/testscommon/networkProviderMock.go +++ b/testscommon/networkProviderMock.go @@ -31,6 +31,7 @@ type networkProviderMock struct { MockCustomCurrencies []resources.Currency MockGenesisBlockHash string MockGenesisTimestamp int64 + MockActivationEpochSpica uint32 MockNetworkConfig *resources.NetworkConfig MockGenesisBalances []*resources.GenesisBalance MockNodeStatus *resources.AggregatedNodeStatus @@ -362,3 +363,8 @@ func (mock *networkProviderMock) GetMempoolTransactionByHash(hash string) (*tran return nil, nil } + +// IsReleaseSpicaActive - +func (mock *networkProviderMock) IsReleaseSpicaActive(epoch uint32) bool { + return epoch >= mock.MockActivationEpochSpica +} From eaef142be12ced3d900228aad82fdbb2c954fc40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Wed, 21 Aug 2024 15:28:42 +0300 Subject: [PATCH 55/75] Move test contract files. --- systemtests/broadcast_samples.py | 5 +++-- systemtests/{ => contracts}/adder.wasm | Bin systemtests/contracts/developer_rewards.wasm | Bin 0 -> 973 bytes systemtests/{ => contracts}/dummy.wasm | Bin 4 files changed, 3 insertions(+), 2 deletions(-) rename systemtests/{ => contracts}/adder.wasm (100%) create mode 100755 systemtests/contracts/developer_rewards.wasm rename systemtests/{ => contracts}/dummy.wasm (100%) diff --git a/systemtests/broadcast_samples.py b/systemtests/broadcast_samples.py index d4334bc3..8e0e7f5f 100644 --- a/systemtests/broadcast_samples.py +++ b/systemtests/broadcast_samples.py @@ -13,8 +13,9 @@ from systemtests.config import CONFIGURATIONS, Configuration -CONTRACT_PATH_ADDER = Path(__file__).parent / "adder.wasm" -CONTRACT_PATH_DUMMY = Path(__file__).parent / "dummy.wasm" +CONTRACT_PATH_ADDER = Path(__file__).parent / "contracts" / "adder.wasm" +CONTRACT_PATH_DUMMY = Path(__file__).parent / "contracts" / "dummy.wasm" +CONTRACT_PATH_DEVELOPER_REWARDS = Path(__file__).parent / "contracts" / "developer_rewards.wasm" def main(): diff --git a/systemtests/adder.wasm b/systemtests/contracts/adder.wasm similarity index 100% rename from systemtests/adder.wasm rename to systemtests/contracts/adder.wasm diff --git a/systemtests/contracts/developer_rewards.wasm b/systemtests/contracts/developer_rewards.wasm new file mode 100755 index 0000000000000000000000000000000000000000..7d8df93e210f88c939641fcb41aaf4a0c0666071 GIT binary patch literal 973 zcmcIjyKdA#6umQkdz`Fy^AG}xJAxuj3YzUGhDB6JNNh>7)_T|nyX(ky5=BUB13^j( zBtC(PDj$&#;3Mey3GUdt5d{qeV~y^eGxu@s(HN3q$pC=$y%Db}7%|%Zx@s4zMz-g|$2?XHpT}!Xd+_$sUte66rLp0IlaG=sO*F^!$K=)F8vdAu8Zj7#86Mp0&YGSNda8Y}~e4fHZgQm_ru8Rt4FwJI)F<3X}mfMa=XyBv!FcYRhd z_3hdg-qdrKdYR=H9Bd+1g6o~0O!6ev;E6OFW=ma8XR8VLq9rz%&K4;I9b(?NTk_$< z@gkWmpX)Qd$WC$d7m=9dzOkrlbPtqI<+In;;K8 z>7#!mgElNQ#+O3X*M?4{>LzT5vUW)!gLO7G#sDJ>aR&wNVh7{+qp5$ttKe`x%5qwxW0O!__p857}% zlr_4xpCEldZ Date: Wed, 21 Aug 2024 16:36:21 +0300 Subject: [PATCH 56/75] Additional unit tests. --- .../services/transactionEventsController.go | 2 +- .../transactionEventsController_test.go | 25 +++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/server/services/transactionEventsController.go b/server/services/transactionEventsController.go index 725cd7a3..15309752 100644 --- a/server/services/transactionEventsController.go +++ b/server/services/transactionEventsController.go @@ -355,7 +355,7 @@ func (controller *transactionEventsController) extractEventsClaimDeveloperReward for _, event := range rawEvents { numTopics := len(event.Topics) - if numTopics != 2 { + if numTopics != numTopicsOfEventClaimDeveloperRewards { return nil, fmt.Errorf("%w: bad number of topics for %s event = %d", errCannotRecognizeEvent, transactionEventClaimDeveloperRewards, numTopics) } diff --git a/server/services/transactionEventsController_test.go b/server/services/transactionEventsController_test.go index b977ad98..e4aa6b8e 100644 --- a/server/services/transactionEventsController_test.go +++ b/server/services/transactionEventsController_test.go @@ -281,4 +281,29 @@ func TestTransactionEventsController_ExtractEvents(t *testing.T) { require.Equal(t, []byte{0x2a}, events[0].nonceAsBytes) require.Equal(t, "100", events[0].value) }) + + t.Run("ClaimDeveloperRewards", func(t *testing.T) { + topic0 := []byte{0x64} + topic1, _ := hex.DecodeString("5cf4abc83e50c5309d807fc3f676988759a1e301001bc9a0265804f42af806b8") + + tx := &transaction.ApiTransactionResult{ + Logs: &transaction.ApiLogs{ + Events: []*transaction.Events{ + { + Identifier: "ClaimDeveloperRewards", + Topics: [][]byte{ + topic0, + topic1, + }, + }, + }, + }, + } + + events, err := controller.extractEventsClaimDeveloperRewards(tx) + require.NoError(t, err) + require.Len(t, events, 1) + require.Equal(t, "100", events[0].value) + require.Equal(t, "erd1tn62hjp72rznp8vq0lplva5csav6rccpqqdungpxtqz0g2hcq6uq9k4cc6", events[0].receiverAddress) + }) } From 1ff56d9636c482bb3d8d20519d0e68a61423e6b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Wed, 21 Aug 2024 16:36:52 +0300 Subject: [PATCH 57/75] New constant. --- server/services/constants.go | 1 + 1 file changed, 1 insertion(+) diff --git a/server/services/constants.go b/server/services/constants.go index 349a671d..1366a33a 100644 --- a/server/services/constants.go +++ b/server/services/constants.go @@ -57,4 +57,5 @@ var ( numTopicsOfEventESDTNFTBurn = 3 numTopicsOfEventESDTNFTAddQuantity = 3 numTopicsOfEventSCDeployBeforeSirius = 2 + numTopicsOfEventClaimDeveloperRewards = 2 ) From 4cf3d5300dd27502aec22bed0e4a328df78d2bd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Thu, 22 Aug 2024 11:59:49 +0300 Subject: [PATCH 58/75] Additional test samples. --- systemtests/broadcast_samples.py | 31 ++++++++++++++++++++++++++++++- systemtests/contracts/dummy.wasm | Bin 115 -> 236 bytes 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/systemtests/broadcast_samples.py b/systemtests/broadcast_samples.py index 8e0e7f5f..63c22141 100644 --- a/systemtests/broadcast_samples.py +++ b/systemtests/broadcast_samples.py @@ -31,7 +31,7 @@ def main(): controller = Controller(configuration, accounts) if mode == "setup": - # controller.send_multiple(controller.create_airdrops()) + controller.send_multiple(controller.create_airdrops()) controller.send_multiple(controller.create_contract_deployments()) elif mode == "run": print("Intra-shard, simple MoveBalance with refund") @@ -176,11 +176,20 @@ def main(): amount=77 )) + print("ClaimDeveloperRewards on directly owned contract") controller.send(controller.create_claim_developer_rewards_on_directly_owned_contract( sender=accounts.get_user(shard=0, index=0), contract=accounts.contracts_by_shard[0][0], )) + print("Intra-shard, relayed v1 transaction with contract call with MoveBalance, with signal error") + controller.send(controller.create_relayed_v1_with_contract_call_with_move_balance_with_signal_error( + relayer=accounts.get_user(shard=0, index=0), + sender=accounts.get_user(shard=0, index=1), + contract=accounts.contracts_by_shard[0][0], + amount=1 + )) + class BunchOfAccounts: def __init__(self, configuration: Configuration) -> None: @@ -440,6 +449,26 @@ def create_claim_developer_rewards_on_directly_owned_contract(self, sender: "Acc return transaction + def create_relayed_v1_with_contract_call_with_move_balance_with_signal_error(self, relayer: "Account", sender: "Account", contract: Address, amount: int) -> Transaction: + inner_transaction = self.contracts_transactions_factory.create_transaction_for_execute( + sender=sender.address, + contract=contract, + function="add", + gas_limit=5000000, + arguments=[1, 2, 3, 4, 5], + native_transfer_amount=amount + ) + + self._apply_nonce(inner_transaction) + self._sign(inner_transaction) + + transaction = self.relayed_transactions_factory.create_relayed_v1_transaction( + inner_transaction=inner_transaction, + relayer_address=relayer.address, + ) + + return transaction + def send_multiple(self, transactions: List[Transaction]): for transaction in transactions: self._apply_nonce(transaction) diff --git a/systemtests/contracts/dummy.wasm b/systemtests/contracts/dummy.wasm index 3c36df962a52b73fa104b51e38b89b635f769a43..3c4efc1233011007777c9a0ffa1f61ec43ff820d 100755 GIT binary patch literal 236 zcmXAj&5nXV5QM8|1~E)FAwGfeC%fx;gJ(?i9lSx}pybKht?x0GNPgtsos6dvCJtw)5o9co?K!i>3%-+ckj)->Ay literal 115 zcmWN{u?~VT6ouh)?yW%RLi7n-oejG_2_zKl(3Sw2uqdOCZU21ZkA4XRz)CuP&;fFy zy7;y}uI+X7(JxOxjgmYlAw|V6Z~M>K@@$IWLYg;$$>R!^ujw)OgM;Xp60|<5z-Be> JmuAnw{sGKc6}kWb From 406b3fc63216f0303ee00075c7edd5adab6b85da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Thu, 22 Aug 2024 13:49:10 +0300 Subject: [PATCH 59/75] Exotic scenario: extractInnerTxOperationsIfBeforeSiriusRelayedCompletelyIntrashardWithSignalError. --- cmd/rosetta/cli.go | 10 ++++ cmd/rosetta/main.go | 1 + server/factory/interface.go | 1 + server/factory/provider.go | 2 + server/provider/networkProvider.go | 11 ++++ server/services/interface.go | 1 + server/services/transactionsTransformer.go | 23 +++++--- .../services/transactionsTransformer_test.go | 55 +++++++++++++++++-- testscommon/networkProviderMock.go | 6 ++ 9 files changed, 98 insertions(+), 12 deletions(-) diff --git a/cmd/rosetta/cli.go b/cmd/rosetta/cli.go index 9c4cbcb9..5ce59cfa 100644 --- a/cmd/rosetta/cli.go +++ b/cmd/rosetta/cli.go @@ -169,6 +169,13 @@ VERSION: Required: false, } + cliFlagActivationEpochSirius = cli.UintFlag{ + Name: "activation-epoch-spica", + Usage: "Specifies the activation epoch for Sirius release.", + Required: false, + Value: 1265, + } + cliFlagActivationEpochSpica = cli.UintFlag{ Name: "activation-epoch-spica", Usage: "Specifies the activation epoch for Spica release.", @@ -203,6 +210,7 @@ func getAllCliFlags() []cli.Flag { cliFlagNumHistoricalEpochs, cliFlagShouldHandleContracts, cliFlagConfigFileCustomCurrencies, + cliFlagActivationEpochSirius, cliFlagActivationEpochSpica, } } @@ -233,6 +241,7 @@ type parsedCliFlags struct { numHistoricalEpochs uint32 shouldHandleContracts bool configFileCustomCurrencies string + activationEpochSirius uint32 activationEpochSpica uint32 } @@ -263,6 +272,7 @@ func getParsedCliFlags(ctx *cli.Context) parsedCliFlags { numHistoricalEpochs: uint32(ctx.GlobalUint(cliFlagNumHistoricalEpochs.Name)), shouldHandleContracts: ctx.GlobalBool(cliFlagShouldHandleContracts.Name), configFileCustomCurrencies: ctx.GlobalString(cliFlagConfigFileCustomCurrencies.Name), + activationEpochSirius: uint32(ctx.GlobalUint(cliFlagActivationEpochSirius.Name)), activationEpochSpica: uint32(ctx.GlobalUint(cliFlagActivationEpochSpica.Name)), } } diff --git a/cmd/rosetta/main.go b/cmd/rosetta/main.go index dbb2d423..c0c338a4 100644 --- a/cmd/rosetta/main.go +++ b/cmd/rosetta/main.go @@ -82,6 +82,7 @@ func startRosetta(ctx *cli.Context) error { FirstHistoricalEpoch: cliFlags.firstHistoricalEpoch, NumHistoricalEpochs: cliFlags.numHistoricalEpochs, ShouldHandleContracts: cliFlags.shouldHandleContracts, + ActivationEpochSirius: cliFlags.activationEpochSpica, ActivationEpochSpica: cliFlags.activationEpochSpica, }) if err != nil { diff --git a/server/factory/interface.go b/server/factory/interface.go index 861af975..0be4f8c9 100644 --- a/server/factory/interface.go +++ b/server/factory/interface.go @@ -35,6 +35,7 @@ type NetworkProvider interface { ComputeReceiptHash(apiReceipt *transaction.ApiReceipt) (string, error) ComputeTransactionFeeForMoveBalance(tx *transaction.ApiTransactionResult) *big.Int GetMempoolTransactionByHash(hash string) (*transaction.ApiTransactionResult, error) + IsReleaseSiriusActive(epoch uint32) bool IsReleaseSpicaActive(epoch uint32) bool LogDescription() } diff --git a/server/factory/provider.go b/server/factory/provider.go index b3acc26d..393a90ca 100644 --- a/server/factory/provider.go +++ b/server/factory/provider.go @@ -49,6 +49,7 @@ type ArgsCreateNetworkProvider struct { FirstHistoricalEpoch uint32 NumHistoricalEpochs uint32 ShouldHandleContracts bool + ActivationEpochSirius uint32 ActivationEpochSpica uint32 } @@ -141,6 +142,7 @@ func CreateNetworkProvider(args ArgsCreateNetworkProvider) (NetworkProvider, err FirstHistoricalEpoch: args.FirstHistoricalEpoch, NumHistoricalEpochs: args.NumHistoricalEpochs, ShouldHandleContracts: args.ShouldHandleContracts, + ActivationEpochSirius: args.ActivationEpochSirius, ActivationEpochSpica: args.ActivationEpochSpica, ObserverFacade: &components.ObserverFacade{ diff --git a/server/provider/networkProvider.go b/server/provider/networkProvider.go index 67ac8b69..cae2bdae 100644 --- a/server/provider/networkProvider.go +++ b/server/provider/networkProvider.go @@ -42,6 +42,7 @@ type ArgsNewNetworkProvider struct { FirstHistoricalEpoch uint32 NumHistoricalEpochs uint32 ShouldHandleContracts bool + ActivationEpochSirius uint32 ActivationEpochSpica uint32 ObserverFacade observerFacade @@ -65,6 +66,7 @@ type networkProvider struct { firstHistoricalEpoch uint32 numHistoricalEpochs uint32 shouldHandleContracts bool + activationEpochSirius uint32 activationEpochSpica uint32 observerFacade observerFacade @@ -106,6 +108,7 @@ func NewNetworkProvider(args ArgsNewNetworkProvider) (*networkProvider, error) { firstHistoricalEpoch: args.FirstHistoricalEpoch, numHistoricalEpochs: args.NumHistoricalEpochs, shouldHandleContracts: args.ShouldHandleContracts, + activationEpochSirius: args.ActivationEpochSirius, activationEpochSpica: args.ActivationEpochSpica, observerFacade: args.ObserverFacade, @@ -465,6 +468,11 @@ func (provider *networkProvider) ComputeTransactionFeeForMoveBalance(tx *transac return fee } +// IsReleaseSiriusActive returns whether the Sirius release is active in the provided epoch +func (provider *networkProvider) IsReleaseSiriusActive(epoch uint32) bool { + return epoch < provider.activationEpochSpica +} + // IsReleaseSpicaActive returns whether the Spica release is active in the provided epoch func (provider *networkProvider) IsReleaseSpicaActive(epoch uint32) bool { return epoch >= provider.activationEpochSpica @@ -482,6 +490,9 @@ func (provider *networkProvider) LogDescription() { "observedProjectedShardIsSet", provider.observedProjectedShardIsSet, "firstHistoricalEpoch", provider.firstHistoricalEpoch, "numHistoricalEpochs", provider.numHistoricalEpochs, + "shouldHandleContracts", provider.shouldHandleContracts, + "activationEpochSirius", provider.activationEpochSirius, + "activationEpochSpica", provider.activationEpochSpica, "nativeCurrency", provider.GetNativeCurrency().Symbol, "customCurrencies", provider.GetCustomCurrenciesSymbols(), ) diff --git a/server/services/interface.go b/server/services/interface.go index ca9630dc..dab3742a 100644 --- a/server/services/interface.go +++ b/server/services/interface.go @@ -34,5 +34,6 @@ type NetworkProvider interface { ComputeReceiptHash(apiReceipt *transaction.ApiReceipt) (string, error) ComputeTransactionFeeForMoveBalance(tx *transaction.ApiTransactionResult) *big.Int GetMempoolTransactionByHash(hash string) (*transaction.ApiTransactionResult, error) + IsReleaseSiriusActive(epoch uint32) bool IsReleaseSpicaActive(epoch uint32) bool } diff --git a/server/services/transactionsTransformer.go b/server/services/transactionsTransformer.go index 73683caf..2e0a8c59 100644 --- a/server/services/transactionsTransformer.go +++ b/server/services/transactionsTransformer.go @@ -214,15 +214,11 @@ func (transformer *transactionsTransformer) normalTxToRosetta(tx *transaction.Ap Amount: transformer.extension.valueToNativeAmount("-" + tx.InitiallyPaidFee), }) - innerTxOperationsIfRelayedCompletelyIntrashardWithSignalError, err := transformer.extractInnerTxOperationsIfRelayedCompletelyIntrashardWithSignalError(tx) + innerTxOperationsIfRelayedCompletelyIntrashardWithSignalError, err := transformer.extractInnerTxOperationsIfBeforeSiriusRelayedCompletelyIntrashardWithSignalError(tx) if err != nil { return nil, err } - if len(innerTxOperationsIfRelayedCompletelyIntrashardWithSignalError) > 0 { - log.Info("normalTxToRosetta(): innerTxOperationsIfRelayedCompletelyIntrashardWithSignalError", "tx", tx.Hash, "block", tx.BlockNonce) - } - operations = append(operations, innerTxOperationsIfRelayedCompletelyIntrashardWithSignalError...) return &types.Transaction{ @@ -232,8 +228,19 @@ func (transformer *transactionsTransformer) normalTxToRosetta(tx *transaction.Ap }, nil } -// This only handles operations for the native balance. -func (transformer *transactionsTransformer) extractInnerTxOperationsIfRelayedCompletelyIntrashardWithSignalError(tx *transaction.ApiTransactionResult) ([]*types.Operation, error) { +// extractInnerTxOperationsIfBeforeSiriusRelayedCompletelyIntrashardWithSignalError recovers the inner transaction operations (native balance transfers) +// for a transaction that is relayed and completely intrashard, and has a signal error, and was processed before the activation of Sirius features. +// Before Sirius, such a transaction is accompanied by an SCR with the value refund, thus Rosetta has to recover the inner transaction operations, as well, +// to show the complete picture (that the operations cancel each other out). +// After Sirius, the protocol does not generate an SCR with the value refund for such transactions, so this workaround is not needed. +// Additional references: +// - https://github.com/multiversx/mx-chain-rosetta/pull/81/files +// - https://console.cloud.google.com/bigquery?sq=667383445384:bfeb7de9aeec453192612ddc7fa9d94e +func (transformer *transactionsTransformer) extractInnerTxOperationsIfBeforeSiriusRelayedCompletelyIntrashardWithSignalError(tx *transaction.ApiTransactionResult) ([]*types.Operation, error) { + if transformer.provider.IsReleaseSiriusActive(tx.Epoch) { + return []*types.Operation{}, nil + } + // Only relayed V1 is handled. Relayed V2 cannot bear native value in the inner transaction. isRelayedTransaction := isRelayedV1Transaction(tx) if !isRelayedTransaction { @@ -253,6 +260,8 @@ func (transformer *transactionsTransformer) extractInnerTxOperationsIfRelayedCom return []*types.Operation{}, nil } + log.Info("extractInnerTxOperationsIfBeforeSiriusRelayedCompletelyIntrashardWithSignalError", "tx", tx.Hash) + senderAddress := transformer.provider.ConvertPubKeyToAddress(innerTx.SenderPubKey) receiverAddress := transformer.provider.ConvertPubKeyToAddress(innerTx.ReceiverPubKey) diff --git a/server/services/transactionsTransformer_test.go b/server/services/transactionsTransformer_test.go index e9056190..160a2c4d 100644 --- a/server/services/transactionsTransformer_test.go +++ b/server/services/transactionsTransformer_test.go @@ -19,6 +19,8 @@ func TestTransactionsTransformer_NormalTxToRosettaTx(t *testing.T) { extension := newNetworkProviderExtension(networkProvider) transformer := newTransactionsTransformer(networkProvider) + networkProvider.MockActivationEpochSirius = 42 + t.Run("move balance tx", func(t *testing.T) { tx := &transaction.ApiTransactionResult{ Hash: "aaaa", @@ -55,8 +57,9 @@ func TestTransactionsTransformer_NormalTxToRosettaTx(t *testing.T) { require.Equal(t, expectedRosettaTx, rosettaTx) }) - t.Run("relayed tx, completely intrashard, with signal error", func(t *testing.T) { + t.Run("relayed tx, completely intrashard, with signal error (before Sirius)", func(t *testing.T) { tx := &transaction.ApiTransactionResult{ + Epoch: 41, Type: string(transaction.TxTypeNormal), ProcessingTypeOnSource: transactionProcessingTypeRelayedV1, ProcessingTypeOnDestination: transactionProcessingTypeRelayedV1, @@ -103,6 +106,46 @@ func TestTransactionsTransformer_NormalTxToRosettaTx(t *testing.T) { require.NoError(t, err) require.Equal(t, expectedRosettaTx, rosettaTx) }) + + t.Run("relayed tx, completely intrashard, with signal error (after Sirius)", func(t *testing.T) { + tx := &transaction.ApiTransactionResult{ + Epoch: 43, + Type: string(transaction.TxTypeNormal), + ProcessingTypeOnSource: transactionProcessingTypeRelayedV1, + ProcessingTypeOnDestination: transactionProcessingTypeRelayedV1, + Hash: "aaaa", + Sender: testscommon.TestUserAShard0.Address, + Receiver: testscommon.TestUserBShard0.Address, + SourceShard: 0, + DestinationShard: 0, + InitiallyPaidFee: "50000000000000", + // Has non-zero value + Data: []byte("relayedTx@7b226e6f6e6365223a372c2273656e646572223a226e69424758747949504349644a78793373796c6c455a474c78506a503148734a45646e43732b6d726577413d222c227265636569766572223a224141414141414141414141464145356c3079623173734a3933504433672f4b396f48384579366d576958513d222c2276616c7565223a313030303030303030303030303030303030302c226761735072696365223a313030303030303030302c226761734c696d6974223a35303030302c2264617461223a22222c227369676e6174757265223a226e6830743338585a77614b6a725878446969716f6d364d6a5671724d612f6b70767474696a33692b5a6d43492f3778626830596762363548424151445744396f7036575567674541755430756e5253595736455341413d3d222c22636861696e4944223a224d513d3d222c2276657273696f6e223a327d"), + Logs: &transaction.ApiLogs{ + Events: []*transaction.Events{ + { + Identifier: transactionEventSignalError, + }, + }, + }, + } + + expectedRosettaTx := &types.Transaction{ + TransactionIdentifier: hashToTransactionIdentifier("aaaa"), + Operations: []*types.Operation{ + { + Type: opFee, + Account: addressToAccountIdentifier(testscommon.TestUserAShard0.Address), + Amount: extension.valueToNativeAmount("-50000000000000"), + }, + }, + Metadata: extractTransactionMetadata(tx), + } + + rosettaTx, err := transformer.normalTxToRosetta(tx) + require.NoError(t, err) + require.Equal(t, expectedRosettaTx, rosettaTx) + }) } func TestTransactionsTransformer_ExtractInnerTxOperationsIfRelayedCompletelyIntrashardWithSignalError(t *testing.T) { @@ -110,12 +153,14 @@ func TestTransactionsTransformer_ExtractInnerTxOperationsIfRelayedCompletelyIntr extension := newNetworkProviderExtension(networkProvider) transformer := newTransactionsTransformer(networkProvider) + networkProvider.MockActivationEpochSirius = 42 + t.Run("non-relayed tx", func(t *testing.T) { tx := &transaction.ApiTransactionResult{ Type: string(transaction.TxTypeNormal), } - operations, err := transformer.extractInnerTxOperationsIfRelayedCompletelyIntrashardWithSignalError(tx) + operations, err := transformer.extractInnerTxOperationsIfBeforeSiriusRelayedCompletelyIntrashardWithSignalError(tx) require.NoError(t, err) require.Len(t, operations, 0) }) @@ -128,7 +173,7 @@ func TestTransactionsTransformer_ExtractInnerTxOperationsIfRelayedCompletelyIntr Data: []byte("bad"), } - operations, err := transformer.extractInnerTxOperationsIfRelayedCompletelyIntrashardWithSignalError(tx) + operations, err := transformer.extractInnerTxOperationsIfBeforeSiriusRelayedCompletelyIntrashardWithSignalError(tx) require.ErrorIs(t, err, errCannotParseRelayedV1) require.Empty(t, operations) }) @@ -150,7 +195,7 @@ func TestTransactionsTransformer_ExtractInnerTxOperationsIfRelayedCompletelyIntr }, } - operations, err := transformer.extractInnerTxOperationsIfRelayedCompletelyIntrashardWithSignalError(tx) + operations, err := transformer.extractInnerTxOperationsIfBeforeSiriusRelayedCompletelyIntrashardWithSignalError(tx) require.NoError(t, err) require.Equal(t, []*types.Operation{ { @@ -183,7 +228,7 @@ func TestTransactionsTransformer_ExtractInnerTxOperationsIfRelayedCompletelyIntr }, } - operations, err := transformer.extractInnerTxOperationsIfRelayedCompletelyIntrashardWithSignalError(tx) + operations, err := transformer.extractInnerTxOperationsIfBeforeSiriusRelayedCompletelyIntrashardWithSignalError(tx) require.NoError(t, err) require.Len(t, operations, 0) }) diff --git a/testscommon/networkProviderMock.go b/testscommon/networkProviderMock.go index 38c5606e..84e8b739 100644 --- a/testscommon/networkProviderMock.go +++ b/testscommon/networkProviderMock.go @@ -31,6 +31,7 @@ type networkProviderMock struct { MockCustomCurrencies []resources.Currency MockGenesisBlockHash string MockGenesisTimestamp int64 + MockActivationEpochSirius uint32 MockActivationEpochSpica uint32 MockNetworkConfig *resources.NetworkConfig MockGenesisBalances []*resources.GenesisBalance @@ -364,6 +365,11 @@ func (mock *networkProviderMock) GetMempoolTransactionByHash(hash string) (*tran return nil, nil } +// IsReleaseSiriusActive - +func (mock *networkProviderMock) IsReleaseSiriusActive(epoch uint32) bool { + return epoch >= mock.MockActivationEpochSirius +} + // IsReleaseSpicaActive - func (mock *networkProviderMock) IsReleaseSpicaActive(epoch uint32) bool { return epoch >= mock.MockActivationEpochSpica From 619eea4cb5a437d88a4cef25dbfef66e7a5ceca8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Thu, 22 Aug 2024 14:05:27 +0300 Subject: [PATCH 60/75] Fix passing of CLI params, add test config params. --- cmd/rosetta/cli.go | 2 +- cmd/rosetta/main.go | 2 +- server/provider/networkProvider.go | 2 +- systemtests/check_with_mesh_cli.py | 2 ++ systemtests/config.py | 10 ++++++++++ 5 files changed, 15 insertions(+), 3 deletions(-) diff --git a/cmd/rosetta/cli.go b/cmd/rosetta/cli.go index 5ce59cfa..1c290c70 100644 --- a/cmd/rosetta/cli.go +++ b/cmd/rosetta/cli.go @@ -170,7 +170,7 @@ VERSION: } cliFlagActivationEpochSirius = cli.UintFlag{ - Name: "activation-epoch-spica", + Name: "activation-epoch-sirius", Usage: "Specifies the activation epoch for Sirius release.", Required: false, Value: 1265, diff --git a/cmd/rosetta/main.go b/cmd/rosetta/main.go index c0c338a4..90965e45 100644 --- a/cmd/rosetta/main.go +++ b/cmd/rosetta/main.go @@ -82,7 +82,7 @@ func startRosetta(ctx *cli.Context) error { FirstHistoricalEpoch: cliFlags.firstHistoricalEpoch, NumHistoricalEpochs: cliFlags.numHistoricalEpochs, ShouldHandleContracts: cliFlags.shouldHandleContracts, - ActivationEpochSirius: cliFlags.activationEpochSpica, + ActivationEpochSirius: cliFlags.activationEpochSirius, ActivationEpochSpica: cliFlags.activationEpochSpica, }) if err != nil { diff --git a/server/provider/networkProvider.go b/server/provider/networkProvider.go index cae2bdae..82b35f56 100644 --- a/server/provider/networkProvider.go +++ b/server/provider/networkProvider.go @@ -470,7 +470,7 @@ func (provider *networkProvider) ComputeTransactionFeeForMoveBalance(tx *transac // IsReleaseSiriusActive returns whether the Sirius release is active in the provided epoch func (provider *networkProvider) IsReleaseSiriusActive(epoch uint32) bool { - return epoch < provider.activationEpochSpica + return epoch >= provider.activationEpochSirius } // IsReleaseSpicaActive returns whether the Spica release is active in the provided epoch diff --git a/systemtests/check_with_mesh_cli.py b/systemtests/check_with_mesh_cli.py index d4272c79..fe94a880 100644 --- a/systemtests/check_with_mesh_cli.py +++ b/systemtests/check_with_mesh_cli.py @@ -70,6 +70,8 @@ def run_rosetta(configuration: Configuration): f"--config-custom-currencies={configuration.config_file_custom_currencies}", f"--first-historical-epoch={current_epoch}", f"--num-historical-epochs={configuration.num_historical_epochs}", + f"--activation-epoch-sirius={configuration.activation_epoch_sirius}", + f"--activation-epoch-spica={configuration.activation_epoch_spica}", ] return subprocess.Popen(command) diff --git a/systemtests/config.py b/systemtests/config.py index 95bd11b0..4e66b6ba 100644 --- a/systemtests/config.py +++ b/systemtests/config.py @@ -13,6 +13,8 @@ class Configuration: num_historical_epochs: int observer_url: str proxy_url: str + activation_epoch_sirius: int + activation_epoch_spica: int check_construction_native_configuration_file: str check_construction_custom_configuration_file: str check_data_configuration_file: str @@ -33,6 +35,8 @@ class Configuration: num_historical_epochs=2, observer_url="", proxy_url="https://gateway.multiversx.com", + activation_epoch_sirius=1265, + activation_epoch_spica=4294967295, check_construction_native_configuration_file="", check_construction_custom_configuration_file="", check_data_configuration_file="systemtests/mesh_cli_config/check-data.json", @@ -50,6 +54,8 @@ class Configuration: num_historical_epochs=2, observer_url="", proxy_url="https://devnet-gateway.multiversx.com", + activation_epoch_sirius=629, + activation_epoch_spica=4294967295, check_construction_native_configuration_file="systemtests/mesh_cli_config/devnet-construction-native.json", check_construction_custom_configuration_file="systemtests/mesh_cli_config/devnet-construction-custom.json", check_data_configuration_file="systemtests/mesh_cli_config/check-data.json", @@ -70,6 +76,8 @@ class Configuration: num_historical_epochs=1, observer_url="", proxy_url="https://testnet-gateway.multiversx.com", + activation_epoch_sirius=1, + activation_epoch_spica=4294967295, check_construction_native_configuration_file="systemtests/mesh_cli_config/testnet-construction-native.json", check_construction_custom_configuration_file="systemtests/mesh_cli_config/testnet-construction-custom.json", check_data_configuration_file="systemtests/mesh_cli_config/check-data.json", @@ -90,6 +98,8 @@ class Configuration: num_historical_epochs=2, observer_url="", proxy_url="http://localhost:7950", + activation_epoch_sirius=1, + activation_epoch_spica=4294967295, check_construction_native_configuration_file="systemtests/mesh_cli_config/localnet-construction-native.json", check_construction_custom_configuration_file="systemtests/mesh_cli_config/localnet-construction-custom.json", check_data_configuration_file="systemtests/mesh_cli_config/check-data.json", From 25181cd90348d2696ef31f7b52640fda585b29ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Thu, 22 Aug 2024 14:36:32 +0300 Subject: [PATCH 61/75] Some TODOs. --- systemtests/broadcast_samples.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/systemtests/broadcast_samples.py b/systemtests/broadcast_samples.py index 63c22141..34643e2a 100644 --- a/systemtests/broadcast_samples.py +++ b/systemtests/broadcast_samples.py @@ -182,6 +182,9 @@ def main(): contract=accounts.contracts_by_shard[0][0], )) + # TODO: claim developer rewards with parent-child contracts + # TODO: claim developer rewards on directly owned contracts, cross shard (use change owner address). + print("Intra-shard, relayed v1 transaction with contract call with MoveBalance, with signal error") controller.send(controller.create_relayed_v1_with_contract_call_with_move_balance_with_signal_error( relayer=accounts.get_user(shard=0, index=0), From c97a67abd5f018f3ebd14b6fd4a633ba6cf4704c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Thu, 22 Aug 2024 15:23:38 +0300 Subject: [PATCH 62/75] Removed not needed code. --- server/services/constants.go | 2 -- server/services/errors.go | 1 - server/services/relayedTransactions.go | 27 -------------------------- 3 files changed, 30 deletions(-) diff --git a/server/services/constants.go b/server/services/constants.go index 1366a33a..f7aec9bf 100644 --- a/server/services/constants.go +++ b/server/services/constants.go @@ -10,8 +10,6 @@ import ( var ( transactionVersion = 1 transactionProcessingTypeRelayedV1 = "RelayedTx" - transactionProcessingTypeRelayedV2 = "RelayedTxV2" - transactionProcessingTypeRelayedV3 = "RelayedTxV3" transactionProcessingTypeBuiltInFunctionCall = "BuiltInFunctionCall" transactionProcessingTypeMoveBalance = "MoveBalance" transactionProcessingTypeContractInvoking = "SCInvoking" diff --git a/server/services/errors.go b/server/services/errors.go index 3f9c76b8..76f3bab8 100644 --- a/server/services/errors.go +++ b/server/services/errors.go @@ -202,4 +202,3 @@ func (factory *errFactory) getPrototypeByCode(code errCode) errPrototype { var errEventNotFound = errors.New("transaction event not found") var errCannotRecognizeEvent = errors.New("cannot recognize transaction event") var errCannotParseRelayedV1 = errors.New("cannot parse relayed V1 transaction") -var errCannotParseRelayedV2 = errors.New("cannot parse relayed V2 transaction") diff --git a/server/services/relayedTransactions.go b/server/services/relayedTransactions.go index 62917eb1..3f530d9f 100644 --- a/server/services/relayedTransactions.go +++ b/server/services/relayedTransactions.go @@ -16,23 +16,12 @@ type innerTransactionOfRelayedV1 struct { SenderPubKey []byte `json:"sender"` } -// innerTransactionOfRelayedV2 is used to parse the inner transaction of a relayed V2 transaction, and holds only the fields handled by Rosetta. -type innerTransactionOfRelayedV2 struct { - ReceiverPubKey []byte `json:"receiver"` -} - func isRelayedV1Transaction(tx *transaction.ApiTransactionResult) bool { return (tx.Type == string(transaction.TxTypeNormal)) && (tx.ProcessingTypeOnSource == transactionProcessingTypeRelayedV1) && (tx.ProcessingTypeOnDestination == transactionProcessingTypeRelayedV1) } -func isRelayedV2Transaction(tx *transaction.ApiTransactionResult) bool { - return (tx.Type == string(transaction.TxTypeNormal)) && - (tx.ProcessingTypeOnSource == transactionProcessingTypeRelayedV2) && - (tx.ProcessingTypeOnDestination == transactionProcessingTypeRelayedV2) -} - func parseInnerTxOfRelayedV1(tx *transaction.ApiTransactionResult) (*innerTransactionOfRelayedV1, error) { subparts := strings.Split(string(tx.Data), argumentsSeparator) if len(subparts) != 2 { @@ -53,19 +42,3 @@ func parseInnerTxOfRelayedV1(tx *transaction.ApiTransactionResult) (*innerTransa return &innerTx, nil } - -func parseInnerTxOfRelayedV2(tx *transaction.ApiTransactionResult) (*innerTransactionOfRelayedV2, error) { - subparts := strings.Split(string(tx.Data), argumentsSeparator) - if len(subparts) != 5 { - return nil, errCannotParseRelayedV2 - } - - receiverPubKey, err := hex.DecodeString(subparts[1]) - if err != nil { - return nil, err - } - - return &innerTransactionOfRelayedV2{ - ReceiverPubKey: receiverPubKey, - }, nil -} From d5270150f34df2741a1cd710d4fe1c168be1848b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Thu, 22 Aug 2024 16:12:33 +0300 Subject: [PATCH 63/75] Handle & dismiss ineffective refunds. --- server/services/networkProviderExtension.go | 4 + .../services/transactionsFeaturesDetector.go | 16 ++- .../transactionsFeaturesDetector_test.go | 23 ++++ server/services/transactionsTransformer.go | 14 +-- .../services/transactionsTransformer_test.go | 106 +++++++++++------- systemtests/check_with_mesh_cli.py | 1 + 6 files changed, 111 insertions(+), 53 deletions(-) diff --git a/server/services/networkProviderExtension.go b/server/services/networkProviderExtension.go index cfff7de6..1ad2ba61 100644 --- a/server/services/networkProviderExtension.go +++ b/server/services/networkProviderExtension.go @@ -78,6 +78,10 @@ func (extension *networkProviderExtension) getGenesisBlockIdentifier() *types.Bl return blockSummaryToIdentifier(summary) } +func (extension *networkProviderExtension) isContractAddress(address string) bool { + return !extension.isUserAddress(address) +} + func (extension *networkProviderExtension) isUserAddress(address string) bool { pubKey, err := extension.provider.ConvertAddressToPubKey(address) if err != nil { diff --git a/server/services/transactionsFeaturesDetector.go b/server/services/transactionsFeaturesDetector.go index 3c33d5bc..e6b91a9e 100644 --- a/server/services/transactionsFeaturesDetector.go +++ b/server/services/transactionsFeaturesDetector.go @@ -5,14 +5,16 @@ import ( ) type transactionsFeaturesDetector struct { - networkProvider NetworkProvider - eventsController *transactionEventsController + networkProvider NetworkProvider + networkProviderExtension *networkProviderExtension + eventsController *transactionEventsController } func newTransactionsFeaturesDetector(provider NetworkProvider) *transactionsFeaturesDetector { return &transactionsFeaturesDetector{ - networkProvider: provider, - eventsController: newTransactionEventsController(provider), + networkProvider: provider, + networkProviderExtension: newNetworkProviderExtension(provider), + eventsController: newTransactionEventsController(provider), } } @@ -106,3 +108,9 @@ func (detector *transactionsFeaturesDetector) isContractCallWithSignalError(tx * func (detector *transactionsFeaturesDetector) isIntrashard(tx *transaction.ApiTransactionResult) bool { return tx.SourceShard == tx.DestinationShard } + +// isSmartContractResultIneffectiveRefund detects smart contract results that are ineffective refunds. +// Also see: https://console.cloud.google.com/bigquery?sq=667383445384:de7a5f3f172f4b50a9aaed353ef79839 +func (detector *transactionsFeaturesDetector) isSmartContractResultIneffectiveRefund(scr *transaction.ApiTransactionResult) bool { + return scr.IsRefund && scr.Sender == scr.Receiver && detector.networkProviderExtension.isContractAddress(scr.Sender) +} diff --git a/server/services/transactionsFeaturesDetector_test.go b/server/services/transactionsFeaturesDetector_test.go index 70af7207..552e47b6 100644 --- a/server/services/transactionsFeaturesDetector_test.go +++ b/server/services/transactionsFeaturesDetector_test.go @@ -235,3 +235,26 @@ func TestTransactionsFeaturesDetector_isIntrashard(t *testing.T) { DestinationShard: 1, })) } + +func TestTransactionsFeaturesDetector_isSmartContractResultIneffectiveRefund(t *testing.T) { + networkProvider := testscommon.NewNetworkProviderMock() + detector := newTransactionsFeaturesDetector(networkProvider) + + require.True(t, detector.isSmartContractResultIneffectiveRefund(&transaction.ApiTransactionResult{ + Sender: testscommon.TestAddressOfContract, + Receiver: testscommon.TestAddressOfContract, + IsRefund: true, + })) + + require.False(t, detector.isSmartContractResultIneffectiveRefund(&transaction.ApiTransactionResult{ + Sender: testscommon.TestAddressOfContract, + Receiver: testscommon.TestAddressOfContract, + IsRefund: false, + })) + + require.False(t, detector.isSmartContractResultIneffectiveRefund(&transaction.ApiTransactionResult{ + Sender: testscommon.TestAddressOfContract, + Receiver: testscommon.TestAddressAlice, + IsRefund: false, + })) +} diff --git a/server/services/transactionsTransformer.go b/server/services/transactionsTransformer.go index 2e0a8c59..9614863b 100644 --- a/server/services/transactionsTransformer.go +++ b/server/services/transactionsTransformer.go @@ -116,16 +116,16 @@ func (transformer *transactionsTransformer) unsignedTxToRosettaTx( scr *transaction.ApiTransactionResult, txsInBlock []*transaction.ApiTransactionResult, ) *types.Transaction { - if scr.IsRefund { - if scr.Sender == scr.Receiver && !transformer.extension.isUserAddress(scr.Sender) { - log.Info("unsignedTxToRosettaTx: dismissed refund", "hash", scr.Hash, "originalTxHash", scr.OriginalTransactionHash) + if transformer.featuresDetector.isSmartContractResultIneffectiveRefund(scr) { + log.Info("unsignedTxToRosettaTx: ineffective refund", "hash", scr.Hash) - return &types.Transaction{ - TransactionIdentifier: hashToTransactionIdentifier(scr.Hash), - Operations: []*types.Operation{}, - } + return &types.Transaction{ + TransactionIdentifier: hashToTransactionIdentifier(scr.Hash), + Operations: []*types.Operation{}, } + } + if scr.IsRefund { return &types.Transaction{ TransactionIdentifier: hashToTransactionIdentifier(scr.Hash), Operations: []*types.Operation{ diff --git a/server/services/transactionsTransformer_test.go b/server/services/transactionsTransformer_test.go index 160a2c4d..cd16ca0a 100644 --- a/server/services/transactionsTransformer_test.go +++ b/server/services/transactionsTransformer_test.go @@ -239,55 +239,77 @@ func TestTransactionsTransformer_UnsignedTxToRosettaTx(t *testing.T) { extension := newNetworkProviderExtension(networkProvider) transformer := newTransactionsTransformer(networkProvider) - refundTx := &transaction.ApiTransactionResult{ - Hash: "aaaa", - Sender: testscommon.TestAddressOfContract, - Receiver: testscommon.TestAddressAlice, - Value: "1234", - IsRefund: true, - } + t.Run("refund SCR", func(t *testing.T) { + tx := &transaction.ApiTransactionResult{ + Hash: "aaaa", + Sender: testscommon.TestAddressOfContract, + Receiver: testscommon.TestAddressAlice, + Value: "1234", + IsRefund: true, + } - expectedRefundTx := &types.Transaction{ - TransactionIdentifier: hashToTransactionIdentifier("aaaa"), - Operations: []*types.Operation{ - { - Type: opFeeRefundAsScResult, - Account: addressToAccountIdentifier(testscommon.TestAddressAlice), - Amount: extension.valueToNativeAmount("1234"), + expectedTx := &types.Transaction{ + TransactionIdentifier: hashToTransactionIdentifier("aaaa"), + Operations: []*types.Operation{ + { + Type: opFeeRefundAsScResult, + Account: addressToAccountIdentifier(testscommon.TestAddressAlice), + Amount: extension.valueToNativeAmount("1234"), + }, }, - }, - } + } - moveBalanceTx := &transaction.ApiTransactionResult{ - Hash: "aaaa", - Sender: testscommon.TestAddressOfContract, - Receiver: testscommon.TestAddressAlice, - Value: "1234", - } + rosettaTx := transformer.unsignedTxToRosettaTx(tx, nil) + require.Equal(t, expectedTx, rosettaTx) + }) - expectedMoveBalanceTx := &types.Transaction{ - TransactionIdentifier: hashToTransactionIdentifier("aaaa"), - Operations: []*types.Operation{ - { - Type: opScResult, - Account: addressToAccountIdentifier(testscommon.TestAddressOfContract), - Amount: extension.valueToNativeAmount("-1234"), - }, - { - Type: opScResult, - Account: addressToAccountIdentifier(testscommon.TestAddressAlice), - Amount: extension.valueToNativeAmount("1234"), + t.Run("move balance SCR", func(t *testing.T) { + tx := &transaction.ApiTransactionResult{ + Hash: "aaaa", + Sender: testscommon.TestAddressOfContract, + Receiver: testscommon.TestAddressAlice, + Value: "1234", + } + + expectedTx := &types.Transaction{ + TransactionIdentifier: hashToTransactionIdentifier("aaaa"), + Operations: []*types.Operation{ + { + Type: opScResult, + Account: addressToAccountIdentifier(testscommon.TestAddressOfContract), + Amount: extension.valueToNativeAmount("-1234"), + }, + { + Type: opScResult, + Account: addressToAccountIdentifier(testscommon.TestAddressAlice), + Amount: extension.valueToNativeAmount("1234"), + }, }, - }, - Metadata: extractTransactionMetadata(moveBalanceTx), - } + Metadata: extractTransactionMetadata(tx), + } + + rosettaTx := transformer.unsignedTxToRosettaTx(tx, []*transaction.ApiTransactionResult{tx}) + require.Equal(t, expectedTx, rosettaTx) + }) - txsInBlock := []*transaction.ApiTransactionResult{refundTx, moveBalanceTx} + t.Run("ineffective refund SCR", func(t *testing.T) { + tx := &transaction.ApiTransactionResult{ + Hash: "aaaa", + Sender: testscommon.TestAddressOfContract, + Receiver: testscommon.TestAddressOfContract, + Value: "1234", + IsRefund: true, + } - rosettaRefundTx := transformer.unsignedTxToRosettaTx(refundTx, txsInBlock) - rosettaMoveBalanceTx := transformer.unsignedTxToRosettaTx(moveBalanceTx, txsInBlock) - require.Equal(t, expectedRefundTx, rosettaRefundTx) - require.Equal(t, expectedMoveBalanceTx, rosettaMoveBalanceTx) + expectedTx := &types.Transaction{ + TransactionIdentifier: hashToTransactionIdentifier("aaaa"), + Operations: []*types.Operation{}, + Metadata: nil, + } + + rosettaTx := transformer.unsignedTxToRosettaTx(tx, []*transaction.ApiTransactionResult{tx}) + require.Equal(t, expectedTx, rosettaTx) + }) } func TestTransactionsTransformer_InvalidTxToRosettaTx(t *testing.T) { diff --git a/systemtests/check_with_mesh_cli.py b/systemtests/check_with_mesh_cli.py index fe94a880..2469ef32 100644 --- a/systemtests/check_with_mesh_cli.py +++ b/systemtests/check_with_mesh_cli.py @@ -66,6 +66,7 @@ def run_rosetta(configuration: Configuration): f"--observer-actual-shard={configuration.network_shard}", f"--network-id={configuration.network_id}", f"--network-name={configuration.network_name}", + f"--handle-contracts", f"--native-currency={configuration.native_currency}", f"--config-custom-currencies={configuration.config_file_custom_currencies}", f"--first-historical-epoch={current_epoch}", From b0383a8c8b1b619c83bc95110c9feded6be1c4e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Fri, 23 Aug 2024 15:05:58 +0300 Subject: [PATCH 64/75] Handle event "transfer value only". --- server/services/constants.go | 5 + .../services/transactionEventsController.go | 136 ++++++++++-------- server/services/transactionsTransformer.go | 12 +- 3 files changed, 84 insertions(+), 69 deletions(-) diff --git a/server/services/constants.go b/server/services/constants.go index f7aec9bf..37a3e156 100644 --- a/server/services/constants.go +++ b/server/services/constants.go @@ -43,6 +43,9 @@ var ( transactionEventClaimDeveloperRewards = core.BuiltInFunctionClaimDeveloperRewards transactionEventTopicInvalidMetaTransaction = "meta transaction is invalid" transactionEventTopicInvalidMetaTransactionNotEnoughGas = "meta transaction is invalid: not enough gas" + + transactionEventDataExecuteOnDestContext = "ExecuteOnDestContext" + transactionEventDataAsyncCall = "AsyncCall" ) var ( @@ -56,4 +59,6 @@ var ( numTopicsOfEventESDTNFTAddQuantity = 3 numTopicsOfEventSCDeployBeforeSirius = 2 numTopicsOfEventClaimDeveloperRewards = 2 + numTopicsOfEventTransferValueOnlyBeforeSirius = 3 + numTopicsOfEventTransferValueOnlyAfterSirius = 2 ) diff --git a/server/services/transactionEventsController.go b/server/services/transactionEventsController.go index 15309752..edb5ad39 100644 --- a/server/services/transactionEventsController.go +++ b/server/services/transactionEventsController.go @@ -45,88 +45,104 @@ func (controller *transactionEventsController) extractEventSCDeploy(tx *transact } func (controller *transactionEventsController) extractEventTransferValueOnly(tx *transaction.ApiTransactionResult) ([]*eventTransferValueOnly, error) { + isBeforeSirius := !controller.provider.IsReleaseSiriusActive(tx.Epoch) rawEvents := controller.findManyEventsByIdentifier(tx, transactionEventTransferValueOnly) - typedEvents := make([]*eventTransferValueOnly, 0) for _, event := range rawEvents { - numTopics := len(event.Topics) - isBeforeSirius := numTopics == 3 - isAfterSirius := numTopics == 2 - - var sender string - var receiver string - var valueBytes []byte + if string(event.Data) != transactionEventDataExecuteOnDestContext && string(event.Data) != transactionEventDataAsyncCall { + continue + } if isBeforeSirius { - senderPubKey := event.Topics[0] - receiverPubKey := event.Topics[1] - valueBytes = event.Topics[2] - - sender = controller.provider.ConvertPubKeyToAddress(senderPubKey) - receiver = controller.provider.ConvertPubKeyToAddress(receiverPubKey) - } else if isAfterSirius { - // https://github.com/multiversx/mx-specs/blob/main/releases/protocol/release-specs-v1.6.0-Sirius.md#17-logs--events-changes-5490 - sender = event.Address - receiverPubKey := event.Topics[1] - valueBytes = event.Topics[0] - - receiver = controller.provider.ConvertPubKeyToAddress(receiverPubKey) - } else { - return nil, fmt.Errorf("%w: bad number of topics for '%s' = %d", errCannotRecognizeEvent, transactionEventTransferValueOnly, numTopics) - } + typedEvent, err := controller.decideEffectiveEventTransferValueOnlyBeforeSirius(event) + if err != nil { + return nil, err + } - value := big.NewInt(0).SetBytes(valueBytes) + if typedEvent != nil { + typedEvents = append(typedEvents, typedEvent) + } + } else { + typedEvent, err := controller.decideEffectiveEventTransferValueOnlyAfterSirius(event) + if err != nil { + return nil, err + } - typedEvents = append(typedEvents, &eventTransferValueOnly{ - sender: sender, - receiver: receiver, - value: value.String(), - }) + if typedEvent != nil { + typedEvents = append(typedEvents, typedEvent) + } + } } return typedEvents, nil } -// TODO: Use a cached "contractResultsSummaries" (this a must)! -func filterOutTransferValueOnlyEventsThatAreAlreadyCapturedAsContractResults( - txOfInterest *transaction.ApiTransactionResult, - events []*eventTransferValueOnly, - txsInBlock []*transaction.ApiTransactionResult, -) []*eventTransferValueOnly { - // First, we find all contract results in this block, and we "summarize" them (in a map). - contractResultsSummaries := make(map[string]struct{}) - - for _, item := range txsInBlock { - isContractResult := item.Type == string(transaction.TxTypeUnsigned) - if !isContractResult { - continue - } +func (controller *transactionEventsController) decideEffectiveEventTransferValueOnlyBeforeSirius(event *transaction.Events) (*eventTransferValueOnly, error) { + numTopics := len(event.Topics) - // If not part of the same transaction graph, then skip it. - if !(item.OriginalTransactionHash == txOfInterest.Hash) && !(item.OriginalTransactionHash == txOfInterest.OriginalTransactionHash) { - continue - } + if numTopics != numTopicsOfEventTransferValueOnlyBeforeSirius { + return nil, fmt.Errorf("%w: bad number of topics for 'transferValueOnly' = %d", errCannotRecognizeEvent, numTopics) + } - summary := fmt.Sprintf("%s-%s-%s", item.Sender, item.Receiver, item.Value) - contractResultsSummaries[summary] = struct{}{} + senderPubKey := event.Topics[0] + receiverPubKey := event.Topics[1] + valueBytes := event.Topics[2] + + if len(valueBytes) == 0 { + return nil, nil } - eventsToKeep := make([]*eventTransferValueOnly, 0, len(events)) + isIntrashard := controller.provider.ComputeShardIdOfPubKey(senderPubKey) == controller.provider.ComputeShardIdOfPubKey(receiverPubKey) + if !isIntrashard { + return nil, nil + } - for _, event := range events { - summary := fmt.Sprintf("%s-%s-%s", event.sender, event.receiver, event.value) + sender := controller.provider.ConvertPubKeyToAddress(senderPubKey) + receiver := controller.provider.ConvertPubKeyToAddress(receiverPubKey) + value := big.NewInt(0).SetBytes(valueBytes) - _, isAlreadyCaptured := contractResultsSummaries[summary] - if isAlreadyCaptured { - continue - } + return &eventTransferValueOnly{ + sender: sender, + receiver: receiver, + value: value.String(), + }, nil +} - // Event not captured as contract result, so we should keep it. - eventsToKeep = append(eventsToKeep, event) +// See: https://github.com/multiversx/mx-specs/blob/main/releases/protocol/release-specs-v1.6.0-Sirius.md#17-logs--events-changes-5490 +func (controller *transactionEventsController) decideEffectiveEventTransferValueOnlyAfterSirius(event *transaction.Events) (*eventTransferValueOnly, error) { + numTopics := len(event.Topics) + + if numTopics != numTopicsOfEventTransferValueOnlyAfterSirius { + return nil, fmt.Errorf("%w: bad number of topics for 'transferValueOnly' = %d", errCannotRecognizeEvent, numTopics) } - return eventsToKeep + valueBytes := event.Topics[0] + receiverPubKey := event.Topics[1] + + if len(valueBytes) == 0 { + return nil, nil + } + + sender := event.Address + senderPubKey, err := controller.provider.ConvertAddressToPubKey(sender) + if err != nil { + return nil, err + } + + isIntrashard := controller.provider.ComputeShardIdOfPubKey(senderPubKey) == controller.provider.ComputeShardIdOfPubKey(receiverPubKey) + if !isIntrashard { + return nil, nil + } + + receiver := controller.provider.ConvertPubKeyToAddress(receiverPubKey) + value := big.NewInt(0).SetBytes(valueBytes) + + return &eventTransferValueOnly{ + sender: sender, + receiver: receiver, + value: value.String(), + }, nil } func (controller *transactionEventsController) hasAnySignalError(tx *transaction.ApiTransactionResult) bool { diff --git a/server/services/transactionsTransformer.go b/server/services/transactionsTransformer.go index 9614863b..5bce9a02 100644 --- a/server/services/transactionsTransformer.go +++ b/server/services/transactionsTransformer.go @@ -104,7 +104,7 @@ func (transformer *transactionsTransformer) txToRosettaTx(tx *transaction.ApiTra return nil, fmt.Errorf("unknown transaction type: %s", tx.Type) } - err = transformer.addOperationsGivenTransactionEvents(tx, txsInBlock, rosettaTx) + err = transformer.addOperationsGivenTransactionEvents(tx, rosettaTx) if err != nil { return nil, err } @@ -117,7 +117,7 @@ func (transformer *transactionsTransformer) unsignedTxToRosettaTx( txsInBlock []*transaction.ApiTransactionResult, ) *types.Transaction { if transformer.featuresDetector.isSmartContractResultIneffectiveRefund(scr) { - log.Info("unsignedTxToRosettaTx: ineffective refund", "hash", scr.Hash) + log.Info("unsignedTxToRosettaTx: ineffective refund", "hash", scr.Hash, "block", scr.BlockNonce) return &types.Transaction{ TransactionIdentifier: hashToTransactionIdentifier(scr.Hash), @@ -360,11 +360,7 @@ func (transformer *transactionsTransformer) mempoolMoveBalanceTxToRosettaTx(tx * } } -func (transformer *transactionsTransformer) addOperationsGivenTransactionEvents( - tx *transaction.ApiTransactionResult, - txsInBlock []*transaction.ApiTransactionResult, - rosettaTx *types.Transaction, -) error { +func (transformer *transactionsTransformer) addOperationsGivenTransactionEvents(tx *transaction.ApiTransactionResult, rosettaTx *types.Transaction) error { hasSignalError := transformer.featuresDetector.eventsController.hasAnySignalError(tx) if hasSignalError { return nil @@ -380,8 +376,6 @@ func (transformer *transactionsTransformer) addOperationsGivenTransactionEvents( return err } - eventsTransferValueOnly = filterOutTransferValueOnlyEventsThatAreAlreadyCapturedAsContractResults(tx, eventsTransferValueOnly, txsInBlock) - eventsESDTTransfer, err := transformer.eventsController.extractEventsESDTOrESDTNFTTransfers(tx) if err != nil { return err From 1354c73dba74fd4eb384af6bc0869165ce6b918b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Fri, 23 Aug 2024 16:58:33 +0300 Subject: [PATCH 65/75] Add tests, refactor. --- .../services/transactionEventsController.go | 15 +-- .../transactionEventsController_test.go | 114 ++++++++++++++++++ testscommon/addresses.go | 19 ++- 3 files changed, 137 insertions(+), 11 deletions(-) diff --git a/server/services/transactionEventsController.go b/server/services/transactionEventsController.go index edb5ad39..e34c7ae5 100644 --- a/server/services/transactionEventsController.go +++ b/server/services/transactionEventsController.go @@ -50,10 +50,6 @@ func (controller *transactionEventsController) extractEventTransferValueOnly(tx typedEvents := make([]*eventTransferValueOnly, 0) for _, event := range rawEvents { - if string(event.Data) != transactionEventDataExecuteOnDestContext && string(event.Data) != transactionEventDataAsyncCall { - continue - } - if isBeforeSirius { typedEvent, err := controller.decideEffectiveEventTransferValueOnlyBeforeSirius(event) if err != nil { @@ -93,11 +89,6 @@ func (controller *transactionEventsController) decideEffectiveEventTransferValue return nil, nil } - isIntrashard := controller.provider.ComputeShardIdOfPubKey(senderPubKey) == controller.provider.ComputeShardIdOfPubKey(receiverPubKey) - if !isIntrashard { - return nil, nil - } - sender := controller.provider.ConvertPubKeyToAddress(senderPubKey) receiver := controller.provider.ConvertPubKeyToAddress(receiverPubKey) value := big.NewInt(0).SetBytes(valueBytes) @@ -124,6 +115,11 @@ func (controller *transactionEventsController) decideEffectiveEventTransferValue return nil, nil } + if string(event.Data) != transactionEventDataExecuteOnDestContext && string(event.Data) != transactionEventDataAsyncCall { + // Ineffective event, since the balance change is already captured by a SCR. + return nil, nil + } + sender := event.Address senderPubKey, err := controller.provider.ConvertAddressToPubKey(sender) if err != nil { @@ -132,6 +128,7 @@ func (controller *transactionEventsController) decideEffectiveEventTransferValue isIntrashard := controller.provider.ComputeShardIdOfPubKey(senderPubKey) == controller.provider.ComputeShardIdOfPubKey(receiverPubKey) if !isIntrashard { + // Ineffective event, since the balance change is already captured by a SCR. return nil, nil } diff --git a/server/services/transactionEventsController_test.go b/server/services/transactionEventsController_test.go index e4aa6b8e..484717f5 100644 --- a/server/services/transactionEventsController_test.go +++ b/server/services/transactionEventsController_test.go @@ -2,6 +2,7 @@ package services import ( "encoding/hex" + "math/big" "testing" "github.com/multiversx/mx-chain-core-go/data/transaction" @@ -175,6 +176,8 @@ func TestTransactionEventsController_ExtractEvents(t *testing.T) { networkProvider := testscommon.NewNetworkProviderMock() controller := newTransactionEventsController(networkProvider) + networkProvider.MockActivationEpochSirius = 42 + t.Run("SCDeploy", func(t *testing.T) { topic0, _ := hex.DecodeString("00000000000000000500def8dad1161f8f0c38f3e6e73515ed81058f0b5606b8") topic1, _ := hex.DecodeString("5cf4abc83e50c5309d807fc3f676988759a1e301001bc9a0265804f42af806b8") @@ -203,6 +206,117 @@ func TestTransactionEventsController_ExtractEvents(t *testing.T) { require.Equal(t, "erd1tn62hjp72rznp8vq0lplva5csav6rccpqqdungpxtqz0g2hcq6uq9k4cc6", events[0].deployerAddress) }) + t.Run("transferValueOnly, before Sirius", func(t *testing.T) { + topic0 := testscommon.TestContractFooShard0.PubKey + topic1 := testscommon.TestContractBarShard0.PubKey + topic2 := big.NewInt(100).Bytes() + + tx := &transaction.ApiTransactionResult{ + Epoch: 41, + Logs: &transaction.ApiLogs{ + Events: []*transaction.Events{ + { + Identifier: "transferValueOnly", + Address: "erd1qqqqqqqqqqqqqpgqmmud45gkr78scw8numnn290dsyzc7z6kq6uqw2jcza", + Topics: [][]byte{ + topic0, + topic1, + topic2, + }, + }, + }, + }, + } + + events, err := controller.extractEventTransferValueOnly(tx) + require.NoError(t, err) + require.Len(t, events, 1) + require.Equal(t, testscommon.TestContractFooShard0.Address, events[0].sender) + require.Equal(t, testscommon.TestContractBarShard0.Address, events[0].receiver) + require.Equal(t, "100", events[0].value) + }) + + t.Run("transferValueOnly, after Sirius, effective (intra-shard ExecuteOnDestContext)", func(t *testing.T) { + topic0 := big.NewInt(100).Bytes() + topic1 := testscommon.TestContractBarShard0.PubKey + + tx := &transaction.ApiTransactionResult{ + Epoch: 43, + Logs: &transaction.ApiLogs{ + Events: []*transaction.Events{ + { + Identifier: "transferValueOnly", + Address: testscommon.TestContractFooShard0.Address, + Topics: [][]byte{ + topic0, + topic1, + }, + Data: []byte("ExecuteOnDestContext"), + }, + }, + }, + } + + events, err := controller.extractEventTransferValueOnly(tx) + require.NoError(t, err) + require.Len(t, events, 1) + require.Equal(t, testscommon.TestContractFooShard0.Address, events[0].sender) + require.Equal(t, testscommon.TestContractBarShard0.Address, events[0].receiver) + require.Equal(t, "100", events[0].value) + }) + + t.Run("transferValueOnly, after Sirius, ineffective (cross-shard AsyncCall)", func(t *testing.T) { + topic0 := big.NewInt(100).Bytes() + topic1 := testscommon.TestContractBarShard1.PubKey + + tx := &transaction.ApiTransactionResult{ + Epoch: 43, + Logs: &transaction.ApiLogs{ + Events: []*transaction.Events{ + { + Identifier: "transferValueOnly", + Address: testscommon.TestContractFooShard0.Address, + Topics: [][]byte{ + topic0, + topic1, + }, + Data: []byte("AsyncCall"), + }, + }, + }, + } + + events, err := controller.extractEventTransferValueOnly(tx) + require.NoError(t, err) + require.Len(t, events, 0) + }) + + t.Run("transferValueOnly, after Sirius, ineffective, intra-shard BackTransfer", func(t *testing.T) { + topic0 := big.NewInt(100).Bytes() + topic1 := testscommon.TestContractBarShard0.PubKey + + tx := &transaction.ApiTransactionResult{ + Epoch: 43, + Logs: &transaction.ApiLogs{ + Events: []*transaction.Events{ + { + Identifier: "transferValueOnly", + Address: testscommon.TestContractFooShard0.Address, + Topics: [][]byte{ + topic0, + topic1, + }, + Data: []byte("BackTransfer"), + }, + }, + }, + } + + events, err := controller.extractEventTransferValueOnly(tx) + require.NoError(t, err) + require.Len(t, events, 0) + }) + t.Run("ESDTNFTCreate", func(t *testing.T) { tx := &transaction.ApiTransactionResult{ Logs: &transaction.ApiLogs{ diff --git a/testscommon/addresses.go b/testscommon/addresses.go index ad5e2a45..c8856cb5 100644 --- a/testscommon/addresses.go +++ b/testscommon/addresses.go @@ -32,8 +32,23 @@ var ( // TestUserCShard0 is a test account (user) TestUserCShard0 = newTestAccount("erd1ncsyvhku3q7zy8f8rjmmx2t9zxgch38cel28kzg3m8pt86dt0vqqecw0gy") - // TestContractShard0 is a test account (contract) - TestContractShard0 = newTestAccount("erd1qqqqqqqqqqqqqpgqfejaxfh4ktp8mh8s77pl90dq0uzvh2vk396qlcwepw") + // TestContractFooShard0 is a test account (contract) + TestContractFooShard0 = newTestAccount("erd1qqqqqqqqqqqqqpgqagjekf5mxv86hy5c62vvtug5vc6jmgcsq6uq8reras") + + // TestContractBarShard0 is a test account (contract) + TestContractBarShard0 = newTestAccount("erd1qqqqqqqqqqqqqpgqdstpe4fepzl4w8683xw88t5kcjkxz0zaq6uquj6ztu") + + // TestContractFooShard1 is a test account (contract) + TestContractFooShard1 = newTestAccount("erd1qqqqqqqqqqqqqpgq89t5xm4x04tnt9lv747wdrsaycf3rcwcggzsa7crse") + + // TestContractBarShard1 is a test account (contract) + TestContractBarShard1 = newTestAccount("erd1qqqqqqqqqqqqqpgq0dtujxcrmwwqdtwzvq5nxuwgjcgaty7fggzse2vmm2") + + // TestContractFooShard2 is a test account (contract) + TestContractFooShard2 = newTestAccount("erd1qqqqqqqqqqqqqpgqeesfamasje5zru7ku79m8p4xqfqnywvqxj0qhtyzdr") + + // TestContractBarShard2 is a test account (contract) + TestContractBarShard2 = newTestAccount("erd1ux2wvqyh8pw8ea26urjqq65mqytfn42dr980pvucztxk9w79xj0q2x98te") // TestUserShard1 is a test account (user) TestUserShard1 = newTestAccount("erd1qyu5wthldzr8wx5c9ucg8kjagg0jfs53s8nr3zpz3hypefsdd8ssycr6th") From 68084277cbfde606e8951ea9b90c10949235f24b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Mon, 26 Aug 2024 09:40:04 +0300 Subject: [PATCH 66/75] Do not handle transferValueOnly events before Sirius. --- .../services/transactionEventsController.go | 55 ++++--------------- .../transactionEventsController_test.go | 7 +-- 2 files changed, 13 insertions(+), 49 deletions(-) diff --git a/server/services/transactionEventsController.go b/server/services/transactionEventsController.go index e34c7ae5..589a53bc 100644 --- a/server/services/transactionEventsController.go +++ b/server/services/transactionEventsController.go @@ -45,59 +45,26 @@ func (controller *transactionEventsController) extractEventSCDeploy(tx *transact } func (controller *transactionEventsController) extractEventTransferValueOnly(tx *transaction.ApiTransactionResult) ([]*eventTransferValueOnly, error) { - isBeforeSirius := !controller.provider.IsReleaseSiriusActive(tx.Epoch) + if !controller.provider.IsReleaseSiriusActive(tx.Epoch) { + return make([]*eventTransferValueOnly, 0), nil + } + rawEvents := controller.findManyEventsByIdentifier(tx, transactionEventTransferValueOnly) typedEvents := make([]*eventTransferValueOnly, 0) for _, event := range rawEvents { - if isBeforeSirius { - typedEvent, err := controller.decideEffectiveEventTransferValueOnlyBeforeSirius(event) - if err != nil { - return nil, err - } - - if typedEvent != nil { - typedEvents = append(typedEvents, typedEvent) - } - } else { - typedEvent, err := controller.decideEffectiveEventTransferValueOnlyAfterSirius(event) - if err != nil { - return nil, err - } - - if typedEvent != nil { - typedEvents = append(typedEvents, typedEvent) - } + typedEvent, err := controller.decideEffectiveEventTransferValueOnlyAfterSirius(event) + if err != nil { + return nil, err } - } - return typedEvents, nil -} - -func (controller *transactionEventsController) decideEffectiveEventTransferValueOnlyBeforeSirius(event *transaction.Events) (*eventTransferValueOnly, error) { - numTopics := len(event.Topics) - - if numTopics != numTopicsOfEventTransferValueOnlyBeforeSirius { - return nil, fmt.Errorf("%w: bad number of topics for 'transferValueOnly' = %d", errCannotRecognizeEvent, numTopics) - } - - senderPubKey := event.Topics[0] - receiverPubKey := event.Topics[1] - valueBytes := event.Topics[2] + if typedEvent != nil { + typedEvents = append(typedEvents, typedEvent) + } - if len(valueBytes) == 0 { - return nil, nil } - sender := controller.provider.ConvertPubKeyToAddress(senderPubKey) - receiver := controller.provider.ConvertPubKeyToAddress(receiverPubKey) - value := big.NewInt(0).SetBytes(valueBytes) - - return &eventTransferValueOnly{ - sender: sender, - receiver: receiver, - value: value.String(), - }, nil + return typedEvents, nil } // See: https://github.com/multiversx/mx-specs/blob/main/releases/protocol/release-specs-v1.6.0-Sirius.md#17-logs--events-changes-5490 diff --git a/server/services/transactionEventsController_test.go b/server/services/transactionEventsController_test.go index 484717f5..b8851d09 100644 --- a/server/services/transactionEventsController_test.go +++ b/server/services/transactionEventsController_test.go @@ -206,7 +206,7 @@ func TestTransactionEventsController_ExtractEvents(t *testing.T) { require.Equal(t, "erd1tn62hjp72rznp8vq0lplva5csav6rccpqqdungpxtqz0g2hcq6uq9k4cc6", events[0].deployerAddress) }) - t.Run("transferValueOnly, before Sirius", func(t *testing.T) { + t.Run("transferValueOnly, before Sirius (not handled at all)", func(t *testing.T) { topic0 := testscommon.TestContractFooShard0.PubKey topic1 := testscommon.TestContractBarShard0.PubKey topic2 := big.NewInt(100).Bytes() @@ -230,10 +230,7 @@ func TestTransactionEventsController_ExtractEvents(t *testing.T) { events, err := controller.extractEventTransferValueOnly(tx) require.NoError(t, err) - require.Len(t, events, 1) - require.Equal(t, testscommon.TestContractFooShard0.Address, events[0].sender) - require.Equal(t, testscommon.TestContractBarShard0.Address, events[0].receiver) - require.Equal(t, "100", events[0].value) + require.Len(t, events, 0) }) t.Run("transferValueOnly, after Sirius, effective (intra-shard ExecuteOnDestContext)", func(t *testing.T) { From b004bf5109f291ae0756f257c013bab134967efc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Mon, 26 Aug 2024 12:12:00 +0300 Subject: [PATCH 67/75] Handle calls & deployments with signal error differently before Sirius. --- server/services/transactionsTransformer.go | 17 +++-- .../services/transactionsTransformer_test.go | 74 +++++++++++++++++++ 2 files changed, 85 insertions(+), 6 deletions(-) diff --git a/server/services/transactionsTransformer.go b/server/services/transactionsTransformer.go index 5bce9a02..50a53ea1 100644 --- a/server/services/transactionsTransformer.go +++ b/server/services/transactionsTransformer.go @@ -188,12 +188,17 @@ func (transformer *transactionsTransformer) rewardTxToRosettaTx(tx *transaction. func (transformer *transactionsTransformer) normalTxToRosetta(tx *transaction.ApiTransactionResult) (*types.Transaction, error) { operations := make([]*types.Operation, 0) - // Special handling of: - // - intra-shard contract calls, bearing value, which fail with signal error - // - direct contract deployments, bearing value, which fail with signal error - // For these, the protocol does not generate an explicit SCR with the value refund (before Sirius, in some cases, it did). - // However, since the value remains at the sender, we don't emit any operations in these circumstances. - transfersValue := isNonZeroAmount(tx.Value) && !transformer.featuresDetector.isContractDeploymentWithSignalErrorOrIntrashardContractCallWithSignalError(tx) + transfersValue := isNonZeroAmount(tx.Value) + + if transformer.provider.IsReleaseSiriusActive(tx.Epoch) { + // Special handling of: + // - intra-shard contract calls, bearing value, which fail with signal error + // - direct contract deployments, bearing value, which fail with signal error + // For these, the protocol does not generate an explicit SCR with the value refund (before Sirius, in some cases, it did). + // However, since the value remains at the sender, we don't emit any operations in these circumstances. + transfersValue = transfersValue && !transformer.featuresDetector.isContractDeploymentWithSignalErrorOrIntrashardContractCallWithSignalError(tx) + } + if transfersValue { operations = append(operations, &types.Operation{ Type: opTransfer, diff --git a/server/services/transactionsTransformer_test.go b/server/services/transactionsTransformer_test.go index cd16ca0a..46e2a0f4 100644 --- a/server/services/transactionsTransformer_test.go +++ b/server/services/transactionsTransformer_test.go @@ -57,6 +57,80 @@ func TestTransactionsTransformer_NormalTxToRosettaTx(t *testing.T) { require.Equal(t, expectedRosettaTx, rosettaTx) }) + t.Run("contract deployment with signal error (after Sirius)", func(t *testing.T) { + tx := &transaction.ApiTransactionResult{ + Epoch: 43, + ProcessingTypeOnSource: "SCDeployment", + ProcessingTypeOnDestination: "SCDeployment", + Hash: "aaaa", + Sender: testscommon.TestAddressAlice, + Receiver: testscommon.TestAddressBob, + Value: "1234", + InitiallyPaidFee: "5000000000000000", + Logs: &transaction.ApiLogs{ + Events: []*transaction.Events{ + { + Identifier: transactionEventSignalError, + }, + }, + }, + } + + expectedRosettaTx := &types.Transaction{ + TransactionIdentifier: hashToTransactionIdentifier("aaaa"), + Operations: []*types.Operation{ + { + Type: opFee, + Account: addressToAccountIdentifier(testscommon.TestAddressAlice), + Amount: extension.valueToNativeAmount("-5000000000000000"), + }, + }, + Metadata: extractTransactionMetadata(tx), + } + + rosettaTx, err := transformer.normalTxToRosetta(tx) + require.NoError(t, err) + require.Equal(t, expectedRosettaTx, rosettaTx) + }) + + t.Run("intra-shard contract call with signal error (after Sirius)", func(t *testing.T) { + tx := &transaction.ApiTransactionResult{ + Epoch: 43, + ProcessingTypeOnSource: "SCInvoking", + ProcessingTypeOnDestination: "SCInvoking", + Hash: "aaaa", + Sender: testscommon.TestUserAShard0.Address, + Receiver: testscommon.TestContractBarShard0.Address, + SourceShard: 0, + DestinationShard: 0, + Value: "1234", + InitiallyPaidFee: "5000000000000000", + Logs: &transaction.ApiLogs{ + Events: []*transaction.Events{ + { + Identifier: transactionEventSignalError, + }, + }, + }, + } + + expectedRosettaTx := &types.Transaction{ + TransactionIdentifier: hashToTransactionIdentifier("aaaa"), + Operations: []*types.Operation{ + { + Type: opFee, + Account: addressToAccountIdentifier(testscommon.TestUserAShard0.Address), + Amount: extension.valueToNativeAmount("-5000000000000000"), + }, + }, + Metadata: extractTransactionMetadata(tx), + } + + rosettaTx, err := transformer.normalTxToRosetta(tx) + require.NoError(t, err) + require.Equal(t, expectedRosettaTx, rosettaTx) + }) + t.Run("relayed tx, completely intrashard, with signal error (before Sirius)", func(t *testing.T) { tx := &transaction.ApiTransactionResult{ Epoch: 41, From 424d4b0cd4ee90e4a84c120d5e0c0859e04b260e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Mon, 26 Aug 2024 13:24:12 +0300 Subject: [PATCH 68/75] Remove unused var. --- server/services/constants.go | 1 - 1 file changed, 1 deletion(-) diff --git a/server/services/constants.go b/server/services/constants.go index 03845823..3f6e146a 100644 --- a/server/services/constants.go +++ b/server/services/constants.go @@ -28,7 +28,6 @@ var ( var ( transactionEventSignalError = core.SignalErrorOperation - transactionEventInternalVMErrors = "internalVMErrors" transactionEventSCDeploy = core.SCDeployIdentifier transactionEventTransferValueOnly = "transferValueOnly" transactionEventESDTTransfer = "ESDTTransfer" From 1dd197fc13d92a9e3c70ce4f7aa9d65b5fd1de6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Mon, 26 Aug 2024 16:50:03 +0300 Subject: [PATCH 69/75] Fix comments. --- server/provider/networkProvider.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/provider/networkProvider.go b/server/provider/networkProvider.go index 82b35f56..65eb5f83 100644 --- a/server/provider/networkProvider.go +++ b/server/provider/networkProvider.go @@ -224,7 +224,8 @@ func (provider *networkProvider) GetBlockByNonce(nonce uint64) (*api.Block, erro return nil, err } - // TODO: returns a mutate copy instead. + // The block (copy) returned by doGetBlockByNonce() is now mutated. + // The mutated copy is not held in a cache (not needed). err = provider.simplifyBlockWithScheduledTransactions(block) if err != nil { return nil, err @@ -259,7 +260,7 @@ func (provider *networkProvider) doGetBlockByNonce(nonce uint64) (*api.Block, er return createBlockCopy(block), nil } -// TOOD: Remove this. Cache the simplified block, instead. +// createBlockCopy creates a somehow shallow copy of a block. func createBlockCopy(block *api.Block) *api.Block { miniblocksCopy := make([]*api.MiniBlock, len(block.MiniBlocks)) From 7ab79e31ec3a666700c56f964b8571c9337b25a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Tue, 27 Aug 2024 12:17:21 +0300 Subject: [PATCH 70/75] Additional explanations, minor refactoring. --- server/provider/constants.go | 1 + server/provider/scheduledMiniblocks.go | 41 ++++++++++++++++++-------- 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/server/provider/constants.go b/server/provider/constants.go index 74cd0c97..edb1b109 100644 --- a/server/provider/constants.go +++ b/server/provider/constants.go @@ -4,4 +4,5 @@ var ( nativeCurrencyNumDecimals = 18 genesisBlockNonce = 0 blocksCacheCapacity = 1024 + miniblockTypeArtificial = "Artificial" ) diff --git a/server/provider/scheduledMiniblocks.go b/server/provider/scheduledMiniblocks.go index a8348c07..7ef18f48 100644 --- a/server/provider/scheduledMiniblocks.go +++ b/server/provider/scheduledMiniblocks.go @@ -23,6 +23,7 @@ func (provider *networkProvider) simplifyBlockWithScheduledTransactions(block *a return nil } +// reportScheduledTransactions logs the number of transactions in miniblocks of types: "Scheduled", "Processed", and "Invalid". func reportScheduledTransactions(block *api.Block) { numScheduled := 0 numProcessed := 0 @@ -47,18 +48,22 @@ func doSimplifyBlockWithScheduledTransactions(previousBlock *api.Block, block *a txs := gatherEffectiveTransactions(block.Shard, previousBlock, block, nextBlock) receipts := gatherAllReceipts(block) + // Downstream, when recovering balance-changing operations, we do not care about the original container (miniblock) of the effective transaction. + // We group all transactions in an "artificial" miniblock, and all receipts in another "artificial" miniblock. block.MiniBlocks = []*api.MiniBlock{ { - Type: "Artificial", + Type: miniblockTypeArtificial, Transactions: txs, }, { - Type: "Artificial", + Type: miniblockTypeArtificial, Receipts: receipts, }, } } +// gatherEffectiveTransactions gathers transactions whose effects (mutation of accounts state) are visible in the current block. +// They are gathered from the previous, current and next block. func gatherEffectiveTransactions(selfShard uint32, previousBlock *api.Block, currentBlock *api.Block, nextBlock *api.Block) []*transaction.ApiTransactionResult { txsInCurrentBlock := gatherAllTransactions(currentBlock) @@ -66,6 +71,7 @@ func gatherEffectiveTransactions(selfShard uint32, previousBlock *api.Block, cur scheduledTxsInCurrentBlock := gatherScheduledTransactions(currentBlock) if len(scheduledTxsInPreviousBlock) == 0 && len(scheduledTxsInCurrentBlock) == 0 { + // Trivial case, no special handling needed. return txsInCurrentBlock } @@ -73,22 +79,24 @@ func gatherEffectiveTransactions(selfShard uint32, previousBlock *api.Block, cur var currentlyExecutedResults []*transaction.ApiTransactionResult if len(scheduledTxsInPreviousBlock) > 0 { + // Look behind, for any contract results that, even if present in the current block, had their effects in the previous block. previouslyExecutedResults = findImmediatelyExecutingContractResults(selfShard, scheduledTxsInPreviousBlock, txsInCurrentBlock) } if len(scheduledTxsInCurrentBlock) > 0 { + // Look ahead, for any contract results that, even if present in the next block, have their effects in the current block. txsInNextBlock := gatherAllTransactions(nextBlock) currentlyExecutedResults = findImmediatelyExecutingContractResults(selfShard, scheduledTxsInCurrentBlock, txsInNextBlock) } // effectiveTxs - // = txsInCurrentBlock - // - txsInPreviousBlock (excludes transactions in "processed" miniblocks, for example) - // - previouslyExecutedResults - // + currentlyExecutedResults + // = txsInCurrentBlock term (a) + // - txsInPreviousBlock (excludes transactions in "processed" miniblocks, for example) term (b) + // - previouslyExecutedResults term (c) + // + currentlyExecutedResults term (d) - effectiveTxs := make([]*transaction.ApiTransactionResult, 0) effectiveTxsByHash := make(map[string]*transaction.ApiTransactionResult) + // term (a) for _, tx := range txsInCurrentBlock { effectiveTxsByHash[tx.Hash] = tx } @@ -96,21 +104,27 @@ func gatherEffectiveTransactions(selfShard uint32, previousBlock *api.Block, cur if len(scheduledTxsInPreviousBlock) > 0 { txsInPreviousBlock := gatherAllTransactions(previousBlock) + // term (b) for _, tx := range txsInPreviousBlock { delete(effectiveTxsByHash, tx.Hash) } + // term (c) for _, tx := range previouslyExecutedResults { delete(effectiveTxsByHash, tx.Hash) } } if len(scheduledTxsInCurrentBlock) > 0 { + // term (d) for _, tx := range currentlyExecutedResults { effectiveTxsByHash[tx.Hash] = tx } } + // Collect the effective transactions in a slice. + effectiveTxs := make([]*transaction.ApiTransactionResult, 0, len(effectiveTxsByHash)) + for _, tx := range effectiveTxsByHash { effectiveTxs = append(effectiveTxs, tx) } @@ -123,7 +137,7 @@ func findImmediatelyExecutingContractResults( transactions []*transaction.ApiTransactionResult, maybeContractResults []*transaction.ApiTransactionResult, ) []*transaction.ApiTransactionResult { - immediateleyExecutingContractResults := make([]*transaction.ApiTransactionResult, 0) + immediatelyExecutingContractResults := make([]*transaction.ApiTransactionResult, 0) nextContractResultsByHash := make(map[string][]*transaction.ApiTransactionResult) for _, item := range maybeContractResults { @@ -131,11 +145,11 @@ func findImmediatelyExecutingContractResults( } for _, tx := range transactions { - immediateleyExecutingContractResultsPart := findImmediatelyExecutingContractResultsOfTransaction(selfShard, tx, nextContractResultsByHash) - immediateleyExecutingContractResults = append(immediateleyExecutingContractResults, immediateleyExecutingContractResultsPart...) + immediatelyExecutingContractResultsPart := findImmediatelyExecutingContractResultsOfTransaction(selfShard, tx, nextContractResultsByHash) + immediatelyExecutingContractResults = append(immediatelyExecutingContractResults, immediatelyExecutingContractResultsPart...) } - return immediateleyExecutingContractResults + return immediatelyExecutingContractResults } func findImmediatelyExecutingContractResultsOfTransaction( @@ -146,8 +160,8 @@ func findImmediatelyExecutingContractResultsOfTransaction( immediatelyExecutingContractResults := make([]*transaction.ApiTransactionResult, 0) for _, nextContractResult := range nextContractResultsByHash[tx.Hash] { - // Not immediately executing. if nextContractResult.SourceShard != selfShard { + // Cross-shard, not immediately executing. continue } @@ -160,6 +174,7 @@ func findImmediatelyExecutingContractResultsOfTransaction( return immediatelyExecutingContractResults } +// gatherScheduledTransactions gathers transactions in miniblocks of type "Scheduled" func gatherScheduledTransactions(block *api.Block) []*transaction.ApiTransactionResult { scheduledTxs := make([]*transaction.ApiTransactionResult, 0) @@ -177,6 +192,7 @@ func gatherScheduledTransactions(block *api.Block) []*transaction.ApiTransaction return scheduledTxs } +// gatherAllTransactions gathers all transactions, from all miniblocks func gatherAllTransactions(block *api.Block) []*transaction.ApiTransactionResult { txs := make([]*transaction.ApiTransactionResult, 0) @@ -189,6 +205,7 @@ func gatherAllTransactions(block *api.Block) []*transaction.ApiTransactionResult return txs } +// gatherAllReceipts gathers all receipts, from all miniblocks func gatherAllReceipts(block *api.Block) []*transaction.ApiReceipt { receipts := make([]*transaction.ApiReceipt, 0) From c7fa86d2f55bd68a6ce632f868934fd97a10eda3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Tue, 27 Aug 2024 14:11:42 +0300 Subject: [PATCH 71/75] Additional comments & explanations. --- server/provider/scheduledMiniblocks.go | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/server/provider/scheduledMiniblocks.go b/server/provider/scheduledMiniblocks.go index 7ef18f48..c36643c0 100644 --- a/server/provider/scheduledMiniblocks.go +++ b/server/provider/scheduledMiniblocks.go @@ -122,7 +122,7 @@ func gatherEffectiveTransactions(selfShard uint32, previousBlock *api.Block, cur } } - // Collect the effective transactions in a slice. + // Collect & return the effective transactions. effectiveTxs := make([]*transaction.ApiTransactionResult, 0, len(effectiveTxsByHash)) for _, tx := range effectiveTxsByHash { @@ -132,42 +132,48 @@ func gatherEffectiveTransactions(selfShard uint32, previousBlock *api.Block, cur return effectiveTxs } +// findImmediatelyExecutingContractResults scans "maybeContractResults" for (immediately executing) contract results of "transactions" func findImmediatelyExecutingContractResults( selfShard uint32, transactions []*transaction.ApiTransactionResult, maybeContractResults []*transaction.ApiTransactionResult, ) []*transaction.ApiTransactionResult { immediatelyExecutingContractResults := make([]*transaction.ApiTransactionResult, 0) - nextContractResultsByHash := make(map[string][]*transaction.ApiTransactionResult) + directContractResultsByHash := make(map[string][]*transaction.ApiTransactionResult) + // Prepare a look-up { transaction or SCR hash } -> { list of direct contract results (direct descendants) }, + // using the "previous transaction hash" link. for _, item := range maybeContractResults { - nextContractResultsByHash[item.PreviousTransactionHash] = append(nextContractResultsByHash[item.PreviousTransactionHash], item) + directContractResultsByHash[item.PreviousTransactionHash] = append(directContractResultsByHash[item.PreviousTransactionHash], item) } + // For each transaction, find (accumulate) all contract results that are immediately executing. for _, tx := range transactions { - immediatelyExecutingContractResultsPart := findImmediatelyExecutingContractResultsOfTransaction(selfShard, tx, nextContractResultsByHash) + immediatelyExecutingContractResultsPart := findImmediatelyExecutingContractResultsOfTransaction(selfShard, tx, directContractResultsByHash) immediatelyExecutingContractResults = append(immediatelyExecutingContractResults, immediatelyExecutingContractResultsPart...) } return immediatelyExecutingContractResults } +// findImmediatelyExecutingContractResultsOfTransaction scans "directContractResultsByHash" for (immediately executing) contract results of "tx" func findImmediatelyExecutingContractResultsOfTransaction( selfShard uint32, tx *transaction.ApiTransactionResult, - nextContractResultsByHash map[string][]*transaction.ApiTransactionResult, + directContractResultsByHash map[string][]*transaction.ApiTransactionResult, ) []*transaction.ApiTransactionResult { immediatelyExecutingContractResults := make([]*transaction.ApiTransactionResult, 0) - for _, nextContractResult := range nextContractResultsByHash[tx.Hash] { - if nextContractResult.SourceShard != selfShard { + for _, directContractResult := range directContractResultsByHash[tx.Hash] { + if directContractResult.SourceShard != selfShard { // Cross-shard, not immediately executing. continue } - immediatelyExecutingContractResults = append(immediatelyExecutingContractResults, nextContractResult) - // Recursive call: - immediatelyExecutingContractResultsPart := findImmediatelyExecutingContractResultsOfTransaction(selfShard, nextContractResult, nextContractResultsByHash) + // Found immediately executing contract result, retain it. + immediatelyExecutingContractResults = append(immediatelyExecutingContractResults, directContractResult) + // Furthermore, recursively find all its (immediately executing) descendants. + immediatelyExecutingContractResultsPart := findImmediatelyExecutingContractResultsOfTransaction(selfShard, directContractResult, directContractResultsByHash) immediatelyExecutingContractResults = append(immediatelyExecutingContractResults, immediatelyExecutingContractResultsPart...) } From a2cc3733c0915b7f820cda90820eab584c55d4b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Tue, 27 Aug 2024 14:57:50 +0300 Subject: [PATCH 72/75] Unit tests --- server/provider/scheduledMiniblocks.go | 2 +- server/provider/scheduledMiniblocks_test.go | 315 +++++++------------- 2 files changed, 102 insertions(+), 215 deletions(-) diff --git a/server/provider/scheduledMiniblocks.go b/server/provider/scheduledMiniblocks.go index c36643c0..3cd87560 100644 --- a/server/provider/scheduledMiniblocks.go +++ b/server/provider/scheduledMiniblocks.go @@ -166,7 +166,7 @@ func findImmediatelyExecutingContractResultsOfTransaction( for _, directContractResult := range directContractResultsByHash[tx.Hash] { if directContractResult.SourceShard != selfShard { - // Cross-shard, not immediately executing. + // Contract result comes from another shard. continue } diff --git a/server/provider/scheduledMiniblocks_test.go b/server/provider/scheduledMiniblocks_test.go index e1e733ee..77d90c88 100644 --- a/server/provider/scheduledMiniblocks_test.go +++ b/server/provider/scheduledMiniblocks_test.go @@ -9,9 +9,10 @@ import ( "github.com/stretchr/testify/require" ) -func TestGatherInvalidTransactions(t *testing.T) { - // Block N-1 - previousBlock := &api.Block{ +func TestGatherEffectiveTransactions(t *testing.T) { + selfShard := uint32(1) + + block_5 := &api.Block{ MiniBlocks: []*api.MiniBlock{ { Type: dataBlock.InvalidBlock.String(), @@ -22,8 +23,7 @@ func TestGatherInvalidTransactions(t *testing.T) { }, } - // Block N - block := &api.Block{ + block_6 := &api.Block{ MiniBlocks: []*api.MiniBlock{ { ProcessingType: dataBlock.Scheduled.String(), @@ -41,8 +41,7 @@ func TestGatherInvalidTransactions(t *testing.T) { }, } - // Block N+1 - nextBlock := &api.Block{ + block_7 := &api.Block{ MiniBlocks: []*api.MiniBlock{ { ProcessingType: dataBlock.Processed.String(), @@ -60,276 +59,164 @@ func TestGatherInvalidTransactions(t *testing.T) { }, } - // "aaaa" is ignored for N, since it produces effects in N-1 - // "eeee" is ignored for N, since it produces effects in N+1 - invalidTxs := gatherInvalidTransactions(previousBlock, block, nextBlock) - require.Len(t, invalidTxs, 1) - require.Equal(t, "bbbb", invalidTxs[0].Hash) -} - -func TestGatherInvalidTransactions_WhenIntraShardIsMissingInPreviousBlock(t *testing.T) { - // Edge case on start of epoch. - - // Block N-1 - previousBlock := &api.Block{ - MiniBlocks: []*api.MiniBlock{ - { - ProcessingType: dataBlock.Scheduled.String(), - Transactions: []*transaction.ApiTransactionResult{ - {Hash: "aaaa"}, - }, - }, - // "aaaa" is an invalid transaction that produces effects in N-1, - // but the intra-shard, "invalid" miniblock is missing (not provided by the API). - }, - } - - // Block N - block := &api.Block{ - MiniBlocks: []*api.MiniBlock{ - { - ProcessingType: dataBlock.Scheduled.String(), - Transactions: []*transaction.ApiTransactionResult{ - {Hash: "abab"}, - {Hash: "cccc"}, - }, - }, - { - Type: dataBlock.InvalidBlock.String(), - Transactions: []*transaction.ApiTransactionResult{ - {Hash: "aaaa"}, - }, - }, - { - // Intra-shard miniblock, holds both "aaaa" (scheduled in N-1, with effects in N) - // and "abab" (scheduled in N, with effects in N) - Type: dataBlock.InvalidBlock.String(), - Transactions: []*transaction.ApiTransactionResult{ - {Hash: "aaaa"}, - {Hash: "abab"}, - }, - }, - }, + block_8 := &api.Block{ + MiniBlocks: []*api.MiniBlock{}, } - // Block N+1 - nextBlock := &api.Block{ - MiniBlocks: []*api.MiniBlock{ - { - ProcessingType: dataBlock.Processed.String(), - Transactions: []*transaction.ApiTransactionResult{ - {Hash: "cccc"}, - }, - }, - { - Type: dataBlock.InvalidBlock.String(), - Transactions: []*transaction.ApiTransactionResult{ - {Hash: "abab"}, - }, - }, - }, - } + txs := gatherEffectiveTransactions(selfShard, block_5, block_6, block_7) + require.Len(t, txs, 2) + require.Contains(t, txs, &transaction.ApiTransactionResult{Hash: "bbbb"}) + require.Contains(t, txs, &transaction.ApiTransactionResult{Hash: "cccc"}) - // "aaaa" is ignored for N, since it produces effects in N-1 - invalidTxs := gatherInvalidTransactions(previousBlock, block, nextBlock) - require.Len(t, invalidTxs, 1) - require.Equal(t, "abab", invalidTxs[0].Hash) + txs = gatherEffectiveTransactions(selfShard, block_6, block_7, block_8) + require.Len(t, txs, 1) + require.Contains(t, txs, &transaction.ApiTransactionResult{Hash: "eeee"}) } -func TestDoSimplifyBlockWithScheduledTransactions_WithRespectToConstructionState(t *testing.T) { - // Edge case on cross-shard miniblocks, both scheduled and final. +func TestFindImmediatelyExecutingContractResults(t *testing.T) { + selfShard := uint32(1) + otherShard := uint32(0) - // Empty, trivial blocks at N-1 and N+1 - previousBlock := &api.Block{MiniBlocks: []*api.MiniBlock{}} - nextBlock := &api.Block{MiniBlocks: []*api.MiniBlock{}} + t.Run("trivial case (no transactions, no smart contract results)", func(t *testing.T) { + transactions := []*transaction.ApiTransactionResult{} + maybeContractResults := []*transaction.ApiTransactionResult{} - // Scheduled & Final (won't be removed) - block := &api.Block{ - MiniBlocks: []*api.MiniBlock{ - { - ProcessingType: dataBlock.Scheduled.String(), - ConstructionState: dataBlock.Final.String(), - Transactions: []*transaction.ApiTransactionResult{ - {Hash: "aaaa"}, - {Hash: "bbbb"}, - }, - }, - }, - } + results := findImmediatelyExecutingContractResults(selfShard, transactions, maybeContractResults) + require.Len(t, results, 0) + }) - doSimplifyBlockWithScheduledTransactions(previousBlock, block, nextBlock) - require.Len(t, block.MiniBlocks, 1) - require.Len(t, block.MiniBlocks[0].Transactions, 2) - require.Equal(t, "aaaa", block.MiniBlocks[0].Transactions[0].Hash) - require.Equal(t, "bbbb", block.MiniBlocks[0].Transactions[1].Hash) + t.Run("trivial case (no smart contract results)", func(t *testing.T) { + transactions := []*transaction.ApiTransactionResult{ + {Hash: "aaaa"}, + {Hash: "bbbb"}, + } - // Scheduled & !Final (will be removed) - block = &api.Block{ - MiniBlocks: []*api.MiniBlock{ - { - ProcessingType: dataBlock.Scheduled.String(), - Transactions: []*transaction.ApiTransactionResult{ - {Hash: "aaaa"}, - {Hash: "bbbb"}, - }, - }, - }, - } + maybeContractResults := []*transaction.ApiTransactionResult{} + + results := findImmediatelyExecutingContractResults(selfShard, transactions, maybeContractResults) + require.Len(t, results, 0) + }) + + t.Run("with contract results (only direct descendants)", func(t *testing.T) { + transactions := []*transaction.ApiTransactionResult{ + {Hash: "aaaa"}, + {Hash: "bbbb"}, + } + + maybeContractResults := []*transaction.ApiTransactionResult{ + {Hash: "aa00", PreviousTransactionHash: "aaaa", SourceShard: selfShard}, + {Hash: "aa11", PreviousTransactionHash: "aaaa", SourceShard: selfShard}, + {Hash: "bb00", PreviousTransactionHash: "bbbb", SourceShard: otherShard}, + {Hash: "bb11", PreviousTransactionHash: "bbbb", SourceShard: selfShard}, + } + + results := findImmediatelyExecutingContractResults(selfShard, transactions, maybeContractResults) + require.Len(t, results, 3) + require.Contains(t, results, &transaction.ApiTransactionResult{Hash: "aa00", PreviousTransactionHash: "aaaa", SourceShard: selfShard}) + require.Contains(t, results, &transaction.ApiTransactionResult{Hash: "aa11", PreviousTransactionHash: "aaaa", SourceShard: selfShard}) + require.Contains(t, results, &transaction.ApiTransactionResult{Hash: "bb11", PreviousTransactionHash: "bbbb", SourceShard: selfShard}) + }) - doSimplifyBlockWithScheduledTransactions(previousBlock, block, nextBlock) - require.Len(t, block.MiniBlocks, 0) + t.Run("with contract results (direct and indirect descendants)", func(t *testing.T) { + transactions := []*transaction.ApiTransactionResult{ + {Hash: "aaaa"}, + {Hash: "bbbb"}, + } + + maybeContractResults := []*transaction.ApiTransactionResult{ + {Hash: "aa00", PreviousTransactionHash: "aaaa", SourceShard: selfShard}, + {Hash: "aa11", PreviousTransactionHash: "aaaa", SourceShard: selfShard}, + {Hash: "bb00", PreviousTransactionHash: "bbbb", SourceShard: otherShard}, + {Hash: "bb11", PreviousTransactionHash: "bbbb", SourceShard: selfShard}, + {Hash: "cc00", PreviousTransactionHash: "aa00", SourceShard: selfShard}, + {Hash: "cc11", PreviousTransactionHash: "aa00", SourceShard: selfShard}, + {Hash: "dd00", PreviousTransactionHash: "bb00", SourceShard: otherShard}, + {Hash: "dd11", PreviousTransactionHash: "bb00", SourceShard: selfShard}, + } + + results := findImmediatelyExecutingContractResults(selfShard, transactions, maybeContractResults) + require.Len(t, results, 5) + require.Contains(t, results, &transaction.ApiTransactionResult{Hash: "aa00", PreviousTransactionHash: "aaaa", SourceShard: selfShard}) + require.Contains(t, results, &transaction.ApiTransactionResult{Hash: "aa11", PreviousTransactionHash: "aaaa", SourceShard: selfShard}) + require.Contains(t, results, &transaction.ApiTransactionResult{Hash: "bb11", PreviousTransactionHash: "bbbb", SourceShard: selfShard}) + require.Contains(t, results, &transaction.ApiTransactionResult{Hash: "cc00", PreviousTransactionHash: "aa00", SourceShard: selfShard}) + require.Contains(t, results, &transaction.ApiTransactionResult{Hash: "cc11", PreviousTransactionHash: "aa00", SourceShard: selfShard}) + }) } -func TestDeduplicatePreviouslyAppearingContractResultsInReceipts(t *testing.T) { - // Block N-1 - previousBlock := &api.Block{ +func TestGatherScheduledTransactions(t *testing.T) { + block := &api.Block{ MiniBlocks: []*api.MiniBlock{ - // Should not be subject to deduplication (not from receipts) { - IsFromReceiptsStorage: false, - Type: dataBlock.SmartContractResultBlock.String(), + ProcessingType: dataBlock.Scheduled.String(), Transactions: []*transaction.ApiTransactionResult{ {Hash: "aaaa"}, - }, - }, - // Should not be subject to deduplication (due to "type") - { - IsFromReceiptsStorage: true, - Type: dataBlock.InvalidBlock.String(), - Transactions: []*transaction.ApiTransactionResult{ {Hash: "bbbb"}, }, }, - // Should be subject to deduplication { - IsFromReceiptsStorage: true, - Type: dataBlock.SmartContractResultBlock.String(), + ProcessingType: dataBlock.Normal.String(), Transactions: []*transaction.ApiTransactionResult{ {Hash: "cccc"}, {Hash: "dddd"}, }, }, - }, - } - - // Block N - block := &api.Block{ - MiniBlocks: []*api.MiniBlock{ - // Should not be subject to deduplication (not from receipts) - { - IsFromReceiptsStorage: false, - Type: dataBlock.SmartContractResultBlock.String(), - Transactions: []*transaction.ApiTransactionResult{ - {Hash: "aaaa"}, - }, - }, - // Should not be subject to deduplication (due to "type") - { - IsFromReceiptsStorage: true, - Type: dataBlock.InvalidBlock.String(), - Transactions: []*transaction.ApiTransactionResult{ - {Hash: "bbbb"}, - }, - }, - // Should be subject to deduplication { - IsFromReceiptsStorage: true, - Type: dataBlock.SmartContractResultBlock.String(), + Type: dataBlock.Processed.String(), Transactions: []*transaction.ApiTransactionResult{ - {Hash: "cccc"}, {Hash: "eeee"}, + {Hash: "ffff"}, }, }, }, } - deduplicatePreviouslyAppearingContractResultsInReceipts(previousBlock, block) - - require.Len(t, block.MiniBlocks, 3) - require.Len(t, block.MiniBlocks[0].Transactions, 1) - require.Len(t, block.MiniBlocks[1].Transactions, 1) - require.Len(t, block.MiniBlocks[2].Transactions, 1) - require.Equal(t, "aaaa", block.MiniBlocks[0].Transactions[0].Hash) - require.Equal(t, "bbbb", block.MiniBlocks[1].Transactions[0].Hash) - require.Equal(t, "eeee", block.MiniBlocks[2].Transactions[0].Hash) + txs := gatherScheduledTransactions(block) + require.Len(t, txs, 2) + require.Equal(t, "aaaa", txs[0].Hash) + require.Equal(t, "bbbb", txs[1].Hash) } -func TestFindContractResultsInReceipts(t *testing.T) { +func TestGatherAllTransactions(t *testing.T) { block := &api.Block{ MiniBlocks: []*api.MiniBlock{ { - IsFromReceiptsStorage: false, - Type: dataBlock.SmartContractResultBlock.String(), Transactions: []*transaction.ApiTransactionResult{ {Hash: "aaaa"}, {Hash: "bbbb"}, }, }, { - IsFromReceiptsStorage: true, - Type: dataBlock.InvalidBlock.String(), Transactions: []*transaction.ApiTransactionResult{ {Hash: "cccc"}, {Hash: "dddd"}, }, }, - { - IsFromReceiptsStorage: true, - Type: dataBlock.SmartContractResultBlock.String(), - Transactions: []*transaction.ApiTransactionResult{ - {Hash: "eeee"}, - {Hash: "ffff"}, - }, - }, }, } - found := findContractResultsInReceipts(block) - require.Len(t, found, 2) - require.Contains(t, found, "eeee") - require.Contains(t, found, "ffff") + txs := gatherAllTransactions(block) + require.Len(t, txs, 4) } -func TestRemoveContractResultsInReceipts(t *testing.T) { +func TestGatherAllReceipts(t *testing.T) { block := &api.Block{ MiniBlocks: []*api.MiniBlock{ { - IsFromReceiptsStorage: false, - Type: dataBlock.SmartContractResultBlock.String(), - Transactions: []*transaction.ApiTransactionResult{ - {Hash: "aaaa"}, - {Hash: "bbbb"}, + Receipts: []*transaction.ApiReceipt{ + {TxHash: "aaaa"}, + {TxHash: "bbbb"}, }, }, { - IsFromReceiptsStorage: true, - Type: dataBlock.InvalidBlock.String(), - Transactions: []*transaction.ApiTransactionResult{ - {Hash: "cccc"}, - {Hash: "dddd"}, - }, - }, - { - IsFromReceiptsStorage: true, - Type: dataBlock.SmartContractResultBlock.String(), - Transactions: []*transaction.ApiTransactionResult{ - {Hash: "eeee"}, - {Hash: "ffff"}, - {Hash: "abba"}, - {Hash: "aabb"}, + Receipts: []*transaction.ApiReceipt{ + {TxHash: "cccc"}, + {TxHash: "dddd"}, }, }, }, } - removeContractResultsInReceipts(block, map[string]struct{}{ - "eeee": {}, - "ffff": {}, - }) - - require.Len(t, block.MiniBlocks[0].Transactions, 2) - require.Len(t, block.MiniBlocks[1].Transactions, 2) - require.Len(t, block.MiniBlocks[2].Transactions, 2) - require.Equal(t, "abba", block.MiniBlocks[2].Transactions[0].Hash) - require.Equal(t, "aabb", block.MiniBlocks[2].Transactions[1].Hash) + receipts := gatherAllReceipts(block) + require.Len(t, receipts, 4) } From f0d79ad7332e902ad907364e66a11f10d61b34e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Tue, 27 Aug 2024 14:58:09 +0300 Subject: [PATCH 73/75] Simplification: omit field "IsFromReceiptsStorage". --- server/provider/networkProvider.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/server/provider/networkProvider.go b/server/provider/networkProvider.go index 65eb5f83..9cf07987 100644 --- a/server/provider/networkProvider.go +++ b/server/provider/networkProvider.go @@ -266,13 +266,12 @@ func createBlockCopy(block *api.Block) *api.Block { for i, miniblock := range block.MiniBlocks { miniblocksCopy[i] = &api.MiniBlock{ - Type: miniblock.Type, - Hash: miniblock.Hash, - ProcessingType: miniblock.ProcessingType, - ConstructionState: miniblock.ConstructionState, - IsFromReceiptsStorage: miniblock.IsFromReceiptsStorage, - SourceShard: miniblock.SourceShard, - DestinationShard: miniblock.DestinationShard, + Type: miniblock.Type, + Hash: miniblock.Hash, + ProcessingType: miniblock.ProcessingType, + ConstructionState: miniblock.ConstructionState, + SourceShard: miniblock.SourceShard, + DestinationShard: miniblock.DestinationShard, // This is sufficient for our purposes (we don't mutate the transactions themselves, we only mutate the list of transactions within a miniblock). Transactions: miniblock.Transactions, Receipts: miniblock.Receipts, From adf50fa8c942941d7d8cbc3decf2c88c5db00dd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Tue, 27 Aug 2024 17:32:28 +0300 Subject: [PATCH 74/75] Additional testing. --- server/provider/scheduledMiniblocks.go | 6 +- server/provider/scheduledMiniblocks_test.go | 69 ++++++++++++++++++--- 2 files changed, 63 insertions(+), 12 deletions(-) diff --git a/server/provider/scheduledMiniblocks.go b/server/provider/scheduledMiniblocks.go index 3cd87560..a6fc142a 100644 --- a/server/provider/scheduledMiniblocks.go +++ b/server/provider/scheduledMiniblocks.go @@ -79,11 +79,13 @@ func gatherEffectiveTransactions(selfShard uint32, previousBlock *api.Block, cur var currentlyExecutedResults []*transaction.ApiTransactionResult if len(scheduledTxsInPreviousBlock) > 0 { - // Look behind, for any contract results that, even if present in the current block, had their effects in the previous block. + // Look behind, for any contract results that, even if present in the current block, had their effects in the previous block, + // where their parent transaction was "scheduled". previouslyExecutedResults = findImmediatelyExecutingContractResults(selfShard, scheduledTxsInPreviousBlock, txsInCurrentBlock) } if len(scheduledTxsInCurrentBlock) > 0 { - // Look ahead, for any contract results that, even if present in the next block, have their effects in the current block. + // Look ahead, for any contract results that, even if present in the next block, have their effects in the current block, + // where their parent transaction is "scheduled". txsInNextBlock := gatherAllTransactions(nextBlock) currentlyExecutedResults = findImmediatelyExecutingContractResults(selfShard, scheduledTxsInCurrentBlock, txsInNextBlock) } diff --git a/server/provider/scheduledMiniblocks_test.go b/server/provider/scheduledMiniblocks_test.go index 77d90c88..315f3c05 100644 --- a/server/provider/scheduledMiniblocks_test.go +++ b/server/provider/scheduledMiniblocks_test.go @@ -12,12 +12,21 @@ import ( func TestGatherEffectiveTransactions(t *testing.T) { selfShard := uint32(1) + tx_a := &transaction.ApiTransactionResult{Hash: "aaaa"} + tx_b := &transaction.ApiTransactionResult{Hash: "bbbb"} + tx_c := &transaction.ApiTransactionResult{Hash: "cccc"} + tx_d := &transaction.ApiTransactionResult{Hash: "dddd"} + tx_e := &transaction.ApiTransactionResult{Hash: "eeee"} + tx_f := &transaction.ApiTransactionResult{Hash: "ffff"} + tx_g := &transaction.ApiTransactionResult{Hash: "ffaa", PreviousTransactionHash: tx_e.Hash, SourceShard: selfShard} + tx_i := &transaction.ApiTransactionResult{Hash: "ffbb", PreviousTransactionHash: tx_f.Hash, SourceShard: selfShard} + block_5 := &api.Block{ MiniBlocks: []*api.MiniBlock{ { Type: dataBlock.InvalidBlock.String(), Transactions: []*transaction.ApiTransactionResult{ - {Hash: "aaaa"}, + tx_a, }, }, }, @@ -28,14 +37,14 @@ func TestGatherEffectiveTransactions(t *testing.T) { { ProcessingType: dataBlock.Scheduled.String(), Transactions: []*transaction.ApiTransactionResult{ - {Hash: "bbbb"}, - {Hash: "cccc"}, + tx_b, + tx_c, }, }, { Type: dataBlock.InvalidBlock.String(), Transactions: []*transaction.ApiTransactionResult{ - {Hash: "bbbb"}, + tx_b, }, }, }, @@ -46,14 +55,14 @@ func TestGatherEffectiveTransactions(t *testing.T) { { ProcessingType: dataBlock.Processed.String(), Transactions: []*transaction.ApiTransactionResult{ - {Hash: "cccc"}, + tx_c, }, }, { Type: dataBlock.InvalidBlock.String(), Transactions: []*transaction.ApiTransactionResult{ - {Hash: "bbbb"}, - {Hash: "eeee"}, + tx_b, + tx_d, }, }, }, @@ -63,14 +72,54 @@ func TestGatherEffectiveTransactions(t *testing.T) { MiniBlocks: []*api.MiniBlock{}, } + block_9 := &api.Block{ + MiniBlocks: []*api.MiniBlock{ + { + ProcessingType: dataBlock.Scheduled.String(), + Transactions: []*transaction.ApiTransactionResult{ + tx_e, + }, + }, + }, + } + + block_10 := &api.Block{ + MiniBlocks: []*api.MiniBlock{ + { + Transactions: []*transaction.ApiTransactionResult{ + tx_f, + }, + }, + { + Transactions: []*transaction.ApiTransactionResult{ + tx_g, + tx_i, + }, + }, + }, + } + + // Current block is 6 txs := gatherEffectiveTransactions(selfShard, block_5, block_6, block_7) require.Len(t, txs, 2) - require.Contains(t, txs, &transaction.ApiTransactionResult{Hash: "bbbb"}) - require.Contains(t, txs, &transaction.ApiTransactionResult{Hash: "cccc"}) + require.Contains(t, txs, tx_b) + require.Contains(t, txs, tx_c) + // Current block is 7 txs = gatherEffectiveTransactions(selfShard, block_6, block_7, block_8) require.Len(t, txs, 1) - require.Contains(t, txs, &transaction.ApiTransactionResult{Hash: "eeee"}) + require.Contains(t, txs, tx_d) + + // Current block is 8 + txs = gatherEffectiveTransactions(selfShard, block_7, block_8, block_9) + require.Len(t, txs, 0) + + // Current block is 9 + txs = gatherEffectiveTransactions(selfShard, block_8, block_9, block_10) + require.Len(t, txs, 2) + require.Contains(t, txs, tx_e) + require.Contains(t, txs, tx_g) + } func TestFindImmediatelyExecutingContractResults(t *testing.T) { From 6476e07d276a5e8675b6d3bc602d20f7b97ce4c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Tue, 27 Aug 2024 18:34:46 +0300 Subject: [PATCH 75/75] Additional tests. --- server/provider/scheduledMiniblocks_test.go | 140 +++++++++++++++++--- 1 file changed, 121 insertions(+), 19 deletions(-) diff --git a/server/provider/scheduledMiniblocks_test.go b/server/provider/scheduledMiniblocks_test.go index 315f3c05..cdd275fc 100644 --- a/server/provider/scheduledMiniblocks_test.go +++ b/server/provider/scheduledMiniblocks_test.go @@ -1,14 +1,107 @@ package provider import ( + "errors" "testing" "github.com/multiversx/mx-chain-core-go/data/api" dataBlock "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/transaction" + "github.com/multiversx/mx-chain-proxy-go/common" + "github.com/multiversx/mx-chain-proxy-go/data" + "github.com/multiversx/mx-chain-rosetta/testscommon" "github.com/stretchr/testify/require" ) +func TestNetworkProviderSimplifyBlockWithScheduledTransactions(t *testing.T) { + observerFacade := testscommon.NewObserverFacadeMock() + args := createDefaultArgsNewNetworkProvider() + args.ObserverFacade = observerFacade + + provider, err := NewNetworkProvider(args) + require.Nil(t, err) + require.NotNil(t, provider) + + tx_a := &transaction.ApiTransactionResult{Hash: "aaaa"} + tx_b := &transaction.ApiTransactionResult{Hash: "bbbb"} + tx_c := &transaction.ApiTransactionResult{Hash: "cccc"} + tx_d := &transaction.ApiTransactionResult{Hash: "dddd"} + + blocks := []*api.Block{ + { + Nonce: 0, + MiniBlocks: []*api.MiniBlock{ + { + Type: dataBlock.InvalidBlock.String(), + Transactions: []*transaction.ApiTransactionResult{ + tx_a, + }, + }, + }, + }, + { + Nonce: 1, + MiniBlocks: []*api.MiniBlock{ + { + ProcessingType: dataBlock.Scheduled.String(), + Transactions: []*transaction.ApiTransactionResult{ + tx_b, + tx_c, + }, + }, + { + Type: dataBlock.InvalidBlock.String(), + Transactions: []*transaction.ApiTransactionResult{ + tx_b, + }, + }, + }, + }, + { + Nonce: 2, + MiniBlocks: []*api.MiniBlock{ + { + ProcessingType: dataBlock.Processed.String(), + Transactions: []*transaction.ApiTransactionResult{ + tx_c, + }, + }, + { + Type: dataBlock.InvalidBlock.String(), + Transactions: []*transaction.ApiTransactionResult{ + tx_b, + tx_d, + }, + }, + }, + }, + } + + observerFacade.GetBlockByNonceCalled = func(shardID uint32, nonce uint64, options common.BlockQueryOptions) (*data.BlockApiResponse, error) { + if int(nonce) < len(blocks) { + return &data.BlockApiResponse{ + Data: data.BlockApiResponsePayload{ + Block: *blocks[nonce], + }, + }, nil + } + + return nil, errors.New("unexpected request") + } + + err = provider.simplifyBlockWithScheduledTransactions(blocks[1]) + require.Nil(t, err) + + require.Len(t, blocks[1].MiniBlocks, 2) + require.Equal(t, miniblockTypeArtificial, blocks[1].MiniBlocks[0].Type) + require.Equal(t, miniblockTypeArtificial, blocks[1].MiniBlocks[1].Type) + + txs := blocks[1].MiniBlocks[0].Transactions + require.Len(t, txs, 2) + require.Contains(t, txs, tx_b) + require.Contains(t, txs, tx_c) +} + func TestGatherEffectiveTransactions(t *testing.T) { selfShard := uint32(1) @@ -18,10 +111,10 @@ func TestGatherEffectiveTransactions(t *testing.T) { tx_d := &transaction.ApiTransactionResult{Hash: "dddd"} tx_e := &transaction.ApiTransactionResult{Hash: "eeee"} tx_f := &transaction.ApiTransactionResult{Hash: "ffff"} - tx_g := &transaction.ApiTransactionResult{Hash: "ffaa", PreviousTransactionHash: tx_e.Hash, SourceShard: selfShard} - tx_i := &transaction.ApiTransactionResult{Hash: "ffbb", PreviousTransactionHash: tx_f.Hash, SourceShard: selfShard} + tx_g_result_of_e := &transaction.ApiTransactionResult{Hash: "ffaa", PreviousTransactionHash: tx_e.Hash, SourceShard: selfShard} + tx_i_result_of_f := &transaction.ApiTransactionResult{Hash: "ffbb", PreviousTransactionHash: tx_f.Hash, SourceShard: selfShard} - block_5 := &api.Block{ + block_0 := &api.Block{ MiniBlocks: []*api.MiniBlock{ { Type: dataBlock.InvalidBlock.String(), @@ -32,7 +125,7 @@ func TestGatherEffectiveTransactions(t *testing.T) { }, } - block_6 := &api.Block{ + block_1 := &api.Block{ MiniBlocks: []*api.MiniBlock{ { ProcessingType: dataBlock.Scheduled.String(), @@ -50,7 +143,7 @@ func TestGatherEffectiveTransactions(t *testing.T) { }, } - block_7 := &api.Block{ + block_2 := &api.Block{ MiniBlocks: []*api.MiniBlock{ { ProcessingType: dataBlock.Processed.String(), @@ -68,11 +161,11 @@ func TestGatherEffectiveTransactions(t *testing.T) { }, } - block_8 := &api.Block{ + block_3 := &api.Block{ MiniBlocks: []*api.MiniBlock{}, } - block_9 := &api.Block{ + block_4 := &api.Block{ MiniBlocks: []*api.MiniBlock{ { ProcessingType: dataBlock.Scheduled.String(), @@ -83,7 +176,7 @@ func TestGatherEffectiveTransactions(t *testing.T) { }, } - block_10 := &api.Block{ + block_5 := &api.Block{ MiniBlocks: []*api.MiniBlock{ { Transactions: []*transaction.ApiTransactionResult{ @@ -92,34 +185,43 @@ func TestGatherEffectiveTransactions(t *testing.T) { }, { Transactions: []*transaction.ApiTransactionResult{ - tx_g, - tx_i, + tx_g_result_of_e, + tx_i_result_of_f, }, }, }, } - // Current block is 6 - txs := gatherEffectiveTransactions(selfShard, block_5, block_6, block_7) + block_6 := &api.Block{ + MiniBlocks: []*api.MiniBlock{}, + } + + // Current block is 1 + txs := gatherEffectiveTransactions(selfShard, block_0, block_1, block_2) require.Len(t, txs, 2) require.Contains(t, txs, tx_b) require.Contains(t, txs, tx_c) - // Current block is 7 - txs = gatherEffectiveTransactions(selfShard, block_6, block_7, block_8) + // Current block is 2 + txs = gatherEffectiveTransactions(selfShard, block_1, block_2, block_3) require.Len(t, txs, 1) require.Contains(t, txs, tx_d) - // Current block is 8 - txs = gatherEffectiveTransactions(selfShard, block_7, block_8, block_9) + // Current block is 3 + txs = gatherEffectiveTransactions(selfShard, block_2, block_3, block_4) require.Len(t, txs, 0) - // Current block is 9 - txs = gatherEffectiveTransactions(selfShard, block_8, block_9, block_10) + // Current block is 4 + txs = gatherEffectiveTransactions(selfShard, block_3, block_4, block_5) require.Len(t, txs, 2) require.Contains(t, txs, tx_e) - require.Contains(t, txs, tx_g) + require.Contains(t, txs, tx_g_result_of_e) + // Current block is 5 + txs = gatherEffectiveTransactions(selfShard, block_4, block_5, block_6) + require.Len(t, txs, 2) + require.Contains(t, txs, tx_f) + require.Contains(t, txs, tx_i_result_of_f) } func TestFindImmediatelyExecutingContractResults(t *testing.T) {