From f5f4d5672edf12bcbb3761e6248ec7cdd22d9d5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Thu, 22 Aug 2024 16:43:39 +0300 Subject: [PATCH 1/2] Handle & dismiss ineffective refunds. --- server/provider/scheduledMiniblocks.go | 231 ++++++++++-------- server/services/networkProviderExtension.go | 4 + .../services/transactionsFeaturesDetector.go | 16 +- .../transactionsFeaturesDetector_test.go | 23 ++ server/services/transactionsTransformer.go | 9 + .../services/transactionsTransformer_test.go | 106 ++++---- systemtests/check_with_mesh_cli.py | 1 + 7 files changed, 243 insertions(+), 147 deletions(-) diff --git a/server/provider/scheduledMiniblocks.go b/server/provider/scheduledMiniblocks.go index 3691e1ed..a8348c07 100644 --- a/server/provider/scheduledMiniblocks.go +++ b/server/provider/scheduledMiniblocks.go @@ -17,157 +17,186 @@ func (provider *networkProvider) simplifyBlockWithScheduledTransactions(block *a return err } + reportScheduledTransactions(block) doSimplifyBlockWithScheduledTransactions(previousBlock, block, nextBlock) - deduplicatePreviouslyAppearingContractResultsInReceipts(previousBlock, block) return nil } -func doSimplifyBlockWithScheduledTransactions(previousBlock *api.Block, block *api.Block, nextBlock *api.Block) { - // Discard "processed" miniblocks in block N, since they already produced effects in N-1 - removeProcessedMiniblocksOfBlock(block) - - // Move "processed" miniblocks from N+1 to N - processedMiniblocksInNextBlock := findProcessedMiniblocks(nextBlock) - appendMiniblocksToBlock(block, processedMiniblocksInNextBlock) +func reportScheduledTransactions(block *api.Block) { + numScheduled := 0 + numProcessed := 0 + numInvalid := 0 - // Build an artificial miniblock holding the "invalid" transactions that produced their effects in block N, - // and replace the existing (one or two "invalid" miniblocks). - invalidTxs := gatherInvalidTransactions(previousBlock, block, nextBlock) - invalidMiniblock := &api.MiniBlock{ - Type: dataBlock.InvalidBlock.String(), - Transactions: invalidTxs, + for _, miniblock := range block.MiniBlocks { + if miniblock.ProcessingType == dataBlock.Scheduled.String() { + numScheduled += len(miniblock.Transactions) + } else if miniblock.ProcessingType == dataBlock.Processed.String() { + numProcessed += len(miniblock.Transactions) + } else if miniblock.Type == dataBlock.InvalidBlock.String() { + numInvalid += len(miniblock.Transactions) + } } - removeInvalidMiniblocks(block) - if len(invalidMiniblock.Transactions) > 0 { - appendMiniblocksToBlock(block, []*api.MiniBlock{invalidMiniblock}) + if numScheduled > 0 || numProcessed > 0 { + log.Info("reportScheduledTransactions()", "scheduled", numScheduled, "processed", numProcessed, "invalid", numInvalid, "block", block.Nonce) } - - // Discard "scheduled" miniblocks of N, since we've already brought the "processed" ones from N+1, - // and also handled the "invalid" ones. - removeScheduledMiniblocks(block) } -func removeProcessedMiniblocksOfBlock(block *api.Block) { - removeMiniblocksFromBlock(block, func(miniblock *api.MiniBlock) bool { - return miniblock.ProcessingType == dataBlock.Processed.String() - }) +func doSimplifyBlockWithScheduledTransactions(previousBlock *api.Block, block *api.Block, nextBlock *api.Block) { + txs := gatherEffectiveTransactions(block.Shard, previousBlock, block, nextBlock) + receipts := gatherAllReceipts(block) + + block.MiniBlocks = []*api.MiniBlock{ + { + Type: "Artificial", + Transactions: txs, + }, + { + Type: "Artificial", + Receipts: receipts, + }, + } } -func removeScheduledMiniblocks(block *api.Block) { - removeMiniblocksFromBlock(block, func(miniblock *api.MiniBlock) bool { - hasProcessingTypeScheduled := miniblock.ProcessingType == dataBlock.Scheduled.String() - hasConstructionStateNotFinal := miniblock.ConstructionState != dataBlock.Final.String() - shouldRemove := hasProcessingTypeScheduled && hasConstructionStateNotFinal - return shouldRemove - }) -} +func gatherEffectiveTransactions(selfShard uint32, previousBlock *api.Block, currentBlock *api.Block, nextBlock *api.Block) []*transaction.ApiTransactionResult { + txsInCurrentBlock := gatherAllTransactions(currentBlock) -func removeInvalidMiniblocks(block *api.Block) { - removeMiniblocksFromBlock(block, func(miniblock *api.MiniBlock) bool { - return miniblock.Type == dataBlock.InvalidBlock.String() - }) -} + scheduledTxsInPreviousBlock := gatherScheduledTransactions(previousBlock) + scheduledTxsInCurrentBlock := gatherScheduledTransactions(currentBlock) -func gatherInvalidTransactions(previousBlock *api.Block, block *api.Block, nextBlock *api.Block) []*transaction.ApiTransactionResult { - // Find "invalid" transactions that are "final" in N - invalidTxsInBlock := findInvalidTransactions(block) - // If also present in N-1, discard them - scheduledTxsHashesPreviousBlock := findScheduledTransactionsHashes(previousBlock) - invalidTxsInBlock = discardTransactions(invalidTxsInBlock, scheduledTxsHashesPreviousBlock) + if len(scheduledTxsInPreviousBlock) == 0 && len(scheduledTxsInCurrentBlock) == 0 { + return txsInCurrentBlock + } - // Find "invalid" transactions in N+1 that are "scheduled" in N - invalidTxsInNextBlock := findInvalidTransactions(nextBlock) - scheduledTxsHashesInBlock := findScheduledTransactionsHashes(block) - invalidTxsScheduledInBlock := filterTransactions(invalidTxsInNextBlock, scheduledTxsHashesInBlock) + var previouslyExecutedResults []*transaction.ApiTransactionResult + var currentlyExecutedResults []*transaction.ApiTransactionResult - // Duplication might occur, since a block can contain two "invalid" miniblocks, - // one added to block body, one saved in the receipts unit (at times, they have different content, different hashes). - invalidTxs := append(invalidTxsInBlock, invalidTxsScheduledInBlock...) - invalidTxs = deduplicateTransactions(invalidTxs) + if len(scheduledTxsInPreviousBlock) > 0 { + previouslyExecutedResults = findImmediatelyExecutingContractResults(selfShard, scheduledTxsInPreviousBlock, txsInCurrentBlock) + } + if len(scheduledTxsInCurrentBlock) > 0 { + txsInNextBlock := gatherAllTransactions(nextBlock) + currentlyExecutedResults = findImmediatelyExecutingContractResults(selfShard, scheduledTxsInCurrentBlock, txsInNextBlock) + } - return invalidTxs -} + // effectiveTxs + // = txsInCurrentBlock + // - txsInPreviousBlock (excludes transactions in "processed" miniblocks, for example) + // - previouslyExecutedResults + // + currentlyExecutedResults -func findScheduledTransactionsHashes(block *api.Block) map[string]struct{} { - txs := make(map[string]struct{}) + effectiveTxs := make([]*transaction.ApiTransactionResult, 0) + effectiveTxsByHash := make(map[string]*transaction.ApiTransactionResult) - for _, miniblock := range block.MiniBlocks { - hasProcessingTypeScheduled := miniblock.ProcessingType == dataBlock.Scheduled.String() - hasConstructionStateNotFinal := miniblock.ConstructionState != dataBlock.Final.String() - shouldAccumulateTxs := hasProcessingTypeScheduled && hasConstructionStateNotFinal - - if shouldAccumulateTxs { - for _, tx := range miniblock.Transactions { - txs[tx.Hash] = struct{}{} - } - } + for _, tx := range txsInCurrentBlock { + effectiveTxsByHash[tx.Hash] = tx } - return txs -} + if len(scheduledTxsInPreviousBlock) > 0 { + txsInPreviousBlock := gatherAllTransactions(previousBlock) -func findProcessedMiniblocks(block *api.Block) []*api.MiniBlock { - foundMiniblocks := make([]*api.MiniBlock, 0, len(block.MiniBlocks)) + for _, tx := range txsInPreviousBlock { + delete(effectiveTxsByHash, tx.Hash) + } - for _, miniblock := range block.MiniBlocks { - if miniblock.ProcessingType == dataBlock.Processed.String() { - foundMiniblocks = append(foundMiniblocks, miniblock) + for _, tx := range previouslyExecutedResults { + delete(effectiveTxsByHash, tx.Hash) } } - return foundMiniblocks + if len(scheduledTxsInCurrentBlock) > 0 { + for _, tx := range currentlyExecutedResults { + effectiveTxsByHash[tx.Hash] = tx + } + } + + for _, tx := range effectiveTxsByHash { + effectiveTxs = append(effectiveTxs, tx) + } + + return effectiveTxs } -func findInvalidTransactions(block *api.Block) []*transaction.ApiTransactionResult { - invalidTxs := make([]*transaction.ApiTransactionResult, 0) +func findImmediatelyExecutingContractResults( + selfShard uint32, + transactions []*transaction.ApiTransactionResult, + maybeContractResults []*transaction.ApiTransactionResult, +) []*transaction.ApiTransactionResult { + immediateleyExecutingContractResults := make([]*transaction.ApiTransactionResult, 0) + nextContractResultsByHash := make(map[string][]*transaction.ApiTransactionResult) - for _, miniblock := range block.MiniBlocks { - if miniblock.Type == dataBlock.InvalidBlock.String() { - for _, tx := range miniblock.Transactions { - invalidTxs = append(invalidTxs, tx) - } - } + for _, item := range maybeContractResults { + nextContractResultsByHash[item.PreviousTransactionHash] = append(nextContractResultsByHash[item.PreviousTransactionHash], item) } - return invalidTxs + for _, tx := range transactions { + immediateleyExecutingContractResultsPart := findImmediatelyExecutingContractResultsOfTransaction(selfShard, tx, nextContractResultsByHash) + immediateleyExecutingContractResults = append(immediateleyExecutingContractResults, immediateleyExecutingContractResultsPart...) + } + + return immediateleyExecutingContractResults } -// Sometimes, an invalid transaction processed in a scheduled miniblock -// might have its smart contract result (if any) saved in the receipts unit of both blocks N and N+1. -// This function ignores the duplicate entries in block N. -func deduplicatePreviouslyAppearingContractResultsInReceipts(previousBlock *api.Block, block *api.Block) { - previouslyAppearing := findContractResultsInReceipts(previousBlock) - removeContractResultsInReceipts(block, previouslyAppearing) +func findImmediatelyExecutingContractResultsOfTransaction( + selfShard uint32, + tx *transaction.ApiTransactionResult, + nextContractResultsByHash map[string][]*transaction.ApiTransactionResult, +) []*transaction.ApiTransactionResult { + immediatelyExecutingContractResults := make([]*transaction.ApiTransactionResult, 0) + + for _, nextContractResult := range nextContractResultsByHash[tx.Hash] { + // Not immediately executing. + if nextContractResult.SourceShard != selfShard { + continue + } + + immediatelyExecutingContractResults = append(immediatelyExecutingContractResults, nextContractResult) + // Recursive call: + immediatelyExecutingContractResultsPart := findImmediatelyExecutingContractResultsOfTransaction(selfShard, nextContractResult, nextContractResultsByHash) + immediatelyExecutingContractResults = append(immediatelyExecutingContractResults, immediatelyExecutingContractResultsPart...) + } + + return immediatelyExecutingContractResults } -func findContractResultsInReceipts(block *api.Block) map[string]struct{} { - txs := make(map[string]struct{}) +func gatherScheduledTransactions(block *api.Block) []*transaction.ApiTransactionResult { + scheduledTxs := make([]*transaction.ApiTransactionResult, 0) for _, miniblock := range block.MiniBlocks { - if !isContractResultsMiniblockInReceipts(miniblock) { + isScheduled := miniblock.ProcessingType == dataBlock.Scheduled.String() + if !isScheduled { continue } for _, tx := range miniblock.Transactions { - txs[tx.Hash] = struct{}{} + scheduledTxs = append(scheduledTxs, tx) } } - return txs + return scheduledTxs } -func removeContractResultsInReceipts(block *api.Block, txsHashes map[string]struct{}) { +func gatherAllTransactions(block *api.Block) []*transaction.ApiTransactionResult { + txs := make([]*transaction.ApiTransactionResult, 0) + for _, miniblock := range block.MiniBlocks { - if !isContractResultsMiniblockInReceipts(miniblock) { - continue + for _, tx := range miniblock.Transactions { + txs = append(txs, tx) } - - miniblock.Transactions = discardTransactions(miniblock.Transactions, txsHashes) } + + return txs } -func isContractResultsMiniblockInReceipts(miniblock *api.MiniBlock) bool { - return miniblock.Type == dataBlock.SmartContractResultBlock.String() && miniblock.IsFromReceiptsStorage +func gatherAllReceipts(block *api.Block) []*transaction.ApiReceipt { + receipts := make([]*transaction.ApiReceipt, 0) + + for _, miniblock := range block.MiniBlocks { + for _, receipt := range miniblock.Receipts { + receipts = append(receipts, receipt) + } + } + + return receipts } diff --git a/server/services/networkProviderExtension.go b/server/services/networkProviderExtension.go index cfff7de6..1ad2ba61 100644 --- a/server/services/networkProviderExtension.go +++ b/server/services/networkProviderExtension.go @@ -78,6 +78,10 @@ func (extension *networkProviderExtension) getGenesisBlockIdentifier() *types.Bl return blockSummaryToIdentifier(summary) } +func (extension *networkProviderExtension) isContractAddress(address string) bool { + return !extension.isUserAddress(address) +} + func (extension *networkProviderExtension) isUserAddress(address string) bool { pubKey, err := extension.provider.ConvertAddressToPubKey(address) if err != nil { diff --git a/server/services/transactionsFeaturesDetector.go b/server/services/transactionsFeaturesDetector.go index 3c33d5bc..e6b91a9e 100644 --- a/server/services/transactionsFeaturesDetector.go +++ b/server/services/transactionsFeaturesDetector.go @@ -5,14 +5,16 @@ import ( ) type transactionsFeaturesDetector struct { - networkProvider NetworkProvider - eventsController *transactionEventsController + networkProvider NetworkProvider + networkProviderExtension *networkProviderExtension + eventsController *transactionEventsController } func newTransactionsFeaturesDetector(provider NetworkProvider) *transactionsFeaturesDetector { return &transactionsFeaturesDetector{ - networkProvider: provider, - eventsController: newTransactionEventsController(provider), + networkProvider: provider, + networkProviderExtension: newNetworkProviderExtension(provider), + eventsController: newTransactionEventsController(provider), } } @@ -106,3 +108,9 @@ func (detector *transactionsFeaturesDetector) isContractCallWithSignalError(tx * func (detector *transactionsFeaturesDetector) isIntrashard(tx *transaction.ApiTransactionResult) bool { return tx.SourceShard == tx.DestinationShard } + +// isSmartContractResultIneffectiveRefund detects smart contract results that are ineffective refunds. +// Also see: https://console.cloud.google.com/bigquery?sq=667383445384:de7a5f3f172f4b50a9aaed353ef79839 +func (detector *transactionsFeaturesDetector) isSmartContractResultIneffectiveRefund(scr *transaction.ApiTransactionResult) bool { + return scr.IsRefund && scr.Sender == scr.Receiver && detector.networkProviderExtension.isContractAddress(scr.Sender) +} diff --git a/server/services/transactionsFeaturesDetector_test.go b/server/services/transactionsFeaturesDetector_test.go index 70af7207..552e47b6 100644 --- a/server/services/transactionsFeaturesDetector_test.go +++ b/server/services/transactionsFeaturesDetector_test.go @@ -235,3 +235,26 @@ func TestTransactionsFeaturesDetector_isIntrashard(t *testing.T) { DestinationShard: 1, })) } + +func TestTransactionsFeaturesDetector_isSmartContractResultIneffectiveRefund(t *testing.T) { + networkProvider := testscommon.NewNetworkProviderMock() + detector := newTransactionsFeaturesDetector(networkProvider) + + require.True(t, detector.isSmartContractResultIneffectiveRefund(&transaction.ApiTransactionResult{ + Sender: testscommon.TestAddressOfContract, + Receiver: testscommon.TestAddressOfContract, + IsRefund: true, + })) + + require.False(t, detector.isSmartContractResultIneffectiveRefund(&transaction.ApiTransactionResult{ + Sender: testscommon.TestAddressOfContract, + Receiver: testscommon.TestAddressOfContract, + IsRefund: false, + })) + + require.False(t, detector.isSmartContractResultIneffectiveRefund(&transaction.ApiTransactionResult{ + Sender: testscommon.TestAddressOfContract, + Receiver: testscommon.TestAddressAlice, + IsRefund: false, + })) +} diff --git a/server/services/transactionsTransformer.go b/server/services/transactionsTransformer.go index 894a00be..5a96708e 100644 --- a/server/services/transactionsTransformer.go +++ b/server/services/transactionsTransformer.go @@ -114,6 +114,15 @@ func (transformer *transactionsTransformer) unsignedTxToRosettaTx( scr *transaction.ApiTransactionResult, txsInBlock []*transaction.ApiTransactionResult, ) *types.Transaction { + if transformer.featuresDetector.isSmartContractResultIneffectiveRefund(scr) { + log.Info("unsignedTxToRosettaTx: ineffective refund", "hash", scr.Hash) + + return &types.Transaction{ + TransactionIdentifier: hashToTransactionIdentifier(scr.Hash), + Operations: []*types.Operation{}, + } + } + if scr.IsRefund { return &types.Transaction{ TransactionIdentifier: hashToTransactionIdentifier(scr.Hash), diff --git a/server/services/transactionsTransformer_test.go b/server/services/transactionsTransformer_test.go index 160a2c4d..cd16ca0a 100644 --- a/server/services/transactionsTransformer_test.go +++ b/server/services/transactionsTransformer_test.go @@ -239,55 +239,77 @@ func TestTransactionsTransformer_UnsignedTxToRosettaTx(t *testing.T) { extension := newNetworkProviderExtension(networkProvider) transformer := newTransactionsTransformer(networkProvider) - refundTx := &transaction.ApiTransactionResult{ - Hash: "aaaa", - Sender: testscommon.TestAddressOfContract, - Receiver: testscommon.TestAddressAlice, - Value: "1234", - IsRefund: true, - } + t.Run("refund SCR", func(t *testing.T) { + tx := &transaction.ApiTransactionResult{ + Hash: "aaaa", + Sender: testscommon.TestAddressOfContract, + Receiver: testscommon.TestAddressAlice, + Value: "1234", + IsRefund: true, + } - expectedRefundTx := &types.Transaction{ - TransactionIdentifier: hashToTransactionIdentifier("aaaa"), - Operations: []*types.Operation{ - { - Type: opFeeRefundAsScResult, - Account: addressToAccountIdentifier(testscommon.TestAddressAlice), - Amount: extension.valueToNativeAmount("1234"), + expectedTx := &types.Transaction{ + TransactionIdentifier: hashToTransactionIdentifier("aaaa"), + Operations: []*types.Operation{ + { + Type: opFeeRefundAsScResult, + Account: addressToAccountIdentifier(testscommon.TestAddressAlice), + Amount: extension.valueToNativeAmount("1234"), + }, }, - }, - } + } - moveBalanceTx := &transaction.ApiTransactionResult{ - Hash: "aaaa", - Sender: testscommon.TestAddressOfContract, - Receiver: testscommon.TestAddressAlice, - Value: "1234", - } + rosettaTx := transformer.unsignedTxToRosettaTx(tx, nil) + require.Equal(t, expectedTx, rosettaTx) + }) - expectedMoveBalanceTx := &types.Transaction{ - TransactionIdentifier: hashToTransactionIdentifier("aaaa"), - Operations: []*types.Operation{ - { - Type: opScResult, - Account: addressToAccountIdentifier(testscommon.TestAddressOfContract), - Amount: extension.valueToNativeAmount("-1234"), - }, - { - Type: opScResult, - Account: addressToAccountIdentifier(testscommon.TestAddressAlice), - Amount: extension.valueToNativeAmount("1234"), + t.Run("move balance SCR", func(t *testing.T) { + tx := &transaction.ApiTransactionResult{ + Hash: "aaaa", + Sender: testscommon.TestAddressOfContract, + Receiver: testscommon.TestAddressAlice, + Value: "1234", + } + + expectedTx := &types.Transaction{ + TransactionIdentifier: hashToTransactionIdentifier("aaaa"), + Operations: []*types.Operation{ + { + Type: opScResult, + Account: addressToAccountIdentifier(testscommon.TestAddressOfContract), + Amount: extension.valueToNativeAmount("-1234"), + }, + { + Type: opScResult, + Account: addressToAccountIdentifier(testscommon.TestAddressAlice), + Amount: extension.valueToNativeAmount("1234"), + }, }, - }, - Metadata: extractTransactionMetadata(moveBalanceTx), - } + Metadata: extractTransactionMetadata(tx), + } + + rosettaTx := transformer.unsignedTxToRosettaTx(tx, []*transaction.ApiTransactionResult{tx}) + require.Equal(t, expectedTx, rosettaTx) + }) - txsInBlock := []*transaction.ApiTransactionResult{refundTx, moveBalanceTx} + t.Run("ineffective refund SCR", func(t *testing.T) { + tx := &transaction.ApiTransactionResult{ + Hash: "aaaa", + Sender: testscommon.TestAddressOfContract, + Receiver: testscommon.TestAddressOfContract, + Value: "1234", + IsRefund: true, + } - rosettaRefundTx := transformer.unsignedTxToRosettaTx(refundTx, txsInBlock) - rosettaMoveBalanceTx := transformer.unsignedTxToRosettaTx(moveBalanceTx, txsInBlock) - require.Equal(t, expectedRefundTx, rosettaRefundTx) - require.Equal(t, expectedMoveBalanceTx, rosettaMoveBalanceTx) + expectedTx := &types.Transaction{ + TransactionIdentifier: hashToTransactionIdentifier("aaaa"), + Operations: []*types.Operation{}, + Metadata: nil, + } + + rosettaTx := transformer.unsignedTxToRosettaTx(tx, []*transaction.ApiTransactionResult{tx}) + require.Equal(t, expectedTx, rosettaTx) + }) } func TestTransactionsTransformer_InvalidTxToRosettaTx(t *testing.T) { diff --git a/systemtests/check_with_mesh_cli.py b/systemtests/check_with_mesh_cli.py index fe94a880..2469ef32 100644 --- a/systemtests/check_with_mesh_cli.py +++ b/systemtests/check_with_mesh_cli.py @@ -66,6 +66,7 @@ def run_rosetta(configuration: Configuration): f"--observer-actual-shard={configuration.network_shard}", f"--network-id={configuration.network_id}", f"--network-name={configuration.network_name}", + f"--handle-contracts", f"--native-currency={configuration.native_currency}", f"--config-custom-currencies={configuration.config_file_custom_currencies}", f"--first-historical-epoch={current_epoch}", From 65e2ab1dec2b1bec8924d3cb7eafb91c4329e072 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Thu, 22 Aug 2024 16:46:13 +0300 Subject: [PATCH 2/2] Undo changes (added by mistake). --- server/provider/scheduledMiniblocks.go | 231 +++++++++++-------------- 1 file changed, 101 insertions(+), 130 deletions(-) diff --git a/server/provider/scheduledMiniblocks.go b/server/provider/scheduledMiniblocks.go index a8348c07..3691e1ed 100644 --- a/server/provider/scheduledMiniblocks.go +++ b/server/provider/scheduledMiniblocks.go @@ -17,186 +17,157 @@ func (provider *networkProvider) simplifyBlockWithScheduledTransactions(block *a return err } - reportScheduledTransactions(block) doSimplifyBlockWithScheduledTransactions(previousBlock, block, nextBlock) + deduplicatePreviouslyAppearingContractResultsInReceipts(previousBlock, block) return nil } -func reportScheduledTransactions(block *api.Block) { - numScheduled := 0 - numProcessed := 0 - numInvalid := 0 - - for _, miniblock := range block.MiniBlocks { - if miniblock.ProcessingType == dataBlock.Scheduled.String() { - numScheduled += len(miniblock.Transactions) - } else if miniblock.ProcessingType == dataBlock.Processed.String() { - numProcessed += len(miniblock.Transactions) - } else if miniblock.Type == dataBlock.InvalidBlock.String() { - numInvalid += len(miniblock.Transactions) - } - } - - if numScheduled > 0 || numProcessed > 0 { - log.Info("reportScheduledTransactions()", "scheduled", numScheduled, "processed", numProcessed, "invalid", numInvalid, "block", block.Nonce) - } -} - func doSimplifyBlockWithScheduledTransactions(previousBlock *api.Block, block *api.Block, nextBlock *api.Block) { - txs := gatherEffectiveTransactions(block.Shard, previousBlock, block, nextBlock) - receipts := gatherAllReceipts(block) - - block.MiniBlocks = []*api.MiniBlock{ - { - Type: "Artificial", - Transactions: txs, - }, - { - Type: "Artificial", - Receipts: receipts, - }, - } -} - -func gatherEffectiveTransactions(selfShard uint32, previousBlock *api.Block, currentBlock *api.Block, nextBlock *api.Block) []*transaction.ApiTransactionResult { - txsInCurrentBlock := gatherAllTransactions(currentBlock) + // Discard "processed" miniblocks in block N, since they already produced effects in N-1 + removeProcessedMiniblocksOfBlock(block) - scheduledTxsInPreviousBlock := gatherScheduledTransactions(previousBlock) - scheduledTxsInCurrentBlock := gatherScheduledTransactions(currentBlock) + // Move "processed" miniblocks from N+1 to N + processedMiniblocksInNextBlock := findProcessedMiniblocks(nextBlock) + appendMiniblocksToBlock(block, processedMiniblocksInNextBlock) - if len(scheduledTxsInPreviousBlock) == 0 && len(scheduledTxsInCurrentBlock) == 0 { - return txsInCurrentBlock + // Build an artificial miniblock holding the "invalid" transactions that produced their effects in block N, + // and replace the existing (one or two "invalid" miniblocks). + invalidTxs := gatherInvalidTransactions(previousBlock, block, nextBlock) + invalidMiniblock := &api.MiniBlock{ + Type: dataBlock.InvalidBlock.String(), + Transactions: invalidTxs, } + removeInvalidMiniblocks(block) - var previouslyExecutedResults []*transaction.ApiTransactionResult - var currentlyExecutedResults []*transaction.ApiTransactionResult - - if len(scheduledTxsInPreviousBlock) > 0 { - previouslyExecutedResults = findImmediatelyExecutingContractResults(selfShard, scheduledTxsInPreviousBlock, txsInCurrentBlock) + if len(invalidMiniblock.Transactions) > 0 { + appendMiniblocksToBlock(block, []*api.MiniBlock{invalidMiniblock}) } - if len(scheduledTxsInCurrentBlock) > 0 { - txsInNextBlock := gatherAllTransactions(nextBlock) - currentlyExecutedResults = findImmediatelyExecutingContractResults(selfShard, scheduledTxsInCurrentBlock, txsInNextBlock) - } - - // effectiveTxs - // = txsInCurrentBlock - // - txsInPreviousBlock (excludes transactions in "processed" miniblocks, for example) - // - previouslyExecutedResults - // + currentlyExecutedResults - effectiveTxs := make([]*transaction.ApiTransactionResult, 0) - effectiveTxsByHash := make(map[string]*transaction.ApiTransactionResult) + // Discard "scheduled" miniblocks of N, since we've already brought the "processed" ones from N+1, + // and also handled the "invalid" ones. + removeScheduledMiniblocks(block) +} - for _, tx := range txsInCurrentBlock { - effectiveTxsByHash[tx.Hash] = tx - } +func removeProcessedMiniblocksOfBlock(block *api.Block) { + removeMiniblocksFromBlock(block, func(miniblock *api.MiniBlock) bool { + return miniblock.ProcessingType == dataBlock.Processed.String() + }) +} - if len(scheduledTxsInPreviousBlock) > 0 { - txsInPreviousBlock := gatherAllTransactions(previousBlock) +func removeScheduledMiniblocks(block *api.Block) { + removeMiniblocksFromBlock(block, func(miniblock *api.MiniBlock) bool { + hasProcessingTypeScheduled := miniblock.ProcessingType == dataBlock.Scheduled.String() + hasConstructionStateNotFinal := miniblock.ConstructionState != dataBlock.Final.String() + shouldRemove := hasProcessingTypeScheduled && hasConstructionStateNotFinal + return shouldRemove + }) +} - for _, tx := range txsInPreviousBlock { - delete(effectiveTxsByHash, tx.Hash) - } +func removeInvalidMiniblocks(block *api.Block) { + removeMiniblocksFromBlock(block, func(miniblock *api.MiniBlock) bool { + return miniblock.Type == dataBlock.InvalidBlock.String() + }) +} - for _, tx := range previouslyExecutedResults { - delete(effectiveTxsByHash, tx.Hash) - } - } +func gatherInvalidTransactions(previousBlock *api.Block, block *api.Block, nextBlock *api.Block) []*transaction.ApiTransactionResult { + // Find "invalid" transactions that are "final" in N + invalidTxsInBlock := findInvalidTransactions(block) + // If also present in N-1, discard them + scheduledTxsHashesPreviousBlock := findScheduledTransactionsHashes(previousBlock) + invalidTxsInBlock = discardTransactions(invalidTxsInBlock, scheduledTxsHashesPreviousBlock) - if len(scheduledTxsInCurrentBlock) > 0 { - for _, tx := range currentlyExecutedResults { - effectiveTxsByHash[tx.Hash] = tx - } - } + // Find "invalid" transactions in N+1 that are "scheduled" in N + invalidTxsInNextBlock := findInvalidTransactions(nextBlock) + scheduledTxsHashesInBlock := findScheduledTransactionsHashes(block) + invalidTxsScheduledInBlock := filterTransactions(invalidTxsInNextBlock, scheduledTxsHashesInBlock) - for _, tx := range effectiveTxsByHash { - effectiveTxs = append(effectiveTxs, tx) - } + // Duplication might occur, since a block can contain two "invalid" miniblocks, + // one added to block body, one saved in the receipts unit (at times, they have different content, different hashes). + invalidTxs := append(invalidTxsInBlock, invalidTxsScheduledInBlock...) + invalidTxs = deduplicateTransactions(invalidTxs) - return effectiveTxs + return invalidTxs } -func findImmediatelyExecutingContractResults( - selfShard uint32, - transactions []*transaction.ApiTransactionResult, - maybeContractResults []*transaction.ApiTransactionResult, -) []*transaction.ApiTransactionResult { - immediateleyExecutingContractResults := make([]*transaction.ApiTransactionResult, 0) - nextContractResultsByHash := make(map[string][]*transaction.ApiTransactionResult) - - for _, item := range maybeContractResults { - nextContractResultsByHash[item.PreviousTransactionHash] = append(nextContractResultsByHash[item.PreviousTransactionHash], item) - } +func findScheduledTransactionsHashes(block *api.Block) map[string]struct{} { + txs := make(map[string]struct{}) - for _, tx := range transactions { - immediateleyExecutingContractResultsPart := findImmediatelyExecutingContractResultsOfTransaction(selfShard, tx, nextContractResultsByHash) - immediateleyExecutingContractResults = append(immediateleyExecutingContractResults, immediateleyExecutingContractResultsPart...) + for _, miniblock := range block.MiniBlocks { + hasProcessingTypeScheduled := miniblock.ProcessingType == dataBlock.Scheduled.String() + hasConstructionStateNotFinal := miniblock.ConstructionState != dataBlock.Final.String() + shouldAccumulateTxs := hasProcessingTypeScheduled && hasConstructionStateNotFinal + + if shouldAccumulateTxs { + for _, tx := range miniblock.Transactions { + txs[tx.Hash] = struct{}{} + } + } } - return immediateleyExecutingContractResults + return txs } -func findImmediatelyExecutingContractResultsOfTransaction( - selfShard uint32, - tx *transaction.ApiTransactionResult, - nextContractResultsByHash map[string][]*transaction.ApiTransactionResult, -) []*transaction.ApiTransactionResult { - immediatelyExecutingContractResults := make([]*transaction.ApiTransactionResult, 0) +func findProcessedMiniblocks(block *api.Block) []*api.MiniBlock { + foundMiniblocks := make([]*api.MiniBlock, 0, len(block.MiniBlocks)) - for _, nextContractResult := range nextContractResultsByHash[tx.Hash] { - // Not immediately executing. - if nextContractResult.SourceShard != selfShard { - continue + for _, miniblock := range block.MiniBlocks { + if miniblock.ProcessingType == dataBlock.Processed.String() { + foundMiniblocks = append(foundMiniblocks, miniblock) } - - immediatelyExecutingContractResults = append(immediatelyExecutingContractResults, nextContractResult) - // Recursive call: - immediatelyExecutingContractResultsPart := findImmediatelyExecutingContractResultsOfTransaction(selfShard, nextContractResult, nextContractResultsByHash) - immediatelyExecutingContractResults = append(immediatelyExecutingContractResults, immediatelyExecutingContractResultsPart...) } - return immediatelyExecutingContractResults + return foundMiniblocks } -func gatherScheduledTransactions(block *api.Block) []*transaction.ApiTransactionResult { - scheduledTxs := make([]*transaction.ApiTransactionResult, 0) +func findInvalidTransactions(block *api.Block) []*transaction.ApiTransactionResult { + invalidTxs := make([]*transaction.ApiTransactionResult, 0) for _, miniblock := range block.MiniBlocks { - isScheduled := miniblock.ProcessingType == dataBlock.Scheduled.String() - if !isScheduled { - continue - } - - for _, tx := range miniblock.Transactions { - scheduledTxs = append(scheduledTxs, tx) + if miniblock.Type == dataBlock.InvalidBlock.String() { + for _, tx := range miniblock.Transactions { + invalidTxs = append(invalidTxs, tx) + } } } - return scheduledTxs + return invalidTxs +} + +// Sometimes, an invalid transaction processed in a scheduled miniblock +// might have its smart contract result (if any) saved in the receipts unit of both blocks N and N+1. +// This function ignores the duplicate entries in block N. +func deduplicatePreviouslyAppearingContractResultsInReceipts(previousBlock *api.Block, block *api.Block) { + previouslyAppearing := findContractResultsInReceipts(previousBlock) + removeContractResultsInReceipts(block, previouslyAppearing) } -func gatherAllTransactions(block *api.Block) []*transaction.ApiTransactionResult { - txs := make([]*transaction.ApiTransactionResult, 0) +func findContractResultsInReceipts(block *api.Block) map[string]struct{} { + txs := make(map[string]struct{}) for _, miniblock := range block.MiniBlocks { + if !isContractResultsMiniblockInReceipts(miniblock) { + continue + } + for _, tx := range miniblock.Transactions { - txs = append(txs, tx) + txs[tx.Hash] = struct{}{} } } return txs } -func gatherAllReceipts(block *api.Block) []*transaction.ApiReceipt { - receipts := make([]*transaction.ApiReceipt, 0) - +func removeContractResultsInReceipts(block *api.Block, txsHashes map[string]struct{}) { for _, miniblock := range block.MiniBlocks { - for _, receipt := range miniblock.Receipts { - receipts = append(receipts, receipt) + if !isContractResultsMiniblockInReceipts(miniblock) { + continue } + + miniblock.Transactions = discardTransactions(miniblock.Transactions, txsHashes) } +} - return receipts +func isContractResultsMiniblockInReceipts(miniblock *api.MiniBlock) bool { + return miniblock.Type == dataBlock.SmartContractResultBlock.String() && miniblock.IsFromReceiptsStorage }