From 31ee4359df47449f9f635803c0998e9156f8f97f Mon Sep 17 00:00:00 2001 From: Volodymyr Kravets Date: Fri, 22 Nov 2024 17:55:33 +0200 Subject: [PATCH 01/10] feat(snap): add extra checks; update snap discovery --- .../src/main/java/co/rsk/RskContext.java | 68 ++++++ .../co/rsk/config/RskSystemProperties.java | 15 -- .../java/co/rsk/core/bc/BlockChainImpl.java | 10 +- .../co/rsk/core/bc/BlockValidatorImpl.java | 6 +- .../java/co/rsk/net/SnapshotProcessor.java | 228 ++++++++++++++++-- .../main/java/co/rsk/net/SyncProcessor.java | 40 ++- .../java/co/rsk/net/sync/BaseSyncState.java | 6 +- .../co/rsk/net/sync/BlockConnectorHelper.java | 21 +- .../net/sync/CheckingBestHeaderSyncState.java | 2 +- .../DownloadingBackwardsHeadersSyncState.java | 2 +- .../net/sync/DownloadingHeadersSyncState.java | 8 +- .../sync/DownloadingSkeletonSyncState.java | 8 +- .../sync/FindingConnectionPointSyncState.java | 19 +- .../sync/PeerAndModeDecidingSyncState.java | 21 +- .../co/rsk/net/sync/PeersInformation.java | 2 +- .../sync/SnapDownloadingHeadersSyncState.java | 92 +++++++ .../SnapDownloadingSkeletonSyncState.java | 40 +++ .../SnapFindingConnectionPointSyncState.java | 43 ++++ .../java/co/rsk/net/sync/SnapSyncState.java | 57 ++++- .../co/rsk/net/sync/SyncConfiguration.java | 2 - .../co/rsk/net/sync/SyncEventsHandler.java | 8 +- .../co/rsk/net/sync/SyncMessageHandler.java | 21 +- .../main/java/co/rsk/net/sync/SyncState.java | 2 +- .../rsk/validators/BlockDifficultyRule.java | 4 +- .../BlockHeaderParentCompositeRule.java | 2 +- ...ckHeaderParentDependantValidationRule.java | 6 +- .../validators/BlockParentGasLimitRule.java | 5 +- .../rsk/validators/BlockParentNumberRule.java | 6 +- .../BlockTimeStampValidationRule.java | 2 +- .../rsk/validators/PrevMinGasPriceRule.java | 2 +- .../java/org/ethereum/core/Blockchain.java | 2 + .../core/genesis/BlockChainLoader.java | 1 + .../org/ethereum/db/BlockHeaderStore.java | 35 +++ .../main/java/org/ethereum/db/BlockStore.java | 2 +- .../org/ethereum/db/IndexedBlockStore.java | 33 +++ .../org/ethereum/net/client/Capability.java | 4 + .../net/client/ConfigCapabilitiesImpl.java | 9 +- .../ethereum/net/message/StaticMessages.java | 11 +- .../java/org/ethereum/net/server/Channel.java | 23 +- .../server/EthereumChannelInitializer.java | 2 +- rskj-core/src/main/resources/reference.conf | 4 +- .../rsk/core/bc/BlockRelayValidatorTest.java | 12 +- .../co/rsk/core/bc/BlockValidatorBuilder.java | 4 +- .../co/rsk/core/bc/BlockValidatorTest.java | 2 +- .../co/rsk/net/SnapshotProcessorTest.java | 60 ++++- .../sync/CheckingBestHeaderSyncStateTest.java | 6 +- ...nloadingBackwardsHeadersSyncStateTest.java | 2 +- .../sync/DownloadingHeadersSyncStateTest.java | 6 +- .../rsk/net/sync/SimpleSyncEventsHandler.java | 13 +- .../co/rsk/net/sync/SnapSyncStateTest.java | 10 +- .../PrevMinGasPriceValidatorTest.java | 4 +- .../org/ethereum/core/ImportLightTest.java | 1 + .../java/org/ethereum/db/BlockStoreDummy.java | 22 +- .../ethereum/jsontestsuite/TestRunner.java | 1 + .../runners/StateTestRunner.java | 1 + 55 files changed, 839 insertions(+), 179 deletions(-) create mode 100644 rskj-core/src/main/java/co/rsk/net/sync/SnapDownloadingHeadersSyncState.java create mode 100644 rskj-core/src/main/java/co/rsk/net/sync/SnapDownloadingSkeletonSyncState.java create mode 100644 rskj-core/src/main/java/co/rsk/net/sync/SnapFindingConnectionPointSyncState.java create mode 100644 rskj-core/src/main/java/org/ethereum/db/BlockHeaderStore.java diff --git a/rskj-core/src/main/java/co/rsk/RskContext.java b/rskj-core/src/main/java/co/rsk/RskContext.java index db130d7069a..8aaaf13f66c 100644 --- a/rskj-core/src/main/java/co/rsk/RskContext.java +++ b/rskj-core/src/main/java/co/rsk/RskContext.java @@ -180,7 +180,9 @@ public class RskContext implements NodeContext, NodeBootstrapper { private ProofOfWorkRule proofOfWorkRule; private ForkDetectionDataRule forkDetectionDataRule; private BlockParentDependantValidationRule blockParentDependantValidationRule; + private BlockParentDependantValidationRule snapBlockParentDependantValidationRule; private BlockValidationRule blockValidationRule; + private BlockValidationRule snapBlockValidationRule; private BlockValidationRule minerServerBlockValidationRule; private BlockValidator blockValidator; private BlockValidator blockHeaderValidator; @@ -1133,6 +1135,33 @@ public synchronized BlockValidationRule getBlockValidationRule() { return blockValidationRule; } + public synchronized BlockValidationRule getSnapBlockValidationRule() { + checkIfNotClosed(); + + if (snapBlockValidationRule == null) { + final RskSystemProperties rskSystemProperties = getRskSystemProperties(); + final Constants commonConstants = rskSystemProperties.getNetworkConstants(); + final BlockTimeStampValidationRule blockTimeStampValidationRule = new BlockTimeStampValidationRule( + commonConstants.getNewBlockMaxSecondsInTheFuture(), + rskSystemProperties.getActivationConfig(), + rskSystemProperties.getNetworkConstants() + ); + snapBlockValidationRule = new BlockCompositeRule( + new TxsMinGasPriceRule(), + new BlockTxsMaxGasPriceRule(rskSystemProperties.getActivationConfig()), + new BlockRootValidationRule(rskSystemProperties.getActivationConfig()), + getProofOfWorkRule(), + new RemascValidationRule(), + blockTimeStampValidationRule, + new GasLimitRule(commonConstants.getMinGasLimit()), + new ExtraDataRule(commonConstants.getMaximumExtraDataSize()), + new ValidTxExecutionSublistsEdgesRule(getRskSystemProperties().getActivationConfig()) + ); + } + + return snapBlockValidationRule; + } + public synchronized BlockParentDependantValidationRule getBlockParentDependantValidationRule() { checkIfNotClosed(); @@ -1151,6 +1180,23 @@ public synchronized BlockParentDependantValidationRule getBlockParentDependantVa return blockParentDependantValidationRule; } + public synchronized BlockParentDependantValidationRule getSnapBlockParentDependantValidationRule() { + checkIfNotClosed(); + + if (snapBlockParentDependantValidationRule == null) { + Constants commonConstants = getRskSystemProperties().getNetworkConstants(); + snapBlockParentDependantValidationRule = new BlockParentCompositeRule( + new BlockTxsFieldsValidationRule(getBlockTxSignatureCache()), + new PrevMinGasPriceRule(), + new BlockParentNumberRule(), + new BlockDifficultyRule(getDifficultyCalculator()), + new BlockParentGasLimitRule(commonConstants.getGasLimitBoundDivisor()) + ); + } + + return snapBlockParentDependantValidationRule; + } + public synchronized org.ethereum.db.BlockStore buildBlockStore(String databaseDir) { checkIfNotClosed(); @@ -2010,12 +2056,34 @@ private SyncPool getSyncPool() { private SnapshotProcessor getSnapshotProcessor() { if (snapshotProcessor == null) { + final RskSystemProperties rskSystemProperties = getRskSystemProperties(); + final Constants commonConstants = rskSystemProperties.getNetworkConstants(); + final BlockTimeStampValidationRule blockTimeStampValidationRule = new BlockTimeStampValidationRule( + commonConstants.getNewBlockMaxSecondsInTheFuture(), + rskSystemProperties.getActivationConfig(), + rskSystemProperties.getNetworkConstants() + ); + snapshotProcessor = new SnapshotProcessor( getBlockchain(), getTrieStore(), getPeersInformation(), getBlockStore(), getTransactionPool(), + getSnapBlockParentDependantValidationRule(), + getSnapBlockValidationRule(), + new BlockHeaderParentCompositeRule( + new PrevMinGasPriceRule(), + new BlockParentNumberRule(), + blockTimeStampValidationRule, + new BlockDifficultyRule(getDifficultyCalculator()), + new BlockParentGasLimitRule(commonConstants.getGasLimitBoundDivisor()) + ), + new BlockHeaderCompositeRule( + getProofOfWorkRule(), + blockTimeStampValidationRule, + new ValidGasUsedRule() + ), getRskSystemProperties().getSnapshotChunkSize(), getRskSystemProperties().isSnapshotParallelEnabled() ); diff --git a/rskj-core/src/main/java/co/rsk/config/RskSystemProperties.java b/rskj-core/src/main/java/co/rsk/config/RskSystemProperties.java index 1638f7f155c..6adaa405eda 100644 --- a/rskj-core/src/main/java/co/rsk/config/RskSystemProperties.java +++ b/rskj-core/src/main/java/co/rsk/config/RskSystemProperties.java @@ -430,17 +430,6 @@ public int getLongSyncLimit() { public boolean isServerSnapshotSyncEnabled() { return configFromFiles.getBoolean("sync.snapshot.server.enabled");} public boolean isClientSnapshotSyncEnabled() { return configFromFiles.getBoolean(PROPERTY_SNAP_CLIENT_ENABLED);} - @Override - public List peerCapabilities() { - List capabilities = super.peerCapabilities(); - - if (isSnapshotSyncEnabled()) { - capabilities.add(Capability.SNAP); - } - - return capabilities; - } - public int getSnapshotChunkTimeout() { return configFromFiles.getInt("sync.snapshot.client.chunkRequestTimeout"); } @@ -570,10 +559,6 @@ public GasPriceCalculator.GasCalculatorType getGasCalculatorType() { return gasCalculatorType; } - public boolean isSnapshotSyncEnabled(){ - return isServerSnapshotSyncEnabled() || isClientSnapshotSyncEnabled(); - } - private void fetchMethodTimeout(Config configElement, Map methodTimeoutMap) { configElement.getObject("methods.timeout") .unwrapped() diff --git a/rskj-core/src/main/java/co/rsk/core/bc/BlockChainImpl.java b/rskj-core/src/main/java/co/rsk/core/bc/BlockChainImpl.java index 6bed3f10649..2755414b8c9 100644 --- a/rskj-core/src/main/java/co/rsk/core/bc/BlockChainImpl.java +++ b/rskj-core/src/main/java/co/rsk/core/bc/BlockChainImpl.java @@ -78,6 +78,7 @@ public class BlockChainImpl implements Blockchain { private static final Logger logger = LoggerFactory.getLogger("blockchain"); private static final PanicProcessor panicProcessor = new PanicProcessor(); + private final Genesis genesis; private final BlockStore blockStore; private final ReceiptStore receiptStore; private final TransactionPool transactionPool; @@ -94,13 +95,15 @@ public class BlockChainImpl implements Blockchain { private final BlockExecutor blockExecutor; private boolean noValidation; - public BlockChainImpl(BlockStore blockStore, + public BlockChainImpl(Genesis genesis, + BlockStore blockStore, ReceiptStore receiptStore, TransactionPool transactionPool, EthereumListener listener, BlockValidator blockValidator, BlockExecutor blockExecutor, StateRootHandler stateRootHandler) { + this.genesis = genesis; this.blockStore = blockStore; this.receiptStore = receiptStore; this.listener = listener; @@ -421,6 +424,11 @@ public Block getBestBlock() { return this.status.getBestBlock(); } + @Override + public Genesis getGenesisBlock() { + return this.genesis; + } + public void setNoValidation(boolean noValidation) { this.noValidation = noValidation; } diff --git a/rskj-core/src/main/java/co/rsk/core/bc/BlockValidatorImpl.java b/rskj-core/src/main/java/co/rsk/core/bc/BlockValidatorImpl.java index 3b80a2f6d54..be5c529307b 100644 --- a/rskj-core/src/main/java/co/rsk/core/bc/BlockValidatorImpl.java +++ b/rskj-core/src/main/java/co/rsk/core/bc/BlockValidatorImpl.java @@ -35,11 +35,11 @@ public class BlockValidatorImpl implements BlockValidator { private static final Logger logger = LoggerFactory.getLogger("blocksyncservice"); - private BlockStore blockStore; + private final BlockStore blockStore; - private BlockParentDependantValidationRule blockParentValidator; + private final BlockParentDependantValidationRule blockParentValidator; - private BlockValidationRule blockValidator; + private final BlockValidationRule blockValidator; public BlockValidatorImpl(BlockStore blockStore, BlockParentDependantValidationRule blockParentValidator, BlockValidationRule blockValidator) { this.blockStore = blockStore; diff --git a/rskj-core/src/main/java/co/rsk/net/SnapshotProcessor.java b/rskj-core/src/main/java/co/rsk/net/SnapshotProcessor.java index 862ddc6fed0..93a88845491 100644 --- a/rskj-core/src/main/java/co/rsk/net/SnapshotProcessor.java +++ b/rskj-core/src/main/java/co/rsk/net/SnapshotProcessor.java @@ -20,18 +20,24 @@ import co.rsk.config.InternalService; import co.rsk.core.BlockDifficulty; +import co.rsk.core.types.bytes.Bytes; +import co.rsk.crypto.Keccak256; import co.rsk.net.messages.*; import co.rsk.net.sync.*; +import co.rsk.scoring.EventType; import co.rsk.trie.TrieDTO; import co.rsk.trie.TrieDTOInOrderIterator; import co.rsk.trie.TrieDTOInOrderRecoverer; import co.rsk.trie.TrieStore; +import co.rsk.validators.BlockHeaderParentDependantValidationRule; +import co.rsk.validators.BlockHeaderValidationRule; +import co.rsk.validators.BlockParentDependantValidationRule; +import co.rsk.validators.BlockValidationRule; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; import org.apache.commons.lang3.tuple.ImmutablePair; -import org.ethereum.core.Block; -import org.ethereum.core.Blockchain; -import org.ethereum.core.TransactionPool; +import org.apache.commons.lang3.tuple.Pair; +import org.ethereum.core.*; import org.ethereum.db.BlockStore; import org.ethereum.util.RLP; import org.ethereum.util.RLPElement; @@ -44,6 +50,7 @@ import java.util.*; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; /** @@ -69,7 +76,14 @@ public class SnapshotProcessor implements InternalService { private final int chunkSize; private final SnapshotPeersInformation peersInformation; private final TransactionPool transactionPool; - private long messageId = 0; + + private final BlockParentDependantValidationRule blockParentValidator; + private final BlockValidationRule blockValidator; + + private final BlockHeaderParentDependantValidationRule blockHeaderParentValidator; + private final BlockHeaderValidationRule blockHeaderValidator; + + private final AtomicLong messageId = new AtomicLong(0); // flag for parallel requests private final boolean parallel; @@ -78,14 +92,21 @@ public class SnapshotProcessor implements InternalService { private volatile Boolean isRunning; private final Thread thread; + public SnapshotProcessor(Blockchain blockchain, TrieStore trieStore, SnapshotPeersInformation peersInformation, BlockStore blockStore, TransactionPool transactionPool, + BlockParentDependantValidationRule blockParentValidator, + BlockValidationRule blockValidator, + BlockHeaderParentDependantValidationRule blockHeaderParentValidator, + BlockHeaderValidationRule blockHeaderValidator, int chunkSize, boolean isParallelEnabled) { - this(blockchain, trieStore, peersInformation, blockStore, transactionPool, chunkSize, isParallelEnabled, null); + this(blockchain, trieStore, peersInformation, blockStore, transactionPool, + blockParentValidator, blockValidator, blockHeaderParentValidator, blockHeaderValidator, + chunkSize, isParallelEnabled, null); } @VisibleForTesting @@ -94,6 +115,10 @@ public SnapshotProcessor(Blockchain blockchain, SnapshotPeersInformation peersInformation, BlockStore blockStore, TransactionPool transactionPool, + BlockParentDependantValidationRule blockParentValidator, + BlockValidationRule blockValidator, + BlockHeaderParentDependantValidationRule blockHeaderParentValidator, + BlockHeaderValidationRule blockHeaderValidator, int chunkSize, boolean isParallelEnabled, @Nullable SyncMessageHandler.Listener listener) { @@ -103,6 +128,13 @@ public SnapshotProcessor(Blockchain blockchain, this.chunkSize = chunkSize; this.blockStore = blockStore; this.transactionPool = transactionPool; + + this.blockParentValidator = blockParentValidator; + this.blockValidator = blockValidator; + + this.blockHeaderParentValidator = blockHeaderParentValidator; + this.blockHeaderValidator = blockHeaderValidator; + this.parallel = isParallelEnabled; this.thread = new Thread(new SyncMessageHandler("SNAP requests", requestQueue, listener) { @@ -113,19 +145,27 @@ public boolean isRunning() { }, "snap sync request handler"); } - public void startSyncing() { + public void startSyncing(SnapSyncState state) { // get more than one peer, use the peer queue // TODO(snap-poc) deal with multiple peers algorithm here - Peer peer = peersInformation.getBestSnapPeerCandidates().get(0); + Optional bestPeerOpt = peersInformation.getBestSnapPeer(); + if (bestPeerOpt.isEmpty()) { + logger.warn("No more valid peer to start snapshot sync against."); + stopSyncing(state); + return; + } logger.info("CLIENT - Starting Snapshot sync."); - requestSnapStatus(peer); + requestSnapStatus(bestPeerOpt.get()); } - // TODO(snap-poc) should be called on errors too private void stopSyncing(SnapSyncState state) { state.finish(); } + private void failSyncing(SnapSyncState state, Peer peer, EventType eventType, String message, Object... arguments) { + state.fail(peer, eventType, message, arguments); + } + /** * STATUS */ @@ -174,35 +214,84 @@ void processSnapStatusRequestInternal(Peer sender, SnapStatusRequestMessage igno byte[] rootHash = checkpointBlock.getStateRoot(); Optional opt = trieStore.retrieveDTO(rootHash); - long trieSize = 0; - if (opt.isPresent()) { - trieSize = opt.get().getTotalSize(); - } else { - logger.debug("SERVER - trie is notPresent"); + if (opt.isEmpty()) { + logger.warn("SERVER - trie is not present for rootHash: {}", Bytes.of(rootHash)); + return; } + + long trieSize = opt.get().getTotalSize(); logger.debug("SERVER - processing snapshot status request - rootHash: {} trieSize: {}", rootHash, trieSize); SnapStatusResponseMessage responseMessage = new SnapStatusResponseMessage(blocks, difficulties, trieSize); sender.sendMessage(responseMessage); } public void processSnapStatusResponse(SnapSyncState state, Peer sender, SnapStatusResponseMessage responseMessage) { + if (!state.isRunning()) { + return; + } + List blocksFromResponse = responseMessage.getBlocks(); List difficultiesFromResponse = responseMessage.getDifficulties(); + if (blocksFromResponse.size() != difficultiesFromResponse.size()) { + failSyncing(state, sender, EventType.INVALID_BLOCK, "Blocks and difficulties size mismatch. Blocks: [{}], Difficulties: [{}]", blocksFromResponse.size(), difficultiesFromResponse.size()); + return; + } + Block lastBlock = blocksFromResponse.get(blocksFromResponse.size() - 1); + BlockDifficulty lastBlockDifficulty = difficultiesFromResponse.get(difficultiesFromResponse.size() - 1); state.setLastBlock(lastBlock); - state.setLastBlockDifficulty(lastBlock.getCumulativeDifficulty()); + state.setLastBlockDifficulty(lastBlockDifficulty); state.setRemoteRootHash(lastBlock.getStateRoot()); state.setRemoteTrieSize(responseMessage.getTrieSize()); - for (int i = 0; i < blocksFromResponse.size(); i++) { - state.addBlock(new ImmutablePair<>(blocksFromResponse.get(i), difficultiesFromResponse.get(i))); + if (!validateAndSaveBlocks(state, sender, blocksFromResponse, difficultiesFromResponse)) { + return; } + logger.debug("CLIENT - Processing snapshot status response - last blockNumber: {} triesize: {}", lastBlock.getNumber(), state.getRemoteTrieSize()); logger.debug("Blocks included in the response: {} from {} to {}", blocksFromResponse.size(), blocksFromResponse.get(0).getNumber(), blocksFromResponse.get(blocksFromResponse.size() - 1).getNumber()); requestBlocksChunk(sender, blocksFromResponse.get(0).getNumber()); - generateChunkRequestTasks(state); - startRequestingChunks(state); +// generateChunkRequestTasks(state); +// startRequestingChunks(state); + } + + private boolean validateAndSaveBlocks(SnapSyncState state, Peer sender, List blocks, List difficulties) { + Pair childBlockPair = state.getLastBlockPair(); + for (int i = blocks.size() - 1; i >= 0; i--) { + Block block = blocks.get(i); + BlockDifficulty totalDifficulty = difficulties.get(i); + + Pair blockPair = new ImmutablePair<>(block, totalDifficulty); + if (!areBlockPairsValid(blockPair, childBlockPair)) { + failSyncing(state, sender, EventType.INVALID_BLOCK, "Block [{}]/[{}] at height: [{}] is not valid", block.getHash(), totalDifficulty, block.getNumber()); + return false; + } + + state.addBlock(blockPair); + childBlockPair = blockPair; + } + + state.setLastVerifiedBlockHeader(childBlockPair.getLeft().getHeader()); + + return true; + } + + private boolean areBlockPairsValid(Pair blockPair, @Nullable Pair childBlockPair) { + if (!blockValidator.isValid(blockPair.getLeft())) { + return false; + } + + if (childBlockPair == null) { + return true; + } + + if (!blockPair.getLeft().isParentOf(childBlockPair.getLeft()) + || !blockPair.getRight().equals(childBlockPair.getRight().subtract(childBlockPair.getLeft().getCumulativeDifficulty()))) { + return false; + } + + return blockParentValidator.isValid(childBlockPair.getLeft(), blockPair.getLeft()); } /** @@ -213,6 +302,73 @@ private void requestBlocksChunk(Peer sender, long blockNumber) { sender.sendMessage(new SnapBlocksRequestMessage(blockNumber)); } + public void processBlockHeaderChunk(SnapSyncState state, Peer sender, List chunk) { + if (!state.isRunning()) { + return; + } + + logger.debug("CLIENT - Processing block headers response - chunk: [{}; {}]", chunk.get(0).getNumber(), chunk.get(chunk.size() - 1).getNumber()); + + if (!validateBlockHeaders(state, sender, chunk)) { + state.fail(sender, EventType.INVALID_HEADER, "Invalid block headers received"); + return; + } + + if (blocksVerified(state)) { + boolean result = rebuildStateAndSave(state); + logger.info("CLIENT - Snapshot sync finished {}! ", result ? "successfully" : "with errors"); + stopSyncing(state); + } + + requestNextBlockHeadersChunk(state, sender); + } + + private boolean validateBlockHeaders(SnapSyncState state, Peer sender, List blockHeaders) { + for (int i = 0; i < blockHeaders.size(); i++) { + BlockHeader blockHeader = blockHeaders.get(i); + BlockHeader lastVerifiedBlockHeader = state.getLastVerifiedBlockHeader(); + + if (!areBlockHeadersValid(blockHeader, lastVerifiedBlockHeader)) { + failSyncing(state, sender, EventType.INVALID_BLOCK, "Block header [{}] at height: [{}] is not valid", blockHeader.getHash(), blockHeader.getNumber()); + return false; + } + + state.setLastVerifiedBlockHeader(blockHeader); + } + + return true; + } + + private boolean areBlockHeadersValid(BlockHeader blockHeader, BlockHeader childBlockHeader) { + if (!blockHeaderValidator.isValid(blockHeader)) { + return false; + } + + if (!blockHeader.isParentOf(childBlockHeader)) { + return false; + } + + return blockHeaderParentValidator.isValid(childBlockHeader, blockHeader); + } + + /** + * BLOCK HEADER CHUNK + */ + private void requestNextBlockHeadersChunk(SnapSyncState state, Peer sender) { + BlockHeader lastVerifiedBlockHeader = state.getLastVerifiedBlockHeader(); + Keccak256 parentHash = lastVerifiedBlockHeader.getParentHash(); + long count = Math.min(state.getBlockHeaderChunkSize(), lastVerifiedBlockHeader.getNumber() - 1); + if (count < 1) { + logger.info("CLIENT - No more block headers to request but no genesis found"); + state.fail(sender, EventType.INVALID_HEADER, "Invalid block headers genesis block"); + return; + } + + logger.debug("CLIENT - Requesting block header chunk to node {} - block hash {}", sender.getPeerNodeID(), parentHash); + + state.getSyncEventsHandler().sendBlockHeadersRequest(sender, new ChunkDescriptor(parentHash.getBytes(), (int) count)); + } + public void processSnapBlocksRequest(Peer sender, SnapBlocksRequestMessage requestMessage) { if (isRunning != Boolean.TRUE) { logger.warn("processSnapBlocksRequest: invalid state, isRunning: [{}]", isRunning); @@ -248,20 +404,30 @@ void processSnapBlocksRequestInternal(Peer sender, SnapBlocksRequestMessage requ } public void processSnapBlocksResponse(SnapSyncState state, Peer sender, SnapBlocksResponseMessage responseMessage) { + if (!state.isRunning()) { + return; + } + long lastRequiredBlock = state.getLastBlock().getNumber() - BLOCKS_REQUIRED; List blocksFromResponse = responseMessage.getBlocks(); logger.debug("CLIENT - Processing snap blocks response. Receiving from block {} to block {} Objective: {}.", blocksFromResponse.get(0).getNumber(), blocksFromResponse.get(blocksFromResponse.size() - 1).getNumber(), lastRequiredBlock); List difficultiesFromResponse = responseMessage.getDifficulties(); - for (int i = 0; i < blocksFromResponse.size(); i++) { - state.addBlock(new ImmutablePair<>(blocksFromResponse.get(i), difficultiesFromResponse.get(i))); + if (!validateAndSaveBlocks(state, sender, blocksFromResponse, difficultiesFromResponse)) { + return; } + long nextChunk = blocksFromResponse.get(0).getNumber(); logger.debug("CLIENT - SnapBlock - nexChunk : {} - lastRequired {}, missing {}", nextChunk, lastRequiredBlock, nextChunk - lastRequiredBlock); if (nextChunk > lastRequiredBlock) { requestBlocksChunk(sender, nextChunk); } else { logger.info("CLIENT - Finished Snap blocks request sending."); + + generateChunkRequestTasks(state); + startRequestingChunks(state); + + requestNextBlockHeadersChunk(state, sender); } } @@ -270,7 +436,7 @@ public void processSnapBlocksResponse(SnapSyncState state, Peer sender, SnapBloc */ private void requestStateChunk(Peer peer, long from, long blockNumber, int chunkSize) { logger.debug("CLIENT - Requesting state chunk to node {} - block {} - chunkNumber {}", peer.getPeerNodeID(), blockNumber, from / chunkSize); - SnapStateChunkRequestMessage message = new SnapStateChunkRequestMessage(messageId++, blockNumber, from, chunkSize); + SnapStateChunkRequestMessage message = new SnapStateChunkRequestMessage(messageId.getAndIncrement(), blockNumber, from, chunkSize); peer.sendMessage(message); } @@ -334,6 +500,10 @@ void processStateChunkRequestInternal(Peer sender, SnapStateChunkRequestMessage } public void processStateChunkResponse(SnapSyncState state, Peer peer, SnapStateChunkResponseMessage responseMessage) { + if (!state.isRunning()) { + return; + } + logger.debug("CLIENT - State chunk received chunkNumber {}. From {} to {} of total size {}", responseMessage.getFrom() / CHUNK_ITEM_SIZE, responseMessage.getFrom(), responseMessage.getTo(), state.getRemoteTrieSize()); PriorityQueue queue = state.getSnapStateChunkQueue(); @@ -373,7 +543,6 @@ void onStateChunkResponseError(Peer peer, SnapStateChunkResponseMessage response requestStateChunk(alternativePeer, responseMessage.getFrom(), responseMessage.getBlockNumber(), chunkSize); } - private void processOrderedStateChunkResponse(SnapSyncState state, Peer peer, SnapStateChunkResponseMessage message) throws Exception { logger.debug("CLIENT - Processing State chunk received from {} to {}", message.getFrom(), message.getTo()); peersInformation.getOrRegisterPeer(peer); @@ -430,9 +599,11 @@ private void processOrderedStateChunkResponse(SnapSyncState state, Peer peer, Sn if (!message.isComplete()) { executeNextChunkRequestTask(state, peer); } else { - boolean result = rebuildStateAndSave(state); - logger.info("CLIENT - Snapshot sync finished {}! ", result ? "successfully" : "with errors"); - stopSyncing(state); + if (blocksVerified(state)) { + boolean result = rebuildStateAndSave(state); + logger.info("CLIENT - Snapshot sync finished {}! ", result ? "successfully" : "with errors"); + stopSyncing(state); + } } } else { logger.error("Error while verifying chunk response: {}", message); @@ -440,6 +611,11 @@ private void processOrderedStateChunkResponse(SnapSyncState state, Peer peer, Sn } } + private boolean blocksVerified(SnapSyncState state) { + BlockHeader lastVerifiedBlockHeader = state.getLastVerifiedBlockHeader(); + return lastVerifiedBlockHeader != null && lastVerifiedBlockHeader.getParentHash().equals(blockchain.getGenesisBlock().getHash()); + } + /** * Once state share is received, rebuild the trie, save it in db and save all the blocks. */ diff --git a/rskj-core/src/main/java/co/rsk/net/SyncProcessor.java b/rskj-core/src/main/java/co/rsk/net/SyncProcessor.java index 7c282118e40..d2f5e4613be 100644 --- a/rskj-core/src/main/java/co/rsk/net/SyncProcessor.java +++ b/rskj-core/src/main/java/co/rsk/net/SyncProcessor.java @@ -178,7 +178,7 @@ public void processBlockHeadersResponse(Peer peer, BlockHeadersResponseMessage m MessageType messageType = message.getMessageType(); if (isPending(messageId, messageType)) { removePendingMessage(messageId, messageType); - syncState.newBlockHeaders(message.getBlockHeaders()); + syncState.newBlockHeaders(peer, message.getBlockHeaders()); } else { notifyUnexpectedMessageToPeerScoring(peer, "block headers"); } @@ -289,8 +289,8 @@ public void startBlockForwardSyncing(Peer peer) { } @Override - public void startSnapSync() { - logger.info("Start Snap syncing"); + public void startSnapSync(Peer peer) { + logger.info("Start Snap syncing with {}", peer.getPeerNodeID()); setSyncState(new SnapSyncState(this, snapshotProcessor, syncConfiguration)); } @@ -329,6 +329,21 @@ public void startDownloadingHeaders(Map> skeletons, connectionPoint)); } + @Override + public void startDownloadingSnapHeaders(Map> skeletons, long connectionPoint, Peer peer) { + setSyncState(new SnapDownloadingHeadersSyncState( + syncConfiguration, + this, + consensusValidationMainchainView, + difficultyRule, + blockHeaderValidationRule, + peer, + skeletons, + connectionPoint, + peersInformation, + blockStore)); + } + @Override public void startDownloadingSkeleton(long connectionPoint, Peer peer) { setSyncState(new DownloadingSkeletonSyncState( @@ -339,6 +354,16 @@ public void startDownloadingSkeleton(long connectionPoint, Peer peer) { connectionPoint)); } + @Override + public void startDownloadingSnapSkeleton(long connectionPoint, Peer peer) { + setSyncState(new SnapDownloadingSkeletonSyncState( + syncConfiguration, + this, + peersInformation, + peer, + connectionPoint)); + } + @Override public void startFindingConnectionPoint(Peer peer) { NodeID peerId = peer.getPeerNodeID(); @@ -348,6 +373,15 @@ public void startFindingConnectionPoint(Peer peer) { syncConfiguration, this, blockStore, peer, bestBlockNumber)); } + @Override + public void startFindingSnapConnectionPoint(Peer peer) { + NodeID peerId = peer.getPeerNodeID(); + logger.debug("Find snap connection point with node {}", peerId); + long bestBlockNumber = peersInformation.getPeer(peer).getStatus().getBestBlockNumber(); + setSyncState(new SnapFindingConnectionPointSyncState( + syncConfiguration, this, blockStore, peer, bestBlockNumber)); + } + @Override public void backwardSyncing(Peer peer) { NodeID peerId = peer.getPeerNodeID(); diff --git a/rskj-core/src/main/java/co/rsk/net/sync/BaseSyncState.java b/rskj-core/src/main/java/co/rsk/net/sync/BaseSyncState.java index a2b0fa2cf9b..ea4fdb85a4b 100644 --- a/rskj-core/src/main/java/co/rsk/net/sync/BaseSyncState.java +++ b/rskj-core/src/main/java/co/rsk/net/sync/BaseSyncState.java @@ -30,8 +30,8 @@ import java.util.List; public abstract class BaseSyncState implements SyncState { - protected SyncConfiguration syncConfiguration; - protected SyncEventsHandler syncEventsHandler; + protected final SyncConfiguration syncConfiguration; + protected final SyncEventsHandler syncEventsHandler; protected Duration timeElapsed; @@ -57,7 +57,7 @@ public void tick(Duration duration) { protected void onMessageTimeOut() { /* empty */ } @Override - public void newBlockHeaders(List chunk) { /* empty */ } + public void newBlockHeaders(Peer peer, List chunk) { /* empty */ } @Override public void newBody(BodyResponseMessage message, Peer peer) { /* empty */ } diff --git a/rskj-core/src/main/java/co/rsk/net/sync/BlockConnectorHelper.java b/rskj-core/src/main/java/co/rsk/net/sync/BlockConnectorHelper.java index 58191074de5..0985f7eb0eb 100644 --- a/rskj-core/src/main/java/co/rsk/net/sync/BlockConnectorHelper.java +++ b/rskj-core/src/main/java/co/rsk/net/sync/BlockConnectorHelper.java @@ -29,6 +29,7 @@ public class BlockConnectorHelper { private static final Logger logger = LoggerFactory.getLogger("SnapBlockConnector"); + private final BlockStore blockStore; public BlockConnectorHelper(BlockStore blockStore) { @@ -42,25 +43,19 @@ public void startConnecting(List> blockAndDifficult } blockAndDifficultiesList.sort(new BlockAndDiffComparator()); - Block child = null; logger.info("Start connecting blocks ranging from {} to {} - Total: {}", blockAndDifficultiesList.get(0).getKey().getNumber(), blockAndDifficultiesList.get(blockAndDifficultiesList.size() - 1).getKey().getNumber(), blockAndDifficultiesList.size()); int blockIndex = blockAndDifficultiesList.size() - 1; - if (blockStore.isEmpty()) { - Pair blockAndDifficulty = blockAndDifficultiesList.get(blockIndex); - child = blockAndDifficulty.getLeft(); - logger.debug("BlockStore is empty, setting child block number the last block from the list: {}", child.getNumber()); - blockStore.saveBlock(child, blockAndDifficulty.getRight(), true); - logger.debug("Block number: {} saved", child.getNumber()); - blockIndex--; - } else { - logger.debug("BlockStore is not empty, getting best block"); - child = blockStore.getBestBlock(); - logger.debug("Best block number: {}", child.getNumber()); - } + Pair blockAndDifficulty = blockAndDifficultiesList.get(blockIndex); + Block child = blockAndDifficulty.getLeft(); + logger.debug("Setting child block number the last block from the list: {}", child.getNumber()); + blockStore.saveBlock(child, blockAndDifficulty.getRight(), true); + logger.debug("Block number: {} saved", child.getNumber()); + blockIndex--; + while (blockIndex >= 0) { Pair currentBlockAndDifficulty = blockAndDifficultiesList.get(blockIndex); Block currentBlock = currentBlockAndDifficulty.getLeft(); diff --git a/rskj-core/src/main/java/co/rsk/net/sync/CheckingBestHeaderSyncState.java b/rskj-core/src/main/java/co/rsk/net/sync/CheckingBestHeaderSyncState.java index 37fd4870bea..44c9fabf22f 100644 --- a/rskj-core/src/main/java/co/rsk/net/sync/CheckingBestHeaderSyncState.java +++ b/rskj-core/src/main/java/co/rsk/net/sync/CheckingBestHeaderSyncState.java @@ -46,7 +46,7 @@ public void onEnter(){ } @Override - public void newBlockHeaders(List chunk){ + public void newBlockHeaders(Peer peer, List chunk){ BlockHeader header = chunk.get(0); boolean unexpectedHeader = !ByteUtil.fastEquals(header.getHash().getBytes(), miniChunk.getHash()); if (unexpectedHeader) { diff --git a/rskj-core/src/main/java/co/rsk/net/sync/DownloadingBackwardsHeadersSyncState.java b/rskj-core/src/main/java/co/rsk/net/sync/DownloadingBackwardsHeadersSyncState.java index 915825a4711..ca52f440933 100644 --- a/rskj-core/src/main/java/co/rsk/net/sync/DownloadingBackwardsHeadersSyncState.java +++ b/rskj-core/src/main/java/co/rsk/net/sync/DownloadingBackwardsHeadersSyncState.java @@ -43,7 +43,7 @@ public DownloadingBackwardsHeadersSyncState( } @Override - public void newBlockHeaders(List toRequest) { + public void newBlockHeaders(Peer peer, List toRequest) { syncEventsHandler.backwardDownloadBodies( child, toRequest.stream().skip(1).collect(Collectors.toList()), selectedPeer ); diff --git a/rskj-core/src/main/java/co/rsk/net/sync/DownloadingHeadersSyncState.java b/rskj-core/src/main/java/co/rsk/net/sync/DownloadingHeadersSyncState.java index 26d64046097..c159cd2d743 100644 --- a/rskj-core/src/main/java/co/rsk/net/sync/DownloadingHeadersSyncState.java +++ b/rskj-core/src/main/java/co/rsk/net/sync/DownloadingHeadersSyncState.java @@ -73,7 +73,7 @@ public DownloadingHeadersSyncState( } @Override - public void newBlockHeaders(List chunk) { + public void newBlockHeaders(Peer peer, List chunk) { Optional currentChunkOpt = chunksDownloadHelper.getCurrentChunk(); if (!currentChunkOpt.isPresent()) { syncEventsHandler.onSyncIssue(selectedPeer, "Current chunk not present on {}", this.getClass()); @@ -124,7 +124,7 @@ public void newBlockHeaders(List chunk) { if (!chunksDownloadHelper.hasNextChunk()) { // Finished verifying headers - syncEventsHandler.startDownloadingBodies(pendingHeaders, skeletons, selectedPeer); + processPendingHeaders(pendingHeaders, skeletons, selectedPeer); return; } @@ -161,4 +161,8 @@ private boolean blockHeaderIsValid(BlockHeader header, BlockHeader parentHeader) return blockParentValidationRule.validate(header, parentHeader); } + + void processPendingHeaders(List> pendingHeaders, Map> skeletons, Peer peer) { + syncEventsHandler.startDownloadingBodies(pendingHeaders, skeletons, peer); + } } diff --git a/rskj-core/src/main/java/co/rsk/net/sync/DownloadingSkeletonSyncState.java b/rskj-core/src/main/java/co/rsk/net/sync/DownloadingSkeletonSyncState.java index 7b20fa4d5db..6028dfed01f 100644 --- a/rskj-core/src/main/java/co/rsk/net/sync/DownloadingSkeletonSyncState.java +++ b/rskj-core/src/main/java/co/rsk/net/sync/DownloadingSkeletonSyncState.java @@ -74,7 +74,7 @@ public void newSkeleton(List skeleton, Peer peer) { syncEventsHandler.stopSyncing(); return; } - syncEventsHandler.startDownloadingHeaders(skeletons, connectionPoint, peer); + startDownloadingHeaders(skeletons, connectionPoint, peer); } } @@ -94,7 +94,7 @@ public void tick(Duration duration) { return; } - syncEventsHandler.startDownloadingHeaders(skeletons, connectionPoint, selectedPeer); + startDownloadingHeaders(skeletons, connectionPoint, selectedPeer); } } @@ -102,4 +102,8 @@ public void tick(Duration duration) { public void onEnter() { peersInformation.getBestPeerCandidates().forEach(p -> syncEventsHandler.sendSkeletonRequest(p, connectionPoint)); } + + protected void startDownloadingHeaders(Map> skeletons, long connectionPoint, Peer peer) { + syncEventsHandler.startDownloadingHeaders(skeletons, connectionPoint, peer); + } } diff --git a/rskj-core/src/main/java/co/rsk/net/sync/FindingConnectionPointSyncState.java b/rskj-core/src/main/java/co/rsk/net/sync/FindingConnectionPointSyncState.java index 6f00c653455..5b472933970 100644 --- a/rskj-core/src/main/java/co/rsk/net/sync/FindingConnectionPointSyncState.java +++ b/rskj-core/src/main/java/co/rsk/net/sync/FindingConnectionPointSyncState.java @@ -17,6 +17,7 @@ */ package co.rsk.net.sync; +import co.rsk.crypto.Keccak256; import co.rsk.net.Peer; import org.ethereum.db.BlockStore; @@ -24,7 +25,7 @@ public class FindingConnectionPointSyncState extends BaseSelectedPeerSyncState { - private final BlockStore blockStore; + protected final BlockStore blockStore; private final ConnectionPointFinder connectionPointFinder; public FindingConnectionPointSyncState(SyncConfiguration syncConfiguration, @@ -43,15 +44,15 @@ public FindingConnectionPointSyncState(SyncConfiguration syncConfiguration, @Override public void newConnectionPointData(byte[] hash) { - boolean knownBlock = isKnownBlock(hash); + boolean knownBlock = isKnown(new Keccak256(hash)); Optional cp = connectionPointFinder.getConnectionPoint(); if (cp.isPresent()) { if (knownBlock) { - syncEventsHandler.startDownloadingSkeleton(cp.get(), selectedPeer); + processConnectionPoint(cp.get(), selectedPeer); } else { syncEventsHandler.onSyncIssue(selectedPeer, "Connection point not found on {}", this.getClass()); } - return; + return; } if (knownBlock) { @@ -63,7 +64,7 @@ public void newConnectionPointData(byte[] hash) { cp = connectionPointFinder.getConnectionPoint(); // No need to ask for genesis hash if (cp.isPresent() && cp.get() == 0L) { - syncEventsHandler.startDownloadingSkeleton(cp.get(), selectedPeer); + processConnectionPoint(cp.get(), selectedPeer); return; } @@ -71,8 +72,8 @@ public void newConnectionPointData(byte[] hash) { trySendRequest(); } - private boolean isKnownBlock(byte[] hash) { - return blockStore.isBlockExist(hash); + protected boolean isKnown(Keccak256 hash) { + return blockStore.isBlockExist(hash.getBytes()); } private void trySendRequest() { @@ -83,4 +84,8 @@ private void trySendRequest() { public void onEnter() { trySendRequest(); } + + protected void processConnectionPoint(long connectionPoint, Peer peer) { + syncEventsHandler.startDownloadingSkeleton(connectionPoint, peer); + } } diff --git a/rskj-core/src/main/java/co/rsk/net/sync/PeerAndModeDecidingSyncState.java b/rskj-core/src/main/java/co/rsk/net/sync/PeerAndModeDecidingSyncState.java index 1389d0e9ad1..777d6ced94b 100644 --- a/rskj-core/src/main/java/co/rsk/net/sync/PeerAndModeDecidingSyncState.java +++ b/rskj-core/src/main/java/co/rsk/net/sync/PeerAndModeDecidingSyncState.java @@ -88,35 +88,30 @@ private void tryStartSyncing() { private boolean tryStartSnapshotSync() { if (!syncConfiguration.isClientSnapSyncEnabled()) { - logger.trace("Snap syncing disabled"); + logger.debug("Snap syncing disabled"); return false; } - // TODO(snap-poc) deal with multiple peers logic here - // TODO: To be handled when we implement the multiple peers - //List bestPeers = peersInformation.getBestPeerCandidates(); - - // TODO: for now, use pre-configured snap boot nodes instead (until snap nodes discovery is implemented) - SnapshotPeersInformation snapPeersInformation = peersInformation; - Optional bestPeerOpt = snapPeersInformation.getBestSnapPeer(); + Optional bestPeerOpt = peersInformation.getBestSnapPeer(); Optional peerBestBlockNumOpt = bestPeerOpt.flatMap(this::getPeerBestBlockNumber); - if (!bestPeerOpt.isPresent() || !peerBestBlockNumOpt.isPresent()) { - logger.trace("Snap syncing not possible, no valid peer"); + if (bestPeerOpt.isEmpty() || peerBestBlockNumOpt.isEmpty()) { + logger.info("Snap syncing not possible, no snap-capable peer available"); return false; } // we consider Snap as part of the Long Sync if (!isValidSnapDistance(peerBestBlockNumOpt.get())) { - logger.debug("Snap syncing not required (long sync not required)"); + logger.info("Snap syncing not required"); return false; } // we consider Snap as part of the Long Sync syncEventsHandler.onLongSyncUpdate(true, peerBestBlockNumOpt.get()); - // send the LIST - syncEventsHandler.startSnapSync(); + // start snap syncing +// syncEventsHandler.startFindingSnapConnectionPoint(bestPeerOpt.get()); + syncEventsHandler.startSnapSync(bestPeerOpt.get()); return true; } diff --git a/rskj-core/src/main/java/co/rsk/net/sync/PeersInformation.java b/rskj-core/src/main/java/co/rsk/net/sync/PeersInformation.java index 593e814beb2..995b3fae8e1 100644 --- a/rskj-core/src/main/java/co/rsk/net/sync/PeersInformation.java +++ b/rskj-core/src/main/java/co/rsk/net/sync/PeersInformation.java @@ -143,7 +143,7 @@ private Optional getBestPeer(Stream> bestC return Optional.of(entriesToConsider.get(randomIndex).getKey()); } - return getBestCandidatesStream() + return bestCandidatesStream .max(this.peerComparator) .map(Map.Entry::getKey); } diff --git a/rskj-core/src/main/java/co/rsk/net/sync/SnapDownloadingHeadersSyncState.java b/rskj-core/src/main/java/co/rsk/net/sync/SnapDownloadingHeadersSyncState.java new file mode 100644 index 00000000000..47e2625de14 --- /dev/null +++ b/rskj-core/src/main/java/co/rsk/net/sync/SnapDownloadingHeadersSyncState.java @@ -0,0 +1,92 @@ +/* + * This file is part of RskJ + * Copyright (C) 2024 RSK Labs Ltd. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ +package co.rsk.net.sync; + +import co.rsk.core.bc.ConsensusValidationMainchainView; +import co.rsk.net.Peer; +import co.rsk.net.Status; +import co.rsk.validators.BlockHeaderValidationRule; +import org.ethereum.core.BlockHeader; +import org.ethereum.core.BlockIdentifier; +import org.ethereum.db.BlockStore; +import org.ethereum.validator.DependentBlockHeaderRule; + +import java.util.Deque; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +public class SnapDownloadingHeadersSyncState extends DownloadingHeadersSyncState { + + private final PeersInformation peersInformation; + private final BlockStore blockStore; + + public SnapDownloadingHeadersSyncState( + SyncConfiguration syncConfiguration, + SyncEventsHandler syncEventsHandler, + ConsensusValidationMainchainView mainchainView, + DependentBlockHeaderRule blockParentValidationRule, + BlockHeaderValidationRule blockHeaderValidationRule, + Peer peer, + Map> skeletons, + long connectionPoint, + PeersInformation peersInformation, + BlockStore blockStore) { + super(syncConfiguration, syncEventsHandler, mainchainView, blockParentValidationRule, blockHeaderValidationRule, peer, skeletons, connectionPoint); + this.peersInformation = peersInformation; + this.blockStore = blockStore; + } + + @Override + void processPendingHeaders(List> pendingHeaders, Map> skeletons, Peer peer) { + pendingHeaders.forEach(headers -> headers.forEach(blockStore::saveBlockHeader)); + + Optional bestPeerOpt = peersInformation.getBestPeer(); + if (bestPeerOpt.isEmpty()) { + syncEventsHandler.stopSyncing(); + return; + } + + Optional bestSnapPeerOpt = peersInformation.getBestSnapPeer(); + if (bestSnapPeerOpt.isEmpty()) { + syncEventsHandler.stopSyncing(); + return; + } + + Optional bestSnapPeerStatusOpt = Optional.ofNullable(peersInformation.getPeer(bestSnapPeerOpt.get())).map(SyncPeerStatus::getStatus); + if (bestSnapPeerStatusOpt.isEmpty()) { + syncEventsHandler.stopSyncing(); + return; + } + + BlockHeader last = pendingHeaders.get(pendingHeaders.size() - 1).getLast(); + long numOfBlocksTillBest = bestSnapPeerStatusOpt.get().getBestBlockNumber() - last.getNumber(); + long syncMaxDistance = (long) syncConfiguration.getChunkSize() * syncConfiguration.getMaxSkeletonChunks(); + if (numOfBlocksTillBest < syncMaxDistance) { + syncEventsHandler.startSnapSync(bestSnapPeerOpt.get()); + return; + } + + Peer bestPeer = bestPeerOpt.get(); + if (bestPeer.getPeerNodeID().equals(peer.getPeerNodeID())) { + syncEventsHandler.startDownloadingSnapSkeleton(last.getNumber(), bestPeer); + } else { + syncEventsHandler.startFindingSnapConnectionPoint(bestPeer); + } + } +} diff --git a/rskj-core/src/main/java/co/rsk/net/sync/SnapDownloadingSkeletonSyncState.java b/rskj-core/src/main/java/co/rsk/net/sync/SnapDownloadingSkeletonSyncState.java new file mode 100644 index 00000000000..31ff762efe8 --- /dev/null +++ b/rskj-core/src/main/java/co/rsk/net/sync/SnapDownloadingSkeletonSyncState.java @@ -0,0 +1,40 @@ +/* + * This file is part of RskJ + * Copyright (C) 2024 RSK Labs Ltd. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ +package co.rsk.net.sync; + +import co.rsk.net.Peer; +import org.ethereum.core.BlockIdentifier; + +import java.util.List; +import java.util.Map; + +public class SnapDownloadingSkeletonSyncState extends DownloadingSkeletonSyncState { + + public SnapDownloadingSkeletonSyncState(SyncConfiguration syncConfiguration, + SyncEventsHandler syncEventsHandler, + PeersInformation peersInformation, + Peer peer, + long connectionPoint) { + super(syncConfiguration, syncEventsHandler, peersInformation, peer, connectionPoint); + } + + @Override + protected void startDownloadingHeaders(Map> skeletons, long connectionPoint, Peer peer) { + syncEventsHandler.startDownloadingSnapHeaders(skeletons, connectionPoint, peer); + } +} diff --git a/rskj-core/src/main/java/co/rsk/net/sync/SnapFindingConnectionPointSyncState.java b/rskj-core/src/main/java/co/rsk/net/sync/SnapFindingConnectionPointSyncState.java new file mode 100644 index 00000000000..c9987c85c8a --- /dev/null +++ b/rskj-core/src/main/java/co/rsk/net/sync/SnapFindingConnectionPointSyncState.java @@ -0,0 +1,43 @@ +/* + * This file is part of RskJ + * Copyright (C) 2024 RSK Labs Ltd. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ +package co.rsk.net.sync; + +import co.rsk.crypto.Keccak256; +import co.rsk.net.Peer; +import org.ethereum.db.BlockStore; + +public class SnapFindingConnectionPointSyncState extends FindingConnectionPointSyncState { + + public SnapFindingConnectionPointSyncState(SyncConfiguration syncConfiguration, + SyncEventsHandler syncEventsHandler, + BlockStore blockStore, + Peer selectedPeer, + long peerBestBlockNumber) { + super(syncConfiguration, syncEventsHandler, blockStore, selectedPeer, peerBestBlockNumber); + } + + @Override + protected boolean isKnown(Keccak256 hash) { + return blockStore.blockHeaderExists(hash) || super.isKnown(hash); + } + + @Override + protected void processConnectionPoint(long connectionPoint, Peer peer) { + syncEventsHandler.startDownloadingSnapSkeleton(connectionPoint, peer); + } +} diff --git a/rskj-core/src/main/java/co/rsk/net/sync/SnapSyncState.java b/rskj-core/src/main/java/co/rsk/net/sync/SnapSyncState.java index 65494b86129..d5d8873563c 100644 --- a/rskj-core/src/main/java/co/rsk/net/sync/SnapSyncState.java +++ b/rskj-core/src/main/java/co/rsk/net/sync/SnapSyncState.java @@ -21,14 +21,17 @@ import co.rsk.core.BlockDifficulty; import co.rsk.net.Peer; import co.rsk.net.SnapshotProcessor; +import co.rsk.net.messages.MessageType; import co.rsk.net.messages.SnapBlocksResponseMessage; import co.rsk.net.messages.SnapStateChunkResponseMessage; import co.rsk.net.messages.SnapStatusResponseMessage; +import co.rsk.scoring.EventType; import co.rsk.trie.TrieDTO; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; import org.apache.commons.lang3.tuple.Pair; import org.ethereum.core.Block; +import org.ethereum.core.BlockHeader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -63,6 +66,7 @@ public class SnapSyncState extends BaseSyncState { private byte[] remoteRootHash; private final List> blocks; private Block lastBlock; + private BlockHeader lastVerifiedBlockHeader; private BlockDifficulty lastBlockDifficulty; private long nextExpectedFrom = 0L; @@ -98,7 +102,7 @@ public void onEnter() { } isRunning = Boolean.TRUE; thread.start(); - snapshotProcessor.startSyncing(); + snapshotProcessor.startSyncing(this); } @Override @@ -146,6 +150,25 @@ public void run() { } } + @Override + public void newBlockHeaders(Peer peer, List chunk) { + try { + responseQueue.put(new SyncMessageHandler.Job(peer, MessageType.BLOCK_HEADERS_RESPONSE_MESSAGE) { + @Override + public void run() { + snapshotProcessor.processBlockHeaderChunk(SnapSyncState.this, peer, chunk); + } + }); + } catch (InterruptedException e) { + logger.warn("SnapStateChunkResponseMessage processing was interrupted", e); + Thread.currentThread().interrupt(); + } + } + + public SyncEventsHandler getSyncEventsHandler() { + return this.syncEventsHandler; + } + public void onNewChunk() { resetTimeElapsed(); } @@ -175,6 +198,14 @@ public void setLastBlock(Block lastBlock) { this.lastBlock = lastBlock; } + public BlockHeader getLastVerifiedBlockHeader() { + return lastVerifiedBlockHeader; + } + + public void setLastVerifiedBlockHeader(BlockHeader lastVerifiedBlockHeader) { + this.lastVerifiedBlockHeader = lastVerifiedBlockHeader; + } + public long getNextExpectedFrom() { return nextExpectedFrom; } @@ -211,8 +242,8 @@ public void addBlock(Pair blockPair) { blocks.add(blockPair); } - public void addAllBlocks(List> blocks) { - this.blocks.addAll(blocks); + public Pair getLastBlockPair() { + return blocks.isEmpty() ? null : blocks.get(blocks.size() - 1); } public void connectBlocks(BlockConnectorHelper blockConnectorHelper) { @@ -247,6 +278,14 @@ public Queue getChunkTaskQueue() { return chunkTaskQueue; } + public int getBlockHeaderChunkSize() { + return syncConfiguration.getChunkSize(); + } + + public boolean isRunning() { + return isRunning == Boolean.TRUE; + } + public void finish() { if (isRunning != Boolean.TRUE) { logger.warn("Invalid state, isRunning: [{}]", isRunning); @@ -259,6 +298,18 @@ public void finish() { syncEventsHandler.stopSyncing(); } + public void fail(Peer peer, EventType eventType, String message, Object... arguments) { + if (isRunning != Boolean.TRUE) { + logger.warn("Invalid state, isRunning: [{}]", isRunning); + return; + } + + isRunning = Boolean.FALSE; + thread.interrupt(); + + syncEventsHandler.onErrorSyncing(peer, eventType, message, arguments); + } + @VisibleForTesting public void setRunning() { isRunning = true; diff --git a/rskj-core/src/main/java/co/rsk/net/sync/SyncConfiguration.java b/rskj-core/src/main/java/co/rsk/net/sync/SyncConfiguration.java index beeb1ac70e7..142de5b1ea5 100644 --- a/rskj-core/src/main/java/co/rsk/net/sync/SyncConfiguration.java +++ b/rskj-core/src/main/java/co/rsk/net/sync/SyncConfiguration.java @@ -127,8 +127,6 @@ public SyncConfiguration( this.timeoutWaitingSnapChunk = Duration.ofSeconds(timeoutWaitingSnapChunk); this.snapshotSyncLimit = snapshotSyncLimit; - - List snapBootNodesList = snapBootNodes != null ? snapBootNodes : Collections.emptyList(); nodeIdToSnapshotTrustedPeerMap = Collections.unmodifiableMap(snapBootNodesList.stream() diff --git a/rskj-core/src/main/java/co/rsk/net/sync/SyncEventsHandler.java b/rskj-core/src/main/java/co/rsk/net/sync/SyncEventsHandler.java index e3cb4d93c83..b57561854e5 100644 --- a/rskj-core/src/main/java/co/rsk/net/sync/SyncEventsHandler.java +++ b/rskj-core/src/main/java/co/rsk/net/sync/SyncEventsHandler.java @@ -41,8 +41,12 @@ public interface SyncEventsHandler { void startDownloadingHeaders(Map> skeletons, long connectionPoint, Peer peer); + void startDownloadingSnapHeaders(Map> skeletons, long connectionPoint, Peer peer); + void startDownloadingSkeleton(long connectionPoint, Peer peer); + void startDownloadingSnapSkeleton(long connectionPoint, Peer peer); + void startBlockForwardSyncing(Peer peer); void backwardDownloadBodies(Block parent, List toRequest, Peer peer); @@ -57,7 +61,9 @@ public interface SyncEventsHandler { void startFindingConnectionPoint(Peer peer); + void startFindingSnapConnectionPoint(Peer peer); + void backwardSyncing(Peer peer); - void startSnapSync(); + void startSnapSync(Peer peer); } diff --git a/rskj-core/src/main/java/co/rsk/net/sync/SyncMessageHandler.java b/rskj-core/src/main/java/co/rsk/net/sync/SyncMessageHandler.java index b2f282cc468..725f3cb9340 100644 --- a/rskj-core/src/main/java/co/rsk/net/sync/SyncMessageHandler.java +++ b/rskj-core/src/main/java/co/rsk/net/sync/SyncMessageHandler.java @@ -20,6 +20,7 @@ import co.rsk.net.Peer; import co.rsk.net.messages.Message; +import co.rsk.net.messages.MessageType; import co.rsk.util.FormatUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -65,7 +66,7 @@ public void run() { job = jobQueue.take(); if (logger.isDebugEnabled()) { - logger.debug("Processing msg: [{}] from: [{}] for: [{}]", job.getMsg().getMessageType(), job.getSender(), name); + logger.debug("Processing msg: [{}] from: [{}] for: [{}]", job.getMsgType(), job.getSender(), name); jobStart = Instant.now(); } @@ -73,7 +74,7 @@ public void run() { if (logger.isDebugEnabled()) { logger.debug("Finished processing of msg: [{}] from: [{}] for: [{}] after [{}] seconds.", - job.getMsg().getMessageType(), job.getSender(), name, + job.getMsgType(), job.getSender(), name, FormatUtils.formatNanosecondsToSeconds(Duration.between(jobStart, Instant.now()).toNanos())); } @@ -116,27 +117,31 @@ public interface Listener { public static abstract class Job implements Runnable { private final Peer sender; - - private final Message msg; + private final MessageType msgType; public Job(Peer sender, Message msg) { this.sender = sender; - this.msg = msg; + this.msgType = msg.getMessageType(); + } + + public Job(Peer sender, MessageType msgType) { + this.sender = sender; + this.msgType = msgType; } public Peer getSender() { return sender; } - public Message getMsg() { - return msg; + public MessageType getMsgType() { + return msgType; } @Override public String toString() { return "SyncMessageHandler{" + "sender=" + sender + - ", msgType=" + msg.getMessageType() + + ", msgType=" + msgType + '}'; } } diff --git a/rskj-core/src/main/java/co/rsk/net/sync/SyncState.java b/rskj-core/src/main/java/co/rsk/net/sync/SyncState.java index e3b9b48fa47..4d0436d45b3 100644 --- a/rskj-core/src/main/java/co/rsk/net/sync/SyncState.java +++ b/rskj-core/src/main/java/co/rsk/net/sync/SyncState.java @@ -29,7 +29,7 @@ import java.util.List; public interface SyncState { - void newBlockHeaders(List chunk); + void newBlockHeaders(Peer peer, List chunk); // TODO(mc) don't receive a full message void newBody(BodyResponseMessage message, Peer peer); diff --git a/rskj-core/src/main/java/co/rsk/validators/BlockDifficultyRule.java b/rskj-core/src/main/java/co/rsk/validators/BlockDifficultyRule.java index 2afb970bb60..2f95ab041ff 100644 --- a/rskj-core/src/main/java/co/rsk/validators/BlockDifficultyRule.java +++ b/rskj-core/src/main/java/co/rsk/validators/BlockDifficultyRule.java @@ -43,12 +43,12 @@ public BlockDifficultyRule(DifficultyCalculator difficultyCalculator) { } @Override - public boolean isValid(BlockHeader header, Block parent) { + public boolean isValid(BlockHeader header, BlockHeader parent) { if (header == null || parent == null) { logger.warn("BlockDifficultyRule - block or parent are null"); return false; } - BlockDifficulty calcDifficulty = difficultyCalculator.calcDifficulty(header, parent.getHeader()); + BlockDifficulty calcDifficulty = difficultyCalculator.calcDifficulty(header, parent); BlockDifficulty difficulty = header.getDifficulty(); if (!difficulty.equals(calcDifficulty)) { diff --git a/rskj-core/src/main/java/co/rsk/validators/BlockHeaderParentCompositeRule.java b/rskj-core/src/main/java/co/rsk/validators/BlockHeaderParentCompositeRule.java index 5726b28138b..d7d24093afd 100644 --- a/rskj-core/src/main/java/co/rsk/validators/BlockHeaderParentCompositeRule.java +++ b/rskj-core/src/main/java/co/rsk/validators/BlockHeaderParentCompositeRule.java @@ -34,7 +34,7 @@ public BlockHeaderParentCompositeRule(BlockHeaderParentDependantValidationRule.. } @Override - public boolean isValid(BlockHeader header, Block parent) { + public boolean isValid(BlockHeader header, BlockHeader parent) { String shortHash = header.getPrintableHash(); long number = header.getNumber(); logger.debug("Validating parent header {} {}", shortHash, number); diff --git a/rskj-core/src/main/java/co/rsk/validators/BlockHeaderParentDependantValidationRule.java b/rskj-core/src/main/java/co/rsk/validators/BlockHeaderParentDependantValidationRule.java index c89c070bedb..8878b977d94 100644 --- a/rskj-core/src/main/java/co/rsk/validators/BlockHeaderParentDependantValidationRule.java +++ b/rskj-core/src/main/java/co/rsk/validators/BlockHeaderParentDependantValidationRule.java @@ -22,5 +22,9 @@ import org.ethereum.core.BlockHeader; public interface BlockHeaderParentDependantValidationRule { - boolean isValid(BlockHeader header, Block parent); + boolean isValid(BlockHeader header, BlockHeader parent); + + default boolean isValid(BlockHeader header, Block parentBlock) { + return isValid(header, parentBlock == null ? null : parentBlock.getHeader()); + } } diff --git a/rskj-core/src/main/java/co/rsk/validators/BlockParentGasLimitRule.java b/rskj-core/src/main/java/co/rsk/validators/BlockParentGasLimitRule.java index a41d101e1ce..5510436ed4b 100644 --- a/rskj-core/src/main/java/co/rsk/validators/BlockParentGasLimitRule.java +++ b/rskj-core/src/main/java/co/rsk/validators/BlockParentGasLimitRule.java @@ -50,15 +50,14 @@ public BlockParentGasLimitRule(int gasLimitBoundDivisor) { @Override - public boolean isValid(BlockHeader header, Block parent) { + public boolean isValid(BlockHeader header, BlockHeader parent) { if (header == null || parent == null) { logger.warn("BlockParentGasLimitRule - block or parent are null"); return false; } - BlockHeader parentHeader = parent.getHeader(); BigInteger headerGasLimit = new BigInteger(1, header.getGasLimit()); - BigInteger parentGasLimit = new BigInteger(1, parentHeader.getGasLimit()); + BigInteger parentGasLimit = new BigInteger(1, parent.getGasLimit()); if (headerGasLimit.compareTo(parentGasLimit.multiply(BigInteger.valueOf(gasLimitBoundDivisor - 1L)).divide(BigInteger.valueOf(gasLimitBoundDivisor))) < 0 || headerGasLimit.compareTo(parentGasLimit.multiply(BigInteger.valueOf(gasLimitBoundDivisor + 1L)).divide(BigInteger.valueOf(gasLimitBoundDivisor))) > 0) { diff --git a/rskj-core/src/main/java/co/rsk/validators/BlockParentNumberRule.java b/rskj-core/src/main/java/co/rsk/validators/BlockParentNumberRule.java index 318ce146b5d..a1484e27de0 100644 --- a/rskj-core/src/main/java/co/rsk/validators/BlockParentNumberRule.java +++ b/rskj-core/src/main/java/co/rsk/validators/BlockParentNumberRule.java @@ -35,13 +35,13 @@ public class BlockParentNumberRule implements BlockParentDependantValidationRule private static final Logger logger = LoggerFactory.getLogger("blockvalidator"); @Override - public boolean isValid(BlockHeader header, Block parent) { + public boolean isValid(BlockHeader header, BlockHeader parent) { if (header == null || parent == null) { logger.warn("BlockParentNumberRule - block or parent are null"); return false; } - BlockHeader parentHeader = parent.getHeader(); - if (header.getNumber() != (parentHeader.getNumber() + 1)) { + + if (header.getNumber() != (parent.getNumber() + 1)) { logger.warn("#{}: block number is not parentBlock number + 1", header.getNumber()); return false; } diff --git a/rskj-core/src/main/java/co/rsk/validators/BlockTimeStampValidationRule.java b/rskj-core/src/main/java/co/rsk/validators/BlockTimeStampValidationRule.java index c9432812af1..2e5cf4aae1f 100644 --- a/rskj-core/src/main/java/co/rsk/validators/BlockTimeStampValidationRule.java +++ b/rskj-core/src/main/java/co/rsk/validators/BlockTimeStampValidationRule.java @@ -88,7 +88,7 @@ public boolean isValid(BlockHeader header) { } @Override - public boolean isValid(BlockHeader header, Block parent) { + public boolean isValid(BlockHeader header, BlockHeader parent) { if (this.validPeriodLength == 0) { return true; } diff --git a/rskj-core/src/main/java/co/rsk/validators/PrevMinGasPriceRule.java b/rskj-core/src/main/java/co/rsk/validators/PrevMinGasPriceRule.java index 95bfa4641bc..1530abdcd36 100644 --- a/rskj-core/src/main/java/co/rsk/validators/PrevMinGasPriceRule.java +++ b/rskj-core/src/main/java/co/rsk/validators/PrevMinGasPriceRule.java @@ -39,7 +39,7 @@ public boolean isValid(Block block, Block parent) { } @Override - public boolean isValid(BlockHeader header, Block parent) { + public boolean isValid(BlockHeader header, BlockHeader parent) { if (header.isGenesis()) { return true; } diff --git a/rskj-core/src/main/java/org/ethereum/core/Blockchain.java b/rskj-core/src/main/java/org/ethereum/core/Blockchain.java index 9870fc063db..59aa9c240fb 100644 --- a/rskj-core/src/main/java/org/ethereum/core/Blockchain.java +++ b/rskj-core/src/main/java/org/ethereum/core/Blockchain.java @@ -56,6 +56,8 @@ public interface Blockchain { */ Block getBestBlock(); + Genesis getGenesisBlock(); + long getSize(); ImportResult tryToConnect(Block block); diff --git a/rskj-core/src/main/java/org/ethereum/core/genesis/BlockChainLoader.java b/rskj-core/src/main/java/org/ethereum/core/genesis/BlockChainLoader.java index c14eb12e018..8e593d5b344 100644 --- a/rskj-core/src/main/java/org/ethereum/core/genesis/BlockChainLoader.java +++ b/rskj-core/src/main/java/org/ethereum/core/genesis/BlockChainLoader.java @@ -106,6 +106,7 @@ public BlockChainImpl loadBlockchain() { } BlockChainImpl blockchain = new BlockChainImpl( + genesis, blockStore, receiptStore, transactionPool, diff --git a/rskj-core/src/main/java/org/ethereum/db/BlockHeaderStore.java b/rskj-core/src/main/java/org/ethereum/db/BlockHeaderStore.java new file mode 100644 index 00000000000..9823aa49a62 --- /dev/null +++ b/rskj-core/src/main/java/org/ethereum/db/BlockHeaderStore.java @@ -0,0 +1,35 @@ +/* + * This file is part of RskJ + * Copyright (C) 2024 RSK Labs Ltd. + * (derived from ethereumJ library, Copyright (c) 2016 ) + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +package org.ethereum.db; + +import co.rsk.crypto.Keccak256; +import org.ethereum.core.BlockHeader; + +public interface BlockHeaderStore { + + boolean blockHeaderExists(Keccak256 hash); + + BlockHeader getBlockHeaderByHash(Keccak256 hash); + + void saveBlockHeader(BlockHeader blockHeader); + + void removeBlockHeader(Keccak256 hash); + +} diff --git a/rskj-core/src/main/java/org/ethereum/db/BlockStore.java b/rskj-core/src/main/java/org/ethereum/db/BlockStore.java index c751f4dd105..75b612f8131 100644 --- a/rskj-core/src/main/java/org/ethereum/db/BlockStore.java +++ b/rskj-core/src/main/java/org/ethereum/db/BlockStore.java @@ -31,7 +31,7 @@ * @author Roman Mandeleil * @since 08.01.2015 */ -public interface BlockStore extends RemascCache { +public interface BlockStore extends BlockHeaderStore, RemascCache { /** * Gets the block hash by its index. diff --git a/rskj-core/src/main/java/org/ethereum/db/IndexedBlockStore.java b/rskj-core/src/main/java/org/ethereum/db/IndexedBlockStore.java index 7e2df441f2d..335323a0444 100644 --- a/rskj-core/src/main/java/org/ethereum/db/IndexedBlockStore.java +++ b/rskj-core/src/main/java/org/ethereum/db/IndexedBlockStore.java @@ -34,6 +34,7 @@ import org.ethereum.core.BlockHeader; import org.ethereum.core.Bloom; import org.ethereum.datasource.KeyValueDataSource; +import org.ethereum.util.ByteUtil; import org.mapdb.DataIO; import org.mapdb.Serializer; import org.slf4j.Logger; @@ -51,6 +52,7 @@ public class IndexedBlockStore implements BlockStore { private static final Logger logger = LoggerFactory.getLogger("general"); private static final Profiler profiler = ProfilerFactory.getInstance(); + private static final byte[] BLOCK_HEADER_KEY_PREFIX = new byte[]{'h'}; private final BlockCache blockCache; private final MaxSizeHashMap>> remascCache; @@ -492,6 +494,37 @@ public synchronized List getChainBlocksByNumber(long number) { return result; } + @Override + public boolean blockHeaderExists(Keccak256 hash) { + byte[] blockHeaderKey = ByteUtil.merge(BLOCK_HEADER_KEY_PREFIX, hash.getBytes()); + return this.blocks.get(blockHeaderKey) != null; + } + + @Override + public BlockHeader getBlockHeaderByHash(Keccak256 hash) { + byte[] blockHeaderKey = ByteUtil.merge(BLOCK_HEADER_KEY_PREFIX, hash.getBytes()); + byte[] blockHeaderRlp = this.blocks.get(blockHeaderKey); + + if (blockHeaderRlp == null) { + return null; + } + + return blockFactory.decodeHeader(blockHeaderRlp, false); + } + + @Override + public void saveBlockHeader(BlockHeader blockHeader) { + Keccak256 hash = blockHeader.getHash(); + byte[] blockHeaderKey = ByteUtil.merge(BLOCK_HEADER_KEY_PREFIX, hash.getBytes()); + this.blocks.put(blockHeaderKey, blockHeader.getFullEncoded()); + } + + @Override + public void removeBlockHeader(Keccak256 hash) { + byte[] blockHeaderKey = ByteUtil.merge(BLOCK_HEADER_KEY_PREFIX, hash.getBytes()); + this.blocks.delete(blockHeaderKey); + } + /** * Deletes from disk storage all blocks with number strictly larger than blockNumber. * Note that this doesn't clean the caches, making it unsuitable for using after initialization. diff --git a/rskj-core/src/main/java/org/ethereum/net/client/Capability.java b/rskj-core/src/main/java/org/ethereum/net/client/Capability.java index c8eb98b1669..50884f01497 100644 --- a/rskj-core/src/main/java/org/ethereum/net/client/Capability.java +++ b/rskj-core/src/main/java/org/ethereum/net/client/Capability.java @@ -51,6 +51,10 @@ public boolean isRSK() { return RSK.equals(name); } + public boolean isSNAP() { + return SNAP.equals(name); + } + @Override public boolean equals(Object obj) { if (this == obj) { diff --git a/rskj-core/src/main/java/org/ethereum/net/client/ConfigCapabilitiesImpl.java b/rskj-core/src/main/java/org/ethereum/net/client/ConfigCapabilitiesImpl.java index eb20c0e2b64..8cfac9b9e29 100644 --- a/rskj-core/src/main/java/org/ethereum/net/client/ConfigCapabilitiesImpl.java +++ b/rskj-core/src/main/java/org/ethereum/net/client/ConfigCapabilitiesImpl.java @@ -40,7 +40,7 @@ public class ConfigCapabilitiesImpl implements ConfigCapabilities{ private final RskSystemProperties config; - private SortedSet allCapabilities = new TreeSet<>(); + private final SortedSet allCapabilities = new TreeSet<>(); public ConfigCapabilitiesImpl(RskSystemProperties config) { if (config.syncVersion() != null) { @@ -54,14 +54,13 @@ public ConfigCapabilitiesImpl(RskSystemProperties config) { } } - if (config.isSnapshotSyncEnabled() && allCapabilities.stream().anyMatch(Capability::isRSK)) { + if (allCapabilities.stream().anyMatch(Capability::isRSK)) { allCapabilities.add(new Capability(SNAP, SNAP_VERSION)); } this.config = config; } - /** * Gets the capabilities listed in 'peer.capabilities' config property * sorted by their names. @@ -83,6 +82,10 @@ public List getConfigCapabilities() { @Override public List getSupportedCapabilities(HelloMessage hello) { List configCaps = getConfigCapabilities(); + if (config.isClientSnapshotSyncEnabled()) { + configCaps.add(new Capability(Capability.SNAP, Capability.SNAP_VERSION)); + } + List supported = new ArrayList<>(); List eths = new ArrayList<>(); diff --git a/rskj-core/src/main/java/org/ethereum/net/message/StaticMessages.java b/rskj-core/src/main/java/org/ethereum/net/message/StaticMessages.java index 10030069022..4c08cd4866c 100644 --- a/rskj-core/src/main/java/org/ethereum/net/message/StaticMessages.java +++ b/rskj-core/src/main/java/org/ethereum/net/message/StaticMessages.java @@ -19,7 +19,7 @@ package org.ethereum.net.message; -import org.ethereum.config.SystemProperties; +import co.rsk.config.RskSystemProperties; import org.ethereum.net.client.Capability; import org.ethereum.net.client.ConfigCapabilities; import org.ethereum.net.p2p.*; @@ -37,7 +37,7 @@ */ public class StaticMessages { - private final SystemProperties config; + private final RskSystemProperties config; private final ConfigCapabilities configCapabilities; public static final PingMessage PING_MESSAGE = new PingMessage(); @@ -45,7 +45,7 @@ public class StaticMessages { public static final GetPeersMessage GET_PEERS_MESSAGE = new GetPeersMessage(); public static final DisconnectMessage DISCONNECT_MESSAGE = new DisconnectMessage(ReasonCode.REQUESTED); - public StaticMessages(SystemProperties config, ConfigCapabilities configCapabilities) { + public StaticMessages(RskSystemProperties config, ConfigCapabilities configCapabilities) { this.config = config; this.configCapabilities = configCapabilities; } @@ -53,11 +53,14 @@ public StaticMessages(SystemProperties config, ConfigCapabilities configCapabili public HelloMessage createHelloMessage(String peerId) { return createHelloMessage(peerId, config.getPeerPort()); } - public HelloMessage createHelloMessage(String peerId, int listenPort) { + public HelloMessage createHelloMessage(String peerId, int listenPort) { String helloAnnouncement = buildHelloAnnouncement(); byte p2pVersion = (byte) config.defaultP2PVersion(); List capabilities = configCapabilities.getConfigCapabilities(); + if (config.isServerSnapshotSyncEnabled()) { + capabilities.add(new Capability(Capability.SNAP, Capability.SNAP_VERSION)); + } return new HelloMessage(p2pVersion, helloAnnouncement, capabilities, listenPort, peerId); diff --git a/rskj-core/src/main/java/org/ethereum/net/server/Channel.java b/rskj-core/src/main/java/org/ethereum/net/server/Channel.java index a79390d7dcd..c390be3318e 100644 --- a/rskj-core/src/main/java/org/ethereum/net/server/Channel.java +++ b/rskj-core/src/main/java/org/ethereum/net/server/Channel.java @@ -19,13 +19,12 @@ package org.ethereum.net.server; -import co.rsk.net.Peer; import co.rsk.net.NodeID; +import co.rsk.net.Peer; import co.rsk.net.eth.RskMessage; import co.rsk.net.eth.RskWireProtocol; import co.rsk.net.messages.Message; import co.rsk.net.messages.MessageType; -import com.google.common.annotations.VisibleForTesting; import io.netty.buffer.ByteBuf; import io.netty.channel.ChannelHandlerContext; import org.ethereum.net.MessageQueue; @@ -51,7 +50,6 @@ import java.math.BigInteger; import java.net.InetAddress; import java.net.InetSocketAddress; -import java.util.ArrayList; import java.util.List; import java.util.Objects; @@ -77,7 +75,7 @@ public class Channel implements Peer { private final PeerStatistics peerStats = new PeerStatistics(); private Stats stats; - private final boolean isSnapCapable; + private boolean isSnapCapable; public Channel(MessageQueue msgQueue, MessageCodec messageCodec, @@ -85,8 +83,7 @@ public Channel(MessageQueue msgQueue, RskWireProtocol.Factory rskWireProtocolFactory, Eth62MessageFactory eth62MessageFactory, StaticMessages staticMessages, - String remoteId, - List capabilities) { + String remoteId) { this.msgQueue = msgQueue; this.messageCodec = messageCodec; this.nodeManager = nodeManager; @@ -95,19 +92,6 @@ public Channel(MessageQueue msgQueue, this.staticMessages = staticMessages; this.isActive = remoteId != null && !remoteId.isEmpty(); this.stats = new Stats(); - this.isSnapCapable = capabilities.stream() - .anyMatch(capability -> Capability.SNAP.equals(capability.getName())); - } - - @VisibleForTesting - public Channel(MessageQueue msgQueue, - MessageCodec messageCodec, - NodeManager nodeManager, - RskWireProtocol.Factory rskWireProtocolFactory, - Eth62MessageFactory eth62MessageFactory, - StaticMessages staticMessages, - String remoteId) { - this(msgQueue, messageCodec, nodeManager, rskWireProtocolFactory, eth62MessageFactory, staticMessages, remoteId, new ArrayList<>()); } public void sendHelloMessage(ChannelHandlerContext ctx, FrameCodec frameCodec, String nodeId, @@ -170,6 +154,7 @@ public Node getNode() { } public void initMessageCodes(List caps) { + isSnapCapable = caps.stream().anyMatch(Capability::isSNAP); messageCodec.initMessageCodes(caps); } diff --git a/rskj-core/src/main/java/org/ethereum/net/server/EthereumChannelInitializer.java b/rskj-core/src/main/java/org/ethereum/net/server/EthereumChannelInitializer.java index f38bd7eacf8..7d0f478f7ff 100644 --- a/rskj-core/src/main/java/org/ethereum/net/server/EthereumChannelInitializer.java +++ b/rskj-core/src/main/java/org/ethereum/net/server/EthereumChannelInitializer.java @@ -111,7 +111,7 @@ public void initChannel(NioSocketChannel ch) { P2pHandler p2pHandler = new P2pHandler(ethereumListener, messageQueue, config.getPeerP2PPingInterval()); MessageCodec messageCodec = new MessageCodec(ethereumListener, config); HandshakeHandler handshakeHandler = new HandshakeHandler(config, peerScoringManager, p2pHandler, messageCodec, configCapabilities); - Channel channel = new Channel(messageQueue, messageCodec, nodeManager, rskWireProtocolFactory, eth62MessageFactory, staticMessages, remoteId, configCapabilities.getConfigCapabilities()); + Channel channel = new Channel(messageQueue, messageCodec, nodeManager, rskWireProtocolFactory, eth62MessageFactory, staticMessages, remoteId); ch.pipeline().addLast("readTimeoutHandler", new ReadTimeoutHandler(config.peerChannelReadTimeout(), TimeUnit.SECONDS)); ch.pipeline().addLast("handshakeHandler", handshakeHandler); diff --git a/rskj-core/src/main/resources/reference.conf b/rskj-core/src/main/resources/reference.conf index 86fa6b74016..75e21c96ec7 100644 --- a/rskj-core/src/main/resources/reference.conf +++ b/rskj-core/src/main/resources/reference.conf @@ -383,9 +383,9 @@ sync { # Request timeout (in seconds) chunkRequestTimeout = 120 # Distance to the tip of the blockchain to start snapshot sync - limit = 1000000 + limit = 10000 # Parallel requests (true, false) - parallel = true + parallel = false # list of SNAP-capable peers to connect to snapBootNodes = [] } diff --git a/rskj-core/src/test/java/co/rsk/core/bc/BlockRelayValidatorTest.java b/rskj-core/src/test/java/co/rsk/core/bc/BlockRelayValidatorTest.java index 03665cfd62c..29ab5d12248 100644 --- a/rskj-core/src/test/java/co/rsk/core/bc/BlockRelayValidatorTest.java +++ b/rskj-core/src/test/java/co/rsk/core/bc/BlockRelayValidatorTest.java @@ -47,7 +47,7 @@ void genesisCheck() { verify(block).isGenesis(); verify(blockValidator, never()).isValid(any()); - verify(blockParentValidator, never()).isValid(any(), any()); + verify(blockParentValidator, never()).isValid(any(), (Block) any()); } @Test @@ -62,7 +62,7 @@ void blockValidatorCheck() { verify(block).isGenesis(); verify(blockValidator).isValid(any()); - verify(blockParentValidator, never()).isValid(any(), any()); + verify(blockParentValidator, never()).isValid(any(), (Block) any()); } @Test @@ -74,7 +74,7 @@ void blockParentValidatorCheck() { when(block.getParentHash()).thenReturn(parentHash); when(blockStore.getBlockByHash(any())).thenReturn(parentBlock); when(blockValidator.isValid(any())).thenReturn(true); - when(blockParentValidator.isValid(any(), any())).thenReturn(false); + when(blockParentValidator.isValid(any(), (Block) any())).thenReturn(false); boolean actualResult = blockRelayValidator.isValid(block); @@ -82,7 +82,7 @@ void blockParentValidatorCheck() { verify(block).isGenesis(); verify(blockValidator).isValid(any()); - verify(blockParentValidator).isValid(any(), any()); + verify(blockParentValidator).isValid(any(), (Block) any()); } @Test @@ -94,7 +94,7 @@ void allValidatorsCheck() { when(block.getParentHash()).thenReturn(parentHash); when(blockStore.getBlockByHash(any())).thenReturn(parentBlock); when(blockValidator.isValid(any())).thenReturn(true); - when(blockParentValidator.isValid(any(), any())).thenReturn(true); + when(blockParentValidator.isValid(any(), (Block) any())).thenReturn(true); boolean actualResult = blockRelayValidator.isValid(block); @@ -102,6 +102,6 @@ void allValidatorsCheck() { verify(block).isGenesis(); verify(blockValidator).isValid(any()); - verify(blockParentValidator).isValid(any(), any()); + verify(blockParentValidator).isValid(any(), (Block) any()); } } diff --git a/rskj-core/src/test/java/co/rsk/core/bc/BlockValidatorBuilder.java b/rskj-core/src/test/java/co/rsk/core/bc/BlockValidatorBuilder.java index fd0b881a61c..34be11c2c6a 100644 --- a/rskj-core/src/test/java/co/rsk/core/bc/BlockValidatorBuilder.java +++ b/rskj-core/src/test/java/co/rsk/core/bc/BlockValidatorBuilder.java @@ -26,9 +26,9 @@ import co.rsk.trie.TrieStore; import co.rsk.util.TimeProvider; import co.rsk.validators.*; +import org.ethereum.core.Block; import org.ethereum.core.BlockTxSignatureCache; import org.ethereum.core.ReceivedTxSignatureCache; -import org.ethereum.core.SignatureCache; import org.ethereum.datasource.HashMapDB; import org.ethereum.db.BlockStore; import org.mockito.Mockito; @@ -99,7 +99,7 @@ public BlockValidatorBuilder addBlockUnclesValidationRule(BlockStore blockStore) Mockito.when(validationRule.isValid(Mockito.any())).thenReturn(true); BlockHeaderParentDependantValidationRule parentValidationRule = Mockito.mock(BlockHeaderParentDependantValidationRule.class); - Mockito.when(parentValidationRule.isValid(Mockito.any(), Mockito.any())).thenReturn(true); + Mockito.when(parentValidationRule.isValid(Mockito.any(), (Block) Mockito.any())).thenReturn(true); this.addBlockUnclesValidationRule(blockStore, validationRule, parentValidationRule); return this; diff --git a/rskj-core/src/test/java/co/rsk/core/bc/BlockValidatorTest.java b/rskj-core/src/test/java/co/rsk/core/bc/BlockValidatorTest.java index 77095d84965..f1f95fd4890 100644 --- a/rskj-core/src/test/java/co/rsk/core/bc/BlockValidatorTest.java +++ b/rskj-core/src/test/java/co/rsk/core/bc/BlockValidatorTest.java @@ -288,7 +288,7 @@ void invalidPOWUncles() { store.saveBlock(uncle1a, TEST_DIFFICULTY, false); BlockHeaderParentDependantValidationRule parentValidationRule = mock(BlockHeaderParentDependantValidationRule.class); - when(parentValidationRule.isValid(Mockito.any(), Mockito.any())).thenReturn(true); + when(parentValidationRule.isValid(Mockito.any(), (Block) Mockito.any())).thenReturn(true); BlockValidatorImpl validator = new BlockValidatorBuilder() .addBlockUnclesValidationRule(store, new ProofOfWorkRule(config).setFallbackMiningEnabled(false), parentValidationRule) diff --git a/rskj-core/src/test/java/co/rsk/net/SnapshotProcessorTest.java b/rskj-core/src/test/java/co/rsk/net/SnapshotProcessorTest.java index 0a6b126bb2d..c9cf24959a9 100644 --- a/rskj-core/src/test/java/co/rsk/net/SnapshotProcessorTest.java +++ b/rskj-core/src/test/java/co/rsk/net/SnapshotProcessorTest.java @@ -25,6 +25,10 @@ import co.rsk.net.sync.SyncMessageHandler; import co.rsk.test.builders.BlockChainBuilder; import co.rsk.trie.TrieStore; +import co.rsk.validators.BlockHeaderParentDependantValidationRule; +import co.rsk.validators.BlockHeaderValidationRule; +import co.rsk.validators.BlockParentDependantValidationRule; +import co.rsk.validators.BlockValidationRule; import org.ethereum.core.Block; import org.ethereum.core.Blockchain; import org.ethereum.core.TransactionPool; @@ -57,6 +61,10 @@ public class SnapshotProcessorTest { private final SnapshotPeersInformation peersInformation = mock(SnapshotPeersInformation.class); private final SnapSyncState snapSyncState = mock(SnapSyncState.class); private final SyncMessageHandler.Listener listener = mock(SyncMessageHandler.Listener.class); + private final BlockParentDependantValidationRule blockParentValidator = mock(BlockParentDependantValidationRule.class); + private final BlockValidationRule blockValidator = mock(BlockValidationRule.class); + private final BlockHeaderParentDependantValidationRule blockHeaderParentValidator = mock(BlockHeaderParentDependantValidationRule.class); + private final BlockHeaderValidationRule blockHeaderValidator = mock(BlockHeaderValidationRule.class); private SnapshotProcessor underTest; @BeforeEach @@ -82,10 +90,14 @@ void givenStartSyncingIsCalled_thenSnapStatusStartToBeRequestedFromPeer() { peersInformation, blockStore, transactionPool, + blockParentValidator, + blockValidator, + blockHeaderParentValidator, + blockHeaderValidator, TEST_CHUNK_SIZE, false); //when - underTest.startSyncing(); + underTest.startSyncing(snapSyncState); //then verify(peer).sendMessage(any(SnapStatusRequestMessage.class)); } @@ -102,6 +114,10 @@ void givenSnapStatusResponseCalled_thenSnapChunkRequestsAreMade() { peersInformation, blockStore, transactionPool, + blockParentValidator, + blockValidator, + blockHeaderParentValidator, + blockHeaderValidator, TEST_CHUNK_SIZE, false); @@ -117,7 +133,7 @@ void givenSnapStatusResponseCalled_thenSnapChunkRequestsAreMade() { doReturn(snapStatusResponseMessage.getTrieSize()).when(snapSyncState).getRemoteTrieSize(); doReturn(new LinkedList<>()).when(snapSyncState).getChunkTaskQueue(); - underTest.startSyncing(); + underTest.startSyncing(snapSyncState); //when underTest.processSnapStatusResponse(snapSyncState, peer, snapStatusResponseMessage); @@ -137,6 +153,10 @@ void givenSnapStatusRequestReceived_thenSnapStatusResponseIsSent() { peersInformation, blockStore, transactionPool, + blockParentValidator, + blockValidator, + blockHeaderParentValidator, + blockHeaderValidator, TEST_CHUNK_SIZE, false); //when @@ -156,6 +176,10 @@ void givenSnapBlockRequestReceived_thenSnapBlocksResponseMessageIsSent() { peersInformation, blockStore, transactionPool, + blockParentValidator, + blockValidator, + blockHeaderParentValidator, + blockHeaderValidator, TEST_CHUNK_SIZE, false); @@ -179,6 +203,10 @@ void givenSnapBlocksResponseReceived_thenSnapBlocksRequestMessageIsSent() { peersInformation, blockStore, transactionPool, + blockParentValidator, + blockValidator, + blockHeaderParentValidator, + blockHeaderValidator, 200, false); @@ -191,7 +219,7 @@ void givenSnapBlocksResponseReceived_thenSnapBlocksRequestMessageIsSent() { SnapStatusResponseMessage snapStatusResponseMessage = new SnapStatusResponseMessage(blocks, difficulties, 100000L); doReturn(new LinkedList<>()).when(snapSyncState).getChunkTaskQueue(); - underTest.startSyncing(); + underTest.startSyncing(snapSyncState); underTest.processSnapStatusResponse(snapSyncState, peer, snapStatusResponseMessage); SnapBlocksResponseMessage snapBlocksResponseMessage = new SnapBlocksResponseMessage(blocks, difficulties); @@ -214,6 +242,10 @@ void givenSnapStateChunkRequest_thenSnapStateChunkResponseMessageIsSent() { peersInformation, blockStore, transactionPool, + blockParentValidator, + blockValidator, + blockHeaderParentValidator, + blockHeaderValidator, TEST_CHUNK_SIZE, false); @@ -239,6 +271,10 @@ void givenProcessSnapStatusRequestIsCalled_thenInternalOneIsCalledLater() throws peersInformation, blockStore, transactionPool, + blockParentValidator, + blockValidator, + blockHeaderParentValidator, + blockHeaderValidator, TEST_CHUNK_SIZE, false, listener) { @@ -259,7 +295,7 @@ void processSnapStatusRequestInternal(Peer sender, SnapStatusRequestMessage requ verify(listener, times(1)).onJobRun(jobArg.capture()); assertEquals(peer, jobArg.getValue().getSender()); - assertEquals(msg, jobArg.getValue().getMsg()); + assertEquals(msg.getMessageType(), jobArg.getValue().getMsgType()); } @Test @@ -275,6 +311,10 @@ void givenProcessSnapBlocksRequestIsCalled_thenInternalOneIsCalledLater() throws peersInformation, blockStore, transactionPool, + blockParentValidator, + blockValidator, + blockHeaderParentValidator, + blockHeaderValidator, TEST_CHUNK_SIZE, false, listener) { @@ -295,7 +335,7 @@ void processSnapBlocksRequestInternal(Peer sender, SnapBlocksRequestMessage requ verify(listener, times(1)).onJobRun(jobArg.capture()); assertEquals(peer, jobArg.getValue().getSender()); - assertEquals(msg, jobArg.getValue().getMsg()); + assertEquals(msg.getMessageType(), jobArg.getValue().getMsgType()); } @Test @@ -311,6 +351,10 @@ void givenProcessStateChunkRequestIsCalled_thenInternalOneIsCalledLater() throws peersInformation, blockStore, transactionPool, + blockParentValidator, + blockValidator, + blockHeaderParentValidator, + blockHeaderValidator, TEST_CHUNK_SIZE, false, listener) { @@ -331,7 +375,7 @@ void processStateChunkRequestInternal(Peer sender, SnapStateChunkRequestMessage verify(listener, times(1)).onJobRun(jobArg.capture()); assertEquals(peer, jobArg.getValue().getSender()); - assertEquals(msg, jobArg.getValue().getMsg()); + assertEquals(msg.getMessageType(), jobArg.getValue().getMsgType()); } @Test @@ -342,6 +386,10 @@ void givenErrorRLPData_thenOnStateChunkErrorIsCalled() { peersInformation, blockStore, transactionPool, + blockParentValidator, + blockValidator, + blockHeaderParentValidator, + blockHeaderValidator, TEST_CHUNK_SIZE, false); diff --git a/rskj-core/src/test/java/co/rsk/net/sync/CheckingBestHeaderSyncStateTest.java b/rskj-core/src/test/java/co/rsk/net/sync/CheckingBestHeaderSyncStateTest.java index bc440807e11..f4b47e0fb78 100644 --- a/rskj-core/src/test/java/co/rsk/net/sync/CheckingBestHeaderSyncStateTest.java +++ b/rskj-core/src/test/java/co/rsk/net/sync/CheckingBestHeaderSyncStateTest.java @@ -68,7 +68,7 @@ void newBlockHeadersWhenValidHeaderContinue() { when(header.getHash().getBytes()).thenReturn(HASH_1); when(blockHeaderValidationRule.isValid(header)).thenReturn(true); - state.newBlockHeaders(Collections.singletonList(header)); + state.newBlockHeaders(null, Collections.singletonList(header)); verify(syncEventsHandler, times(1)).startFindingConnectionPoint(peer); } @@ -79,7 +79,7 @@ void newBlockHeadersWhenInValidHeaderOnErrorSyncing() { when(header.getHash().getBytes()).thenReturn(HASH_1); when(blockHeaderValidationRule.isValid(header)).thenReturn(false); - state.newBlockHeaders(Collections.singletonList(header)); + state.newBlockHeaders(null, Collections.singletonList(header)); verify(syncEventsHandler, times(1)) .onErrorSyncing(peer, EventType.INVALID_HEADER, @@ -92,7 +92,7 @@ void newBlockHeadersWhenDifferentHeaderOnErrorSyncing() { when(header.getHash().getBytes()).thenReturn(HashUtil.sha256(new byte[]{5})); when(blockHeaderValidationRule.isValid(header)).thenReturn(true); - state.newBlockHeaders(Collections.singletonList(header)); + state.newBlockHeaders(null, Collections.singletonList(header)); verify(syncEventsHandler, times(1)) .onErrorSyncing(peer, EventType.INVALID_HEADER, diff --git a/rskj-core/src/test/java/co/rsk/net/sync/DownloadingBackwardsHeadersSyncStateTest.java b/rskj-core/src/test/java/co/rsk/net/sync/DownloadingBackwardsHeadersSyncStateTest.java index 4d1885521a6..d78a5d989f6 100644 --- a/rskj-core/src/test/java/co/rsk/net/sync/DownloadingBackwardsHeadersSyncStateTest.java +++ b/rskj-core/src/test/java/co/rsk/net/sync/DownloadingBackwardsHeadersSyncStateTest.java @@ -96,7 +96,7 @@ void newHeaders() { List receivedHeaders = new LinkedList<>(); - target.newBlockHeaders(receivedHeaders); + target.newBlockHeaders(selectedPeer, receivedHeaders); verify(syncEventsHandler).backwardDownloadBodies(child, receivedHeaders, selectedPeer); diff --git a/rskj-core/src/test/java/co/rsk/net/sync/DownloadingHeadersSyncStateTest.java b/rskj-core/src/test/java/co/rsk/net/sync/DownloadingHeadersSyncStateTest.java index 6f27aa9234c..9c76e9d9afb 100644 --- a/rskj-core/src/test/java/co/rsk/net/sync/DownloadingHeadersSyncStateTest.java +++ b/rskj-core/src/test/java/co/rsk/net/sync/DownloadingHeadersSyncStateTest.java @@ -126,7 +126,7 @@ void newBlockHeadersWhenNoCurrentChunkThenSyncIssue() { when(chunksDownloadHelper.getCurrentChunk()).thenReturn(Optional.empty()); - syncState.newBlockHeaders(new ArrayList<>()); + syncState.newBlockHeaders(selectedPeer, new ArrayList<>()); verify(syncEventsHandler, times(1)).onSyncIssue(selectedPeer, "Current chunk not present on {}", DownloadingHeadersSyncState.class); @@ -157,7 +157,7 @@ void newBlockHeadersWhenUnexpectedChunkSizeThenInvalidMessage() { List chunk = new ArrayList<>(); chunk.add(mock(BlockHeader.class)); - syncState.newBlockHeaders(chunk); + syncState.newBlockHeaders(selectedPeer, chunk); verify(syncEventsHandler, times(1)).onErrorSyncing(selectedPeer, EventType.INVALID_MESSAGE, "Unexpected chunk size received on {}: hash: {}", DownloadingHeadersSyncState.class, HashUtil.toPrintableHash(currentChunk.getHash())); @@ -191,7 +191,7 @@ void newBlockHeadersWhenUnexpectedHeaderThenInvalidMessage() { byte[] headerHash = TestUtils.generateBytes(DownloadingHeadersSyncStateTest.class,"headerHash",32); when(header.getHash().getBytes()).thenReturn(headerHash);; // different from chunkHash chunk.add(header); - syncState.newBlockHeaders(chunk); + syncState.newBlockHeaders(selectedPeer, chunk); verify(syncEventsHandler, times(1)).onErrorSyncing(selectedPeer, EventType.INVALID_MESSAGE, "Unexpected chunk header hash received on {}: hash: {}", DownloadingHeadersSyncState.class, HashUtil.toPrintableHash(currentChunk.getHash())); diff --git a/rskj-core/src/test/java/co/rsk/net/sync/SimpleSyncEventsHandler.java b/rskj-core/src/test/java/co/rsk/net/sync/SimpleSyncEventsHandler.java index 246ee4f56fc..71b00dcaa1b 100644 --- a/rskj-core/src/test/java/co/rsk/net/sync/SimpleSyncEventsHandler.java +++ b/rskj-core/src/test/java/co/rsk/net/sync/SimpleSyncEventsHandler.java @@ -42,6 +42,11 @@ public void startFindingConnectionPoint(Peer peer) { } + @Override + public void startFindingSnapConnectionPoint(Peer peer) { + + } + @Override public void backwardSyncing(Peer peer) { } @@ -65,6 +70,9 @@ public void sendBlockHashRequest(Peer peer, long height) { @Override public void startDownloadingHeaders(Map> skeletons, long connectionPoint, Peer peer) { } + @Override + public void startDownloadingSnapHeaders(Map> skeletons, long connectionPoint, Peer peer) {} + @Override public void startBlockForwardSyncing(Peer peer) { this.startSyncingWasCalled_ = true; @@ -78,6 +86,9 @@ public void backwardDownloadBodies(Block parent, List toRequest, Pe @Override public void startDownloadingSkeleton(long connectionPoint, Peer peer) { } + @Override + public void startDownloadingSnapSkeleton(long connectionPoint, Peer peer) {} + @Override public void stopSyncing() { this.stopSyncingWasCalled_ = true; } @@ -105,5 +116,5 @@ public boolean stopSyncingWasCalled() { } @Override - public void startSnapSync() { } + public void startSnapSync(Peer peer) { } } diff --git a/rskj-core/src/test/java/co/rsk/net/sync/SnapSyncStateTest.java b/rskj-core/src/test/java/co/rsk/net/sync/SnapSyncStateTest.java index 8ce31717683..42319223dc3 100644 --- a/rskj-core/src/test/java/co/rsk/net/sync/SnapSyncStateTest.java +++ b/rskj-core/src/test/java/co/rsk/net/sync/SnapSyncStateTest.java @@ -76,7 +76,7 @@ void givenOnEnterWasCalledAndNotRunningYet_thenSyncingStartsWithTestObjectAsPara //given-when underTest.onEnter(); //then - verify(snapshotProcessor, times(1)).startSyncing(); + verify(snapshotProcessor, times(1)).startSyncing(underTest); } @Test @@ -95,7 +95,7 @@ void givenOnEnterWasCalledTwice_thenSyncingStartsOnlyOnce() { underTest.onEnter(); underTest.onEnter(); //then - verify(snapshotProcessor, times(1)).startSyncing(); + verify(snapshotProcessor, times(1)).startSyncing(underTest); } @Test @@ -178,7 +178,7 @@ void givenOnSnapStatusIsCalled_thenJobIsAddedAndRun() throws InterruptedExceptio verify(listener, times(1)).onJobRun(jobArg.capture()); assertEquals(peer, jobArg.getValue().getSender()); - assertEquals(msg, jobArg.getValue().getMsg()); + assertEquals(msg.getMessageType(), jobArg.getValue().getMsgType()); } @Test @@ -200,7 +200,7 @@ void givenOnSnapBlocksIsCalled_thenJobIsAddedAndRun() throws InterruptedExceptio verify(listener, times(1)).onJobRun(jobArg.capture()); assertEquals(peer, jobArg.getValue().getSender()); - assertEquals(msg, jobArg.getValue().getMsg()); + assertEquals(msg.getMessageType(), jobArg.getValue().getMsgType()); } @Test @@ -222,7 +222,7 @@ void givenOnSnapStateChunkIsCalled_thenJobIsAddedAndRun() throws InterruptedExce verify(listener, times(1)).onJobRun(jobArg.capture()); assertEquals(peer, jobArg.getValue().getSender()); - assertEquals(msg, jobArg.getValue().getMsg()); + assertEquals(msg.getMessageType(), jobArg.getValue().getMsgType()); } @Test diff --git a/rskj-core/src/test/java/co/rsk/validators/PrevMinGasPriceValidatorTest.java b/rskj-core/src/test/java/co/rsk/validators/PrevMinGasPriceValidatorTest.java index bb38e5d54a1..ef846888151 100644 --- a/rskj-core/src/test/java/co/rsk/validators/PrevMinGasPriceValidatorTest.java +++ b/rskj-core/src/test/java/co/rsk/validators/PrevMinGasPriceValidatorTest.java @@ -47,7 +47,7 @@ void noParentBlock() { PrevMinGasPriceRule pmgpv = new PrevMinGasPriceRule(); - Assertions.assertFalse(pmgpv.isValid(header, null)); + Assertions.assertFalse(pmgpv.isValid(header, (Block) null)); } @Test @@ -61,7 +61,7 @@ void genesisBlock() { PrevMinGasPriceRule pmgpv = new PrevMinGasPriceRule(); - Assertions.assertTrue(pmgpv.isValid(header, null)); + Assertions.assertTrue(pmgpv.isValid(header, (Block) null)); } @Test diff --git a/rskj-core/src/test/java/org/ethereum/core/ImportLightTest.java b/rskj-core/src/test/java/org/ethereum/core/ImportLightTest.java index 3d1489a4680..e3c1884f4ca 100644 --- a/rskj-core/src/test/java/org/ethereum/core/ImportLightTest.java +++ b/rskj-core/src/test/java/org/ethereum/core/ImportLightTest.java @@ -80,6 +80,7 @@ public static BlockChainImpl createBlockchain( TransactionPoolImpl transactionPool = new TransactionPoolImpl(config, repositoryLocator, null, blockFactory, listener, transactionExecutorFactory, receivedTxSignatureCache, 10, 100, Mockito.mock(TxQuotaChecker.class), Mockito.mock(GasPriceTracker.class)); BlockChainImpl blockchain = new BlockChainImpl( + genesis, blockStore, receiptStore, transactionPool, diff --git a/rskj-core/src/test/java/org/ethereum/db/BlockStoreDummy.java b/rskj-core/src/test/java/org/ethereum/db/BlockStoreDummy.java index a646ff44f8f..080fc3f281e 100644 --- a/rskj-core/src/test/java/org/ethereum/db/BlockStoreDummy.java +++ b/rskj-core/src/test/java/org/ethereum/db/BlockStoreDummy.java @@ -23,6 +23,7 @@ import co.rsk.crypto.Keccak256; import co.rsk.remasc.Sibling; import org.ethereum.core.Block; +import org.ethereum.core.BlockHeader; import org.ethereum.core.Bloom; import org.ethereum.crypto.HashUtil; @@ -31,7 +32,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Optional; /** * @author Roman Mandeleil @@ -124,6 +124,26 @@ public boolean isEmpty() { return false; } + @Override + public boolean blockHeaderExists(Keccak256 hash) { + return false; + } + + @Override + public BlockHeader getBlockHeaderByHash(Keccak256 hash) { + return null; + } + + @Override + public void saveBlockHeader(BlockHeader blockHeader) { + + } + + @Override + public void removeBlockHeader(Keccak256 hash) { + + } + @Override public void close() { diff --git a/rskj-core/src/test/java/org/ethereum/jsontestsuite/TestRunner.java b/rskj-core/src/test/java/org/ethereum/jsontestsuite/TestRunner.java index 1109dcec96c..352c26d21d7 100644 --- a/rskj-core/src/test/java/org/ethereum/jsontestsuite/TestRunner.java +++ b/rskj-core/src/test/java/org/ethereum/jsontestsuite/TestRunner.java @@ -157,6 +157,7 @@ public List runTestCase(BlockTestingCase testCase) { TransactionPoolImpl transactionPool = new TransactionPoolImpl(config, repositoryLocator, null, blockFactory, listener, transactionExecutorFactory, new ReceivedTxSignatureCache(), 10, 100, Mockito.mock(TxQuotaChecker.class), Mockito.mock(GasPriceTracker.class)); BlockChainImpl blockchain = new BlockChainImpl( + null, blockStore, receiptStore, transactionPool, diff --git a/rskj-core/src/test/java/org/ethereum/jsontestsuite/runners/StateTestRunner.java b/rskj-core/src/test/java/org/ethereum/jsontestsuite/runners/StateTestRunner.java index ea1d8339986..411f886c1bf 100644 --- a/rskj-core/src/test/java/org/ethereum/jsontestsuite/runners/StateTestRunner.java +++ b/rskj-core/src/test/java/org/ethereum/jsontestsuite/runners/StateTestRunner.java @@ -152,6 +152,7 @@ public List runImpl() { BlockStore blockStore = new IndexedBlockStore(blockFactory, new HashMapDB(), new HashMapBlocksIndex()); StateRootHandler stateRootHandler = new StateRootHandler(config.getActivationConfig(), new StateRootsStoreImpl(new HashMapDB())); blockchain = new BlockChainImpl( + null, blockStore, null, null, From 3161ac686f9bb5d8f9c193e601ae31f453287ad3 Mon Sep 17 00:00:00 2001 From: Volodymyr Kravets Date: Mon, 25 Nov 2024 13:30:00 +0200 Subject: [PATCH 02/10] feat(snap): change headers featching stop condition; cleanup --- .../java/co/rsk/core/bc/BlockChainImpl.java | 10 +- .../java/co/rsk/net/SnapshotProcessor.java | 18 ++-- .../main/java/co/rsk/net/SyncProcessor.java | 34 ------- .../co/rsk/net/sync/BlockConnectorHelper.java | 8 +- .../net/sync/DownloadingHeadersSyncState.java | 6 +- .../sync/DownloadingSkeletonSyncState.java | 8 +- .../sync/FindingConnectionPointSyncState.java | 19 ++-- .../sync/SnapDownloadingHeadersSyncState.java | 92 ------------------- .../SnapDownloadingSkeletonSyncState.java | 40 -------- .../SnapFindingConnectionPointSyncState.java | 43 --------- .../co/rsk/net/sync/SyncEventsHandler.java | 6 -- .../java/org/ethereum/core/Blockchain.java | 2 - .../core/genesis/BlockChainLoader.java | 1 - .../org/ethereum/db/BlockHeaderStore.java | 35 ------- .../main/java/org/ethereum/db/BlockStore.java | 2 +- .../org/ethereum/db/IndexedBlockStore.java | 33 ------- .../rsk/net/sync/SimpleSyncEventsHandler.java | 11 --- .../org/ethereum/core/ImportLightTest.java | 2 - .../java/org/ethereum/db/BlockStoreDummy.java | 22 +---- .../ethereum/jsontestsuite/TestRunner.java | 1 - .../runners/StateTestRunner.java | 3 +- 21 files changed, 32 insertions(+), 364 deletions(-) delete mode 100644 rskj-core/src/main/java/co/rsk/net/sync/SnapDownloadingHeadersSyncState.java delete mode 100644 rskj-core/src/main/java/co/rsk/net/sync/SnapDownloadingSkeletonSyncState.java delete mode 100644 rskj-core/src/main/java/co/rsk/net/sync/SnapFindingConnectionPointSyncState.java delete mode 100644 rskj-core/src/main/java/org/ethereum/db/BlockHeaderStore.java diff --git a/rskj-core/src/main/java/co/rsk/core/bc/BlockChainImpl.java b/rskj-core/src/main/java/co/rsk/core/bc/BlockChainImpl.java index 2755414b8c9..6bed3f10649 100644 --- a/rskj-core/src/main/java/co/rsk/core/bc/BlockChainImpl.java +++ b/rskj-core/src/main/java/co/rsk/core/bc/BlockChainImpl.java @@ -78,7 +78,6 @@ public class BlockChainImpl implements Blockchain { private static final Logger logger = LoggerFactory.getLogger("blockchain"); private static final PanicProcessor panicProcessor = new PanicProcessor(); - private final Genesis genesis; private final BlockStore blockStore; private final ReceiptStore receiptStore; private final TransactionPool transactionPool; @@ -95,15 +94,13 @@ public class BlockChainImpl implements Blockchain { private final BlockExecutor blockExecutor; private boolean noValidation; - public BlockChainImpl(Genesis genesis, - BlockStore blockStore, + public BlockChainImpl(BlockStore blockStore, ReceiptStore receiptStore, TransactionPool transactionPool, EthereumListener listener, BlockValidator blockValidator, BlockExecutor blockExecutor, StateRootHandler stateRootHandler) { - this.genesis = genesis; this.blockStore = blockStore; this.receiptStore = receiptStore; this.listener = listener; @@ -424,11 +421,6 @@ public Block getBestBlock() { return this.status.getBestBlock(); } - @Override - public Genesis getGenesisBlock() { - return this.genesis; - } - public void setNoValidation(boolean noValidation) { this.noValidation = noValidation; } diff --git a/rskj-core/src/main/java/co/rsk/net/SnapshotProcessor.java b/rskj-core/src/main/java/co/rsk/net/SnapshotProcessor.java index 93a88845491..bb263d358b3 100644 --- a/rskj-core/src/main/java/co/rsk/net/SnapshotProcessor.java +++ b/rskj-core/src/main/java/co/rsk/net/SnapshotProcessor.java @@ -37,7 +37,10 @@ import com.google.common.collect.Lists; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; -import org.ethereum.core.*; +import org.ethereum.core.Block; +import org.ethereum.core.BlockHeader; +import org.ethereum.core.Blockchain; +import org.ethereum.core.TransactionPool; import org.ethereum.db.BlockStore; import org.ethereum.util.RLP; import org.ethereum.util.RLPElement; @@ -251,9 +254,8 @@ public void processSnapStatusResponse(SnapSyncState state, Peer sender, SnapStat logger.debug("CLIENT - Processing snapshot status response - last blockNumber: {} triesize: {}", lastBlock.getNumber(), state.getRemoteTrieSize()); logger.debug("Blocks included in the response: {} from {} to {}", blocksFromResponse.size(), blocksFromResponse.get(0).getNumber(), blocksFromResponse.get(blocksFromResponse.size() - 1).getNumber()); + requestBlocksChunk(sender, blocksFromResponse.get(0).getNumber()); -// generateChunkRequestTasks(state); -// startRequestingChunks(state); } private boolean validateAndSaveBlocks(SnapSyncState state, Peer sender, List blocks, List difficulties) { @@ -269,11 +271,11 @@ private boolean validateAndSaveBlocks(SnapSyncState state, Peer sender, List> skeletons, connectionPoint)); } - @Override - public void startDownloadingSnapHeaders(Map> skeletons, long connectionPoint, Peer peer) { - setSyncState(new SnapDownloadingHeadersSyncState( - syncConfiguration, - this, - consensusValidationMainchainView, - difficultyRule, - blockHeaderValidationRule, - peer, - skeletons, - connectionPoint, - peersInformation, - blockStore)); - } - @Override public void startDownloadingSkeleton(long connectionPoint, Peer peer) { setSyncState(new DownloadingSkeletonSyncState( @@ -354,16 +339,6 @@ public void startDownloadingSkeleton(long connectionPoint, Peer peer) { connectionPoint)); } - @Override - public void startDownloadingSnapSkeleton(long connectionPoint, Peer peer) { - setSyncState(new SnapDownloadingSkeletonSyncState( - syncConfiguration, - this, - peersInformation, - peer, - connectionPoint)); - } - @Override public void startFindingConnectionPoint(Peer peer) { NodeID peerId = peer.getPeerNodeID(); @@ -373,15 +348,6 @@ public void startFindingConnectionPoint(Peer peer) { syncConfiguration, this, blockStore, peer, bestBlockNumber)); } - @Override - public void startFindingSnapConnectionPoint(Peer peer) { - NodeID peerId = peer.getPeerNodeID(); - logger.debug("Find snap connection point with node {}", peerId); - long bestBlockNumber = peersInformation.getPeer(peer).getStatus().getBestBlockNumber(); - setSyncState(new SnapFindingConnectionPointSyncState( - syncConfiguration, this, blockStore, peer, bestBlockNumber)); - } - @Override public void backwardSyncing(Peer peer) { NodeID peerId = peer.getPeerNodeID(); diff --git a/rskj-core/src/main/java/co/rsk/net/sync/BlockConnectorHelper.java b/rskj-core/src/main/java/co/rsk/net/sync/BlockConnectorHelper.java index 0985f7eb0eb..30c69a7f346 100644 --- a/rskj-core/src/main/java/co/rsk/net/sync/BlockConnectorHelper.java +++ b/rskj-core/src/main/java/co/rsk/net/sync/BlockConnectorHelper.java @@ -64,11 +64,15 @@ public void startConnecting(List> blockAndDifficult if (!currentBlock.isParentOf(child)) { throw new BlockConnectorException(currentBlock.getNumber(), child.getNumber()); } - blockStore.saveBlock(currentBlock, currentBlockAndDifficulty.getRight(), true); + if (!blockStore.isBlockExist(currentBlock.getHash().getBytes())) { + blockStore.saveBlock(currentBlock, currentBlockAndDifficulty.getRight(), true); + } else { + logger.warn("Block: [{}/{}] already exists. Skipping", currentBlock.getNumber(), currentBlock.getHash()); + } child = currentBlock; blockIndex--; } - logger.info("Finished connecting blocks. Last saved block: {}",child.getNumber()); + logger.info("Finished connecting blocks. Last saved block: [{}/{}]", child.getNumber(), child.getHash()); } static class BlockAndDiffComparator implements java.util.Comparator> { diff --git a/rskj-core/src/main/java/co/rsk/net/sync/DownloadingHeadersSyncState.java b/rskj-core/src/main/java/co/rsk/net/sync/DownloadingHeadersSyncState.java index c159cd2d743..e0658eb338c 100644 --- a/rskj-core/src/main/java/co/rsk/net/sync/DownloadingHeadersSyncState.java +++ b/rskj-core/src/main/java/co/rsk/net/sync/DownloadingHeadersSyncState.java @@ -124,7 +124,7 @@ public void newBlockHeaders(Peer peer, List chunk) { if (!chunksDownloadHelper.hasNextChunk()) { // Finished verifying headers - processPendingHeaders(pendingHeaders, skeletons, selectedPeer); + syncEventsHandler.startDownloadingBodies(pendingHeaders, skeletons, selectedPeer); return; } @@ -161,8 +161,4 @@ private boolean blockHeaderIsValid(BlockHeader header, BlockHeader parentHeader) return blockParentValidationRule.validate(header, parentHeader); } - - void processPendingHeaders(List> pendingHeaders, Map> skeletons, Peer peer) { - syncEventsHandler.startDownloadingBodies(pendingHeaders, skeletons, peer); - } } diff --git a/rskj-core/src/main/java/co/rsk/net/sync/DownloadingSkeletonSyncState.java b/rskj-core/src/main/java/co/rsk/net/sync/DownloadingSkeletonSyncState.java index 6028dfed01f..7b20fa4d5db 100644 --- a/rskj-core/src/main/java/co/rsk/net/sync/DownloadingSkeletonSyncState.java +++ b/rskj-core/src/main/java/co/rsk/net/sync/DownloadingSkeletonSyncState.java @@ -74,7 +74,7 @@ public void newSkeleton(List skeleton, Peer peer) { syncEventsHandler.stopSyncing(); return; } - startDownloadingHeaders(skeletons, connectionPoint, peer); + syncEventsHandler.startDownloadingHeaders(skeletons, connectionPoint, peer); } } @@ -94,7 +94,7 @@ public void tick(Duration duration) { return; } - startDownloadingHeaders(skeletons, connectionPoint, selectedPeer); + syncEventsHandler.startDownloadingHeaders(skeletons, connectionPoint, selectedPeer); } } @@ -102,8 +102,4 @@ public void tick(Duration duration) { public void onEnter() { peersInformation.getBestPeerCandidates().forEach(p -> syncEventsHandler.sendSkeletonRequest(p, connectionPoint)); } - - protected void startDownloadingHeaders(Map> skeletons, long connectionPoint, Peer peer) { - syncEventsHandler.startDownloadingHeaders(skeletons, connectionPoint, peer); - } } diff --git a/rskj-core/src/main/java/co/rsk/net/sync/FindingConnectionPointSyncState.java b/rskj-core/src/main/java/co/rsk/net/sync/FindingConnectionPointSyncState.java index 5b472933970..6f00c653455 100644 --- a/rskj-core/src/main/java/co/rsk/net/sync/FindingConnectionPointSyncState.java +++ b/rskj-core/src/main/java/co/rsk/net/sync/FindingConnectionPointSyncState.java @@ -17,7 +17,6 @@ */ package co.rsk.net.sync; -import co.rsk.crypto.Keccak256; import co.rsk.net.Peer; import org.ethereum.db.BlockStore; @@ -25,7 +24,7 @@ public class FindingConnectionPointSyncState extends BaseSelectedPeerSyncState { - protected final BlockStore blockStore; + private final BlockStore blockStore; private final ConnectionPointFinder connectionPointFinder; public FindingConnectionPointSyncState(SyncConfiguration syncConfiguration, @@ -44,15 +43,15 @@ public FindingConnectionPointSyncState(SyncConfiguration syncConfiguration, @Override public void newConnectionPointData(byte[] hash) { - boolean knownBlock = isKnown(new Keccak256(hash)); + boolean knownBlock = isKnownBlock(hash); Optional cp = connectionPointFinder.getConnectionPoint(); if (cp.isPresent()) { if (knownBlock) { - processConnectionPoint(cp.get(), selectedPeer); + syncEventsHandler.startDownloadingSkeleton(cp.get(), selectedPeer); } else { syncEventsHandler.onSyncIssue(selectedPeer, "Connection point not found on {}", this.getClass()); } - return; + return; } if (knownBlock) { @@ -64,7 +63,7 @@ public void newConnectionPointData(byte[] hash) { cp = connectionPointFinder.getConnectionPoint(); // No need to ask for genesis hash if (cp.isPresent() && cp.get() == 0L) { - processConnectionPoint(cp.get(), selectedPeer); + syncEventsHandler.startDownloadingSkeleton(cp.get(), selectedPeer); return; } @@ -72,8 +71,8 @@ public void newConnectionPointData(byte[] hash) { trySendRequest(); } - protected boolean isKnown(Keccak256 hash) { - return blockStore.isBlockExist(hash.getBytes()); + private boolean isKnownBlock(byte[] hash) { + return blockStore.isBlockExist(hash); } private void trySendRequest() { @@ -84,8 +83,4 @@ private void trySendRequest() { public void onEnter() { trySendRequest(); } - - protected void processConnectionPoint(long connectionPoint, Peer peer) { - syncEventsHandler.startDownloadingSkeleton(connectionPoint, peer); - } } diff --git a/rskj-core/src/main/java/co/rsk/net/sync/SnapDownloadingHeadersSyncState.java b/rskj-core/src/main/java/co/rsk/net/sync/SnapDownloadingHeadersSyncState.java deleted file mode 100644 index 47e2625de14..00000000000 --- a/rskj-core/src/main/java/co/rsk/net/sync/SnapDownloadingHeadersSyncState.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * This file is part of RskJ - * Copyright (C) 2024 RSK Labs Ltd. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ -package co.rsk.net.sync; - -import co.rsk.core.bc.ConsensusValidationMainchainView; -import co.rsk.net.Peer; -import co.rsk.net.Status; -import co.rsk.validators.BlockHeaderValidationRule; -import org.ethereum.core.BlockHeader; -import org.ethereum.core.BlockIdentifier; -import org.ethereum.db.BlockStore; -import org.ethereum.validator.DependentBlockHeaderRule; - -import java.util.Deque; -import java.util.List; -import java.util.Map; -import java.util.Optional; - -public class SnapDownloadingHeadersSyncState extends DownloadingHeadersSyncState { - - private final PeersInformation peersInformation; - private final BlockStore blockStore; - - public SnapDownloadingHeadersSyncState( - SyncConfiguration syncConfiguration, - SyncEventsHandler syncEventsHandler, - ConsensusValidationMainchainView mainchainView, - DependentBlockHeaderRule blockParentValidationRule, - BlockHeaderValidationRule blockHeaderValidationRule, - Peer peer, - Map> skeletons, - long connectionPoint, - PeersInformation peersInformation, - BlockStore blockStore) { - super(syncConfiguration, syncEventsHandler, mainchainView, blockParentValidationRule, blockHeaderValidationRule, peer, skeletons, connectionPoint); - this.peersInformation = peersInformation; - this.blockStore = blockStore; - } - - @Override - void processPendingHeaders(List> pendingHeaders, Map> skeletons, Peer peer) { - pendingHeaders.forEach(headers -> headers.forEach(blockStore::saveBlockHeader)); - - Optional bestPeerOpt = peersInformation.getBestPeer(); - if (bestPeerOpt.isEmpty()) { - syncEventsHandler.stopSyncing(); - return; - } - - Optional bestSnapPeerOpt = peersInformation.getBestSnapPeer(); - if (bestSnapPeerOpt.isEmpty()) { - syncEventsHandler.stopSyncing(); - return; - } - - Optional bestSnapPeerStatusOpt = Optional.ofNullable(peersInformation.getPeer(bestSnapPeerOpt.get())).map(SyncPeerStatus::getStatus); - if (bestSnapPeerStatusOpt.isEmpty()) { - syncEventsHandler.stopSyncing(); - return; - } - - BlockHeader last = pendingHeaders.get(pendingHeaders.size() - 1).getLast(); - long numOfBlocksTillBest = bestSnapPeerStatusOpt.get().getBestBlockNumber() - last.getNumber(); - long syncMaxDistance = (long) syncConfiguration.getChunkSize() * syncConfiguration.getMaxSkeletonChunks(); - if (numOfBlocksTillBest < syncMaxDistance) { - syncEventsHandler.startSnapSync(bestSnapPeerOpt.get()); - return; - } - - Peer bestPeer = bestPeerOpt.get(); - if (bestPeer.getPeerNodeID().equals(peer.getPeerNodeID())) { - syncEventsHandler.startDownloadingSnapSkeleton(last.getNumber(), bestPeer); - } else { - syncEventsHandler.startFindingSnapConnectionPoint(bestPeer); - } - } -} diff --git a/rskj-core/src/main/java/co/rsk/net/sync/SnapDownloadingSkeletonSyncState.java b/rskj-core/src/main/java/co/rsk/net/sync/SnapDownloadingSkeletonSyncState.java deleted file mode 100644 index 31ff762efe8..00000000000 --- a/rskj-core/src/main/java/co/rsk/net/sync/SnapDownloadingSkeletonSyncState.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * This file is part of RskJ - * Copyright (C) 2024 RSK Labs Ltd. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ -package co.rsk.net.sync; - -import co.rsk.net.Peer; -import org.ethereum.core.BlockIdentifier; - -import java.util.List; -import java.util.Map; - -public class SnapDownloadingSkeletonSyncState extends DownloadingSkeletonSyncState { - - public SnapDownloadingSkeletonSyncState(SyncConfiguration syncConfiguration, - SyncEventsHandler syncEventsHandler, - PeersInformation peersInformation, - Peer peer, - long connectionPoint) { - super(syncConfiguration, syncEventsHandler, peersInformation, peer, connectionPoint); - } - - @Override - protected void startDownloadingHeaders(Map> skeletons, long connectionPoint, Peer peer) { - syncEventsHandler.startDownloadingSnapHeaders(skeletons, connectionPoint, peer); - } -} diff --git a/rskj-core/src/main/java/co/rsk/net/sync/SnapFindingConnectionPointSyncState.java b/rskj-core/src/main/java/co/rsk/net/sync/SnapFindingConnectionPointSyncState.java deleted file mode 100644 index c9987c85c8a..00000000000 --- a/rskj-core/src/main/java/co/rsk/net/sync/SnapFindingConnectionPointSyncState.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * This file is part of RskJ - * Copyright (C) 2024 RSK Labs Ltd. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ -package co.rsk.net.sync; - -import co.rsk.crypto.Keccak256; -import co.rsk.net.Peer; -import org.ethereum.db.BlockStore; - -public class SnapFindingConnectionPointSyncState extends FindingConnectionPointSyncState { - - public SnapFindingConnectionPointSyncState(SyncConfiguration syncConfiguration, - SyncEventsHandler syncEventsHandler, - BlockStore blockStore, - Peer selectedPeer, - long peerBestBlockNumber) { - super(syncConfiguration, syncEventsHandler, blockStore, selectedPeer, peerBestBlockNumber); - } - - @Override - protected boolean isKnown(Keccak256 hash) { - return blockStore.blockHeaderExists(hash) || super.isKnown(hash); - } - - @Override - protected void processConnectionPoint(long connectionPoint, Peer peer) { - syncEventsHandler.startDownloadingSnapSkeleton(connectionPoint, peer); - } -} diff --git a/rskj-core/src/main/java/co/rsk/net/sync/SyncEventsHandler.java b/rskj-core/src/main/java/co/rsk/net/sync/SyncEventsHandler.java index b57561854e5..e0bd7be4e9a 100644 --- a/rskj-core/src/main/java/co/rsk/net/sync/SyncEventsHandler.java +++ b/rskj-core/src/main/java/co/rsk/net/sync/SyncEventsHandler.java @@ -41,12 +41,8 @@ public interface SyncEventsHandler { void startDownloadingHeaders(Map> skeletons, long connectionPoint, Peer peer); - void startDownloadingSnapHeaders(Map> skeletons, long connectionPoint, Peer peer); - void startDownloadingSkeleton(long connectionPoint, Peer peer); - void startDownloadingSnapSkeleton(long connectionPoint, Peer peer); - void startBlockForwardSyncing(Peer peer); void backwardDownloadBodies(Block parent, List toRequest, Peer peer); @@ -61,8 +57,6 @@ public interface SyncEventsHandler { void startFindingConnectionPoint(Peer peer); - void startFindingSnapConnectionPoint(Peer peer); - void backwardSyncing(Peer peer); void startSnapSync(Peer peer); diff --git a/rskj-core/src/main/java/org/ethereum/core/Blockchain.java b/rskj-core/src/main/java/org/ethereum/core/Blockchain.java index 59aa9c240fb..9870fc063db 100644 --- a/rskj-core/src/main/java/org/ethereum/core/Blockchain.java +++ b/rskj-core/src/main/java/org/ethereum/core/Blockchain.java @@ -56,8 +56,6 @@ public interface Blockchain { */ Block getBestBlock(); - Genesis getGenesisBlock(); - long getSize(); ImportResult tryToConnect(Block block); diff --git a/rskj-core/src/main/java/org/ethereum/core/genesis/BlockChainLoader.java b/rskj-core/src/main/java/org/ethereum/core/genesis/BlockChainLoader.java index 8e593d5b344..c14eb12e018 100644 --- a/rskj-core/src/main/java/org/ethereum/core/genesis/BlockChainLoader.java +++ b/rskj-core/src/main/java/org/ethereum/core/genesis/BlockChainLoader.java @@ -106,7 +106,6 @@ public BlockChainImpl loadBlockchain() { } BlockChainImpl blockchain = new BlockChainImpl( - genesis, blockStore, receiptStore, transactionPool, diff --git a/rskj-core/src/main/java/org/ethereum/db/BlockHeaderStore.java b/rskj-core/src/main/java/org/ethereum/db/BlockHeaderStore.java deleted file mode 100644 index 9823aa49a62..00000000000 --- a/rskj-core/src/main/java/org/ethereum/db/BlockHeaderStore.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * This file is part of RskJ - * Copyright (C) 2024 RSK Labs Ltd. - * (derived from ethereumJ library, Copyright (c) 2016 ) - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package org.ethereum.db; - -import co.rsk.crypto.Keccak256; -import org.ethereum.core.BlockHeader; - -public interface BlockHeaderStore { - - boolean blockHeaderExists(Keccak256 hash); - - BlockHeader getBlockHeaderByHash(Keccak256 hash); - - void saveBlockHeader(BlockHeader blockHeader); - - void removeBlockHeader(Keccak256 hash); - -} diff --git a/rskj-core/src/main/java/org/ethereum/db/BlockStore.java b/rskj-core/src/main/java/org/ethereum/db/BlockStore.java index 75b612f8131..c751f4dd105 100644 --- a/rskj-core/src/main/java/org/ethereum/db/BlockStore.java +++ b/rskj-core/src/main/java/org/ethereum/db/BlockStore.java @@ -31,7 +31,7 @@ * @author Roman Mandeleil * @since 08.01.2015 */ -public interface BlockStore extends BlockHeaderStore, RemascCache { +public interface BlockStore extends RemascCache { /** * Gets the block hash by its index. diff --git a/rskj-core/src/main/java/org/ethereum/db/IndexedBlockStore.java b/rskj-core/src/main/java/org/ethereum/db/IndexedBlockStore.java index 335323a0444..7e2df441f2d 100644 --- a/rskj-core/src/main/java/org/ethereum/db/IndexedBlockStore.java +++ b/rskj-core/src/main/java/org/ethereum/db/IndexedBlockStore.java @@ -34,7 +34,6 @@ import org.ethereum.core.BlockHeader; import org.ethereum.core.Bloom; import org.ethereum.datasource.KeyValueDataSource; -import org.ethereum.util.ByteUtil; import org.mapdb.DataIO; import org.mapdb.Serializer; import org.slf4j.Logger; @@ -52,7 +51,6 @@ public class IndexedBlockStore implements BlockStore { private static final Logger logger = LoggerFactory.getLogger("general"); private static final Profiler profiler = ProfilerFactory.getInstance(); - private static final byte[] BLOCK_HEADER_KEY_PREFIX = new byte[]{'h'}; private final BlockCache blockCache; private final MaxSizeHashMap>> remascCache; @@ -494,37 +492,6 @@ public synchronized List getChainBlocksByNumber(long number) { return result; } - @Override - public boolean blockHeaderExists(Keccak256 hash) { - byte[] blockHeaderKey = ByteUtil.merge(BLOCK_HEADER_KEY_PREFIX, hash.getBytes()); - return this.blocks.get(blockHeaderKey) != null; - } - - @Override - public BlockHeader getBlockHeaderByHash(Keccak256 hash) { - byte[] blockHeaderKey = ByteUtil.merge(BLOCK_HEADER_KEY_PREFIX, hash.getBytes()); - byte[] blockHeaderRlp = this.blocks.get(blockHeaderKey); - - if (blockHeaderRlp == null) { - return null; - } - - return blockFactory.decodeHeader(blockHeaderRlp, false); - } - - @Override - public void saveBlockHeader(BlockHeader blockHeader) { - Keccak256 hash = blockHeader.getHash(); - byte[] blockHeaderKey = ByteUtil.merge(BLOCK_HEADER_KEY_PREFIX, hash.getBytes()); - this.blocks.put(blockHeaderKey, blockHeader.getFullEncoded()); - } - - @Override - public void removeBlockHeader(Keccak256 hash) { - byte[] blockHeaderKey = ByteUtil.merge(BLOCK_HEADER_KEY_PREFIX, hash.getBytes()); - this.blocks.delete(blockHeaderKey); - } - /** * Deletes from disk storage all blocks with number strictly larger than blockNumber. * Note that this doesn't clean the caches, making it unsuitable for using after initialization. diff --git a/rskj-core/src/test/java/co/rsk/net/sync/SimpleSyncEventsHandler.java b/rskj-core/src/test/java/co/rsk/net/sync/SimpleSyncEventsHandler.java index 71b00dcaa1b..d6694ecebe4 100644 --- a/rskj-core/src/test/java/co/rsk/net/sync/SimpleSyncEventsHandler.java +++ b/rskj-core/src/test/java/co/rsk/net/sync/SimpleSyncEventsHandler.java @@ -42,11 +42,6 @@ public void startFindingConnectionPoint(Peer peer) { } - @Override - public void startFindingSnapConnectionPoint(Peer peer) { - - } - @Override public void backwardSyncing(Peer peer) { } @@ -70,9 +65,6 @@ public void sendBlockHashRequest(Peer peer, long height) { @Override public void startDownloadingHeaders(Map> skeletons, long connectionPoint, Peer peer) { } - @Override - public void startDownloadingSnapHeaders(Map> skeletons, long connectionPoint, Peer peer) {} - @Override public void startBlockForwardSyncing(Peer peer) { this.startSyncingWasCalled_ = true; @@ -86,9 +78,6 @@ public void backwardDownloadBodies(Block parent, List toRequest, Pe @Override public void startDownloadingSkeleton(long connectionPoint, Peer peer) { } - @Override - public void startDownloadingSnapSkeleton(long connectionPoint, Peer peer) {} - @Override public void stopSyncing() { this.stopSyncingWasCalled_ = true; } diff --git a/rskj-core/src/test/java/org/ethereum/core/ImportLightTest.java b/rskj-core/src/test/java/org/ethereum/core/ImportLightTest.java index e3c1884f4ca..0ede5ed08a7 100644 --- a/rskj-core/src/test/java/org/ethereum/core/ImportLightTest.java +++ b/rskj-core/src/test/java/org/ethereum/core/ImportLightTest.java @@ -43,7 +43,6 @@ import org.mockito.Mockito; import java.util.Map; -import java.util.function.Supplier; /** * Created by Anton Nashatyrev on 29.12.2015. @@ -80,7 +79,6 @@ public static BlockChainImpl createBlockchain( TransactionPoolImpl transactionPool = new TransactionPoolImpl(config, repositoryLocator, null, blockFactory, listener, transactionExecutorFactory, receivedTxSignatureCache, 10, 100, Mockito.mock(TxQuotaChecker.class), Mockito.mock(GasPriceTracker.class)); BlockChainImpl blockchain = new BlockChainImpl( - genesis, blockStore, receiptStore, transactionPool, diff --git a/rskj-core/src/test/java/org/ethereum/db/BlockStoreDummy.java b/rskj-core/src/test/java/org/ethereum/db/BlockStoreDummy.java index 080fc3f281e..a646ff44f8f 100644 --- a/rskj-core/src/test/java/org/ethereum/db/BlockStoreDummy.java +++ b/rskj-core/src/test/java/org/ethereum/db/BlockStoreDummy.java @@ -23,7 +23,6 @@ import co.rsk.crypto.Keccak256; import co.rsk.remasc.Sibling; import org.ethereum.core.Block; -import org.ethereum.core.BlockHeader; import org.ethereum.core.Bloom; import org.ethereum.crypto.HashUtil; @@ -32,6 +31,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Optional; /** * @author Roman Mandeleil @@ -124,26 +124,6 @@ public boolean isEmpty() { return false; } - @Override - public boolean blockHeaderExists(Keccak256 hash) { - return false; - } - - @Override - public BlockHeader getBlockHeaderByHash(Keccak256 hash) { - return null; - } - - @Override - public void saveBlockHeader(BlockHeader blockHeader) { - - } - - @Override - public void removeBlockHeader(Keccak256 hash) { - - } - @Override public void close() { diff --git a/rskj-core/src/test/java/org/ethereum/jsontestsuite/TestRunner.java b/rskj-core/src/test/java/org/ethereum/jsontestsuite/TestRunner.java index 352c26d21d7..1109dcec96c 100644 --- a/rskj-core/src/test/java/org/ethereum/jsontestsuite/TestRunner.java +++ b/rskj-core/src/test/java/org/ethereum/jsontestsuite/TestRunner.java @@ -157,7 +157,6 @@ public List runTestCase(BlockTestingCase testCase) { TransactionPoolImpl transactionPool = new TransactionPoolImpl(config, repositoryLocator, null, blockFactory, listener, transactionExecutorFactory, new ReceivedTxSignatureCache(), 10, 100, Mockito.mock(TxQuotaChecker.class), Mockito.mock(GasPriceTracker.class)); BlockChainImpl blockchain = new BlockChainImpl( - null, blockStore, receiptStore, transactionPool, diff --git a/rskj-core/src/test/java/org/ethereum/jsontestsuite/runners/StateTestRunner.java b/rskj-core/src/test/java/org/ethereum/jsontestsuite/runners/StateTestRunner.java index 411f886c1bf..3bf9dd6f2b9 100644 --- a/rskj-core/src/test/java/org/ethereum/jsontestsuite/runners/StateTestRunner.java +++ b/rskj-core/src/test/java/org/ethereum/jsontestsuite/runners/StateTestRunner.java @@ -18,7 +18,6 @@ */ package org.ethereum.jsontestsuite.runners; -import co.rsk.peg.constants.BridgeRegTestConstants; import co.rsk.config.TestSystemProperties; import co.rsk.core.Coin; import co.rsk.core.RskAddress; @@ -31,6 +30,7 @@ import co.rsk.db.StateRootsStoreImpl; import co.rsk.peg.BridgeSupportFactory; import co.rsk.peg.RepositoryBtcBlockStoreWithCache; +import co.rsk.peg.constants.BridgeRegTestConstants; import co.rsk.trie.TrieStoreImpl; import org.ethereum.core.*; import org.ethereum.datasource.HashMapDB; @@ -152,7 +152,6 @@ public List runImpl() { BlockStore blockStore = new IndexedBlockStore(blockFactory, new HashMapDB(), new HashMapBlocksIndex()); StateRootHandler stateRootHandler = new StateRootHandler(config.getActivationConfig(), new StateRootsStoreImpl(new HashMapDB())); blockchain = new BlockChainImpl( - null, blockStore, null, null, From bb0801adf0eb9692cb94896f7c7aa270e8e527d0 Mon Sep 17 00:00:00 2001 From: Volodymyr Kravets Date: Wed, 27 Nov 2024 18:22:14 +0200 Subject: [PATCH 03/10] feat(snap): fixed tests; more refactoring and cleanup --- .../java/co/rsk/net/SnapshotProcessor.java | 80 ++++++++--- .../rsk/net/sync/BlockConnectorException.java | 38 ------ .../co/rsk/net/sync/BlockConnectorHelper.java | 35 ++--- .../co/rsk/net/sync/PeersInformation.java | 5 + .../java/co/rsk/net/sync/SnapSyncState.java | 16 ++- .../net/sync/SnapshotPeersInformation.java | 3 + .../co/rsk/net/sync/SyncMessageHandler.java | 2 +- rskj-core/src/main/resources/reference.conf | 2 +- .../co/rsk/net/SnapshotProcessorTest.java | 15 +- .../net/sync/BlockConnectorHelperTest.java | 129 ++---------------- .../co/rsk/net/sync/SnapSyncStateTest.java | 15 +- .../BlockTimeStampValidationRuleTest.java | 54 +++++++- 12 files changed, 174 insertions(+), 220 deletions(-) delete mode 100644 rskj-core/src/main/java/co/rsk/net/sync/BlockConnectorException.java diff --git a/rskj-core/src/main/java/co/rsk/net/SnapshotProcessor.java b/rskj-core/src/main/java/co/rsk/net/SnapshotProcessor.java index bb263d358b3..afb1dd9beaf 100644 --- a/rskj-core/src/main/java/co/rsk/net/SnapshotProcessor.java +++ b/rskj-core/src/main/java/co/rsk/net/SnapshotProcessor.java @@ -243,8 +243,7 @@ public void processSnapStatusResponse(SnapSyncState state, Peer sender, SnapStat Block lastBlock = blocksFromResponse.get(blocksFromResponse.size() - 1); BlockDifficulty lastBlockDifficulty = difficultiesFromResponse.get(difficultiesFromResponse.size() - 1); - state.setLastBlock(lastBlock); - state.setLastBlockDifficulty(lastBlockDifficulty); + state.setLastBlock(lastBlock, lastBlockDifficulty, sender); state.setRemoteRootHash(lastBlock.getStateRoot()); state.setRemoteTrieSize(responseMessage.getTrieSize()); @@ -255,7 +254,14 @@ public void processSnapStatusResponse(SnapSyncState state, Peer sender, SnapStat logger.debug("CLIENT - Processing snapshot status response - last blockNumber: {} triesize: {}", lastBlock.getNumber(), state.getRemoteTrieSize()); logger.debug("Blocks included in the response: {} from {} to {}", blocksFromResponse.size(), blocksFromResponse.get(0).getNumber(), blocksFromResponse.get(blocksFromResponse.size() - 1).getNumber()); - requestBlocksChunk(sender, blocksFromResponse.get(0).getNumber()); + if (blocksVerified(state)) { + logger.info("CLIENT - Finished Snap blocks request sending."); + + generateChunkRequestTasks(state); + startRequestingChunks(state); + } else { + requestBlocksChunk(sender, blocksFromResponse.get(0).getNumber()); + } } private boolean validateAndSaveBlocks(SnapSyncState state, Peer sender, List blocks, List difficulties) { @@ -265,22 +271,34 @@ private boolean validateAndSaveBlocks(SnapSyncState state, Peer sender, List blockPair = new ImmutablePair<>(block, totalDifficulty); - if (!areBlockPairsValid(blockPair, childBlockPair)) { + if (!areBlockPairsValid(blockPair, childBlockPair, true)) { failSyncing(state, sender, EventType.INVALID_BLOCK, "Block [{}]/[{}] at height: [{}] is not valid", block.getHash(), totalDifficulty, block.getNumber()); return false; } + Block parentBlock = blockStore.getBlockByHash(block.getParentHash().getBytes()); + if (parentBlock != null) { + if (areBlockPairsValid(new ImmutablePair<>(parentBlock, blockStore.getTotalDifficultyForHash(parentBlock.getHash().getBytes())), blockPair, false)) { + state.addBlock(blockPair); + state.setLastVerifiedBlockHeader(blockPair.getLeft().getHeader()); + return true; + } else { + failSyncing(state, sender, EventType.INVALID_BLOCK, "Block [{}]/[{}] at height: [{}] is not valid", block.getHash(), totalDifficulty, block.getNumber()); + return false; + } + } + state.addBlock(blockPair); + state.setLastVerifiedBlockHeader(blockPair.getLeft().getHeader()); childBlockPair = blockPair; - state.setLastVerifiedBlockHeader(childBlockPair.getLeft().getHeader()); } return true; } - private boolean areBlockPairsValid(Pair blockPair, @Nullable Pair childBlockPair) { - if (!blockValidator.isValid(blockPair.getLeft())) { + private boolean areBlockPairsValid(Pair blockPair, @Nullable Pair childBlockPair, boolean validateParent) { + if (validateParent && !blockValidator.isValid(blockPair.getLeft())) { return false; } @@ -330,8 +348,14 @@ private boolean validateBlockHeaders(SnapSyncState state, Peer sender, List blocks = Lists.newArrayList(); List difficulties = Lists.newArrayList(); - long startingBlockNumber = requestMessage.getBlockNumber() - BLOCK_CHUNK_SIZE; + + if (requestMessage.getBlockNumber() < 2) { + logger.debug("Snap blocks request from {} failed because of invalid block number {}", sender.getPeerNodeID(), requestMessage.getBlockNumber()); + return; + } + + long startingBlockNumber = Math.max(1, requestMessage.getBlockNumber() - BLOCK_CHUNK_SIZE); for (long i = startingBlockNumber; i < requestMessage.getBlockNumber(); i++) { Block block = blockchain.getBlockByNumber(i); + if (block == null) { + break; + } blocks.add(block); difficulties.add(blockStore.getTotalDifficultyForHash(block.getHash().getBytes())); } @@ -425,7 +466,13 @@ public void processSnapBlocksResponse(SnapSyncState state, Peer sender, SnapBloc long nextChunk = blocksFromResponse.get(0).getNumber(); logger.debug("CLIENT - SnapBlock - nexChunk : {} - lastRequired {}, missing {}", nextChunk, lastRequiredBlock, nextChunk - lastRequiredBlock); - if (nextChunk > lastRequiredBlock) { + + if (blocksVerified(state)) { + logger.info("CLIENT - Finished Snap blocks request sending."); + + generateChunkRequestTasks(state); + startRequestingChunks(state); + } else if (nextChunk > lastRequiredBlock) { requestBlocksChunk(sender, nextChunk); } else { logger.info("CLIENT - Finished Snap blocks request sending."); @@ -564,7 +611,6 @@ private void processOrderedStateChunkResponse(SnapSyncState state, Peer peer, Sn List nodes = new ArrayList<>(); List postRootNodes = new ArrayList<>(); - for (int i = 0; i < preRootElements.size(); i++) { final RLPList trieElement = (RLPList) preRootElements.get(i); final byte[] value = trieElement.get(0).getRLPData(); diff --git a/rskj-core/src/main/java/co/rsk/net/sync/BlockConnectorException.java b/rskj-core/src/main/java/co/rsk/net/sync/BlockConnectorException.java deleted file mode 100644 index 67c297a81ca..00000000000 --- a/rskj-core/src/main/java/co/rsk/net/sync/BlockConnectorException.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * This file is part of RskJ - * Copyright (C) 2023 RSK Labs Ltd. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package co.rsk.net.sync; - -public class BlockConnectorException extends RuntimeException { - private final long blockNumber; - private final long childBlockNumber; - - public BlockConnectorException(final long blockNumber, final long childBlockNumber) { - super(String.format("Block with number %s is not child's (%s) parent.", blockNumber, childBlockNumber)); - this.blockNumber = blockNumber; - this.childBlockNumber = childBlockNumber; - } - - public long getBlockNumber() { - return blockNumber; - } - - public long getChildBlockNumber() { - return childBlockNumber; - } -} diff --git a/rskj-core/src/main/java/co/rsk/net/sync/BlockConnectorHelper.java b/rskj-core/src/main/java/co/rsk/net/sync/BlockConnectorHelper.java index 30c69a7f346..c406eedba85 100644 --- a/rskj-core/src/main/java/co/rsk/net/sync/BlockConnectorHelper.java +++ b/rskj-core/src/main/java/co/rsk/net/sync/BlockConnectorHelper.java @@ -38,47 +38,28 @@ public BlockConnectorHelper(BlockStore blockStore) { public void startConnecting(List> blockAndDifficultiesList) { if (blockAndDifficultiesList.isEmpty()) { - logger.debug("Block list is empty, nothing to connect"); + logger.warn("Block list is empty, nothing to connect"); return; } - blockAndDifficultiesList.sort(new BlockAndDiffComparator()); - logger.info("Start connecting blocks ranging from {} to {} - Total: {}", + logger.info("Start connecting blocks ranging from [{}] to [{}] - Total: [{}]", blockAndDifficultiesList.get(0).getKey().getNumber(), blockAndDifficultiesList.get(blockAndDifficultiesList.size() - 1).getKey().getNumber(), blockAndDifficultiesList.size()); - int blockIndex = blockAndDifficultiesList.size() - 1; - Pair blockAndDifficulty = blockAndDifficultiesList.get(blockIndex); - Block child = blockAndDifficulty.getLeft(); - logger.debug("Setting child block number the last block from the list: {}", child.getNumber()); - blockStore.saveBlock(child, blockAndDifficulty.getRight(), true); - logger.debug("Block number: {} saved", child.getNumber()); - blockIndex--; - - while (blockIndex >= 0) { - Pair currentBlockAndDifficulty = blockAndDifficultiesList.get(blockIndex); - Block currentBlock = currentBlockAndDifficulty.getLeft(); + int totalSaved = 0; + for (Pair pair : blockAndDifficultiesList) { + Block currentBlock = pair.getLeft(); logger.trace("Connecting block number: {}", currentBlock.getNumber()); - if (!currentBlock.isParentOf(child)) { - throw new BlockConnectorException(currentBlock.getNumber(), child.getNumber()); - } if (!blockStore.isBlockExist(currentBlock.getHash().getBytes())) { - blockStore.saveBlock(currentBlock, currentBlockAndDifficulty.getRight(), true); + blockStore.saveBlock(currentBlock, pair.getRight(), true); + totalSaved++; } else { logger.warn("Block: [{}/{}] already exists. Skipping", currentBlock.getNumber(), currentBlock.getHash()); } - child = currentBlock; - blockIndex--; } - logger.info("Finished connecting blocks. Last saved block: [{}/{}]", child.getNumber(), child.getHash()); - } - static class BlockAndDiffComparator implements java.util.Comparator> { - @Override - public int compare(Pair o1, Pair o2) { - return Long.compare(o1.getLeft().getNumber(), o2.getLeft().getNumber()); - } + logger.info("Finished connecting blocks. Total saved: [{}]. ", totalSaved); } } diff --git a/rskj-core/src/main/java/co/rsk/net/sync/PeersInformation.java b/rskj-core/src/main/java/co/rsk/net/sync/PeersInformation.java index 995b3fae8e1..99ebc821496 100644 --- a/rskj-core/src/main/java/co/rsk/net/sync/PeersInformation.java +++ b/rskj-core/src/main/java/co/rsk/net/sync/PeersInformation.java @@ -204,6 +204,11 @@ public List getBestSnapPeerCandidates() { .collect(Collectors.toList()); } + @Override + public Optional getBestPeer(Set exclude) { + return getBestPeer(getBestCandidatesStream().filter(p -> !exclude.contains(p.getKey().getPeerNodeID()))); + } + public Set knownNodeIds() { return peerStatuses.keySet().stream() .map(Peer::getPeerNodeID) diff --git a/rskj-core/src/main/java/co/rsk/net/sync/SnapSyncState.java b/rskj-core/src/main/java/co/rsk/net/sync/SnapSyncState.java index d5d8873563c..16be3646dbf 100644 --- a/rskj-core/src/main/java/co/rsk/net/sync/SnapSyncState.java +++ b/rskj-core/src/main/java/co/rsk/net/sync/SnapSyncState.java @@ -65,9 +65,12 @@ public class SnapSyncState extends BaseSyncState { private long remoteTrieSize; private byte[] remoteRootHash; private final List> blocks; + private Block lastBlock; - private BlockHeader lastVerifiedBlockHeader; private BlockDifficulty lastBlockDifficulty; + private Peer lastBlockSender; + + private BlockHeader lastVerifiedBlockHeader; private long nextExpectedFrom = 0L; @@ -188,14 +191,17 @@ protected void onMessageTimeOut() { // TODO: call syncEventsHandler.onErrorSyncing() and punish peers after SNAP feature discovery is implemented finish(); +// resetTimeElapsed(); } public Block getLastBlock() { return lastBlock; } - public void setLastBlock(Block lastBlock) { + public void setLastBlock(Block lastBlock, BlockDifficulty lastBlockDifficulty, Peer lastBlockSender) { this.lastBlock = lastBlock; + this.lastBlockDifficulty = lastBlockDifficulty; + this.lastBlockSender = lastBlockSender; } public BlockHeader getLastVerifiedBlockHeader() { @@ -218,8 +224,8 @@ public BlockDifficulty getLastBlockDifficulty() { return lastBlockDifficulty; } - public void setLastBlockDifficulty(BlockDifficulty lastBlockDifficulty) { - this.lastBlockDifficulty = lastBlockDifficulty; + public Peer getLastBlockSender() { + return lastBlockSender; } public byte[] getRemoteRootHash() { @@ -314,6 +320,4 @@ public void fail(Peer peer, EventType eventType, String message, Object... argum public void setRunning() { isRunning = true; } - - } diff --git a/rskj-core/src/main/java/co/rsk/net/sync/SnapshotPeersInformation.java b/rskj-core/src/main/java/co/rsk/net/sync/SnapshotPeersInformation.java index 52bed4900e4..1a9fad26a8a 100644 --- a/rskj-core/src/main/java/co/rsk/net/sync/SnapshotPeersInformation.java +++ b/rskj-core/src/main/java/co/rsk/net/sync/SnapshotPeersInformation.java @@ -17,10 +17,12 @@ */ package co.rsk.net.sync; +import co.rsk.net.NodeID; import co.rsk.net.Peer; import java.util.List; import java.util.Optional; +import java.util.Set; /** * This is mostly a workaround because SyncProcessor needs to access Peer instances. @@ -30,5 +32,6 @@ public interface SnapshotPeersInformation { Optional getBestSnapPeer(); List getBestSnapPeerCandidates(); + Optional getBestPeer(Set exclude); SyncPeerStatus getOrRegisterPeer(Peer peer); } diff --git a/rskj-core/src/main/java/co/rsk/net/sync/SyncMessageHandler.java b/rskj-core/src/main/java/co/rsk/net/sync/SyncMessageHandler.java index 725f3cb9340..2f10b9e7d89 100644 --- a/rskj-core/src/main/java/co/rsk/net/sync/SyncMessageHandler.java +++ b/rskj-core/src/main/java/co/rsk/net/sync/SyncMessageHandler.java @@ -66,7 +66,7 @@ public void run() { job = jobQueue.take(); if (logger.isDebugEnabled()) { - logger.debug("Processing msg: [{}] from: [{}] for: [{}]", job.getMsgType(), job.getSender(), name); + logger.debug("Processing msg: [{}] from: [{}] for: [{}]. Pending count: [{}]", job.getMsgType(), job.getSender(), name, jobQueue.size()); jobStart = Instant.now(); } diff --git a/rskj-core/src/main/resources/reference.conf b/rskj-core/src/main/resources/reference.conf index 75e21c96ec7..79d84126d3b 100644 --- a/rskj-core/src/main/resources/reference.conf +++ b/rskj-core/src/main/resources/reference.conf @@ -383,7 +383,7 @@ sync { # Request timeout (in seconds) chunkRequestTimeout = 120 # Distance to the tip of the blockchain to start snapshot sync - limit = 10000 + limit = 1000 # Parallel requests (true, false) parallel = false # list of SNAP-capable peers to connect to diff --git a/rskj-core/src/test/java/co/rsk/net/SnapshotProcessorTest.java b/rskj-core/src/test/java/co/rsk/net/SnapshotProcessorTest.java index c9cf24959a9..82a7c075938 100644 --- a/rskj-core/src/test/java/co/rsk/net/SnapshotProcessorTest.java +++ b/rskj-core/src/test/java/co/rsk/net/SnapshotProcessorTest.java @@ -96,6 +96,7 @@ void givenStartSyncingIsCalled_thenSnapStatusStartToBeRequestedFromPeer() { blockHeaderValidator, TEST_CHUNK_SIZE, false); + doReturn(Optional.of(peer)).when(peersInformation).getBestSnapPeer(); //when underTest.startSyncing(snapSyncState); //then @@ -132,6 +133,10 @@ void givenSnapStatusResponseCalled_thenSnapChunkRequestsAreMade() { doReturn(blocks.get(blocks.size() - 1)).when(snapSyncState).getLastBlock(); doReturn(snapStatusResponseMessage.getTrieSize()).when(snapSyncState).getRemoteTrieSize(); doReturn(new LinkedList<>()).when(snapSyncState).getChunkTaskQueue(); + doReturn(Optional.of(peer)).when(peersInformation).getBestSnapPeer(); + doReturn(true).when(snapSyncState).isRunning(); + doReturn(true).when(blockValidator).isValid(any()); + doReturn(true).when(blockParentValidator).isValid(any(), any()); underTest.startSyncing(snapSyncState); @@ -139,8 +144,8 @@ void givenSnapStatusResponseCalled_thenSnapChunkRequestsAreMade() { underTest.processSnapStatusResponse(snapSyncState, peer, snapStatusResponseMessage); //then - verify(peer, atLeast(3)).sendMessage(any()); // 1 for SnapStatusRequestMessage, 1 for SnapBlocksRequestMessage and 1 for SnapStateChunkRequestMessage - verify(peersInformation, times(2)).getBestSnapPeerCandidates(); + verify(peer, times(2)).sendMessage(any()); // 1 for SnapStatusRequestMessage, 1 for SnapBlocksRequestMessage and 1 for SnapStateChunkRequestMessage + verify(peersInformation, times(1)).getBestSnapPeer(); } @Test @@ -217,6 +222,9 @@ void givenSnapBlocksResponseReceived_thenSnapBlocksRequestMessageIsSent() { } SnapStatusResponseMessage snapStatusResponseMessage = new SnapStatusResponseMessage(blocks, difficulties, 100000L); + doReturn(true).when(snapSyncState).isRunning(); + doReturn(true).when(blockValidator).isValid(any()); + doReturn(true).when(blockParentValidator).isValid(any(), any()); doReturn(new LinkedList<>()).when(snapSyncState).getChunkTaskQueue(); underTest.startSyncing(snapSyncState); @@ -401,6 +409,9 @@ void givenErrorRLPData_thenOnStateChunkErrorIsCalled() { when(snapSyncState.getNextExpectedFrom()).thenReturn(1L); when(responseMessage.getFrom()).thenReturn(1L); when(responseMessage.getChunkOfTrieKeyValue()).thenReturn(RLP.encodedEmptyList()); + doReturn(true).when(snapSyncState).isRunning(); + doReturn(true).when(blockValidator).isValid(any()); + doReturn(true).when(blockParentValidator).isValid(any(), any()); underTest = spy(underTest); underTest.processStateChunkResponse(snapSyncState, peer, responseMessage); diff --git a/rskj-core/src/test/java/co/rsk/net/sync/BlockConnectorHelperTest.java b/rskj-core/src/test/java/co/rsk/net/sync/BlockConnectorHelperTest.java index 4f80a509a12..ebf4ce91835 100644 --- a/rskj-core/src/test/java/co/rsk/net/sync/BlockConnectorHelperTest.java +++ b/rskj-core/src/test/java/co/rsk/net/sync/BlockConnectorHelperTest.java @@ -1,6 +1,7 @@ package co.rsk.net.sync; import co.rsk.core.BlockDifficulty; +import co.rsk.crypto.Keccak256; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.ethereum.core.Block; @@ -13,14 +14,11 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import java.math.BigInteger; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.Mockito.*; @@ -50,128 +48,26 @@ void testStartConnectingWhenBlockListIsEmpty() { } @Test - void testStartConnectingWhenBlockStoreIsEmpty() { - when(blockStore.isEmpty()).thenReturn(true); - - Block block1 = mock(Block.class); - Block block2 = mock(Block.class); - Block block3 = mock(Block.class); - when(block1.getNumber()).thenReturn(1L); - when(block2.getNumber()).thenReturn(2L); - when(block3.getNumber()).thenReturn(3L); - when(block1.isParentOf(block2)).thenReturn(true); - when(block2.isParentOf(block3)).thenReturn(true); - - - BlockDifficulty diff1 = new BlockDifficulty(BigInteger.valueOf(1)); - BlockDifficulty diff2 = new BlockDifficulty(BigInteger.valueOf(2)); - BlockDifficulty diff3 = new BlockDifficulty(BigInteger.valueOf(3)); - blockAndDifficultiesList = buildBlockDifficulties(Arrays.asList(block1, block2,block3), - Arrays.asList(diff1, diff2,diff3)); - - blockConnectorHelper = new BlockConnectorHelper(blockStore); - - blockConnectorHelper.startConnecting(blockAndDifficultiesList); - - verify(blockStore, times(3)).saveBlock(blockCaptor.capture(), difficultyCaptor.capture(), anyBoolean()); - verify(blockStore, times(0)).getBestBlock(); - List savedBlocks = blockCaptor.getAllValues(); - List savedDifficulties = difficultyCaptor.getAllValues(); - assertEquals(block3, savedBlocks.get(0)); - assertEquals(diff3, savedDifficulties.get(0)); - assertEquals(block2, savedBlocks.get(1)); - assertEquals(diff2, savedDifficulties.get(1)); - assertEquals(block1, savedBlocks.get(2)); - assertEquals(diff1, savedDifficulties.get(2)); - - } - - @Test - void testStartConnectingWhenBlockStoreIsEmptyAndNotOrderedList() { - when(blockStore.isEmpty()).thenReturn(true); - - Block block1 = mock(Block.class); - Block block2 = mock(Block.class); - when(block1.getNumber()).thenReturn(1L); - when(block2.getNumber()).thenReturn(2L); - when(block1.isParentOf(block2)).thenReturn(true); - - BlockDifficulty diff1 = new BlockDifficulty(BigInteger.valueOf(1)); - BlockDifficulty diff2 = new BlockDifficulty(BigInteger.valueOf(2)); - blockAndDifficultiesList = buildBlockDifficulties(Arrays.asList(block2, block1), - Arrays.asList(diff2, diff1)); - - blockConnectorHelper = new BlockConnectorHelper(blockStore); - - blockConnectorHelper.startConnecting(blockAndDifficultiesList); - - verify(blockStore, times(2)).saveBlock(blockCaptor.capture(), difficultyCaptor.capture(), anyBoolean()); - verify(blockStore, times(0)).getBestBlock(); - List savedBlocks = blockCaptor.getAllValues(); - List savedDifficulties = difficultyCaptor.getAllValues(); - assertEquals(block2, savedBlocks.get(0)); - assertEquals(diff2, savedDifficulties.get(0)); - assertEquals(block1, savedBlocks.get(1)); - assertEquals(diff1, savedDifficulties.get(1)); - } - - @Test - void testStartConnectingWhenBlockStoreIsNotEmpty() { - Block block1 = mock(Block.class); - Block block2 = mock(Block.class); - Block block3 = mock(Block.class); - - when(block1.getNumber()).thenReturn(1L); - when(block2.getNumber()).thenReturn(2L); - when(block3.getNumber()).thenReturn(3L); - when(block1.isParentOf(block2)).thenReturn(true); - when(block2.isParentOf(block3)).thenReturn(true); - - when(blockStore.isEmpty()).thenReturn(false); - when(blockStore.getBestBlock()).thenReturn(block3); + void testStartConnectingWhenBlockListIsNotEmpty() { + Block block1 = mockBlock( new byte[] { 1 }); + Block block2 = mockBlock(new byte[] { 2 }); blockAndDifficultiesList = buildBlockDifficulties(Arrays.asList(block1, block2), Arrays.asList(mock(BlockDifficulty.class), mock(BlockDifficulty.class))); blockConnectorHelper = new BlockConnectorHelper(blockStore); blockConnectorHelper.startConnecting(blockAndDifficultiesList); - verify(blockStore, times(1)).getBestBlock(); verify(blockStore, times(2)).saveBlock(any(), any(), anyBoolean()); } - @Test - void whenBlockIsNotParentOfExistingBestBlock() { - Block block2 = mock(Block.class); - Block block3 = mock(Block.class); - when(block2.getNumber()).thenReturn(2L); - when(block3.getNumber()).thenReturn(3L); - when(block2.isParentOf(block3)).thenReturn(false); - blockAndDifficultiesList = buildBlockDifficulties(Collections.singletonList(block2), - Collections.singletonList(mock(BlockDifficulty.class))); - - blockConnectorHelper = new BlockConnectorHelper(blockStore); - - when(blockStore.isEmpty()).thenReturn(false); - when(blockStore.getBestBlock()).thenReturn(block3); - - assertThrows(BlockConnectorException.class, () -> blockConnectorHelper.startConnecting(blockAndDifficultiesList)); - } - - @Test - void testStartConnectingWhenBlockIsNotParentOfChild() { - Block block1 = mock(Block.class); - Block block2 = mock(Block.class); - when(block1.getNumber()).thenReturn(1L); - when(block2.getNumber()).thenReturn(2L); - when(block1.isParentOf(block2)).thenReturn(false); - when(blockStore.isEmpty()).thenReturn(true); - blockAndDifficultiesList = buildBlockDifficulties(Arrays.asList(block1, block2), - Arrays.asList(mock(BlockDifficulty.class), mock(BlockDifficulty.class))); - blockConnectorHelper = new BlockConnectorHelper(blockStore); - - assertThrows(BlockConnectorException.class, () -> blockConnectorHelper.startConnecting(blockAndDifficultiesList)); + private Block mockBlock(byte[] hashBytes) { + Block block = mock(Block.class); + Keccak256 hash = mock(Keccak256.class); + when(hash.getBytes()).thenReturn(hashBytes); + when(block.getHash()).thenReturn(hash); + return block; } - List> buildBlockDifficulties(List blocks, List difficulties) { + private List> buildBlockDifficulties(List blocks, List difficulties) { int i = 0; List> list = new ArrayList<>(); for (Block block : blocks) { @@ -180,5 +76,4 @@ List> buildBlockDifficulties(List blocks, Lis } return list; } - -} \ No newline at end of file +} diff --git a/rskj-core/src/test/java/co/rsk/net/sync/SnapSyncStateTest.java b/rskj-core/src/test/java/co/rsk/net/sync/SnapSyncStateTest.java index 42319223dc3..5b0039e6415 100644 --- a/rskj-core/src/test/java/co/rsk/net/sync/SnapSyncStateTest.java +++ b/rskj-core/src/test/java/co/rsk/net/sync/SnapSyncStateTest.java @@ -228,8 +228,14 @@ void givenOnSnapStateChunkIsCalled_thenJobIsAddedAndRun() throws InterruptedExce @Test void testSetAndGetLastBlock() { Block mockBlock = mock(Block.class); - underTest.setLastBlock(mockBlock); + BlockDifficulty mockBlockDifficulty = mock(BlockDifficulty.class); + Peer mockPeer = mock(Peer.class); + + underTest.setLastBlock(mockBlock, mockBlockDifficulty, mockPeer); + assertEquals(mockBlock, underTest.getLastBlock()); + assertEquals(mockBlockDifficulty, underTest.getLastBlockDifficulty()); + assertEquals(mockPeer, underTest.getLastBlockSender()); } @Test @@ -272,13 +278,6 @@ void testGetSnapStateChunkQueue() { assertNotNull(queue); } - @Test - void testSetAndGetLastBlockDifficulty() { - BlockDifficulty mockBlockDifficulty = mock(BlockDifficulty.class); - underTest.setLastBlockDifficulty(mockBlockDifficulty); - assertEquals(mockBlockDifficulty, underTest.getLastBlockDifficulty()); - } - @Test void testSetAndGetRemoteRootHash() { byte[] mockRootHash = new byte[]{1, 2, 3}; diff --git a/rskj-core/src/test/java/co/rsk/validators/BlockTimeStampValidationRuleTest.java b/rskj-core/src/test/java/co/rsk/validators/BlockTimeStampValidationRuleTest.java index 700f6d1b2dc..6c4e792d36b 100644 --- a/rskj-core/src/test/java/co/rsk/validators/BlockTimeStampValidationRuleTest.java +++ b/rskj-core/src/test/java/co/rsk/validators/BlockTimeStampValidationRuleTest.java @@ -129,48 +129,96 @@ void blockInTheFuture() { } @Test - void blockTimeLowerThanParentTime() { + void blockTimeLowerThanParentBlockTime() { int validPeriod = 540; BlockTimeStampValidationRule validationRule = new BlockTimeStampValidationRule(validPeriod, preRskip179Config, Constants.regtest(), timeProvider); when(timeProvider.currentTimeMillis()).thenReturn(10_000_000L); BlockHeader header = mock(BlockHeader.class); Block parent = mock(Block.class); + BlockHeader parentHeader = mock(BlockHeader.class); when(header.getTimestamp()).thenReturn(10_000L); + when(parent.getHeader()).thenReturn(parentHeader); + when(parentHeader.getTimestamp()).thenReturn(10_000L + 1000); + assertFalse(validationRule.isValid(header, parent)); + } + + @Test + void blockTimeLowerThanParentBlockHeaderTime() { + int validPeriod = 540; + BlockTimeStampValidationRule validationRule = new BlockTimeStampValidationRule(validPeriod, preRskip179Config, Constants.regtest(), timeProvider); + + when(timeProvider.currentTimeMillis()).thenReturn(10_000_000L); + BlockHeader header = mock(BlockHeader.class); + BlockHeader parent = mock(BlockHeader.class); + + when(header.getTimestamp()).thenReturn(10_000L); when(parent.getTimestamp()).thenReturn(10_000L + 1000); assertFalse(validationRule.isValid(header, parent)); } @Test - void blockTimeGreaterThanParentTime() { + void blockTimeGreaterThanParentBlockTime() { int validPeriod = 540; BlockTimeStampValidationRule validationRule = new BlockTimeStampValidationRule(validPeriod, preRskip179Config, Constants.regtest(), timeProvider); when(timeProvider.currentTimeMillis()).thenReturn(10_000_000L); BlockHeader header = mock(BlockHeader.class); Block parent = mock(Block.class); + BlockHeader parentHeader = mock(BlockHeader.class); when(header.getTimestamp()).thenReturn(10_000L); + when(parent.getHeader()).thenReturn(parentHeader); + when(parentHeader.getTimestamp()).thenReturn(10_000L - 1000); + + assertTrue(validationRule.isValid(header, parent)); + } + @Test + void blockTimeGreaterThanParentBlockHeaderTime() { + int validPeriod = 540; + BlockTimeStampValidationRule validationRule = new BlockTimeStampValidationRule(validPeriod, preRskip179Config, Constants.regtest(), timeProvider); + + when(timeProvider.currentTimeMillis()).thenReturn(10_000_000L); + BlockHeader header = mock(BlockHeader.class); + BlockHeader parent = mock(BlockHeader.class); + + when(header.getTimestamp()).thenReturn(10_000L); when(parent.getTimestamp()).thenReturn(10_000L - 1000); assertTrue(validationRule.isValid(header, parent)); } @Test - void blockTimeEqualsParentTime() { + void blockTimeEqualsParentBlockTime() { int validPeriod = 540; BlockTimeStampValidationRule validationRule = new BlockTimeStampValidationRule(validPeriod, preRskip179Config, Constants.regtest(), timeProvider); when(timeProvider.currentTimeMillis()).thenReturn(10_000_000L); BlockHeader header = mock(BlockHeader.class); Block parent = mock(Block.class); + BlockHeader parentHeader = mock(BlockHeader.class); when(header.getTimestamp()).thenReturn(10_000L); + when(parent.getHeader()).thenReturn(parentHeader); + when(parentHeader.getTimestamp()).thenReturn(10_000L); + + assertFalse(validationRule.isValid(header, parent)); + } + @Test + void blockTimeEqualsParentBlockHeaderTime() { + int validPeriod = 540; + BlockTimeStampValidationRule validationRule = new BlockTimeStampValidationRule(validPeriod, preRskip179Config, Constants.regtest(), timeProvider); + + when(timeProvider.currentTimeMillis()).thenReturn(10_000_000L); + BlockHeader header = mock(BlockHeader.class); + BlockHeader parent = mock(BlockHeader.class); + + when(header.getTimestamp()).thenReturn(10_000L); when(parent.getTimestamp()).thenReturn(10_000L); assertFalse(validationRule.isValid(header, parent)); From 6d41374bb4248c2f4e0e611b524ec5348ac3151e Mon Sep 17 00:00:00 2001 From: Volodymyr Kravets Date: Fri, 29 Nov 2024 14:29:49 +0200 Subject: [PATCH 04/10] feat(snap): fix block header fetching and snap client config --- rskj-core/src/main/java/co/rsk/net/SnapshotProcessor.java | 4 ++-- rskj-core/src/main/resources/reference.conf | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rskj-core/src/main/java/co/rsk/net/SnapshotProcessor.java b/rskj-core/src/main/java/co/rsk/net/SnapshotProcessor.java index afb1dd9beaf..665f9ef17a6 100644 --- a/rskj-core/src/main/java/co/rsk/net/SnapshotProcessor.java +++ b/rskj-core/src/main/java/co/rsk/net/SnapshotProcessor.java @@ -338,9 +338,9 @@ public void processBlockHeaderChunk(SnapSyncState state, Peer sender, List blockHeaders) { diff --git a/rskj-core/src/main/resources/reference.conf b/rskj-core/src/main/resources/reference.conf index 79d84126d3b..75e21c96ec7 100644 --- a/rskj-core/src/main/resources/reference.conf +++ b/rskj-core/src/main/resources/reference.conf @@ -383,7 +383,7 @@ sync { # Request timeout (in seconds) chunkRequestTimeout = 120 # Distance to the tip of the blockchain to start snapshot sync - limit = 1000 + limit = 10000 # Parallel requests (true, false) parallel = false # list of SNAP-capable peers to connect to From 8fe4cc9d8c32e96b9c86d794051ff58b6c633059 Mon Sep 17 00:00:00 2001 From: Volodymyr Kravets Date: Mon, 2 Dec 2024 15:20:38 +0200 Subject: [PATCH 05/10] feat(snap): more refactoring --- .../rsk/snap/SnapshotSyncIntegrationTest.java | 2 +- .../java/co/rsk/net/SnapshotProcessor.java | 89 ++++++++++--------- .../java/co/rsk/net/sync/SnapSyncState.java | 13 ++- .../co/rsk/net/sync/SyncMessageHandler.java | 9 +- 4 files changed, 65 insertions(+), 48 deletions(-) diff --git a/rskj-core/src/integrationTest/java/co/rsk/snap/SnapshotSyncIntegrationTest.java b/rskj-core/src/integrationTest/java/co/rsk/snap/SnapshotSyncIntegrationTest.java index a28d3655da2..81079f4b3ce 100644 --- a/rskj-core/src/integrationTest/java/co/rsk/snap/SnapshotSyncIntegrationTest.java +++ b/rskj-core/src/integrationTest/java/co/rsk/snap/SnapshotSyncIntegrationTest.java @@ -105,7 +105,7 @@ public void whenStartTheServerAndClientNodes_thenTheClientWillSynchWithServer() boolean isClientSynced = false; while (System.currentTimeMillis() < endTime) { - if (clientNode.getOutput().contains("CLIENT - Starting Snapshot sync.") && clientNode.getOutput().contains("CLIENT - Snapshot sync finished successfully!")) { + if (clientNode.getOutput().contains("Starting Snap sync") && clientNode.getOutput().contains("Snap sync finished successfully")) { try { JsonNode jsonResponse = OkHttpClientTestFixture.getJsonResponseForGetBestBlockMessage(portClientRpc, serverBestBlockNumber); JsonNode jsonResult = jsonResponse.get(0).get("result"); diff --git a/rskj-core/src/main/java/co/rsk/net/SnapshotProcessor.java b/rskj-core/src/main/java/co/rsk/net/SnapshotProcessor.java index 665f9ef17a6..41ec275d6cd 100644 --- a/rskj-core/src/main/java/co/rsk/net/SnapshotProcessor.java +++ b/rskj-core/src/main/java/co/rsk/net/SnapshotProcessor.java @@ -139,28 +139,33 @@ public SnapshotProcessor(Blockchain blockchain, this.blockHeaderValidator = blockHeaderValidator; this.parallel = isParallelEnabled; - this.thread = new Thread(new SyncMessageHandler("SNAP requests", requestQueue, listener) { + this.thread = new Thread(new SyncMessageHandler("SNAP/server", requestQueue, listener) { @Override public boolean isRunning() { return isRunning; } - }, "snap sync request handler"); + }, "snap sync server handler"); } public void startSyncing(SnapSyncState state) { - // get more than one peer, use the peer queue - // TODO(snap-poc) deal with multiple peers algorithm here Optional bestPeerOpt = peersInformation.getBestSnapPeer(); if (bestPeerOpt.isEmpty()) { - logger.warn("No more valid peer to start snapshot sync against."); + logger.warn("No snap-capable peer to start sync against"); stopSyncing(state); return; } - logger.info("CLIENT - Starting Snapshot sync."); + + logger.info("Starting Snap sync"); requestSnapStatus(bestPeerOpt.get()); } + private void completeSyncing(SnapSyncState state) { + boolean result = rebuildStateAndSave(state); + logger.info("Snap sync finished {}", result ? "successfully" : "with errors"); + stopSyncing(state); + } + private void stopSyncing(SnapSyncState state) { state.finish(); } @@ -197,10 +202,9 @@ public void run() { } void processSnapStatusRequestInternal(Peer sender, SnapStatusRequestMessage ignoredRequestMessage) { - logger.debug("SERVER - Processing snapshot status request."); long bestBlockNumber = blockchain.getBestBlock().getNumber(); long checkpointBlockNumber = bestBlockNumber - (bestBlockNumber % BLOCK_NUMBER_CHECKPOINT); - logger.debug("SERVER - checkpointBlockNumber: {}, bestBlockNumber: {}", checkpointBlockNumber, bestBlockNumber); + logger.debug("Processing snapshot status request, checkpointBlockNumber: {}, bestBlockNumber: {}", checkpointBlockNumber, bestBlockNumber); List blocks = Lists.newArrayList(); List difficulties = Lists.newArrayList(); for (long i = checkpointBlockNumber - BLOCK_CHUNK_SIZE; i < checkpointBlockNumber; i++) { @@ -209,21 +213,19 @@ void processSnapStatusRequestInternal(Peer sender, SnapStatusRequestMessage igno difficulties.add(blockStore.getTotalDifficultyForHash(block.getHash().getBytes())); } - logger.trace("SERVER - Sending snapshot status response. From block {} to block {} - chunksize {}", blocks.get(0).getNumber(), blocks.get(blocks.size() - 1).getNumber(), BLOCK_CHUNK_SIZE); Block checkpointBlock = blockchain.getBlockByNumber(checkpointBlockNumber); blocks.add(checkpointBlock); - logger.trace("SERVER - adding checkpoint block: {}", checkpointBlock.getNumber()); difficulties.add(blockStore.getTotalDifficultyForHash(checkpointBlock.getHash().getBytes())); byte[] rootHash = checkpointBlock.getStateRoot(); Optional opt = trieStore.retrieveDTO(rootHash); if (opt.isEmpty()) { - logger.warn("SERVER - trie is not present for rootHash: {}", Bytes.of(rootHash)); + logger.warn("trie is not present for rootHash: {}", Bytes.of(rootHash)); return; } long trieSize = opt.get().getTotalSize(); - logger.debug("SERVER - processing snapshot status request - rootHash: {} trieSize: {}", rootHash, trieSize); + logger.debug("Processing snapshot status request - rootHash: {} trieSize: {}", rootHash, trieSize); SnapStatusResponseMessage responseMessage = new SnapStatusResponseMessage(blocks, difficulties, trieSize); sender.sendMessage(responseMessage); } @@ -251,11 +253,10 @@ public void processSnapStatusResponse(SnapSyncState state, Peer sender, SnapStat return; } - logger.debug("CLIENT - Processing snapshot status response - last blockNumber: {} triesize: {}", lastBlock.getNumber(), state.getRemoteTrieSize()); - logger.debug("Blocks included in the response: {} from {} to {}", blocksFromResponse.size(), blocksFromResponse.get(0).getNumber(), blocksFromResponse.get(blocksFromResponse.size() - 1).getNumber()); + logger.debug("Processing snapshot status response: {} from {} to {} - last blockNumber: {} trieSize: {}", blocksFromResponse.size(), blocksFromResponse.get(0).getNumber(), blocksFromResponse.get(blocksFromResponse.size() - 1).getNumber(), lastBlock.getNumber(), state.getRemoteTrieSize()); if (blocksVerified(state)) { - logger.info("CLIENT - Finished Snap blocks request sending."); + logger.info("Finished Snap blocks request sending"); generateChunkRequestTasks(state); startRequestingChunks(state); @@ -318,7 +319,7 @@ private boolean areBlockPairsValid(Pair blockPair, @Null * BLOCK CHUNK */ private void requestBlocksChunk(Peer sender, long blockNumber) { - logger.debug("CLIENT - Requesting block chunk to node {} - block {}", sender.getPeerNodeID(), blockNumber); + logger.debug("Requesting block chunk to node {} - block {}", sender.getPeerNodeID(), blockNumber); sender.sendMessage(new SnapBlocksRequestMessage(blockNumber)); } @@ -327,7 +328,7 @@ public void processBlockHeaderChunk(SnapSyncState state, Peer sender, List blocks = Lists.newArrayList(); List difficulties = Lists.newArrayList(); @@ -445,7 +446,7 @@ void processSnapBlocksRequestInternal(Peer sender, SnapBlocksRequestMessage requ blocks.add(block); difficulties.add(blockStore.getTotalDifficultyForHash(block.getHash().getBytes())); } - logger.debug("SERVER - Sending snap blocks response. From block {} to block {} - chunksize {}", blocks.get(0).getNumber(), blocks.get(blocks.size() - 1).getNumber(), BLOCK_CHUNK_SIZE); + logger.debug("Sending snap blocks response. From block {} to block {} - chunksize {}", blocks.get(0).getNumber(), blocks.get(blocks.size() - 1).getNumber(), BLOCK_CHUNK_SIZE); SnapBlocksResponseMessage responseMessage = new SnapBlocksResponseMessage(blocks, difficulties); sender.sendMessage(responseMessage); } @@ -457,7 +458,7 @@ public void processSnapBlocksResponse(SnapSyncState state, Peer sender, SnapBloc long lastRequiredBlock = state.getLastBlock().getNumber() - BLOCKS_REQUIRED; List blocksFromResponse = responseMessage.getBlocks(); - logger.debug("CLIENT - Processing snap blocks response. Receiving from block {} to block {} Objective: {}.", blocksFromResponse.get(0).getNumber(), blocksFromResponse.get(blocksFromResponse.size() - 1).getNumber(), lastRequiredBlock); + logger.debug("Processing snap blocks response. Receiving from block {} to block {} Objective: {}.", blocksFromResponse.get(0).getNumber(), blocksFromResponse.get(blocksFromResponse.size() - 1).getNumber(), lastRequiredBlock); List difficultiesFromResponse = responseMessage.getDifficulties(); if (!validateAndSaveBlocks(state, sender, blocksFromResponse, difficultiesFromResponse)) { @@ -465,17 +466,17 @@ public void processSnapBlocksResponse(SnapSyncState state, Peer sender, SnapBloc } long nextChunk = blocksFromResponse.get(0).getNumber(); - logger.debug("CLIENT - SnapBlock - nexChunk : {} - lastRequired {}, missing {}", nextChunk, lastRequiredBlock, nextChunk - lastRequiredBlock); + logger.debug("SnapBlock - nexChunk : {} - lastRequired {}, missing {}", nextChunk, lastRequiredBlock, nextChunk - lastRequiredBlock); if (blocksVerified(state)) { - logger.info("CLIENT - Finished Snap blocks request sending."); + logger.info("Finished Snap blocks request sending. Start requesting state chunks"); generateChunkRequestTasks(state); startRequestingChunks(state); } else if (nextChunk > lastRequiredBlock) { requestBlocksChunk(sender, nextChunk); } else { - logger.info("CLIENT - Finished Snap blocks request sending."); + logger.info("Finished Snap blocks request sending. Start requesting state chunks and block headers"); generateChunkRequestTasks(state); startRequestingChunks(state); @@ -488,7 +489,7 @@ public void processSnapBlocksResponse(SnapSyncState state, Peer sender, SnapBloc * STATE CHUNK */ private void requestStateChunk(Peer peer, long from, long blockNumber, int chunkSize) { - logger.debug("CLIENT - Requesting state chunk to node {} - block {} - chunkNumber {}", peer.getPeerNodeID(), blockNumber, from / chunkSize); + logger.debug("Requesting state chunk to node {} - block {} - chunkNumber {}", peer.getPeerNodeID(), blockNumber, from / chunkSize); SnapStateChunkRequestMessage message = new SnapStateChunkRequestMessage(messageId.getAndIncrement(), blockNumber, from, chunkSize); peer.sendMessage(message); } @@ -518,8 +519,8 @@ void processStateChunkRequestInternal(Peer sender, SnapStateChunkRequestMessage List trieEncoded = new ArrayList<>(); Block block = blockchain.getBlockByNumber(request.getBlockNumber()); final long to = request.getFrom() + (request.getChunkSize() * CHUNK_ITEM_SIZE); - logger.debug("SERVER - Processing state chunk request from node {}. From {} to calculated {} being chunksize {}", sender.getPeerNodeID(), request.getFrom(), to, request.getChunkSize()); - logger.debug("SERVER - Sending state chunk from {} to {}", request.getFrom(), to); + logger.debug("Processing state chunk request from node {}. From {} to calculated {} being chunksize {}", sender.getPeerNodeID(), request.getFrom(), to, request.getChunkSize()); + logger.debug("Sending state chunk from {} to {}", request.getFrom(), to); TrieDTOInOrderIterator it = new TrieDTOInOrderIterator(trieStore, block.getStateRoot(), request.getFrom(), to); // First we add the root nodes on the left of the current node. They are used to validate the chunk. @@ -548,7 +549,7 @@ void processStateChunkRequestInternal(Peer sender, SnapStateChunkRequestMessage long totalChunkTime = System.currentTimeMillis() - startChunk; - logger.debug("SERVER - Sending state chunk from {} of {} bytes to node {}, totalTime {}ms", request.getFrom(), chunkBytes.length, sender.getPeerNodeID(), totalChunkTime); + logger.debug("Sending state chunk from {} of {} bytes to node {}, totalTime {}ms", request.getFrom(), chunkBytes.length, sender.getPeerNodeID(), totalChunkTime); sender.sendMessage(responseMessage); } @@ -557,7 +558,7 @@ public void processStateChunkResponse(SnapSyncState state, Peer peer, SnapStateC return; } - logger.debug("CLIENT - State chunk received chunkNumber {}. From {} to {} of total size {}", responseMessage.getFrom() / CHUNK_ITEM_SIZE, responseMessage.getFrom(), responseMessage.getTo(), state.getRemoteTrieSize()); + logger.debug("State chunk received chunkNumber {}. From {} to {} of total size {}", responseMessage.getFrom() / CHUNK_ITEM_SIZE, responseMessage.getFrom(), responseMessage.getTo(), state.getRemoteTrieSize()); PriorityQueue queue = state.getSnapStateChunkQueue(); queue.add(responseMessage); @@ -565,7 +566,7 @@ public void processStateChunkResponse(SnapSyncState state, Peer peer, SnapStateC while (!queue.isEmpty()) { SnapStateChunkResponseMessage nextMessage = queue.peek(); long nextExpectedFrom = state.getNextExpectedFrom(); - logger.debug("CLIENT - State chunk dequeued from: {} - expected: {}", nextMessage.getFrom(), nextExpectedFrom); + logger.debug("State chunk dequeued from: {} - expected: {}", nextMessage.getFrom(), nextExpectedFrom); if (nextMessage.getFrom() == nextExpectedFrom) { try { processOrderedStateChunkResponse(state, peer, queue.poll()); @@ -580,7 +581,7 @@ public void processStateChunkResponse(SnapSyncState state, Peer peer, SnapStateC } if (!responseMessage.isComplete()) { - logger.debug("CLIENT - State chunk response not complete. Requesting next chunk."); + logger.debug("State chunk response not complete. Requesting next chunk."); executeNextChunkRequestTask(state, peer); } } @@ -597,7 +598,7 @@ void onStateChunkResponseError(Peer peer, SnapStateChunkResponseMessage response } private void processOrderedStateChunkResponse(SnapSyncState state, Peer peer, SnapStateChunkResponseMessage message) throws Exception { - logger.debug("CLIENT - Processing State chunk received from {} to {}", message.getFrom(), message.getTo()); + logger.debug("Processing State chunk received from {} to {}", message.getFrom(), message.getTo()); peersInformation.getOrRegisterPeer(peer); state.onNewChunk(); @@ -652,9 +653,9 @@ private void processOrderedStateChunkResponse(SnapSyncState state, Peer peer, Sn executeNextChunkRequestTask(state, peer); } else { if (blocksVerified(state)) { - boolean result = rebuildStateAndSave(state); - logger.info("CLIENT - Snapshot sync finished {}! ", result ? "successfully" : "with errors"); - stopSyncing(state); + completeSyncing(state); + } else { + state.setStateFetched(); } } } else { @@ -672,24 +673,24 @@ private boolean blocksVerified(SnapSyncState state) { * Once state share is received, rebuild the trie, save it in db and save all the blocks. */ private boolean rebuildStateAndSave(SnapSyncState state) { - logger.info("CLIENT - Recovering trie..."); + logger.info("Recovering trie..."); final TrieDTO[] nodeArray = state.getAllNodes().toArray(new TrieDTO[0]); Optional result = TrieDTOInOrderRecoverer.recoverTrie(nodeArray, this.trieStore::saveDTO); if (result.isPresent() && Arrays.equals(state.getRemoteRootHash(), result.get().calculateHash())) { - logger.info("CLIENT - State final validation OK!"); + logger.info("State final validation OK!"); this.blockchain.removeBlocksByNumber(0); //genesis is removed so backwards sync will always start. BlockConnectorHelper blockConnector = new BlockConnectorHelper(this.blockStore); state.connectBlocks(blockConnector); - logger.info("CLIENT - Setting last block as best block..."); + logger.info("Setting last block as best block..."); this.blockchain.setStatus(state.getLastBlock(), state.getLastBlockDifficulty()); this.transactionPool.setBestBlock(state.getLastBlock()); return true; } - logger.error("CLIENT - State final validation FAILED"); + logger.error("State final validation FAILED"); return false; } diff --git a/rskj-core/src/main/java/co/rsk/net/sync/SnapSyncState.java b/rskj-core/src/main/java/co/rsk/net/sync/SnapSyncState.java index 16be3646dbf..0c00f8a069e 100644 --- a/rskj-core/src/main/java/co/rsk/net/sync/SnapSyncState.java +++ b/rskj-core/src/main/java/co/rsk/net/sync/SnapSyncState.java @@ -60,6 +60,7 @@ public class SnapSyncState extends BaseSyncState { private BigInteger stateSize = BigInteger.ZERO; private BigInteger stateChunkSize = BigInteger.ZERO; + private boolean stateFetched; private final List allNodes; private long remoteTrieSize; @@ -88,13 +89,13 @@ public SnapSyncState(SyncEventsHandler syncEventsHandler, SnapshotProcessor snap this.snapshotProcessor = snapshotProcessor; // TODO(snap-poc) code in SnapshotProcessor should be moved here probably this.allNodes = Lists.newArrayList(); this.blocks = Lists.newArrayList(); - this.thread = new Thread(new SyncMessageHandler("SNAP responses", responseQueue, listener) { + this.thread = new Thread(new SyncMessageHandler("SNAP/client", responseQueue, listener) { @Override public boolean isRunning() { return isRunning; } - }, "snap sync response handler"); + }, "snap sync client handler"); } @Override @@ -268,6 +269,14 @@ public void setStateSize(BigInteger stateSize) { this.stateSize = stateSize; } + public boolean isStateFetched() { + return stateFetched; + } + + public void setStateFetched() { + this.stateFetched = true; + } + public BigInteger getStateChunkSize() { return stateChunkSize; } diff --git a/rskj-core/src/main/java/co/rsk/net/sync/SyncMessageHandler.java b/rskj-core/src/main/java/co/rsk/net/sync/SyncMessageHandler.java index 2f10b9e7d89..37796dfb700 100644 --- a/rskj-core/src/main/java/co/rsk/net/sync/SyncMessageHandler.java +++ b/rskj-core/src/main/java/co/rsk/net/sync/SyncMessageHandler.java @@ -24,6 +24,7 @@ import co.rsk.util.FormatUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.slf4j.MDC; import java.time.Duration; import java.time.Instant; @@ -31,7 +32,9 @@ public abstract class SyncMessageHandler implements Runnable { - private static final Logger logger = LoggerFactory.getLogger("syncprocessor"); + private static final Logger logger = LoggerFactory.getLogger("syncmessagehandler"); + + public static final String QUEUE_NAME = "queue"; private final String name; @@ -65,6 +68,8 @@ public void run() { try { job = jobQueue.take(); + MDC.put(QUEUE_NAME, name); + if (logger.isDebugEnabled()) { logger.debug("Processing msg: [{}] from: [{}] for: [{}]. Pending count: [{}]", job.getMsgType(), job.getSender(), name, jobQueue.size()); jobStart = Instant.now(); @@ -96,6 +101,8 @@ public void run() { if (listener != null) { listener.onException(e); } + } finally { + MDC.remove(QUEUE_NAME); } } From f9ee5547a051171e25535f515e0d17a3c501ee95 Mon Sep 17 00:00:00 2001 From: Volodymyr Kravets Date: Tue, 3 Dec 2024 14:36:34 +0200 Subject: [PATCH 06/10] feat(snap): update logging and error handling in SnapSyncState --- .../java/co/rsk/net/sync/SnapSyncState.java | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/rskj-core/src/main/java/co/rsk/net/sync/SnapSyncState.java b/rskj-core/src/main/java/co/rsk/net/sync/SnapSyncState.java index 0c00f8a069e..516fac3f22a 100644 --- a/rskj-core/src/main/java/co/rsk/net/sync/SnapSyncState.java +++ b/rskj-core/src/main/java/co/rsk/net/sync/SnapSyncState.java @@ -112,6 +112,7 @@ public void onEnter() { @Override public void onSnapStatus(Peer sender, SnapStatusResponseMessage responseMessage) { try { + resetTimeElapsed(); responseQueue.put(new SyncMessageHandler.Job(sender, responseMessage) { @Override public void run() { @@ -119,7 +120,7 @@ public void run() { } }); } catch (InterruptedException e) { - logger.warn("SnapStatusResponseMessage processing was interrupted", e); + logger.warn("{} processing was interrupted", MessageType.SNAP_STATUS_RESPONSE_MESSAGE, e); Thread.currentThread().interrupt(); } } @@ -127,6 +128,7 @@ public void run() { @Override public void onSnapBlocks(Peer sender, SnapBlocksResponseMessage responseMessage) { try { + resetTimeElapsed(); responseQueue.put(new SyncMessageHandler.Job(sender, responseMessage) { @Override public void run() { @@ -134,7 +136,7 @@ public void run() { } }); } catch (InterruptedException e) { - logger.warn("SnapBlocksResponseMessage processing was interrupted", e); + logger.warn("{} processing was interrupted", MessageType.SNAP_BLOCKS_RESPONSE_MESSAGE, e); Thread.currentThread().interrupt(); } } @@ -142,6 +144,7 @@ public void run() { @Override public void onSnapStateChunk(Peer sender, SnapStateChunkResponseMessage responseMessage) { try { + resetTimeElapsed(); responseQueue.put(new SyncMessageHandler.Job(sender, responseMessage) { @Override public void run() { @@ -149,7 +152,7 @@ public void run() { } }); } catch (InterruptedException e) { - logger.warn("SnapStateChunkResponseMessage processing was interrupted", e); + logger.warn("{} processing was interrupted", MessageType.SNAP_STATE_CHUNK_RESPONSE_MESSAGE, e); Thread.currentThread().interrupt(); } } @@ -157,6 +160,7 @@ public void run() { @Override public void newBlockHeaders(Peer peer, List chunk) { try { + resetTimeElapsed(); responseQueue.put(new SyncMessageHandler.Job(peer, MessageType.BLOCK_HEADERS_RESPONSE_MESSAGE) { @Override public void run() { @@ -164,7 +168,7 @@ public void run() { } }); } catch (InterruptedException e) { - logger.warn("SnapStateChunkResponseMessage processing was interrupted", e); + logger.warn("{} processing was interrupted", MessageType.BLOCK_HEADERS_RESPONSE_MESSAGE, e); Thread.currentThread().interrupt(); } } @@ -189,10 +193,7 @@ public void tick(Duration duration) { @Override protected void onMessageTimeOut() { - // TODO: call syncEventsHandler.onErrorSyncing() and punish peers after SNAP feature discovery is implemented - - finish(); -// resetTimeElapsed(); + fail(getLastBlockSender(), EventType.TIMEOUT_MESSAGE, "Snap sync timed out"); } public Block getLastBlock() { @@ -310,6 +311,8 @@ public void finish() { isRunning = Boolean.FALSE; thread.interrupt(); + logger.debug("Stopping Snap Sync"); + syncEventsHandler.stopSyncing(); } @@ -319,6 +322,8 @@ public void fail(Peer peer, EventType eventType, String message, Object... argum return; } + logger.debug("Snap Sync failed due to: {}", eventType); + isRunning = Boolean.FALSE; thread.interrupt(); From 58edd8eeccee88a799adaff49498e667668cd053 Mon Sep 17 00:00:00 2001 From: Volodymyr Kravets Date: Tue, 3 Dec 2024 15:00:32 +0200 Subject: [PATCH 07/10] feat(snap): remove snap-state-chunk-specific timeout config --- .../src/main/java/co/rsk/RskContext.java | 1 - .../co/rsk/config/RskSystemProperties.java | 5 --- .../java/co/rsk/net/SnapshotProcessor.java | 1 - .../java/co/rsk/net/sync/SnapSyncState.java | 15 ------- .../co/rsk/net/sync/SyncConfiguration.java | 10 ----- rskj-core/src/main/resources/expected.conf | 1 - rskj-core/src/main/resources/reference.conf | 2 - .../co/rsk/net/SnapshotProcessorTest.java | 2 - .../co/rsk/net/sync/SnapSyncStateTest.java | 44 ------------------- 9 files changed, 81 deletions(-) diff --git a/rskj-core/src/main/java/co/rsk/RskContext.java b/rskj-core/src/main/java/co/rsk/RskContext.java index 8aaaf13f66c..d176812a7af 100644 --- a/rskj-core/src/main/java/co/rsk/RskContext.java +++ b/rskj-core/src/main/java/co/rsk/RskContext.java @@ -1521,7 +1521,6 @@ protected synchronized SyncConfiguration buildSyncConfiguration() { rskSystemProperties.getTopBest(), rskSystemProperties.isServerSnapshotSyncEnabled(), rskSystemProperties.isClientSnapshotSyncEnabled(), - rskSystemProperties.getSnapshotChunkTimeout(), rskSystemProperties.getSnapshotSyncLimit(), rskSystemProperties.getSnapBootNodes()); } diff --git a/rskj-core/src/main/java/co/rsk/config/RskSystemProperties.java b/rskj-core/src/main/java/co/rsk/config/RskSystemProperties.java index 6adaa405eda..9e2da036ee8 100644 --- a/rskj-core/src/main/java/co/rsk/config/RskSystemProperties.java +++ b/rskj-core/src/main/java/co/rsk/config/RskSystemProperties.java @@ -32,7 +32,6 @@ import org.ethereum.crypto.ECKey; import org.ethereum.crypto.HashUtil; import org.ethereum.listener.GasPriceCalculator; -import org.ethereum.net.client.Capability; import org.ethereum.vm.PrecompiledContracts; import javax.annotation.Nonnull; @@ -430,10 +429,6 @@ public int getLongSyncLimit() { public boolean isServerSnapshotSyncEnabled() { return configFromFiles.getBoolean("sync.snapshot.server.enabled");} public boolean isClientSnapshotSyncEnabled() { return configFromFiles.getBoolean(PROPERTY_SNAP_CLIENT_ENABLED);} - public int getSnapshotChunkTimeout() { - return configFromFiles.getInt("sync.snapshot.client.chunkRequestTimeout"); - } - public boolean isSnapshotParallelEnabled() { return configFromFiles.getBoolean("sync.snapshot.client.parallel");} public int getSnapshotChunkSize() { return configFromFiles.getInt("sync.snapshot.client.chunkSize");} diff --git a/rskj-core/src/main/java/co/rsk/net/SnapshotProcessor.java b/rskj-core/src/main/java/co/rsk/net/SnapshotProcessor.java index 41ec275d6cd..dddb79f1fe1 100644 --- a/rskj-core/src/main/java/co/rsk/net/SnapshotProcessor.java +++ b/rskj-core/src/main/java/co/rsk/net/SnapshotProcessor.java @@ -600,7 +600,6 @@ void onStateChunkResponseError(Peer peer, SnapStateChunkResponseMessage response private void processOrderedStateChunkResponse(SnapSyncState state, Peer peer, SnapStateChunkResponseMessage message) throws Exception { logger.debug("Processing State chunk received from {} to {}", message.getFrom(), message.getTo()); peersInformation.getOrRegisterPeer(peer); - state.onNewChunk(); RLPList nodeLists = RLP.decodeList(message.getChunkOfTrieKeyValue()); final RLPList preRootElements = RLP.decodeList(nodeLists.get(0).getRLPData()); diff --git a/rskj-core/src/main/java/co/rsk/net/sync/SnapSyncState.java b/rskj-core/src/main/java/co/rsk/net/sync/SnapSyncState.java index 516fac3f22a..694ec166526 100644 --- a/rskj-core/src/main/java/co/rsk/net/sync/SnapSyncState.java +++ b/rskj-core/src/main/java/co/rsk/net/sync/SnapSyncState.java @@ -37,7 +37,6 @@ import javax.annotation.Nullable; import java.math.BigInteger; -import java.time.Duration; import java.util.*; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; @@ -177,20 +176,6 @@ public SyncEventsHandler getSyncEventsHandler() { return this.syncEventsHandler; } - public void onNewChunk() { - resetTimeElapsed(); - } - - @Override - public void tick(Duration duration) { - // TODO(snap-poc) handle multiple peers casuistry, similarly to co.rsk.net.sync.DownloadingBodiesSyncState.tick - - timeElapsed = timeElapsed.plus(duration); - if (timeElapsed.compareTo(syncConfiguration.getTimeoutWaitingSnapChunk()) >= 0) { - onMessageTimeOut(); - } - } - @Override protected void onMessageTimeOut() { fail(getLastBlockSender(), EventType.TIMEOUT_MESSAGE, "Snap sync timed out"); diff --git a/rskj-core/src/main/java/co/rsk/net/sync/SyncConfiguration.java b/rskj-core/src/main/java/co/rsk/net/sync/SyncConfiguration.java index 142de5b1ea5..722f1329a30 100644 --- a/rskj-core/src/main/java/co/rsk/net/sync/SyncConfiguration.java +++ b/rskj-core/src/main/java/co/rsk/net/sync/SyncConfiguration.java @@ -47,8 +47,6 @@ public final class SyncConfiguration { private final boolean isServerSnapSyncEnabled; private final boolean isClientSnapSyncEnabled; - private final Duration timeoutWaitingSnapChunk; - private final int snapshotSyncLimit; private final Map nodeIdToSnapshotTrustedPeerMap; @@ -92,7 +90,6 @@ public SyncConfiguration( topBest, isServerSnapSyncEnabled, isClientSnapSyncEnabled, - timeoutWaitingSnapChunk, snapshotSyncLimit, Collections.emptyList()); } @@ -109,7 +106,6 @@ public SyncConfiguration( double topBest, boolean isServerSnapSyncEnabled, boolean isClientSnapSyncEnabled, - int timeoutWaitingSnapChunk, int snapshotSyncLimit, List snapBootNodes) { this.expectedPeers = expectedPeers; @@ -123,8 +119,6 @@ public SyncConfiguration( this.topBest = topBest; this.isServerSnapSyncEnabled = isServerSnapSyncEnabled; this.isClientSnapSyncEnabled = isClientSnapSyncEnabled; - // TODO(snap-poc) re-visit the need of this specific timeout as the algorithm evolves - this.timeoutWaitingSnapChunk = Duration.ofSeconds(timeoutWaitingSnapChunk); this.snapshotSyncLimit = snapshotSyncLimit; List snapBootNodesList = snapBootNodes != null ? snapBootNodes : Collections.emptyList(); @@ -177,10 +171,6 @@ public boolean isClientSnapSyncEnabled() { return isClientSnapSyncEnabled; } - public Duration getTimeoutWaitingSnapChunk() { - return timeoutWaitingSnapChunk; - } - public int getSnapshotSyncLimit() { return snapshotSyncLimit; } diff --git a/rskj-core/src/main/resources/expected.conf b/rskj-core/src/main/resources/expected.conf index 236d60c62e1..c4680601de2 100644 --- a/rskj-core/src/main/resources/expected.conf +++ b/rskj-core/src/main/resources/expected.conf @@ -281,7 +281,6 @@ sync = { enabled = parallel = chunkSize = - chunkRequestTimeout = limit = snapBootNodes = [ { diff --git a/rskj-core/src/main/resources/reference.conf b/rskj-core/src/main/resources/reference.conf index 75e21c96ec7..20a17d2ba5c 100644 --- a/rskj-core/src/main/resources/reference.conf +++ b/rskj-core/src/main/resources/reference.conf @@ -380,8 +380,6 @@ sync { enabled = false # Server / chunk size chunkSize = 50 - # Request timeout (in seconds) - chunkRequestTimeout = 120 # Distance to the tip of the blockchain to start snapshot sync limit = 10000 # Parallel requests (true, false) diff --git a/rskj-core/src/test/java/co/rsk/net/SnapshotProcessorTest.java b/rskj-core/src/test/java/co/rsk/net/SnapshotProcessorTest.java index 82a7c075938..58068feafd0 100644 --- a/rskj-core/src/test/java/co/rsk/net/SnapshotProcessorTest.java +++ b/rskj-core/src/test/java/co/rsk/net/SnapshotProcessorTest.java @@ -416,10 +416,8 @@ void givenErrorRLPData_thenOnStateChunkErrorIsCalled() { underTest.processStateChunkResponse(snapSyncState, peer, responseMessage); - verify(snapSyncState, times(1)).onNewChunk(); verify(underTest, times(1)).onStateChunkResponseError(peer, responseMessage); verify(peer, times(1)).sendMessage(any(SnapStateChunkRequestMessage.class)); - } private void initializeBlockchainWithAmountOfBlocks(int numberOfBlocks) { diff --git a/rskj-core/src/test/java/co/rsk/net/sync/SnapSyncStateTest.java b/rskj-core/src/test/java/co/rsk/net/sync/SnapSyncStateTest.java index 5b0039e6415..02a8c220b25 100644 --- a/rskj-core/src/test/java/co/rsk/net/sync/SnapSyncStateTest.java +++ b/rskj-core/src/test/java/co/rsk/net/sync/SnapSyncStateTest.java @@ -19,7 +19,6 @@ package co.rsk.net.sync; import co.rsk.core.BlockDifficulty; -import co.rsk.net.NodeID; import co.rsk.net.Peer; import co.rsk.net.SnapshotProcessor; import co.rsk.net.messages.SnapBlocksResponseMessage; @@ -33,11 +32,8 @@ import org.mockito.ArgumentCaptor; import java.math.BigInteger; -import java.net.InetAddress; -import java.net.UnknownHostException; import java.time.Duration; import java.util.List; -import java.util.Optional; import java.util.PriorityQueue; import java.util.Queue; import java.util.concurrent.CountDownLatch; @@ -45,7 +41,6 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; import static org.junit.jupiter.api.Assertions.*; import static org.mockito.Mockito.*; @@ -98,27 +93,6 @@ void givenOnEnterWasCalledTwice_thenSyncingStartsOnlyOnce() { verify(snapshotProcessor, times(1)).startSyncing(underTest); } - @Test - void givenOnMessageTimeOutCalled_thenSyncingStops() { - //given-when - underTest.setRunning(); - underTest.onMessageTimeOut(); - //then - verify(syncEventsHandler, times(1)).stopSyncing(); - } - - @Test - void givenNewChunk_thenTimerIsReset() { - //given - underTest.timeElapsed = Duration.ofMinutes(1); - assertThat(underTest.timeElapsed, greaterThan(Duration.ZERO)); - - // when - underTest.onNewChunk(); - //then - assertThat(underTest.timeElapsed, equalTo(Duration.ZERO)); - } - @Test void givenTickIsCalledBeforeTimeout_thenTimerIsUpdated_andNoTimeoutHappens() { //given @@ -132,24 +106,6 @@ void givenTickIsCalledBeforeTimeout_thenTimerIsUpdated_andNoTimeoutHappens() { verify(syncEventsHandler, never()).onErrorSyncing(any(), any(), any(), any()); } - @Test - void givenTickIsCalledAfterTimeout_thenTimerIsUpdated_andTimeoutHappens() throws UnknownHostException { - //given - Duration elapsedTime = Duration.ofMinutes(1); - underTest.timeElapsed = Duration.ZERO; - Peer mockedPeer = mock(Peer.class); - NodeID nodeID = mock(NodeID.class); - when(mockedPeer.getPeerNodeID()).thenReturn(nodeID); - when(mockedPeer.getAddress()).thenReturn(InetAddress.getByName("127.0.0.1")); - when(peersInformation.getBestSnapPeer()).thenReturn(Optional.of(mockedPeer)); - underTest.setRunning(); - // when - underTest.tick(elapsedTime); - //then - assertThat(underTest.timeElapsed, equalTo(elapsedTime)); - verify(syncEventsHandler, times(1)).stopSyncing(); - } - @Test void givenFinishIsCalled_thenSyncEventHandlerStopsSync() { //given-when From a4acb49e36b8d1064a6619194b4b3e7c972208f5 Mon Sep 17 00:00:00 2001 From: Volodymyr Kravets Date: Thu, 5 Dec 2024 18:00:08 +0200 Subject: [PATCH 08/10] feat(snap): address comments; add some tests --- .../rsk/core/bc/BlockRelayValidatorTest.java | 12 +++--- .../co/rsk/core/bc/BlockValidatorBuilder.java | 13 +++--- .../co/rsk/core/bc/BlockValidatorTest.java | 18 +++++---- .../co/rsk/net/sync/SnapSyncStateTest.java | 40 +++++++++++++++++++ 4 files changed, 65 insertions(+), 18 deletions(-) diff --git a/rskj-core/src/test/java/co/rsk/core/bc/BlockRelayValidatorTest.java b/rskj-core/src/test/java/co/rsk/core/bc/BlockRelayValidatorTest.java index 29ab5d12248..95c386be84f 100644 --- a/rskj-core/src/test/java/co/rsk/core/bc/BlockRelayValidatorTest.java +++ b/rskj-core/src/test/java/co/rsk/core/bc/BlockRelayValidatorTest.java @@ -47,7 +47,7 @@ void genesisCheck() { verify(block).isGenesis(); verify(blockValidator, never()).isValid(any()); - verify(blockParentValidator, never()).isValid(any(), (Block) any()); + verify(blockParentValidator, never()).isValid(any(), any(Block.class)); } @Test @@ -62,7 +62,7 @@ void blockValidatorCheck() { verify(block).isGenesis(); verify(blockValidator).isValid(any()); - verify(blockParentValidator, never()).isValid(any(), (Block) any()); + verify(blockParentValidator, never()).isValid(any(), any(Block.class)); } @Test @@ -74,7 +74,7 @@ void blockParentValidatorCheck() { when(block.getParentHash()).thenReturn(parentHash); when(blockStore.getBlockByHash(any())).thenReturn(parentBlock); when(blockValidator.isValid(any())).thenReturn(true); - when(blockParentValidator.isValid(any(), (Block) any())).thenReturn(false); + when(blockParentValidator.isValid(any(), any(Block.class))).thenReturn(false); boolean actualResult = blockRelayValidator.isValid(block); @@ -82,7 +82,7 @@ void blockParentValidatorCheck() { verify(block).isGenesis(); verify(blockValidator).isValid(any()); - verify(blockParentValidator).isValid(any(), (Block) any()); + verify(blockParentValidator).isValid(any(), any(Block.class)); } @Test @@ -94,7 +94,7 @@ void allValidatorsCheck() { when(block.getParentHash()).thenReturn(parentHash); when(blockStore.getBlockByHash(any())).thenReturn(parentBlock); when(blockValidator.isValid(any())).thenReturn(true); - when(blockParentValidator.isValid(any(), (Block) any())).thenReturn(true); + when(blockParentValidator.isValid(any(), any(Block.class))).thenReturn(true); boolean actualResult = blockRelayValidator.isValid(block); @@ -102,6 +102,6 @@ void allValidatorsCheck() { verify(block).isGenesis(); verify(blockValidator).isValid(any()); - verify(blockParentValidator).isValid(any(), (Block) any()); + verify(blockParentValidator).isValid(any(), any(Block.class)); } } diff --git a/rskj-core/src/test/java/co/rsk/core/bc/BlockValidatorBuilder.java b/rskj-core/src/test/java/co/rsk/core/bc/BlockValidatorBuilder.java index 34be11c2c6a..5fb5e4d3e14 100644 --- a/rskj-core/src/test/java/co/rsk/core/bc/BlockValidatorBuilder.java +++ b/rskj-core/src/test/java/co/rsk/core/bc/BlockValidatorBuilder.java @@ -31,7 +31,10 @@ import org.ethereum.core.ReceivedTxSignatureCache; import org.ethereum.datasource.HashMapDB; import org.ethereum.db.BlockStore; -import org.mockito.Mockito; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; /** * Created by mario on 19/01/17. @@ -95,11 +98,11 @@ public BlockValidatorBuilder addTxsMinGasPriceRule() { } public BlockValidatorBuilder addBlockUnclesValidationRule(BlockStore blockStore) { - BlockHeaderValidationRule validationRule = Mockito.mock(BlockHeaderValidationRule.class); - Mockito.when(validationRule.isValid(Mockito.any())).thenReturn(true); + BlockHeaderValidationRule validationRule = mock(BlockHeaderValidationRule.class); + when(validationRule.isValid(any())).thenReturn(true); - BlockHeaderParentDependantValidationRule parentValidationRule = Mockito.mock(BlockHeaderParentDependantValidationRule.class); - Mockito.when(parentValidationRule.isValid(Mockito.any(), (Block) Mockito.any())).thenReturn(true); + BlockHeaderParentDependantValidationRule parentValidationRule = mock(BlockHeaderParentDependantValidationRule.class); + when(parentValidationRule.isValid(any(), any(Block.class))).thenReturn(true); this.addBlockUnclesValidationRule(blockStore, validationRule, parentValidationRule); return this; diff --git a/rskj-core/src/test/java/co/rsk/core/bc/BlockValidatorTest.java b/rskj-core/src/test/java/co/rsk/core/bc/BlockValidatorTest.java index f1f95fd4890..215b1542848 100644 --- a/rskj-core/src/test/java/co/rsk/core/bc/BlockValidatorTest.java +++ b/rskj-core/src/test/java/co/rsk/core/bc/BlockValidatorTest.java @@ -43,12 +43,16 @@ import org.ethereum.db.IndexedBlockStore; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import org.mockito.Mockito; import java.math.BigInteger; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; -import static org.mockito.Mockito.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; /** * Created by ajlopez on 04/08/2016. @@ -288,7 +292,7 @@ void invalidPOWUncles() { store.saveBlock(uncle1a, TEST_DIFFICULTY, false); BlockHeaderParentDependantValidationRule parentValidationRule = mock(BlockHeaderParentDependantValidationRule.class); - when(parentValidationRule.isValid(Mockito.any(), (Block) Mockito.any())).thenReturn(true); + when(parentValidationRule.isValid(any(), any(Block.class))).thenReturn(true); BlockValidatorImpl validator = new BlockValidatorBuilder() .addBlockUnclesValidationRule(store, new ProofOfWorkRule(config).setFallbackMiningEnabled(false), parentValidationRule) @@ -497,7 +501,7 @@ void processBlockWithInvalidMGPTxs() { BlockStore blockStore = mock(org.ethereum.db.BlockStore.class); Repository repository = mock(Repository.class); - when(repository.getNonce(Mockito.any())).thenReturn(BigInteger.ZERO); + when(repository.getNonce(any())).thenReturn(BigInteger.ZERO); Block parent = new BlockBuilder(null, null, null).minGasPrice(BigInteger.ZERO) .parent(new BlockGenerator().getGenesisBlock()).build(); @@ -533,7 +537,7 @@ void processBlockWithInvalidPrevMGP() { BlockStore blockStore = mock(org.ethereum.db.BlockStore.class); Repository repository = mock(Repository.class); - when(repository.getNonce(Mockito.any())).thenReturn(BigInteger.ZERO); + when(repository.getNonce(any())).thenReturn(BigInteger.ZERO); Block parent = new BlockBuilder(null, null, null).minGasPrice(BigInteger.ZERO) .parent(new BlockGenerator().getGenesisBlock()).build(); @@ -556,7 +560,7 @@ void processValidMGPBlock() { BlockStore blockStore = mock(org.ethereum.db.BlockStore.class); Repository repository = mock(Repository.class); - when(repository.getNonce(Mockito.any())).thenReturn(BigInteger.ZERO); + when(repository.getNonce(any())).thenReturn(BigInteger.ZERO); Block parent = new BlockBuilder(null, null, null).minGasPrice(BigInteger.TEN) .parent(new BlockGenerator().getGenesisBlock()).build(); diff --git a/rskj-core/src/test/java/co/rsk/net/sync/SnapSyncStateTest.java b/rskj-core/src/test/java/co/rsk/net/sync/SnapSyncStateTest.java index 02a8c220b25..f12e5080483 100644 --- a/rskj-core/src/test/java/co/rsk/net/sync/SnapSyncStateTest.java +++ b/rskj-core/src/test/java/co/rsk/net/sync/SnapSyncStateTest.java @@ -21,11 +21,14 @@ import co.rsk.core.BlockDifficulty; import co.rsk.net.Peer; import co.rsk.net.SnapshotProcessor; +import co.rsk.net.messages.MessageType; import co.rsk.net.messages.SnapBlocksResponseMessage; import co.rsk.net.messages.SnapStateChunkResponseMessage; import co.rsk.net.messages.SnapStatusResponseMessage; +import co.rsk.scoring.EventType; import org.apache.commons.lang3.tuple.Pair; import org.ethereum.core.Block; +import org.ethereum.core.BlockHeader; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -159,6 +162,29 @@ void givenOnSnapBlocksIsCalled_thenJobIsAddedAndRun() throws InterruptedExceptio assertEquals(msg.getMessageType(), jobArg.getValue().getMsgType()); } + @Test + void givenNewBlockHeadersIsCalled_thenJobIsAddedAndRun() throws InterruptedException { + //given + Peer peer = mock(Peer.class); + //noinspection unchecked + List msg = mock(List.class); + CountDownLatch latch = new CountDownLatch(1); + doCountDownOnQueueEmpty(listener, latch); + underTest.onEnter(); + + //when + underTest.newBlockHeaders(peer, msg); + + //then + assertTrue(latch.await(THREAD_JOIN_TIMEOUT, TimeUnit.MILLISECONDS)); + + ArgumentCaptor jobArg = ArgumentCaptor.forClass(SyncMessageHandler.Job.class); + verify(listener, times(1)).onJobRun(jobArg.capture()); + + assertEquals(peer, jobArg.getValue().getSender()); + assertEquals(MessageType.BLOCK_HEADERS_RESPONSE_MESSAGE, jobArg.getValue().getMsgType()); + } + @Test void givenOnSnapStateChunkIsCalled_thenJobIsAddedAndRun() throws InterruptedException { //given @@ -181,6 +207,20 @@ void givenOnSnapStateChunkIsCalled_thenJobIsAddedAndRun() throws InterruptedExce assertEquals(msg.getMessageType(), jobArg.getValue().getMsgType()); } + @Test + void givenOnMessageTimeOut_thenShouldFail() throws InterruptedException { + //given + Peer peer = mock(Peer.class); + underTest.setLastBlock(mock(Block.class), mock(BlockDifficulty.class), peer); + underTest.setRunning(); + + //when + underTest.onMessageTimeOut(); + + //then + verify(syncEventsHandler, times(1)).onErrorSyncing(eq(peer), eq(EventType.TIMEOUT_MESSAGE), any()); + } + @Test void testSetAndGetLastBlock() { Block mockBlock = mock(Block.class); From 68091e2024482cd96388a122e054b515a46ac21c Mon Sep 17 00:00:00 2001 From: Volodymyr Kravets Date: Fri, 20 Dec 2024 13:59:32 +0200 Subject: [PATCH 09/10] feat(snap): add retry mechanism; add extra configurability; fix snap state chunk msg encoding; other minor fixes --- .../src/main/java/co/rsk/RskContext.java | 1 + .../co/rsk/config/RskSystemProperties.java | 9 +- .../java/co/rsk/net/NodeMessageHandler.java | 11 +- .../java/co/rsk/net/SnapshotProcessor.java | 80 ++++--- .../main/java/co/rsk/net/SyncProcessor.java | 65 ++++-- .../java/co/rsk/net/messages/MessageType.java | 8 +- .../messages/SnapBlocksRequestMessage.java | 26 ++- .../messages/SnapBlocksResponseMessage.java | 25 ++- .../SnapStateChunkRequestMessage.java | 32 +-- .../SnapStateChunkResponseMessage.java | 55 +++-- .../messages/SnapStatusRequestMessage.java | 29 ++- .../messages/SnapStatusResponseMessage.java | 25 ++- .../co/rsk/net/sync/PeersInformation.java | 9 + .../rsk/net/sync/SnapSyncRequestManager.java | 197 ++++++++++++++++++ .../java/co/rsk/net/sync/SnapSyncState.java | 68 ++++-- .../net/sync/SnapshotPeersInformation.java | 2 + .../co/rsk/net/sync/SyncConfiguration.java | 18 +- .../co/rsk/net/sync/SyncEventsHandler.java | 6 + .../main/java/co/rsk/net/sync/SyncState.java | 9 +- .../java/org/ethereum/net/server/Channel.java | 17 +- .../server/EthereumChannelInitializer.java | 3 +- .../java/org/ethereum/net/server/Stats.java | 24 ++- rskj-core/src/main/resources/expected.conf | 2 + rskj-core/src/main/resources/reference.conf | 5 + .../co/rsk/net/NodeMessageHandlerTest.java | 3 +- .../co/rsk/net/SnapshotProcessorTest.java | 30 ++- .../ThreeAsyncNodeUsingSyncProcessorTest.java | 10 +- .../SnapBlocksRequestMessageTest.java | 34 ++- .../SnapBlocksResponseMessageTest.java | 31 ++- .../SnapStateChunkRequestMessageTest.java | 30 +++ .../SnapStateChunkResponseMessageTest.java | 41 +++- .../SnapStatusRequestMessageTest.java | 33 ++- .../SnapStatusResponseMessageTest.java | 33 ++- .../rsk/net/sync/SimpleSyncEventsHandler.java | 8 + .../co/rsk/net/sync/SnapSyncStateTest.java | 54 ++--- 35 files changed, 794 insertions(+), 239 deletions(-) create mode 100644 rskj-core/src/main/java/co/rsk/net/sync/SnapSyncRequestManager.java diff --git a/rskj-core/src/main/java/co/rsk/RskContext.java b/rskj-core/src/main/java/co/rsk/RskContext.java index 1ac4727b8df..7d6ecd7aec5 100644 --- a/rskj-core/src/main/java/co/rsk/RskContext.java +++ b/rskj-core/src/main/java/co/rsk/RskContext.java @@ -2089,6 +2089,7 @@ private SnapshotProcessor getSnapshotProcessor() { new ValidGasUsedRule() ), getRskSystemProperties().getSnapshotChunkSize(), + getRskSystemProperties().checkHistoricalHeaders(), getRskSystemProperties().isSnapshotParallelEnabled() ); } diff --git a/rskj-core/src/main/java/co/rsk/config/RskSystemProperties.java b/rskj-core/src/main/java/co/rsk/config/RskSystemProperties.java index 9e2da036ee8..a868152b511 100644 --- a/rskj-core/src/main/java/co/rsk/config/RskSystemProperties.java +++ b/rskj-core/src/main/java/co/rsk/config/RskSystemProperties.java @@ -73,6 +73,7 @@ public class RskSystemProperties extends SystemProperties { public static final String USE_PEERS_FROM_LAST_SESSION = "peer.discovery.usePeersFromLastSession"; public static final String PROPERTY_SNAP_CLIENT_ENABLED = "sync.snapshot.client.enabled"; + public static final String PROPERTY_SNAP_CLIENT_CHECK_HISTORICAL_HEADERS = "sync.snapshot.client.checkHistoricalHeaders"; public static final String PROPERTY_SNAP_NODES = "sync.snapshot.client.snapBootNodes"; //TODO: REMOVE THIS WHEN THE LocalBLockTests starts working with REMASC @@ -429,6 +430,8 @@ public int getLongSyncLimit() { public boolean isServerSnapshotSyncEnabled() { return configFromFiles.getBoolean("sync.snapshot.server.enabled");} public boolean isClientSnapshotSyncEnabled() { return configFromFiles.getBoolean(PROPERTY_SNAP_CLIENT_ENABLED);} + public boolean checkHistoricalHeaders() { return configFromFiles.getBoolean(PROPERTY_SNAP_CLIENT_CHECK_HISTORICAL_HEADERS);} + public boolean isSnapshotParallelEnabled() { return configFromFiles.getBoolean("sync.snapshot.client.parallel");} public int getSnapshotChunkSize() { return configFromFiles.getInt("sync.snapshot.client.chunkSize");} @@ -512,10 +515,14 @@ public boolean fastBlockPropagation() { return configFromFiles.getBoolean("peer.fastBlockPropagation"); } - public Integer getMessageQueueMaxSize() { + public int getMessageQueueMaxSize() { return configFromFiles.getInt("peer.messageQueue.maxSizePerPeer"); } + public int getMessageQueuePerMinuteThreshold() { + return configFromFiles.getInt("peer.messageQueue.thresholdPerMinutePerPeer"); + } + public boolean rpcZeroSignatureIfRemasc() { return configFromFiles.getBoolean("rpc.zeroSignatureIfRemasc"); } diff --git a/rskj-core/src/main/java/co/rsk/net/NodeMessageHandler.java b/rskj-core/src/main/java/co/rsk/net/NodeMessageHandler.java index 31825bd67d9..66d0c15fd22 100644 --- a/rskj-core/src/main/java/co/rsk/net/NodeMessageHandler.java +++ b/rskj-core/src/main/java/co/rsk/net/NodeMessageHandler.java @@ -211,7 +211,7 @@ private void tryAddMessage(Peer sender, Message message, NodeMsgTraceInfo nodeMs */ private boolean controlMessageIngress(Peer sender, Message message, double score) { return - allowByScore(score) && + allowByScore(sender, message, score) && allowByMessageCount(sender) && allowByMinerNotBanned(sender, message) && allowByMessageUniqueness(sender, message); // prevent repeated is the most expensive and MUST be the last @@ -221,8 +221,13 @@ private boolean controlMessageIngress(Peer sender, Message message, double score /** * assert score is acceptable */ - private boolean allowByScore(double score) { - return score >= 0; + private boolean allowByScore(Peer sender, Message message, double score) { + boolean allow = score >= 0; + if (!allow) { + logger.debug("Message: [{}] from: [{}] with score: [{}] was not allowed", message.getMessageType(), sender, score); + } + + return allow; } /** diff --git a/rskj-core/src/main/java/co/rsk/net/SnapshotProcessor.java b/rskj-core/src/main/java/co/rsk/net/SnapshotProcessor.java index dddb79f1fe1..d9381759558 100644 --- a/rskj-core/src/main/java/co/rsk/net/SnapshotProcessor.java +++ b/rskj-core/src/main/java/co/rsk/net/SnapshotProcessor.java @@ -53,9 +53,10 @@ import java.util.*; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; +import static co.rsk.net.sync.SnapSyncRequestManager.PeerSelector; + /** * Snapshot Synchronization consist in 3 steps: * 1. Status: exchange message with the server, to know which block we are going to sync and what the size of the Unitrie of that block is. @@ -86,8 +87,7 @@ public class SnapshotProcessor implements InternalService { private final BlockHeaderParentDependantValidationRule blockHeaderParentValidator; private final BlockHeaderValidationRule blockHeaderValidator; - private final AtomicLong messageId = new AtomicLong(0); - + private final boolean checkHistoricalHeaders; // flag for parallel requests private final boolean parallel; @@ -106,10 +106,11 @@ public SnapshotProcessor(Blockchain blockchain, BlockHeaderParentDependantValidationRule blockHeaderParentValidator, BlockHeaderValidationRule blockHeaderValidator, int chunkSize, + boolean checkHistoricalHeaders, boolean isParallelEnabled) { this(blockchain, trieStore, peersInformation, blockStore, transactionPool, blockParentValidator, blockValidator, blockHeaderParentValidator, blockHeaderValidator, - chunkSize, isParallelEnabled, null); + chunkSize, checkHistoricalHeaders, isParallelEnabled, null); } @VisibleForTesting @@ -123,6 +124,7 @@ public SnapshotProcessor(Blockchain blockchain, BlockHeaderParentDependantValidationRule blockHeaderParentValidator, BlockHeaderValidationRule blockHeaderValidator, int chunkSize, + boolean checkHistoricalHeaders, boolean isParallelEnabled, @Nullable SyncMessageHandler.Listener listener) { this.blockchain = blockchain; @@ -138,6 +140,7 @@ public SnapshotProcessor(Blockchain blockchain, this.blockHeaderParentValidator = blockHeaderParentValidator; this.blockHeaderValidator = blockHeaderValidator; + this.checkHistoricalHeaders = checkHistoricalHeaders; this.parallel = isParallelEnabled; this.thread = new Thread(new SyncMessageHandler("SNAP/server", requestQueue, listener) { @@ -157,7 +160,7 @@ public void startSyncing(SnapSyncState state) { } logger.info("Starting Snap sync"); - requestSnapStatus(bestPeerOpt.get()); + requestSnapStatus(state, bestPeerOpt.get()); } private void completeSyncing(SnapSyncState state) { @@ -177,9 +180,22 @@ private void failSyncing(SnapSyncState state, Peer peer, EventType eventType, St /** * STATUS */ - private void requestSnapStatus(Peer peer) { - SnapStatusRequestMessage message = new SnapStatusRequestMessage(); - peer.sendMessage(message); + private void requestSnapStatus(SnapSyncState state, Peer peer) { + state.submitRequest(snapPeerSelector(peer), SnapStatusRequestMessage::new); + } + + private PeerSelector peerSelector(@Nullable Peer peer) { + return PeerSelector.builder() + .withDefaultPeer(() -> peer) + .withAltPeer(peersInformation::getBestPeer) + .build(); + } + + private PeerSelector snapPeerSelector(@Nullable Peer snapPeer) { + return PeerSelector.builder() + .withDefaultPeer(() -> snapPeer) + .withAltPeer(peersInformation::getBestSnapPeer) + .build(); } public void processSnapStatusRequest(Peer sender, SnapStatusRequestMessage requestMessage) { @@ -201,7 +217,7 @@ public void run() { } } - void processSnapStatusRequestInternal(Peer sender, SnapStatusRequestMessage ignoredRequestMessage) { + void processSnapStatusRequestInternal(Peer sender, SnapStatusRequestMessage requestMessage) { long bestBlockNumber = blockchain.getBestBlock().getNumber(); long checkpointBlockNumber = bestBlockNumber - (bestBlockNumber % BLOCK_NUMBER_CHECKPOINT); logger.debug("Processing snapshot status request, checkpointBlockNumber: {}, bestBlockNumber: {}", checkpointBlockNumber, bestBlockNumber); @@ -226,7 +242,7 @@ void processSnapStatusRequestInternal(Peer sender, SnapStatusRequestMessage igno long trieSize = opt.get().getTotalSize(); logger.debug("Processing snapshot status request - rootHash: {} trieSize: {}", rootHash, trieSize); - SnapStatusResponseMessage responseMessage = new SnapStatusResponseMessage(blocks, difficulties, trieSize); + SnapStatusResponseMessage responseMessage = new SnapStatusResponseMessage(requestMessage.getId(), blocks, difficulties, trieSize); sender.sendMessage(responseMessage); } @@ -261,7 +277,7 @@ public void processSnapStatusResponse(SnapSyncState state, Peer sender, SnapStat generateChunkRequestTasks(state); startRequestingChunks(state); } else { - requestBlocksChunk(sender, blocksFromResponse.get(0).getNumber()); + requestBlocksChunk(state, blocksFromResponse.get(0).getNumber()); } } @@ -318,9 +334,11 @@ private boolean areBlockPairsValid(Pair blockPair, @Null /** * BLOCK CHUNK */ - private void requestBlocksChunk(Peer sender, long blockNumber) { - logger.debug("Requesting block chunk to node {} - block {}", sender.getPeerNodeID(), blockNumber); - sender.sendMessage(new SnapBlocksRequestMessage(blockNumber)); + private void requestBlocksChunk(SnapSyncState state, long blockNumber) { + state.submitRequest( + peerSelector(null), + messageId -> new SnapBlocksRequestMessage(messageId, blockNumber) + ); } public void processBlockHeaderChunk(SnapSyncState state, Peer sender, List chunk) { @@ -405,7 +423,10 @@ private void requestNextBlockHeadersChunk(SnapSyncState state, Peer sender) { logger.debug("Requesting block header chunk to node {} - block [{}/{}]", peer.getPeerNodeID(), lastVerifiedBlockHeader.getNumber() - 1, parentHash); - state.getSyncEventsHandler().sendBlockHeadersRequest(peer, new ChunkDescriptor(parentHash.getBytes(), (int) count)); + state.submitRequest( + peerSelector(sender), + messageId -> new BlockHeadersRequestMessage(messageId, parentHash.getBytes(), (int) count) + ); } public void processSnapBlocksRequest(Peer sender, SnapBlocksRequestMessage requestMessage) { @@ -447,7 +468,7 @@ void processSnapBlocksRequestInternal(Peer sender, SnapBlocksRequestMessage requ difficulties.add(blockStore.getTotalDifficultyForHash(block.getHash().getBytes())); } logger.debug("Sending snap blocks response. From block {} to block {} - chunksize {}", blocks.get(0).getNumber(), blocks.get(blocks.size() - 1).getNumber(), BLOCK_CHUNK_SIZE); - SnapBlocksResponseMessage responseMessage = new SnapBlocksResponseMessage(blocks, difficulties); + SnapBlocksResponseMessage responseMessage = new SnapBlocksResponseMessage(requestMessage.getId(), blocks, difficulties); sender.sendMessage(responseMessage); } @@ -474,7 +495,12 @@ public void processSnapBlocksResponse(SnapSyncState state, Peer sender, SnapBloc generateChunkRequestTasks(state); startRequestingChunks(state); } else if (nextChunk > lastRequiredBlock) { - requestBlocksChunk(sender, nextChunk); + requestBlocksChunk(state, nextChunk); + } else if (!this.checkHistoricalHeaders) { + logger.info("Finished Snap blocks request sending. Start requesting state chunks without historical headers check"); + + generateChunkRequestTasks(state); + startRequestingChunks(state); } else { logger.info("Finished Snap blocks request sending. Start requesting state chunks and block headers"); @@ -488,10 +514,11 @@ public void processSnapBlocksResponse(SnapSyncState state, Peer sender, SnapBloc /** * STATE CHUNK */ - private void requestStateChunk(Peer peer, long from, long blockNumber, int chunkSize) { - logger.debug("Requesting state chunk to node {} - block {} - chunkNumber {}", peer.getPeerNodeID(), blockNumber, from / chunkSize); - SnapStateChunkRequestMessage message = new SnapStateChunkRequestMessage(messageId.getAndIncrement(), blockNumber, from, chunkSize); - peer.sendMessage(message); + private void requestStateChunk(SnapSyncState state, Peer peer, long from, long blockNumber, int chunkSize) { + state.submitRequest( + snapPeerSelector(peer), + messageId -> new SnapStateChunkRequestMessage(messageId, blockNumber, from, chunkSize) + ); } public void processStateChunkRequest(Peer sender, SnapStateChunkRequestMessage requestMessage) { @@ -573,7 +600,7 @@ public void processStateChunkResponse(SnapSyncState state, Peer peer, SnapStateC state.setNextExpectedFrom(nextExpectedFrom + chunkSize * CHUNK_ITEM_SIZE); } catch (Exception e) { logger.error("Error while processing chunk response. {}", e.getMessage(), e); - onStateChunkResponseError(peer, nextMessage); + onStateChunkResponseError(state, peer, nextMessage); } } else { break; @@ -587,19 +614,18 @@ public void processStateChunkResponse(SnapSyncState state, Peer peer, SnapStateC } @VisibleForTesting - void onStateChunkResponseError(Peer peer, SnapStateChunkResponseMessage responseMessage) { + void onStateChunkResponseError(SnapSyncState state, Peer peer, SnapStateChunkResponseMessage responseMessage) { logger.error("Error while processing chunk response from {} of peer {}. Asking for chunk again.", responseMessage.getFrom(), peer.getPeerNodeID()); Peer alternativePeer = peersInformation.getBestSnapPeerCandidates().stream() .filter(listedPeer -> !listedPeer.getPeerNodeID().equals(peer.getPeerNodeID())) .findFirst() .orElse(peer); logger.debug("Requesting state chunk \"from\" {} to peer {}", responseMessage.getFrom(), peer.getPeerNodeID()); - requestStateChunk(alternativePeer, responseMessage.getFrom(), responseMessage.getBlockNumber(), chunkSize); + requestStateChunk(state, alternativePeer, responseMessage.getFrom(), responseMessage.getBlockNumber(), chunkSize); } private void processOrderedStateChunkResponse(SnapSyncState state, Peer peer, SnapStateChunkResponseMessage message) throws Exception { logger.debug("Processing State chunk received from {} to {}", message.getFrom(), message.getTo()); - peersInformation.getOrRegisterPeer(peer); RLPList nodeLists = RLP.decodeList(message.getChunkOfTrieKeyValue()); final RLPList preRootElements = RLP.decodeList(nodeLists.get(0).getRLPData()); @@ -651,7 +677,7 @@ private void processOrderedStateChunkResponse(SnapSyncState state, Peer peer, Sn if (!message.isComplete()) { executeNextChunkRequestTask(state, peer); } else { - if (blocksVerified(state)) { + if (!this.checkHistoricalHeaders || blocksVerified(state)) { completeSyncing(state); } else { state.setStateFetched(); @@ -716,7 +742,7 @@ private void executeNextChunkRequestTask(SnapSyncState state, Peer peer) { if (!taskQueue.isEmpty()) { ChunkTask task = taskQueue.poll(); - requestStateChunk(peer, task.getFrom(), task.getBlockNumber(), chunkSize); + requestStateChunk(state, peer, task.getFrom(), task.getBlockNumber(), chunkSize); } else { logger.warn("No more chunk request tasks."); } diff --git a/rskj-core/src/main/java/co/rsk/net/SyncProcessor.java b/rskj-core/src/main/java/co/rsk/net/SyncProcessor.java index 0c5c2477eae..8fa931ec2e9 100644 --- a/rskj-core/src/main/java/co/rsk/net/SyncProcessor.java +++ b/rskj-core/src/main/java/co/rsk/net/SyncProcessor.java @@ -41,6 +41,7 @@ import java.time.Duration; import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; /** * This class' methods are executed one at a time because NodeMessageHandler is synchronized. @@ -69,6 +70,7 @@ public class SyncProcessor implements SyncEventsHandler { private final Map pendingMessages; private final AtomicBoolean isSyncing = new AtomicBoolean(); private final SnapshotProcessor snapshotProcessor; + private final AtomicLong lastRequestId = new AtomicLong(); private volatile long initialBlockNumber; private volatile long highestBlockNumber; @@ -76,7 +78,6 @@ public class SyncProcessor implements SyncEventsHandler { private volatile long lastDelayWarn = System.currentTimeMillis(); private SyncState syncState; - private long lastRequestId; @VisibleForTesting public SyncProcessor(Blockchain blockchain, @@ -119,7 +120,7 @@ public SyncProcessor(Blockchain blockchain, this.difficultyRule = new DifficultyRule(difficultyCalculator); this.genesis = genesis; this.ethereumListener = ethereumListener; - this.pendingMessages = new LinkedHashMap() { + this.pendingMessages = Collections.synchronizedMap(new LinkedHashMap<>() { @Override protected boolean removeEldestEntry(Map.Entry eldest) { boolean shouldDiscard = size() > MAX_PENDING_MESSAGES; @@ -128,7 +129,7 @@ protected boolean removeEldestEntry(Map.Entry eldest) { } return shouldDiscard; } - }; + }); this.peersInformation = peersInformation; this.snapshotProcessor = snapshotProcessor; @@ -178,7 +179,7 @@ public void processBlockHeadersResponse(Peer peer, BlockHeadersResponseMessage m MessageType messageType = message.getMessageType(); if (isPending(messageId, messageType)) { removePendingMessage(messageId, messageType); - syncState.newBlockHeaders(peer, message.getBlockHeaders()); + syncState.newBlockHeaders(peer, message); } else { notifyUnexpectedMessageToPeerScoring(peer, "block headers"); } @@ -205,7 +206,7 @@ public void processNewBlockHash(Peer peer, NewBlockHashMessage message) { if (syncState instanceof PeerAndModeDecidingSyncState && blockSyncService.getBlockFromStoreOrBlockchain(hash) == null) { peersInformation.getOrRegisterPeer(peer); - sendMessage(peer, new BlockRequestMessage(++lastRequestId, hash)); + sendMessage(peer, new BlockRequestMessage(nextMessageId(), hash)); } } @@ -224,29 +225,56 @@ public void processBlockResponse(Peer peer, BlockResponseMessage message) { } } - public void processSnapStatusResponse(Peer sender, SnapStatusResponseMessage responseMessage) { - syncState.onSnapStatus(sender, responseMessage); + public void processSnapStatusResponse(Peer peer, SnapStatusResponseMessage responseMessage) { + peersInformation.getOrRegisterPeer(peer); + + long messageId = responseMessage.getId(); + MessageType messageType = responseMessage.getMessageType(); + if (isPending(messageId, messageType)) { + removePendingMessage(messageId, messageType); + syncState.onSnapStatus(peer, responseMessage); + } else { + notifyUnexpectedMessageToPeerScoring(peer, "snap status"); + } } - public void processSnapBlocksResponse(Peer sender, SnapBlocksResponseMessage responseMessage) { - syncState.onSnapBlocks(sender, responseMessage); + public void processSnapBlocksResponse(Peer peer, SnapBlocksResponseMessage responseMessage) { + peersInformation.getOrRegisterPeer(peer); + + long messageId = responseMessage.getId(); + MessageType messageType = responseMessage.getMessageType(); + if (isPending(messageId, messageType)) { + removePendingMessage(messageId, messageType); + syncState.onSnapBlocks(peer, responseMessage); + } else { + notifyUnexpectedMessageToPeerScoring(peer, "snap blocks"); + } } public void processStateChunkResponse(Peer peer, SnapStateChunkResponseMessage responseMessage) { - syncState.onSnapStateChunk(peer, responseMessage); + peersInformation.getOrRegisterPeer(peer); + + long messageId = responseMessage.getId(); + MessageType messageType = responseMessage.getMessageType(); + if (isPending(messageId, messageType)) { + removePendingMessage(messageId, messageType); + syncState.onSnapStateChunk(peer, responseMessage); + } else { + notifyUnexpectedMessageToPeerScoring(peer, "snap state chunk"); + } } @Override public void sendSkeletonRequest(Peer peer, long height) { logger.debug("Send skeleton request to node {} height {}", peer.getPeerNodeID(), height); - MessageWithId message = new SkeletonRequestMessage(++lastRequestId, height); + MessageWithId message = new SkeletonRequestMessage(nextMessageId(), height); sendMessage(peer, message); } @Override public void sendBlockHashRequest(Peer peer, long height) { logger.debug("Send hash request to node {} height {}", peer.getPeerNodeID(), height); - BlockHashRequestMessage message = new BlockHashRequestMessage(++lastRequestId, height); + BlockHashRequestMessage message = new BlockHashRequestMessage(nextMessageId(), height); sendMessage(peer, message); } @@ -255,7 +283,7 @@ public void sendBlockHeadersRequest(Peer peer, ChunkDescriptor chunk) { logger.debug("Send headers request to node {}", peer.getPeerNodeID()); BlockHeadersRequestMessage message = - new BlockHeadersRequestMessage(++lastRequestId, chunk.getHash(), chunk.getCount()); + new BlockHeadersRequestMessage(nextMessageId(), chunk.getHash(), chunk.getCount()); sendMessage(peer, message); } @@ -264,7 +292,7 @@ public long sendBodyRequest(Peer peer, @Nonnull BlockHeader header) { logger.debug("Send body request block {} hash {} to peer {}", header.getNumber(), HashUtil.toPrintableHash(header.getHash().getBytes()), peer.getPeerNodeID()); - BodyRequestMessage message = new BodyRequestMessage(++lastRequestId, header.getHash().getBytes()); + BodyRequestMessage message = new BodyRequestMessage(nextMessageId(), header.getHash().getBytes()); sendMessage(peer, message); return message.getId(); } @@ -464,6 +492,15 @@ int getNoAdvancedPeers() { return this.peersInformation.countIf(s -> chainStatus.hasLowerTotalDifficultyThan(s.getStatus())); } + public long nextMessageId() { + return lastRequestId.incrementAndGet(); + } + + @Override + public void registerPendingMessage(@Nonnull MessageWithId message) { + pendingMessages.put(message.getId(), new MessageInfo(message.getResponseMessageType())); + } + @VisibleForTesting public void registerExpectedMessage(MessageWithId message) { pendingMessages.put(message.getId(), new MessageInfo(message.getMessageType())); diff --git a/rskj-core/src/main/java/co/rsk/net/messages/MessageType.java b/rskj-core/src/main/java/co/rsk/net/messages/MessageType.java index 2b8da7a78d0..bf7b902b95f 100644 --- a/rskj-core/src/main/java/co/rsk/net/messages/MessageType.java +++ b/rskj-core/src/main/java/co/rsk/net/messages/MessageType.java @@ -21,11 +21,11 @@ import co.rsk.core.BlockDifficulty; import co.rsk.net.Status; import co.rsk.remasc.RemascTransaction; +import org.bouncycastle.util.BigIntegers; import org.ethereum.core.*; import org.ethereum.util.RLP; import org.ethereum.util.RLPElement; import org.ethereum.util.RLPList; -import org.bouncycastle.util.BigIntegers; import java.util.ArrayList; import java.util.List; @@ -260,19 +260,19 @@ public Message createMessage(BlockFactory blockFactory, RLPList list) { SNAP_STATE_CHUNK_REQUEST_MESSAGE(20) { @Override public Message createMessage(BlockFactory blockFactory, RLPList list) { - return SnapStateChunkRequestMessage.create(blockFactory, list); + return SnapStateChunkRequestMessage.decodeMessage(blockFactory, list); } }, SNAP_STATE_CHUNK_RESPONSE_MESSAGE(21) { @Override public Message createMessage(BlockFactory blockFactory, RLPList list) { - return SnapStateChunkResponseMessage.create(blockFactory, list); + return SnapStateChunkResponseMessage.decodeMessage(blockFactory, list); } }, SNAP_STATUS_REQUEST_MESSAGE(22) { @Override public Message createMessage(BlockFactory blockFactory, RLPList list) { - return new SnapStatusRequestMessage(); + return SnapStatusRequestMessage.decodeMessage(blockFactory, list); } }, SNAP_STATUS_RESPONSE_MESSAGE(23) { diff --git a/rskj-core/src/main/java/co/rsk/net/messages/SnapBlocksRequestMessage.java b/rskj-core/src/main/java/co/rsk/net/messages/SnapBlocksRequestMessage.java index b5437477383..8cd58698885 100644 --- a/rskj-core/src/main/java/co/rsk/net/messages/SnapBlocksRequestMessage.java +++ b/rskj-core/src/main/java/co/rsk/net/messages/SnapBlocksRequestMessage.java @@ -25,10 +25,13 @@ import java.math.BigInteger; -public class SnapBlocksRequestMessage extends Message { +public class SnapBlocksRequestMessage extends MessageWithId { + private final long id; + private final long blockNumber; - public SnapBlocksRequestMessage(long blockNumber) { + public SnapBlocksRequestMessage(long id, long blockNumber) { + this.id = id; this.blockNumber = blockNumber; } @@ -38,17 +41,30 @@ public MessageType getMessageType() { } @Override - public byte[] getEncodedMessage() { + public MessageType getResponseMessageType() { + return MessageType.SNAP_BLOCKS_RESPONSE_MESSAGE; + } + + @Override + public long getId() { + return this.id; + } + + @Override + protected byte[] getEncodedMessageWithoutId() { byte[] encodedBlockNumber = RLP.encodeBigInteger(BigInteger.valueOf(blockNumber)); return RLP.encodeList(encodedBlockNumber); } public static Message decodeMessage(BlockFactory blockFactory, RLPList list) { - byte[] rlpBlockNumber = list.get(0).getRLPData(); + byte[] rlpId = list.get(0).getRLPData(); + long id = rlpId == null ? 0 : BigIntegers.fromUnsignedByteArray(rlpId).longValue(); + RLPList message = (RLPList)RLP.decode2(list.get(1).getRLPData()).get(0); + byte[] rlpBlockNumber = message.get(0).getRLPData(); long blockNumber = rlpBlockNumber == null ? 0 : BigIntegers.fromUnsignedByteArray(rlpBlockNumber).longValue(); - return new SnapBlocksRequestMessage(blockNumber); + return new SnapBlocksRequestMessage(id, blockNumber); } public long getBlockNumber() { diff --git a/rskj-core/src/main/java/co/rsk/net/messages/SnapBlocksResponseMessage.java b/rskj-core/src/main/java/co/rsk/net/messages/SnapBlocksResponseMessage.java index df0185c458c..b3b4c736342 100644 --- a/rskj-core/src/main/java/co/rsk/net/messages/SnapBlocksResponseMessage.java +++ b/rskj-core/src/main/java/co/rsk/net/messages/SnapBlocksResponseMessage.java @@ -20,6 +20,7 @@ import co.rsk.core.BlockDifficulty; import com.google.common.collect.Lists; +import org.bouncycastle.util.BigIntegers; import org.ethereum.core.Block; import org.ethereum.core.BlockFactory; import org.ethereum.util.RLP; @@ -29,11 +30,14 @@ import java.util.List; import java.util.stream.Collectors; -public class SnapBlocksResponseMessage extends Message { +public class SnapBlocksResponseMessage extends MessageWithId { + private final long id; + private final List blocks; private final List difficulties; - public SnapBlocksResponseMessage(List blocks, List difficulties) { + public SnapBlocksResponseMessage(long id, List blocks, List difficulties) { + this.id = id; this.blocks = blocks; this.difficulties = difficulties; } @@ -52,7 +56,12 @@ public List getBlocks() { } @Override - public byte[] getEncodedMessage() { + public long getId() { + return this.id; + } + + @Override + protected byte[] getEncodedMessageWithoutId() { List rlpBlocks = this.blocks.stream().map(Block::getEncoded).map(RLP::encode).collect(Collectors.toList()); List rlpDifficulties = this.difficulties.stream().map(BlockDifficulty::getBytes).map(RLP::encode).collect(Collectors.toList()); return RLP.encodeList(RLP.encodeList(rlpBlocks.toArray(new byte[][]{})), @@ -60,17 +69,21 @@ public byte[] getEncodedMessage() { } public static Message decodeMessage(BlockFactory blockFactory, RLPList list) { + byte[] rlpId = list.get(0).getRLPData(); + long id = rlpId == null ? 0 : BigIntegers.fromUnsignedByteArray(rlpId).longValue(); + + RLPList message = (RLPList)RLP.decode2(list.get(1).getRLPData()).get(0); List blocks = Lists.newArrayList(); List blockDifficulties = Lists.newArrayList(); - RLPList blocksRLP = RLP.decodeList(list.get(0).getRLPData()); + RLPList blocksRLP = RLP.decodeList(message.get(0).getRLPData()); for (int i = 0; i < blocksRLP.size(); i++) { blocks.add(blockFactory.decodeBlock(blocksRLP.get(i).getRLPData())); } - RLPList difficultiesRLP = RLP.decodeList(list.get(1).getRLPData()); + RLPList difficultiesRLP = RLP.decodeList(message.get(1).getRLPData()); for (int i = 0; i < difficultiesRLP.size(); i++) { blockDifficulties.add(new BlockDifficulty(new BigInteger(difficultiesRLP.get(i).getRLPData()))); } - return new SnapBlocksResponseMessage(blocks, blockDifficulties); + return new SnapBlocksResponseMessage(id, blocks, blockDifficulties); } @Override diff --git a/rskj-core/src/main/java/co/rsk/net/messages/SnapStateChunkRequestMessage.java b/rskj-core/src/main/java/co/rsk/net/messages/SnapStateChunkRequestMessage.java index 3195ff22aa8..fefc467dadb 100644 --- a/rskj-core/src/main/java/co/rsk/net/messages/SnapStateChunkRequestMessage.java +++ b/rskj-core/src/main/java/co/rsk/net/messages/SnapStateChunkRequestMessage.java @@ -43,6 +43,11 @@ public MessageType getMessageType() { return MessageType.SNAP_STATE_CHUNK_REQUEST_MESSAGE; } + @Override + public MessageType getResponseMessageType() { + return MessageType.SNAP_STATE_CHUNK_RESPONSE_MESSAGE; + } + @Override public void accept(MessageVisitor v) { v.apply(this); @@ -61,21 +66,18 @@ protected byte[] getEncodedMessageWithoutId() { return RLP.encodeList(rlpBlockNumber, rlpFrom, rlpChunkSize); } - public static Message create(BlockFactory blockFactory, RLPList list) { - try { - byte[] rlpId = list.get(0).getRLPData(); - RLPList message = (RLPList) RLP.decode2(list.get(1).getRLPData()).get(0); - byte[] rlpBlockNumber = message.get(0).getRLPData(); - byte[] rlpFrom = message.get(1).getRLPData(); - byte[] rlpChunkSize = message.get(2).getRLPData(); - long id = rlpId == null ? 0 : BigIntegers.fromUnsignedByteArray(rlpId).longValue(); - long blockNumber = rlpBlockNumber == null ? 0 : BigIntegers.fromUnsignedByteArray(rlpBlockNumber).longValue(); - long from = rlpFrom == null ? 0 : BigIntegers.fromUnsignedByteArray(rlpFrom).longValue(); - long chunkSize = rlpChunkSize == null ? 0 : BigIntegers.fromUnsignedByteArray(rlpChunkSize).longValue(); - return new SnapStateChunkRequestMessage(id, blockNumber, from, chunkSize); - } catch (Exception e) { - throw e; - } + public static Message decodeMessage(BlockFactory blockFactory, RLPList list) { + byte[] rlpId = list.get(0).getRLPData(); + long id = rlpId == null ? 0 : BigIntegers.fromUnsignedByteArray(rlpId).longValue(); + + RLPList message = (RLPList) RLP.decode2(list.get(1).getRLPData()).get(0); + byte[] rlpBlockNumber = message.get(0).getRLPData(); + byte[] rlpFrom = message.get(1).getRLPData(); + byte[] rlpChunkSize = message.get(2).getRLPData(); + long blockNumber = rlpBlockNumber == null ? 0 : BigIntegers.fromUnsignedByteArray(rlpBlockNumber).longValue(); + long from = rlpFrom == null ? 0 : BigIntegers.fromUnsignedByteArray(rlpFrom).longValue(); + long chunkSize = rlpChunkSize == null ? 0 : BigIntegers.fromUnsignedByteArray(rlpChunkSize).longValue(); + return new SnapStateChunkRequestMessage(id, blockNumber, from, chunkSize); } public long getFrom() { diff --git a/rskj-core/src/main/java/co/rsk/net/messages/SnapStateChunkResponseMessage.java b/rskj-core/src/main/java/co/rsk/net/messages/SnapStateChunkResponseMessage.java index f0af95538cf..23262f24636 100644 --- a/rskj-core/src/main/java/co/rsk/net/messages/SnapStateChunkResponseMessage.java +++ b/rskj-core/src/main/java/co/rsk/net/messages/SnapStateChunkResponseMessage.java @@ -26,8 +26,9 @@ import java.math.BigInteger; public class SnapStateChunkResponseMessage extends MessageWithId { - private final long to; private final long id; + + private final long to; private final byte[] chunkOfTrieKeyValue; private final long from; @@ -59,38 +60,34 @@ public long getId() { return this.id; } - @Override protected byte[] getEncodedMessageWithoutId() { - try { - byte[] rlpBlockNumber = RLP.encodeBigInteger(BigInteger.valueOf(this.blockNumber)); - byte[] rlpFrom = RLP.encodeBigInteger(BigInteger.valueOf(this.from)); - byte[] rlpTo = RLP.encodeBigInteger(BigInteger.valueOf(this.to)); - byte[] rlpComplete = new byte[]{this.complete ? (byte) 1 : (byte) 0}; - return RLP.encodeList(chunkOfTrieKeyValue, rlpBlockNumber, rlpFrom, rlpTo, rlpComplete); - } catch (Exception e) { - throw e; - } + byte[] rlpChunkOfTrieKeyValue = RLP.encodeElement(chunkOfTrieKeyValue); + byte[] rlpBlockNumber = RLP.encodeBigInteger(BigInteger.valueOf(this.blockNumber)); + byte[] rlpFrom = RLP.encodeBigInteger(BigInteger.valueOf(this.from)); + byte[] rlpTo = RLP.encodeBigInteger(BigInteger.valueOf(this.to)); + byte[] rlpComplete = RLP.encodeInt(this.complete ? 1 : 0); + + return RLP.encodeList(rlpChunkOfTrieKeyValue, rlpBlockNumber, rlpFrom, rlpTo, rlpComplete); } - public static Message create(BlockFactory blockFactory, RLPList list) { - try { - byte[] rlpId = list.get(0).getRLPData(); - RLPList message = (RLPList) RLP.decode2(list.get(1).getRLPData()).get(0); - byte[] chunkOfTrieKeys = message.get(0).getRLPData(); - byte[] rlpBlockNumber = message.get(1).getRLPData(); - byte[] rlpFrom = message.get(2).getRLPData(); - byte[] rlpTo = message.get(3).getRLPData(); - byte[] rlpComplete = message.get(4).getRLPData(); - long id = rlpId == null ? 0 : BigIntegers.fromUnsignedByteArray(rlpId).longValue(); - long blockNumber = rlpBlockNumber == null ? 0 : BigIntegers.fromUnsignedByteArray(rlpBlockNumber).longValue(); - long from = rlpFrom == null ? 0 : BigIntegers.fromUnsignedByteArray(rlpFrom).longValue(); - long to = rlpTo == null ? 0 : BigIntegers.fromUnsignedByteArray(rlpTo).longValue(); - boolean complete = rlpComplete == null ? Boolean.FALSE : rlpComplete[0] != 0; - return new SnapStateChunkResponseMessage(id, chunkOfTrieKeys, blockNumber, from, to, complete); - } catch (Exception e) { - throw e; - } + public static Message decodeMessage(BlockFactory blockFactory, RLPList list) { + byte[] rlpId = list.get(0).getRLPData(); + long id = rlpId == null ? 0 : BigIntegers.fromUnsignedByteArray(rlpId).longValue(); + + RLPList message = (RLPList) RLP.decode2(list.get(1).getRLPData()).get(0); + byte[] rlpChunkOfTrieKeys = message.get(0).getRLPData(); + byte[] rlpBlockNumber = message.get(1).getRLPData(); + byte[] rlpFrom = message.get(2).getRLPData(); + byte[] rlpTo = message.get(3).getRLPData(); + byte[] rlpComplete = message.get(4).getRLPData(); + + byte[] chunkOfTrieKeys = rlpChunkOfTrieKeys == null ? new byte[0] : rlpChunkOfTrieKeys; + long blockNumber = rlpBlockNumber == null ? 0 : BigIntegers.fromUnsignedByteArray(rlpBlockNumber).longValue(); + long from = rlpFrom == null ? 0 : BigIntegers.fromUnsignedByteArray(rlpFrom).longValue(); + long to = rlpTo == null ? 0 : BigIntegers.fromUnsignedByteArray(rlpTo).longValue(); + boolean complete = rlpComplete != null && rlpComplete.length != 0 && rlpComplete[0] != 0; + return new SnapStateChunkResponseMessage(id, chunkOfTrieKeys, blockNumber, from, to, complete); } public byte[] getChunkOfTrieKeyValue() { diff --git a/rskj-core/src/main/java/co/rsk/net/messages/SnapStatusRequestMessage.java b/rskj-core/src/main/java/co/rsk/net/messages/SnapStatusRequestMessage.java index 39e822ba788..25bd84dac80 100644 --- a/rskj-core/src/main/java/co/rsk/net/messages/SnapStatusRequestMessage.java +++ b/rskj-core/src/main/java/co/rsk/net/messages/SnapStatusRequestMessage.java @@ -18,11 +18,17 @@ package co.rsk.net.messages; +import org.bouncycastle.util.BigIntegers; +import org.ethereum.core.BlockFactory; import org.ethereum.util.RLP; +import org.ethereum.util.RLPList; -public class SnapStatusRequestMessage extends Message { +public class SnapStatusRequestMessage extends MessageWithId { - public SnapStatusRequestMessage() { + private final long id; + + public SnapStatusRequestMessage(long id) { + this.id = id; } @Override @@ -31,7 +37,17 @@ public MessageType getMessageType() { } @Override - public byte[] getEncodedMessage() { + public MessageType getResponseMessageType() { + return MessageType.SNAP_STATUS_RESPONSE_MESSAGE; + } + + @Override + public long getId() { + return this.id; + } + + @Override + protected byte[] getEncodedMessageWithoutId() { return RLP.encodedEmptyList(); } @@ -39,4 +55,11 @@ public byte[] getEncodedMessage() { public void accept(MessageVisitor v) { v.apply(this); } + + public static Message decodeMessage(BlockFactory blockFactory, RLPList list) { + byte[] rlpId = list.get(0).getRLPData(); + long id = rlpId == null ? 0 : BigIntegers.fromUnsignedByteArray(rlpId).longValue(); + + return new SnapStatusRequestMessage(id); + } } diff --git a/rskj-core/src/main/java/co/rsk/net/messages/SnapStatusResponseMessage.java b/rskj-core/src/main/java/co/rsk/net/messages/SnapStatusResponseMessage.java index e70851edc69..520e2e4fdf8 100644 --- a/rskj-core/src/main/java/co/rsk/net/messages/SnapStatusResponseMessage.java +++ b/rskj-core/src/main/java/co/rsk/net/messages/SnapStatusResponseMessage.java @@ -30,7 +30,8 @@ import java.util.List; import java.util.stream.Collectors; -public class SnapStatusResponseMessage extends Message { +public class SnapStatusResponseMessage extends MessageWithId { + private final long id; private final List blocks; private final List difficulties; private final long trieSize; @@ -43,7 +44,8 @@ public long getTrieSize() { return this.trieSize; } - public SnapStatusResponseMessage(List blocks, List difficulties, long trieSize) { + public SnapStatusResponseMessage(long id, List blocks, List difficulties, long trieSize) { + this.id = id; this.blocks = blocks; this.difficulties = difficulties; this.trieSize = trieSize; @@ -59,7 +61,12 @@ public List getDifficulties() { } @Override - public byte[] getEncodedMessage() { + public long getId() { + return this.id; + } + + @Override + protected byte[] getEncodedMessageWithoutId() { List rlpBlocks = this.blocks.stream().map(Block::getEncoded).map(RLP::encode).collect(Collectors.toList()); List rlpDifficulties = this.difficulties.stream().map(BlockDifficulty::getBytes).map(RLP::encode).collect(Collectors.toList()); byte[] rlpTrieSize = RLP.encodeBigInteger(BigInteger.valueOf(this.trieSize)); @@ -68,8 +75,12 @@ public byte[] getEncodedMessage() { } public static Message decodeMessage(BlockFactory blockFactory, RLPList list) { - RLPList rlpBlocks = RLP.decodeList(list.get(0).getRLPData()); - RLPList rlpDifficulties = RLP.decodeList(list.get(1).getRLPData()); + byte[] rlpId = list.get(0).getRLPData(); + long id = rlpId == null ? 0 : BigIntegers.fromUnsignedByteArray(rlpId).longValue(); + + RLPList message = (RLPList)RLP.decode2(list.get(1).getRLPData()).get(0); + RLPList rlpBlocks = RLP.decodeList(message.get(0).getRLPData()); + RLPList rlpDifficulties = RLP.decodeList(message.get(1).getRLPData()); List blocks = Lists.newArrayList(); List difficulties = Lists.newArrayList(); for (int i = 0; i < rlpBlocks.size(); i++) { @@ -79,10 +90,10 @@ public static Message decodeMessage(BlockFactory blockFactory, RLPList list) { difficulties.add(new BlockDifficulty(new BigInteger(rlpDifficulties.get(i).getRLPData()))); } - byte[] rlpTrieSize = list.get(2).getRLPData(); + byte[] rlpTrieSize = message.get(2).getRLPData(); long trieSize = rlpTrieSize == null ? 0 : BigIntegers.fromUnsignedByteArray(rlpTrieSize).longValue(); - return new SnapStatusResponseMessage(blocks, difficulties, trieSize); + return new SnapStatusResponseMessage(id, blocks, difficulties, trieSize); } @Override diff --git a/rskj-core/src/main/java/co/rsk/net/sync/PeersInformation.java b/rskj-core/src/main/java/co/rsk/net/sync/PeersInformation.java index 99ebc821496..e1dc4f1aee1 100644 --- a/rskj-core/src/main/java/co/rsk/net/sync/PeersInformation.java +++ b/rskj-core/src/main/java/co/rsk/net/sync/PeersInformation.java @@ -209,6 +209,15 @@ public Optional getBestPeer(Set exclude) { return getBestPeer(getBestCandidatesStream().filter(p -> !exclude.contains(p.getKey().getPeerNodeID()))); } + @Override + public Optional getBestSnapPeer(Set exclude) { + return getBestPeer( + getBestCandidatesStream() + .filter(this::isSnapPeerCandidateOrCapable) + .filter(p -> !exclude.contains(p.getKey().getPeerNodeID())) + ); + } + public Set knownNodeIds() { return peerStatuses.keySet().stream() .map(Peer::getPeerNodeID) diff --git a/rskj-core/src/main/java/co/rsk/net/sync/SnapSyncRequestManager.java b/rskj-core/src/main/java/co/rsk/net/sync/SnapSyncRequestManager.java new file mode 100644 index 00000000000..bce97735dbc --- /dev/null +++ b/rskj-core/src/main/java/co/rsk/net/sync/SnapSyncRequestManager.java @@ -0,0 +1,197 @@ +/* + * This file is part of RskJ + * Copyright (C) 2024 RSK Labs Ltd. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +package co.rsk.net.sync; + +import co.rsk.net.NodeID; +import co.rsk.net.Peer; +import co.rsk.net.messages.MessageWithId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.*; +import java.util.function.Function; +import java.util.function.Supplier; + +public class SnapSyncRequestManager { + + private static final Logger logger = LoggerFactory.getLogger("snapshotprocessor"); + + private static final long MAX_RETRY_NUM = 2; + + private final SyncConfiguration syncConfiguration; + private final SyncEventsHandler syncEventsHandler; + + private final Map pendingRequests = new HashMap<>(); + + public SnapSyncRequestManager(@Nonnull SyncConfiguration syncConfiguration, @Nonnull SyncEventsHandler syncEventsHandler) { + this.syncConfiguration = Objects.requireNonNull(syncConfiguration); + this.syncEventsHandler = Objects.requireNonNull(syncEventsHandler); + } + + synchronized void submitRequest(@Nonnull PeerSelector peerSelector, @Nonnull RequestFactory requestFactory) throws SendRequestException { + long messageId = syncEventsHandler.nextMessageId(); + PendingRequest pendingRequest = new PendingRequest(peerSelector, requestFactory); + pendingRequests.put(messageId, pendingRequest); + + MessageWithId messageWithId = pendingRequest.send(messageId, System.currentTimeMillis()); + syncEventsHandler.registerPendingMessage(messageWithId); + } + + synchronized boolean processResponse(@Nonnull MessageWithId responseMessage) { + return pendingRequests.remove(responseMessage.getId()) != null; + } + + synchronized void resendExpiredRequests() throws SendRequestException { + long requestTimeout = syncConfiguration.getTimeoutWaitingRequest().toMillis(); + long now = System.currentTimeMillis(); + long exp = now - requestTimeout; + Map resentRequests = null; + + for (Iterator> iter = pendingRequests.entrySet().iterator(); iter.hasNext(); ) { + Map.Entry msgEntry = iter.next(); + PendingRequest pendingRequest = msgEntry.getValue(); + if (pendingRequest.isExpired(exp)) { + iter.remove(); + + long messageId = syncEventsHandler.nextMessageId(); + MessageWithId messageWithId = pendingRequest.reSend(msgEntry.getKey(), messageId, now); + syncEventsHandler.registerPendingMessage(messageWithId); + if (resentRequests == null) { + resentRequests = new HashMap<>(); + } + resentRequests.put(messageId, pendingRequest); + } + } + + if (resentRequests != null) { + pendingRequests.putAll(resentRequests); + } + } + + @FunctionalInterface + public interface RequestFactory { + MessageWithId createRequest(long messageId); + } + + @FunctionalInterface + public interface PeerSelector { + Optional selectPeer(@Nullable NodeID failedPeerIds); + + static Builder builder() { + return new Builder(); + } + + class Builder { + private Supplier> defaultPeerSupplier = Optional::empty; + private Function, Optional> altPeerSupplier = failedPeerIds -> Optional.empty(); + + public Builder withDefaultPeerOption(Supplier> defaultPeerOptionSupplier) { + this.defaultPeerSupplier = Objects.requireNonNull(defaultPeerOptionSupplier); + return this; + } + + public Builder withDefaultPeer(Supplier defaultPeerSupplier) { + Objects.requireNonNull(defaultPeerSupplier); + this.defaultPeerSupplier = () -> Optional.ofNullable(defaultPeerSupplier.get()); + return this; + } + + public Builder withAltPeer(Function, Optional> altPeerSupplier) { + this.altPeerSupplier = Objects.requireNonNull(altPeerSupplier); + return this; + } + + public PeerSelector build() { + return failedPeerId -> Optional.ofNullable(failedPeerId) + .flatMap(peerId -> altPeerSupplier.apply(Collections.singleton(peerId))) + .or(defaultPeerSupplier) + .or(() -> altPeerSupplier.apply(Collections.emptySet())); + } + } + } + + private static class PendingRequest { + private final PeerSelector peerSelector; + private final RequestFactory requestFactory; + + private Peer selectedPeer; + private long started; + private int retries; + + PendingRequest(@Nonnull PeerSelector peerSelector, @Nonnull RequestFactory requestFactory) { + this.peerSelector = Objects.requireNonNull(peerSelector); + this.requestFactory = Objects.requireNonNull(requestFactory); + } + + MessageWithId send(long messageId, long now) throws SendRequestException { + this.started = now; + + Optional selectedPeerOpt = this.peerSelector.selectPeer(null); + if (selectedPeerOpt.isEmpty()) { + throw new SendRequestException("Failed to send request - no peer available"); + } + + this.selectedPeer = selectedPeerOpt.get(); + + MessageWithId msg = requestFactory.createRequest(messageId); + + logger.debug("Sending request: [{}] with id: [{}] to: [{}]", msg.getMessageType(), msg.getId(), this.selectedPeer.getPeerNodeID()); + selectedPeer.sendMessage(msg); + + return msg; + } + + MessageWithId reSend(long previousMessageId, long newMessageId, long now) throws SendRequestException { + this.started = now; + + if (this.retries >= MAX_RETRY_NUM) { + throw new SendRequestException("Failed to re-send expired request with previous messageId: [" + previousMessageId + "] - max retries reached"); + } + + Optional selectedPeerOpt = this.peerSelector.selectPeer(this.selectedPeer.getPeerNodeID()); + if (selectedPeerOpt.isEmpty()) { + throw new SendRequestException("Failed to re-send expired request with previous messageId: [" + previousMessageId + "] - no peer available"); + } + + this.selectedPeer = selectedPeerOpt.get(); + + MessageWithId msg = this.requestFactory.createRequest(newMessageId); + + logger.debug("Re-sending expired request: [{}] with old id: [{}] to: [{}] with new id: [{}]", msg.getMessageType(), previousMessageId, this.selectedPeer.getPeerNodeID(), msg.getId()); + this.selectedPeer.sendMessage(msg); + + this.retries++; + + return msg; + } + + boolean isExpired(long exp) { + return started <= exp; + } + } + + public static class SendRequestException extends Exception { + + public SendRequestException(String message) { + super(message); + } + } +} diff --git a/rskj-core/src/main/java/co/rsk/net/sync/SnapSyncState.java b/rskj-core/src/main/java/co/rsk/net/sync/SnapSyncState.java index 694ec166526..2234187c56e 100644 --- a/rskj-core/src/main/java/co/rsk/net/sync/SnapSyncState.java +++ b/rskj-core/src/main/java/co/rsk/net/sync/SnapSyncState.java @@ -21,10 +21,7 @@ import co.rsk.core.BlockDifficulty; import co.rsk.net.Peer; import co.rsk.net.SnapshotProcessor; -import co.rsk.net.messages.MessageType; -import co.rsk.net.messages.SnapBlocksResponseMessage; -import co.rsk.net.messages.SnapStateChunkResponseMessage; -import co.rsk.net.messages.SnapStatusResponseMessage; +import co.rsk.net.messages.*; import co.rsk.scoring.EventType; import co.rsk.trie.TrieDTO; import com.google.common.annotations.VisibleForTesting; @@ -35,17 +32,24 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.math.BigInteger; +import java.time.Duration; import java.util.*; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; +import static co.rsk.net.sync.SnapSyncRequestManager.PeerSelector; +import static co.rsk.net.sync.SnapSyncRequestManager.RequestFactory; +import static co.rsk.net.sync.SnapSyncRequestManager.SendRequestException; + public class SnapSyncState extends BaseSyncState { private static final Logger logger = LoggerFactory.getLogger("SnapSyncState"); private final SnapshotProcessor snapshotProcessor; + private final SnapSyncRequestManager snapRequestManager; // queue for processing of SNAP responses private final BlockingQueue responseQueue = new LinkedBlockingQueue<>(); @@ -78,14 +82,16 @@ public class SnapSyncState extends BaseSyncState { private final Thread thread; public SnapSyncState(SyncEventsHandler syncEventsHandler, SnapshotProcessor snapshotProcessor, SyncConfiguration syncConfiguration) { - this(syncEventsHandler, snapshotProcessor, syncConfiguration, null); + this(syncEventsHandler, snapshotProcessor, new SnapSyncRequestManager(syncConfiguration, syncEventsHandler), syncConfiguration, null); } @VisibleForTesting SnapSyncState(SyncEventsHandler syncEventsHandler, SnapshotProcessor snapshotProcessor, - SyncConfiguration syncConfiguration, @Nullable SyncMessageHandler.Listener listener) { + SnapSyncRequestManager snapRequestManager, SyncConfiguration syncConfiguration, + @Nullable SyncMessageHandler.Listener listener) { super(syncEventsHandler, syncConfiguration); - this.snapshotProcessor = snapshotProcessor; // TODO(snap-poc) code in SnapshotProcessor should be moved here probably + this.snapshotProcessor = snapshotProcessor; + this.snapRequestManager = snapRequestManager; this.allNodes = Lists.newArrayList(); this.blocks = Lists.newArrayList(); this.thread = new Thread(new SyncMessageHandler("SNAP/client", responseQueue, listener) { @@ -110,8 +116,12 @@ public void onEnter() { @Override public void onSnapStatus(Peer sender, SnapStatusResponseMessage responseMessage) { + if (!snapRequestManager.processResponse(responseMessage)) { + logger.warn("Unexpected response: [{}] received with id: [{}]. Ignoring", responseMessage.getMessageType(), responseMessage.getId()); + return; + } + try { - resetTimeElapsed(); responseQueue.put(new SyncMessageHandler.Job(sender, responseMessage) { @Override public void run() { @@ -126,8 +136,12 @@ public void run() { @Override public void onSnapBlocks(Peer sender, SnapBlocksResponseMessage responseMessage) { + if (!snapRequestManager.processResponse(responseMessage)) { + logger.warn("Unexpected response: [{}] received with id: [{}]. Ignoring", responseMessage.getMessageType(), responseMessage.getId()); + return; + } + try { - resetTimeElapsed(); responseQueue.put(new SyncMessageHandler.Job(sender, responseMessage) { @Override public void run() { @@ -142,8 +156,12 @@ public void run() { @Override public void onSnapStateChunk(Peer sender, SnapStateChunkResponseMessage responseMessage) { + if (!snapRequestManager.processResponse(responseMessage)) { + logger.warn("Unexpected response: [{}] received with id: [{}]. Ignoring", responseMessage.getMessageType(), responseMessage.getId()); + return; + } + try { - resetTimeElapsed(); responseQueue.put(new SyncMessageHandler.Job(sender, responseMessage) { @Override public void run() { @@ -157,13 +175,17 @@ public void run() { } @Override - public void newBlockHeaders(Peer peer, List chunk) { + public void newBlockHeaders(Peer sender, BlockHeadersResponseMessage responseMessage) { + if (!snapRequestManager.processResponse(responseMessage)) { + logger.warn("Unexpected response: [{}] received with id: [{}]. Ignoring", responseMessage.getMessageType(), responseMessage.getId()); + return; + } + try { - resetTimeElapsed(); - responseQueue.put(new SyncMessageHandler.Job(peer, MessageType.BLOCK_HEADERS_RESPONSE_MESSAGE) { + responseQueue.put(new SyncMessageHandler.Job(sender, responseMessage) { @Override public void run() { - snapshotProcessor.processBlockHeaderChunk(SnapSyncState.this, peer, chunk); + snapshotProcessor.processBlockHeaderChunk(SnapSyncState.this, sender, responseMessage.getBlockHeaders()); } }); } catch (InterruptedException e) { @@ -176,9 +198,23 @@ public SyncEventsHandler getSyncEventsHandler() { return this.syncEventsHandler; } + public synchronized void submitRequest(@Nonnull PeerSelector peerSelector, @Nonnull RequestFactory requestFactory) { + try { + snapRequestManager.submitRequest(peerSelector, requestFactory); + } catch (SendRequestException e) { + logger.warn("Failed to submit expired requests. Stopping snap syncing", e); + finish(); + } + } + @Override - protected void onMessageTimeOut() { - fail(getLastBlockSender(), EventType.TIMEOUT_MESSAGE, "Snap sync timed out"); + public void tick(Duration duration) { + try { + this.snapRequestManager.resendExpiredRequests(); + } catch (SendRequestException e) { + logger.warn("Failed to re-submit expired requests. Stopping snap syncing", e); + finish(); + } } public Block getLastBlock() { diff --git a/rskj-core/src/main/java/co/rsk/net/sync/SnapshotPeersInformation.java b/rskj-core/src/main/java/co/rsk/net/sync/SnapshotPeersInformation.java index 1a9fad26a8a..992842d1499 100644 --- a/rskj-core/src/main/java/co/rsk/net/sync/SnapshotPeersInformation.java +++ b/rskj-core/src/main/java/co/rsk/net/sync/SnapshotPeersInformation.java @@ -30,8 +30,10 @@ * things such as the underlying communication channel. */ public interface SnapshotPeersInformation { + Optional getBestPeer(); Optional getBestSnapPeer(); List getBestSnapPeerCandidates(); Optional getBestPeer(Set exclude); + Optional getBestSnapPeer(Set exclude); SyncPeerStatus getOrRegisterPeer(Peer peer); } diff --git a/rskj-core/src/main/java/co/rsk/net/sync/SyncConfiguration.java b/rskj-core/src/main/java/co/rsk/net/sync/SyncConfiguration.java index 722f1329a30..ecf104b4f6d 100644 --- a/rskj-core/src/main/java/co/rsk/net/sync/SyncConfiguration.java +++ b/rskj-core/src/main/java/co/rsk/net/sync/SyncConfiguration.java @@ -30,10 +30,10 @@ @Immutable public final class SyncConfiguration { @VisibleForTesting - public static final SyncConfiguration DEFAULT = new SyncConfiguration(5, 60, 30, 5, 20, 192, 20, 10, 0, false, false, 60, 0); + public static final SyncConfiguration DEFAULT = new SyncConfiguration(5, 60, 30, 5, 20, 192, 20, 10, 0, false, false, 0); @VisibleForTesting - public static final SyncConfiguration IMMEDIATE_FOR_TESTING = new SyncConfiguration(1, 1, 3, 1, 5, 192, 20, 10, 0, false, false, 60, 0); + public static final SyncConfiguration IMMEDIATE_FOR_TESTING = new SyncConfiguration(1, 1, 3, 1, 5, 192, 20, 10, 0, false, false, 0); private final int expectedPeers; private final Duration timeoutWaitingPeers; @@ -62,7 +62,6 @@ public final class SyncConfiguration { * @param topBest % of top best nodes that will be considered for random selection. * @param isServerSnapSyncEnabled Flag that indicates if server-side snap sync is enabled * @param isClientSnapSyncEnabled Flag that indicates if client-side snap sync is enabled - * @param timeoutWaitingSnapChunk Specific request timeout for snap sync * @param snapshotSyncLimit Distance to the tip of the peer's blockchain to enable snap synchronization. */ public SyncConfiguration( @@ -77,7 +76,6 @@ public SyncConfiguration( double topBest, boolean isServerSnapSyncEnabled, boolean isClientSnapSyncEnabled, - int timeoutWaitingSnapChunk, int snapshotSyncLimit) { this(expectedPeers, timeoutWaitingPeers, @@ -127,27 +125,27 @@ public SyncConfiguration( .collect(Collectors.toMap(peer -> peer.getId().toString(), peer -> peer))); } - public final int getExpectedPeers() { + public int getExpectedPeers() { return expectedPeers; } - public final int getMaxSkeletonChunks() { + public int getMaxSkeletonChunks() { return maxSkeletonChunks; } - public final Duration getTimeoutWaitingPeers() { + public Duration getTimeoutWaitingPeers() { return timeoutWaitingPeers; } - public final Duration getTimeoutWaitingRequest() { + public Duration getTimeoutWaitingRequest() { return timeoutWaitingRequest; } - public final Duration getExpirationTimePeerStatus() { + public Duration getExpirationTimePeerStatus() { return expirationTimePeerStatus; } - public final int getChunkSize() { + public int getChunkSize() { return chunkSize; } diff --git a/rskj-core/src/main/java/co/rsk/net/sync/SyncEventsHandler.java b/rskj-core/src/main/java/co/rsk/net/sync/SyncEventsHandler.java index e0bd7be4e9a..a79905b7620 100644 --- a/rskj-core/src/main/java/co/rsk/net/sync/SyncEventsHandler.java +++ b/rskj-core/src/main/java/co/rsk/net/sync/SyncEventsHandler.java @@ -18,11 +18,13 @@ package co.rsk.net.sync; import co.rsk.net.Peer; +import co.rsk.net.messages.MessageWithId; import co.rsk.scoring.EventType; import org.ethereum.core.Block; import org.ethereum.core.BlockHeader; import org.ethereum.core.BlockIdentifier; +import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.util.Deque; import java.util.List; @@ -60,4 +62,8 @@ public interface SyncEventsHandler { void backwardSyncing(Peer peer); void startSnapSync(Peer peer); + + void registerPendingMessage(@Nonnull MessageWithId message); + + long nextMessageId(); } diff --git a/rskj-core/src/main/java/co/rsk/net/sync/SyncState.java b/rskj-core/src/main/java/co/rsk/net/sync/SyncState.java index 4d0436d45b3..f0dcbe67caa 100644 --- a/rskj-core/src/main/java/co/rsk/net/sync/SyncState.java +++ b/rskj-core/src/main/java/co/rsk/net/sync/SyncState.java @@ -18,10 +18,7 @@ package co.rsk.net.sync; import co.rsk.net.Peer; -import co.rsk.net.messages.BodyResponseMessage; -import co.rsk.net.messages.SnapBlocksResponseMessage; -import co.rsk.net.messages.SnapStateChunkResponseMessage; -import co.rsk.net.messages.SnapStatusResponseMessage; +import co.rsk.net.messages.*; import org.ethereum.core.BlockHeader; import org.ethereum.core.BlockIdentifier; @@ -31,6 +28,10 @@ public interface SyncState { void newBlockHeaders(Peer peer, List chunk); + default void newBlockHeaders(Peer peer, BlockHeadersResponseMessage message) { + newBlockHeaders(peer, message.getBlockHeaders()); + } + // TODO(mc) don't receive a full message void newBody(BodyResponseMessage message, Peer peer); diff --git a/rskj-core/src/main/java/org/ethereum/net/server/Channel.java b/rskj-core/src/main/java/org/ethereum/net/server/Channel.java index c390be3318e..fbe82616564 100644 --- a/rskj-core/src/main/java/org/ethereum/net/server/Channel.java +++ b/rskj-core/src/main/java/org/ethereum/net/server/Channel.java @@ -25,6 +25,7 @@ import co.rsk.net.eth.RskWireProtocol; import co.rsk.net.messages.Message; import co.rsk.net.messages.MessageType; +import com.google.common.annotations.VisibleForTesting; import io.netty.buffer.ByteBuf; import io.netty.channel.ChannelHandlerContext; import org.ethereum.net.MessageQueue; @@ -74,9 +75,10 @@ public class Channel implements Peer { private final PeerStatistics peerStats = new PeerStatistics(); - private Stats stats; + private final Stats stats; private boolean isSnapCapable; + @VisibleForTesting public Channel(MessageQueue msgQueue, MessageCodec messageCodec, NodeManager nodeManager, @@ -84,6 +86,17 @@ public Channel(MessageQueue msgQueue, Eth62MessageFactory eth62MessageFactory, StaticMessages staticMessages, String remoteId) { + this(msgQueue, messageCodec, nodeManager, rskWireProtocolFactory, eth62MessageFactory, staticMessages, remoteId, new Stats()); + } + + public Channel(MessageQueue msgQueue, + MessageCodec messageCodec, + NodeManager nodeManager, + RskWireProtocol.Factory rskWireProtocolFactory, + Eth62MessageFactory eth62MessageFactory, + StaticMessages staticMessages, + String remoteId, + Stats stats) { this.msgQueue = msgQueue; this.messageCodec = messageCodec; this.nodeManager = nodeManager; @@ -91,7 +104,7 @@ public Channel(MessageQueue msgQueue, this.eth62MessageFactory = eth62MessageFactory; this.staticMessages = staticMessages; this.isActive = remoteId != null && !remoteId.isEmpty(); - this.stats = new Stats(); + this.stats = stats; } public void sendHelloMessage(ChannelHandlerContext ctx, FrameCodec frameCodec, String nodeId, diff --git a/rskj-core/src/main/java/org/ethereum/net/server/EthereumChannelInitializer.java b/rskj-core/src/main/java/org/ethereum/net/server/EthereumChannelInitializer.java index 7d0f478f7ff..ae25fcc1f65 100644 --- a/rskj-core/src/main/java/org/ethereum/net/server/EthereumChannelInitializer.java +++ b/rskj-core/src/main/java/org/ethereum/net/server/EthereumChannelInitializer.java @@ -111,7 +111,8 @@ public void initChannel(NioSocketChannel ch) { P2pHandler p2pHandler = new P2pHandler(ethereumListener, messageQueue, config.getPeerP2PPingInterval()); MessageCodec messageCodec = new MessageCodec(ethereumListener, config); HandshakeHandler handshakeHandler = new HandshakeHandler(config, peerScoringManager, p2pHandler, messageCodec, configCapabilities); - Channel channel = new Channel(messageQueue, messageCodec, nodeManager, rskWireProtocolFactory, eth62MessageFactory, staticMessages, remoteId); + Stats stats = new Stats(config.getMessageQueuePerMinuteThreshold()); + Channel channel = new Channel(messageQueue, messageCodec, nodeManager, rskWireProtocolFactory, eth62MessageFactory, staticMessages, remoteId, stats); ch.pipeline().addLast("readTimeoutHandler", new ReadTimeoutHandler(config.peerChannelReadTimeout(), TimeUnit.SECONDS)); ch.pipeline().addLast("handshakeHandler", handshakeHandler); diff --git a/rskj-core/src/main/java/org/ethereum/net/server/Stats.java b/rskj-core/src/main/java/org/ethereum/net/server/Stats.java index 1cdcd9929fa..9dc5c2b9676 100644 --- a/rskj-core/src/main/java/org/ethereum/net/server/Stats.java +++ b/rskj-core/src/main/java/org/ethereum/net/server/Stats.java @@ -29,9 +29,10 @@ public class Stats { // Current minute messages counter private long minute; - // Reject messages over this treshold + + // Reject messages over this threshold // Is calculated using Exponential moving average - private long perMinuteThreshold; + private final long perMinuteThreshold; // events counters // 100% heuristics @@ -44,26 +45,29 @@ public class Stats { private double avg; //in ms // how fast avg and mpm update - private double alpha_m; - private double alpha_a; + private final double alpha_m; + private final double alpha_a; // scores for blocks and others - private double maxBlock; - private double maxOther; + private final double maxBlock; + private final double maxOther; public Stats() { + this(1000); + } + + public Stats(long perMinuteThreshold) { avg = 500; alpha_m = 0.3; alpha_a = 0.03; - perMinuteThreshold = 1000; + this.perMinuteThreshold = perMinuteThreshold; maxBlock = 200; maxOther = 100; mpm = 1; } - public synchronized double update(long timestamp, MessageType type) { long min = timestamp / 60000; long delta = timestamp - lastMessage; @@ -166,6 +170,7 @@ private double priority(MessageType type) { return 0.0; } } + public synchronized void imported(boolean best) { if (best) { importedBest++; @@ -174,7 +179,6 @@ public synchronized void imported(boolean best) { } } - @Override public String toString() { return "Stats{" + @@ -197,7 +201,6 @@ public double getMpm() { return mpm; } - @VisibleForTesting public long getMinute() { return minute; @@ -217,5 +220,4 @@ public void setImportedBest(int importedBest) { public void setImportedNotBest(int importedNotBest) { this.importedNotBest = importedNotBest; } - } diff --git a/rskj-core/src/main/resources/expected.conf b/rskj-core/src/main/resources/expected.conf index a62a7c7a5e9..79d221be2f7 100644 --- a/rskj-core/src/main/resources/expected.conf +++ b/rskj-core/src/main/resources/expected.conf @@ -162,6 +162,7 @@ peer = { bannedPeerIDs = [] bannedMiners = [] messageQueue.maxSizePerPeer = + messageQueue.thresholdPerMinutePerPeer = } genesis = genesis_constants.federationPublicKeys = [] @@ -283,6 +284,7 @@ sync = { } client = { enabled = + checkHistoricalHeaders = parallel = chunkSize = limit = diff --git a/rskj-core/src/main/resources/reference.conf b/rskj-core/src/main/resources/reference.conf index a81984b8b78..6bd24d8fca3 100644 --- a/rskj-core/src/main/resources/reference.conf +++ b/rskj-core/src/main/resources/reference.conf @@ -178,6 +178,9 @@ peer { # Max number of pending messages that will be allowed per peer messageQueue.maxSizePerPeer = 2000 + # Reject peer's messages over this threshold + # It's calculated using exponential moving average + messageQueue.thresholdPerMinutePerPeer = 1000 } miner { @@ -382,6 +385,8 @@ sync { client = { # Client / snapshot sync enabled enabled = false + # Flat that determines if the client should check the historical headers + checkHistoricalHeaders = true # Server / chunk size chunkSize = 50 # Distance to the tip of the blockchain to start snapshot sync diff --git a/rskj-core/src/test/java/co/rsk/net/NodeMessageHandlerTest.java b/rskj-core/src/test/java/co/rsk/net/NodeMessageHandlerTest.java index 09cbf6f5cca..54aa38f0f18 100644 --- a/rskj-core/src/test/java/co/rsk/net/NodeMessageHandlerTest.java +++ b/rskj-core/src/test/java/co/rsk/net/NodeMessageHandlerTest.java @@ -993,8 +993,7 @@ void fillMessageQueue_thenBlockNewMessages() { } // assert that the surplus was not added - Assertions.assertEquals(config.getMessageQueueMaxSize(), (Integer) handler.getMessageQueueSize(sender)); - + Assertions.assertEquals(config.getMessageQueueMaxSize(), handler.getMessageQueueSize(sender)); } @Test diff --git a/rskj-core/src/test/java/co/rsk/net/SnapshotProcessorTest.java b/rskj-core/src/test/java/co/rsk/net/SnapshotProcessorTest.java index 58068feafd0..6457d070f3e 100644 --- a/rskj-core/src/test/java/co/rsk/net/SnapshotProcessorTest.java +++ b/rskj-core/src/test/java/co/rsk/net/SnapshotProcessorTest.java @@ -45,6 +45,8 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import static co.rsk.net.sync.SnapSyncRequestManager.PeerSelector; +import static co.rsk.net.sync.SnapSyncRequestManager.RequestFactory; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.*; @@ -95,12 +97,13 @@ void givenStartSyncingIsCalled_thenSnapStatusStartToBeRequestedFromPeer() { blockHeaderParentValidator, blockHeaderValidator, TEST_CHUNK_SIZE, + true, false); doReturn(Optional.of(peer)).when(peersInformation).getBestSnapPeer(); //when underTest.startSyncing(snapSyncState); //then - verify(peer).sendMessage(any(SnapStatusRequestMessage.class)); + verify(snapSyncState).submitRequest(any(PeerSelector.class), any(RequestFactory.class)); } @Test @@ -120,6 +123,7 @@ void givenSnapStatusResponseCalled_thenSnapChunkRequestsAreMade() { blockHeaderParentValidator, blockHeaderValidator, TEST_CHUNK_SIZE, + true, false); for (long blockNumber = 0; blockNumber < blockchain.getSize(); blockNumber++) { @@ -128,7 +132,7 @@ void givenSnapStatusResponseCalled_thenSnapChunkRequestsAreMade() { difficulties.add(blockStore.getTotalDifficultyForHash(currentBlock.getHash().getBytes())); } - SnapStatusResponseMessage snapStatusResponseMessage = new SnapStatusResponseMessage(blocks, difficulties, 100000L); + SnapStatusResponseMessage snapStatusResponseMessage = new SnapStatusResponseMessage(1, blocks, difficulties, 100000L); doReturn(blocks.get(blocks.size() - 1)).when(snapSyncState).getLastBlock(); doReturn(snapStatusResponseMessage.getTrieSize()).when(snapSyncState).getRemoteTrieSize(); @@ -144,7 +148,7 @@ void givenSnapStatusResponseCalled_thenSnapChunkRequestsAreMade() { underTest.processSnapStatusResponse(snapSyncState, peer, snapStatusResponseMessage); //then - verify(peer, times(2)).sendMessage(any()); // 1 for SnapStatusRequestMessage, 1 for SnapBlocksRequestMessage and 1 for SnapStateChunkRequestMessage + verify(snapSyncState, times(2)).submitRequest(any(PeerSelector.class), any(RequestFactory.class)); // 1 for SnapStatusRequestMessage, 1 for SnapBlocksRequestMessage and 1 for SnapStateChunkRequestMessage verify(peersInformation, times(1)).getBestSnapPeer(); } @@ -163,6 +167,7 @@ void givenSnapStatusRequestReceived_thenSnapStatusResponseIsSent() { blockHeaderParentValidator, blockHeaderValidator, TEST_CHUNK_SIZE, + true, false); //when underTest.processSnapStatusRequestInternal(peer, mock(SnapStatusRequestMessage.class)); @@ -186,9 +191,10 @@ void givenSnapBlockRequestReceived_thenSnapBlocksResponseMessageIsSent() { blockHeaderParentValidator, blockHeaderValidator, TEST_CHUNK_SIZE, + true, false); - SnapBlocksRequestMessage snapBlocksRequestMessage = new SnapBlocksRequestMessage(460); + SnapBlocksRequestMessage snapBlocksRequestMessage = new SnapBlocksRequestMessage(1, 460); //when underTest.processSnapBlocksRequestInternal(peer, snapBlocksRequestMessage); @@ -213,6 +219,7 @@ void givenSnapBlocksResponseReceived_thenSnapBlocksRequestMessageIsSent() { blockHeaderParentValidator, blockHeaderValidator, 200, + true, false); for (long blockNumber = 0; blockNumber < blockchain.getSize(); blockNumber++) { @@ -221,7 +228,7 @@ void givenSnapBlocksResponseReceived_thenSnapBlocksRequestMessageIsSent() { difficulties.add(blockStore.getTotalDifficultyForHash(currentBlock.getHash().getBytes())); } - SnapStatusResponseMessage snapStatusResponseMessage = new SnapStatusResponseMessage(blocks, difficulties, 100000L); + SnapStatusResponseMessage snapStatusResponseMessage = new SnapStatusResponseMessage(1, blocks, difficulties, 100000L); doReturn(true).when(snapSyncState).isRunning(); doReturn(true).when(blockValidator).isValid(any()); doReturn(true).when(blockParentValidator).isValid(any(), any()); @@ -230,14 +237,14 @@ void givenSnapBlocksResponseReceived_thenSnapBlocksRequestMessageIsSent() { underTest.startSyncing(snapSyncState); underTest.processSnapStatusResponse(snapSyncState, peer, snapStatusResponseMessage); - SnapBlocksResponseMessage snapBlocksResponseMessage = new SnapBlocksResponseMessage(blocks, difficulties); + SnapBlocksResponseMessage snapBlocksResponseMessage = new SnapBlocksResponseMessage(1, blocks, difficulties); when(snapSyncState.getLastBlock()).thenReturn(blocks.get(blocks.size() - 1)); //when underTest.processSnapBlocksResponse(snapSyncState, peer, snapBlocksResponseMessage); //then - verify(peer, atLeast(2)).sendMessage(any(SnapBlocksRequestMessage.class)); + verify(snapSyncState, atLeast(2)).submitRequest(any(PeerSelector.class), any(RequestFactory.class)); } @Test @@ -255,6 +262,7 @@ void givenSnapStateChunkRequest_thenSnapStateChunkResponseMessageIsSent() { blockHeaderParentValidator, blockHeaderValidator, TEST_CHUNK_SIZE, + true, false); SnapStateChunkRequestMessage snapStateChunkRequestMessage = new SnapStateChunkRequestMessage(1L, 1L, 1, TEST_CHUNK_SIZE); @@ -284,6 +292,7 @@ void givenProcessSnapStatusRequestIsCalled_thenInternalOneIsCalledLater() throws blockHeaderParentValidator, blockHeaderValidator, TEST_CHUNK_SIZE, + true, false, listener) { @Override @@ -324,6 +333,7 @@ void givenProcessSnapBlocksRequestIsCalled_thenInternalOneIsCalledLater() throws blockHeaderParentValidator, blockHeaderValidator, TEST_CHUNK_SIZE, + true, false, listener) { @Override @@ -364,6 +374,7 @@ void givenProcessStateChunkRequestIsCalled_thenInternalOneIsCalledLater() throws blockHeaderParentValidator, blockHeaderValidator, TEST_CHUNK_SIZE, + true, false, listener) { @Override @@ -399,6 +410,7 @@ void givenErrorRLPData_thenOnStateChunkErrorIsCalled() { blockHeaderParentValidator, blockHeaderValidator, TEST_CHUNK_SIZE, + true, false); PriorityQueue queue = new PriorityQueue<>( @@ -416,8 +428,8 @@ void givenErrorRLPData_thenOnStateChunkErrorIsCalled() { underTest.processStateChunkResponse(snapSyncState, peer, responseMessage); - verify(underTest, times(1)).onStateChunkResponseError(peer, responseMessage); - verify(peer, times(1)).sendMessage(any(SnapStateChunkRequestMessage.class)); + verify(underTest, times(1)).onStateChunkResponseError(snapSyncState, peer, responseMessage); + verify(snapSyncState, times(1)).submitRequest(any(PeerSelector.class), any(RequestFactory.class)); } private void initializeBlockchainWithAmountOfBlocks(int numberOfBlocks) { diff --git a/rskj-core/src/test/java/co/rsk/net/ThreeAsyncNodeUsingSyncProcessorTest.java b/rskj-core/src/test/java/co/rsk/net/ThreeAsyncNodeUsingSyncProcessorTest.java index 1472821f78a..beaca29f2cf 100644 --- a/rskj-core/src/test/java/co/rsk/net/ThreeAsyncNodeUsingSyncProcessorTest.java +++ b/rskj-core/src/test/java/co/rsk/net/ThreeAsyncNodeUsingSyncProcessorTest.java @@ -190,7 +190,7 @@ public void synchronizeNewNodeWithTwoPeersDefault() { SimpleAsyncNode node1 = SimpleAsyncNode.createDefaultNode(b1); SimpleAsyncNode node2 = SimpleAsyncNode.createDefaultNode(b1); - SyncConfiguration syncConfiguration = new SyncConfiguration(2,1,1,1,20,192, 20, 10, 0, false, false, 60, 0); + SyncConfiguration syncConfiguration = new SyncConfiguration(2,1,1,1,20,192, 20, 10, 0, false, false, 0); SimpleAsyncNode node3 = SimpleAsyncNode.createNode(b2, syncConfiguration); Assertions.assertEquals(50, node1.getBestBlock().getNumber()); @@ -231,7 +231,7 @@ public void synchronizeNewNodeWithTwoPeers200Default() { SimpleAsyncNode node1 = SimpleAsyncNode.createDefaultNode(b1); SimpleAsyncNode node2 = SimpleAsyncNode.createDefaultNode(b1); - SyncConfiguration syncConfiguration = new SyncConfiguration(2,1,1,1,20,192, 20, 10, 0, false, false, 60, 0); + SyncConfiguration syncConfiguration = new SyncConfiguration(2,1,1,1,20,192, 20, 10, 0, false, false, 0); SimpleAsyncNode node3 = SimpleAsyncNode.createNode(b2, syncConfiguration); Assertions.assertEquals(200, node1.getBestBlock().getNumber()); @@ -272,7 +272,7 @@ public void synchronizeWithTwoPeers200AndOneFails() { SimpleAsyncNode node1 = SimpleAsyncNode.createDefaultNode(b1); SimpleAsyncNode node2 = SimpleAsyncNode.createDefaultNode(b1); - SyncConfiguration syncConfiguration = new SyncConfiguration(2,1,0,1,20,192, 20, 10, 0, false, false, 60, 0); + SyncConfiguration syncConfiguration = new SyncConfiguration(2,1,0,1,20,192, 20, 10, 0, false, false, 0); SimpleAsyncNode node3 = SimpleAsyncNode.createNode(b2, syncConfiguration); Assertions.assertEquals(200, node1.getBestBlock().getNumber()); @@ -319,7 +319,7 @@ public void synchronizeNewNodeWithTwoPeers200Different() { SimpleAsyncNode node1 = SimpleAsyncNode.createDefaultNode(b1); SimpleAsyncNode node2 = SimpleAsyncNode.createDefaultNode(b2); - SyncConfiguration syncConfiguration = new SyncConfiguration(2,1,1,1,20,192, 20, 10, 0, false, false, 60, 0); + SyncConfiguration syncConfiguration = new SyncConfiguration(2,1,1,1,20,192, 20, 10, 0, false, false, 0); SimpleAsyncNode node3 = SimpleAsyncNode.createNode(b3, syncConfiguration); Assertions.assertEquals(193, node1.getBestBlock().getNumber()); @@ -363,7 +363,7 @@ public void synchronizeNewNodeWithThreePeers400Different() { SimpleAsyncNode node1 = SimpleAsyncNode.createDefaultNode(b2); SimpleAsyncNode node2 = SimpleAsyncNode.createDefaultNode(b2); SimpleAsyncNode node3 = SimpleAsyncNode.createDefaultNode(b3); - SyncConfiguration syncConfiguration = new SyncConfiguration(3,1,10,100,20,192, 20, 10, 0, false, false, 60, 0); + SyncConfiguration syncConfiguration = new SyncConfiguration(3,1,10,100,20,192, 20, 10, 0, false, false, 0); SimpleAsyncNode node4 = SimpleAsyncNode.createNode(b1, syncConfiguration); Assertions.assertEquals(200, node1.getBestBlock().getNumber()); diff --git a/rskj-core/src/test/java/co/rsk/net/messages/SnapBlocksRequestMessageTest.java b/rskj-core/src/test/java/co/rsk/net/messages/SnapBlocksRequestMessageTest.java index e5443746b85..21f3f86d527 100644 --- a/rskj-core/src/test/java/co/rsk/net/messages/SnapBlocksRequestMessageTest.java +++ b/rskj-core/src/test/java/co/rsk/net/messages/SnapBlocksRequestMessageTest.java @@ -19,8 +19,11 @@ package co.rsk.net.messages; import co.rsk.blockchain.utils.BlockGenerator; +import co.rsk.config.TestSystemProperties; import org.ethereum.core.Block; +import org.ethereum.core.BlockFactory; import org.ethereum.util.RLP; +import org.ethereum.util.RLPList; import org.junit.jupiter.api.Test; import java.math.BigInteger; @@ -28,12 +31,15 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.mockito.Mockito.*; class SnapBlocksRequestMessageTest { + private final TestSystemProperties config = new TestSystemProperties(); + private final BlockFactory blockFactory = new BlockFactory(config.getActivationConfig()); private final Block block4Test = new BlockGenerator().getBlock(1); - private final SnapBlocksRequestMessage underTest = new SnapBlocksRequestMessage(block4Test.getNumber()); + private final SnapBlocksRequestMessage underTest = new SnapBlocksRequestMessage(1, block4Test.getNumber()); @Test @@ -52,8 +58,26 @@ void getEncodedMessage_returnExpectedByteArray() { //when byte[] encodedMessage = underTest.getEncodedMessage(); + byte[] expectedEncodedMessage = RLP.encodeList( + RLP.encodeBigInteger(BigInteger.valueOf(underTest.getId())), + RLP.encodeList(RLP.encodeBigInteger(BigInteger.ONE))); + + //then + assertThat(encodedMessage, equalTo(expectedEncodedMessage)); + } + + @Test + void decodeMessage_returnExpectedMessage() { + //given default block 4 test + RLPList encodedRLPList = (RLPList) RLP.decode2(underTest.getEncodedMessage()).get(0); + + //when + Message decodedMessage = SnapBlocksRequestMessage.decodeMessage(blockFactory, encodedRLPList); + //then - assertThat(encodedMessage, equalTo(RLP.encodeList(RLP.encodeBigInteger(BigInteger.ONE)))); + assertInstanceOf(SnapBlocksRequestMessage.class, decodedMessage); + assertThat(underTest.getId(), equalTo(((SnapBlocksRequestMessage) decodedMessage).getId())); + assertEquals(1, ((SnapBlocksRequestMessage) decodedMessage).getBlockNumber()); } @Test @@ -68,10 +92,10 @@ void getBlockNumber_returnTheExpectedValue() { } @Test - void givenAcceptIsCalled_messageVisitorIsAppliedFormessage() { + void givenAcceptIsCalled_messageVisitorIsAppliedForMessage() { //given Block block = new BlockGenerator().getBlock(1); - SnapBlocksRequestMessage message = new SnapBlocksRequestMessage(block.getNumber()); + SnapBlocksRequestMessage message = new SnapBlocksRequestMessage(1, block.getNumber()); MessageVisitor visitor = mock(MessageVisitor.class); //when @@ -80,4 +104,4 @@ void givenAcceptIsCalled_messageVisitorIsAppliedFormessage() { //then verify(visitor, times(1)).apply(message); } -} \ No newline at end of file +} diff --git a/rskj-core/src/test/java/co/rsk/net/messages/SnapBlocksResponseMessageTest.java b/rskj-core/src/test/java/co/rsk/net/messages/SnapBlocksResponseMessageTest.java index ca2268576d2..0e2d2aa0efe 100644 --- a/rskj-core/src/test/java/co/rsk/net/messages/SnapBlocksResponseMessageTest.java +++ b/rskj-core/src/test/java/co/rsk/net/messages/SnapBlocksResponseMessageTest.java @@ -28,14 +28,18 @@ import org.ethereum.db.BlockStore; import org.ethereum.db.IndexedBlockStore; import org.ethereum.util.RLP; +import org.ethereum.util.RLPList; import org.junit.jupiter.api.Test; +import java.math.BigInteger; import java.util.Collections; import java.util.List; +import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.mockito.Mockito.*; class SnapBlocksResponseMessageTest { @@ -46,7 +50,7 @@ class SnapBlocksResponseMessageTest { private final Block block4Test = new BlockGenerator().getBlock(1); private final List blockList = Collections.singletonList(new BlockGenerator().getBlock(1)); private final List blockDifficulties = Collections.singletonList(indexedBlockStore.getTotalDifficultyForHash(block4Test.getHash().getBytes())); - private final SnapBlocksResponseMessage underTest = new SnapBlocksResponseMessage(blockList, blockDifficulties); + private final SnapBlocksResponseMessage underTest = new SnapBlocksResponseMessage(1, blockList, blockDifficulties); @Test @@ -62,8 +66,10 @@ void getMessageType_returnCorrectMessageType() { void getEncodedMessage_returnExpectedByteArray() { //given default block 4 test byte[] expectedEncodedMessage = RLP.encodeList( - RLP.encodeList(RLP.encode(block4Test.getEncoded())), - RLP.encodeList(RLP.encode(blockDifficulties.get(0).getBytes()))); + RLP.encodeBigInteger(BigInteger.valueOf(underTest.getId())), + RLP.encodeList( + RLP.encodeList(RLP.encode(block4Test.getEncoded())), + RLP.encodeList(RLP.encode(blockDifficulties.get(0).getBytes())))); //when byte[] encodedMessage = underTest.getEncodedMessage(); @@ -71,6 +77,23 @@ void getEncodedMessage_returnExpectedByteArray() { assertThat(encodedMessage, equalTo(expectedEncodedMessage)); } + @Test + void decodeMessage_returnExpectedMessage() { + //given default block 4 test + RLPList encodedRLPList = (RLPList) RLP.decode2(underTest.getEncodedMessage()).get(0); + + //when + Message decodedMessage = SnapBlocksResponseMessage.decodeMessage(blockFactory, encodedRLPList); + + //then + assertInstanceOf(SnapBlocksResponseMessage.class, decodedMessage); + assertThat(underTest.getId(), equalTo(((SnapBlocksResponseMessage) decodedMessage).getId())); + assertThat(1, is(((SnapBlocksResponseMessage) decodedMessage).getBlocks().size())); + assertThat(block4Test.getHash(), is(((SnapBlocksResponseMessage) decodedMessage).getBlocks().get(0).getHash())); + assertThat(1, is(((SnapBlocksResponseMessage) decodedMessage).getDifficulties().size())); + assertThat(blockDifficulties.get(0), is(((SnapBlocksResponseMessage) decodedMessage).getDifficulties().get(0))); + } + @Test void getDifficulties_returnTheExpectedValue() { //given default block 4 test @@ -94,7 +117,7 @@ void getBlocks_returnTheExpectedValue() { @Test void givenAcceptIsCalled_messageVisitorIsAppliedForMessage() { //given - SnapBlocksResponseMessage message = new SnapBlocksResponseMessage(blockList, blockDifficulties); + SnapBlocksResponseMessage message = new SnapBlocksResponseMessage(1, blockList, blockDifficulties); MessageVisitor visitor = mock(MessageVisitor.class); //when diff --git a/rskj-core/src/test/java/co/rsk/net/messages/SnapStateChunkRequestMessageTest.java b/rskj-core/src/test/java/co/rsk/net/messages/SnapStateChunkRequestMessageTest.java index e8f5280560c..7629c82dd10 100644 --- a/rskj-core/src/test/java/co/rsk/net/messages/SnapStateChunkRequestMessageTest.java +++ b/rskj-core/src/test/java/co/rsk/net/messages/SnapStateChunkRequestMessageTest.java @@ -19,8 +19,11 @@ package co.rsk.net.messages; import co.rsk.blockchain.utils.BlockGenerator; +import co.rsk.config.TestSystemProperties; import org.ethereum.core.Block; +import org.ethereum.core.BlockFactory; import org.ethereum.util.RLP; +import org.ethereum.util.RLPList; import org.junit.jupiter.api.Test; import java.math.BigInteger; @@ -28,10 +31,14 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.mockito.Mockito.*; public class SnapStateChunkRequestMessageTest { + private final TestSystemProperties config = new TestSystemProperties(); + private final BlockFactory blockFactory = new BlockFactory(config.getActivationConfig()); + @Test void getMessageType_returnCorrectMessageType() { //given @@ -45,6 +52,7 @@ void getMessageType_returnCorrectMessageType() { //then assertThat(messageType, equalTo(MessageType.SNAP_STATE_CHUNK_REQUEST_MESSAGE)); } + @Test void givenParameters4Test_assureExpectedValues() { //given @@ -104,6 +112,28 @@ void getEncodedMessageWithId_returnExpectedByteArray() { assertThat(encodedMessage, equalTo(expectedEncodedMessage)); } + @Test + void decodeMessage_returnExpectedMessage() { + //given default block 4 test + long blockNumber = 1L; + long id4Test = 42L; + long from = 1L; + long chunkSize = 20L; + + SnapStateChunkRequestMessage message = new SnapStateChunkRequestMessage(id4Test, blockNumber, from, chunkSize); + RLPList encodedRLPList = (RLPList) RLP.decode2(message.getEncodedMessage()).get(0); + + //when + Message decodedMessage = SnapStateChunkRequestMessage.decodeMessage(blockFactory, encodedRLPList); + + //then + assertInstanceOf(SnapStateChunkRequestMessage.class, decodedMessage); + assertEquals(id4Test,((SnapStateChunkRequestMessage) decodedMessage).getId()); + assertEquals(from,((SnapStateChunkRequestMessage) decodedMessage).getFrom()); + assertEquals(blockNumber,((SnapStateChunkRequestMessage) decodedMessage).getBlockNumber()); + assertEquals(chunkSize,((SnapStateChunkRequestMessage) decodedMessage).getChunkSize()); + } + @Test void givenAcceptIsCalled_messageVisitorIsAppliedForMessage() { //given diff --git a/rskj-core/src/test/java/co/rsk/net/messages/SnapStateChunkResponseMessageTest.java b/rskj-core/src/test/java/co/rsk/net/messages/SnapStateChunkResponseMessageTest.java index 2ce50f71156..fe8f651ab2e 100644 --- a/rskj-core/src/test/java/co/rsk/net/messages/SnapStateChunkResponseMessageTest.java +++ b/rskj-core/src/test/java/co/rsk/net/messages/SnapStateChunkResponseMessageTest.java @@ -19,19 +19,26 @@ package co.rsk.net.messages; import co.rsk.blockchain.utils.BlockGenerator; +import co.rsk.config.TestSystemProperties; import org.ethereum.core.Block; +import org.ethereum.core.BlockFactory; import org.ethereum.util.RLP; +import org.ethereum.util.RLPList; import org.junit.jupiter.api.Test; import java.math.BigInteger; +import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; -import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.*; import static org.mockito.Mockito.*; public class SnapStateChunkResponseMessageTest { + private final TestSystemProperties config = new TestSystemProperties(); + private final BlockFactory blockFactory = new BlockFactory(config.getActivationConfig()); + @Test void getMessageType_returnCorrectMessageType() { //given @@ -81,11 +88,11 @@ void getEncodedMessageWithoutId_returnExpectedByteArray() { boolean complete = true; byte[] expectedEncodedMessage = RLP.encodeList( - trieValueBytes, + RLP.encodeElement(trieValueBytes), RLP.encodeBigInteger(BigInteger.valueOf(blockNumber)), RLP.encodeBigInteger(BigInteger.valueOf(from)), RLP.encodeBigInteger(BigInteger.valueOf(to)), - new byte[]{(byte) 1}); + RLP.encodeInt(complete ? 1 : 0)); SnapStateChunkResponseMessage message = new SnapStateChunkResponseMessage(id4Test, trieValueBytes, blockNumber, from, to, complete); @@ -114,7 +121,33 @@ void getEncodedMessageWithId_returnExpectedByteArray() { byte[] encodedMessage = message.getEncodedMessage(); //then - assertThat(encodedMessage, equalTo(expectedEncodedMessage)); + assertArrayEquals(encodedMessage, expectedEncodedMessage); + } + + @Test + void decodeMessage_returnExpectedMessage() { + //given default block 4 test + long blockNumber = 111L; + long id4Test = 42L; + byte[] trieValueBytes = "any random data".getBytes(); + long from = 5L; + long to = 20L; + boolean complete = false; + + SnapStateChunkResponseMessage message = new SnapStateChunkResponseMessage(id4Test, trieValueBytes, blockNumber, from, to, complete); + RLPList encodedRLPList = (RLPList) RLP.decode2(message.getEncodedMessage()).get(0); + + //when + Message decodedMessage = SnapStateChunkResponseMessage.decodeMessage(blockFactory, encodedRLPList); + + //then + assertInstanceOf(SnapStateChunkResponseMessage.class, decodedMessage); + assertEquals(id4Test,((SnapStateChunkResponseMessage) decodedMessage).getId()); + assertEquals(from,((SnapStateChunkResponseMessage) decodedMessage).getFrom()); + assertEquals(to,((SnapStateChunkResponseMessage) decodedMessage).getTo()); + assertEquals(blockNumber,((SnapStateChunkResponseMessage) decodedMessage).getBlockNumber()); + assertEquals(complete, ((SnapStateChunkResponseMessage) decodedMessage).isComplete()); + assertThat(trieValueBytes, is(((SnapStateChunkResponseMessage) decodedMessage).getChunkOfTrieKeyValue())); } @Test diff --git a/rskj-core/src/test/java/co/rsk/net/messages/SnapStatusRequestMessageTest.java b/rskj-core/src/test/java/co/rsk/net/messages/SnapStatusRequestMessageTest.java index ff34d0a463c..967406dd48a 100644 --- a/rskj-core/src/test/java/co/rsk/net/messages/SnapStatusRequestMessageTest.java +++ b/rskj-core/src/test/java/co/rsk/net/messages/SnapStatusRequestMessageTest.java @@ -18,18 +18,29 @@ */ package co.rsk.net.messages; +import co.rsk.config.TestSystemProperties; +import org.ethereum.core.BlockFactory; import org.ethereum.util.RLP; +import org.ethereum.util.RLPList; import org.junit.jupiter.api.Test; +import java.math.BigInteger; + import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.mockito.Mockito.*; class SnapStatusRequestMessageTest { + + private final TestSystemProperties config = new TestSystemProperties(); + private final BlockFactory blockFactory = new BlockFactory(config.getActivationConfig()); + @Test void getMessageType_returnCorrectMessageType() { //given - SnapStatusRequestMessage message = new SnapStatusRequestMessage(); + SnapStatusRequestMessage message = new SnapStatusRequestMessage(1); //when MessageType messageType = message.getMessageType(); @@ -41,8 +52,8 @@ void getMessageType_returnCorrectMessageType() { @Test void getEncodedMessage_returnExpectedByteArray() { //given - SnapStatusRequestMessage message = new SnapStatusRequestMessage(); - byte[] expectedEncodedMessage = RLP.encodedEmptyList(); + SnapStatusRequestMessage message = new SnapStatusRequestMessage(1); + byte[] expectedEncodedMessage = RLP.encodeList(RLP.encodeBigInteger(BigInteger.valueOf(1)), RLP.encodedEmptyList()); //when byte[] encodedMessage = message.getEncodedMessage(); @@ -50,10 +61,24 @@ void getEncodedMessage_returnExpectedByteArray() { assertThat(encodedMessage, equalTo(expectedEncodedMessage)); } + @Test + void decodeMessage_returnExpectedMessage() { + //given default block 4 test + SnapStatusRequestMessage message = new SnapStatusRequestMessage(111); + RLPList encodedRLPList = (RLPList) RLP.decode2(message.getEncodedMessage()).get(0); + + //when + Message decodedMessage = SnapStatusRequestMessage.decodeMessage(blockFactory, encodedRLPList); + + //then + assertInstanceOf(SnapStatusRequestMessage.class, decodedMessage); + assertEquals(111, ((SnapStatusRequestMessage) decodedMessage).getId()); + } + @Test void givenAcceptIsCalled_messageVisitorIsAppliedForMessage() { //given - SnapStatusRequestMessage message = new SnapStatusRequestMessage(); + SnapStatusRequestMessage message = new SnapStatusRequestMessage(1); MessageVisitor visitor = mock(MessageVisitor.class); //when diff --git a/rskj-core/src/test/java/co/rsk/net/messages/SnapStatusResponseMessageTest.java b/rskj-core/src/test/java/co/rsk/net/messages/SnapStatusResponseMessageTest.java index 623d0c42325..25e308edb0d 100644 --- a/rskj-core/src/test/java/co/rsk/net/messages/SnapStatusResponseMessageTest.java +++ b/rskj-core/src/test/java/co/rsk/net/messages/SnapStatusResponseMessageTest.java @@ -28,6 +28,7 @@ import org.ethereum.db.BlockStore; import org.ethereum.db.IndexedBlockStore; import org.ethereum.util.RLP; +import org.ethereum.util.RLPList; import org.junit.jupiter.api.Test; import java.math.BigInteger; @@ -36,7 +37,8 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.mockito.Mockito.*; class SnapStatusResponseMessageTest { @@ -48,7 +50,7 @@ class SnapStatusResponseMessageTest { private final List blockList = Collections.singletonList(new BlockGenerator().getBlock(1)); private final List blockDifficulties = Collections.singletonList(indexedBlockStore.getTotalDifficultyForHash(block4Test.getHash().getBytes())); private final long trieSize = 1L; - private final SnapStatusResponseMessage underTest = new SnapStatusResponseMessage(blockList, blockDifficulties, trieSize); + private final SnapStatusResponseMessage underTest = new SnapStatusResponseMessage(1, blockList, blockDifficulties, trieSize); @Test @@ -64,9 +66,11 @@ void getMessageType_returnCorrectMessageType() { void getEncodedMessage_returnExpectedByteArray() { //given default block 4 test byte[] expectedEncodedMessage = RLP.encodeList( - RLP.encodeList(RLP.encode(block4Test.getEncoded())), - RLP.encodeList(RLP.encode(blockDifficulties.get(0).getBytes())), - RLP.encodeBigInteger(BigInteger.valueOf(this.trieSize))); + RLP.encodeBigInteger(BigInteger.valueOf(underTest.getId())), + RLP.encodeList( + RLP.encodeList(RLP.encode(block4Test.getEncoded())), + RLP.encodeList(RLP.encode(blockDifficulties.get(0).getBytes())), + RLP.encodeBigInteger(BigInteger.valueOf(this.trieSize)))); //when byte[] encodedMessage = underTest.getEncodedMessage(); @@ -74,6 +78,23 @@ void getEncodedMessage_returnExpectedByteArray() { assertThat(encodedMessage, equalTo(expectedEncodedMessage)); } + @Test + void decodeMessage_returnExpectedMessage() { + //given default block 4 test + RLPList encodedRLPList = (RLPList) RLP.decode2(underTest.getEncodedMessage()).get(0); + + //when + Message decodedMessage = SnapStatusResponseMessage.decodeMessage(blockFactory, encodedRLPList); + + //then + assertInstanceOf(SnapStatusResponseMessage.class, decodedMessage); + assertEquals(underTest.getId(), ((SnapStatusResponseMessage) decodedMessage).getId()); + assertEquals(1, ((SnapStatusResponseMessage) decodedMessage).getBlocks().size()); + assertEquals(underTest.getBlocks().get(0).getHash(), ((SnapStatusResponseMessage) decodedMessage).getBlocks().get(0).getHash()); + assertEquals(1, ((SnapStatusResponseMessage) decodedMessage).getDifficulties().size()); + assertEquals(underTest.getDifficulties().get(0), ((SnapStatusResponseMessage) decodedMessage).getDifficulties().get(0)); + } + @Test void getDifficulties_returnTheExpectedValue() { //given default block 4 test @@ -107,7 +128,7 @@ void getTrieSize_returnTheExpectedValue() { @Test void givenAcceptIsCalled_messageVisitorIsAppliedForMessage() { //given - SnapStatusResponseMessage message = new SnapStatusResponseMessage(blockList, blockDifficulties, trieSize); + SnapStatusResponseMessage message = new SnapStatusResponseMessage(1, blockList, blockDifficulties, trieSize); MessageVisitor visitor = mock(MessageVisitor.class); //when diff --git a/rskj-core/src/test/java/co/rsk/net/sync/SimpleSyncEventsHandler.java b/rskj-core/src/test/java/co/rsk/net/sync/SimpleSyncEventsHandler.java index d6694ecebe4..b8a735fe02d 100644 --- a/rskj-core/src/test/java/co/rsk/net/sync/SimpleSyncEventsHandler.java +++ b/rskj-core/src/test/java/co/rsk/net/sync/SimpleSyncEventsHandler.java @@ -19,11 +19,13 @@ package co.rsk.net.sync; import co.rsk.net.Peer; +import co.rsk.net.messages.MessageWithId; import co.rsk.scoring.EventType; import org.ethereum.core.Block; import org.ethereum.core.BlockHeader; import org.ethereum.core.BlockIdentifier; +import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.util.Deque; import java.util.List; @@ -106,4 +108,10 @@ public boolean stopSyncingWasCalled() { @Override public void startSnapSync(Peer peer) { } + + @Override + public long nextMessageId() { return 0; } + + @Override + public void registerPendingMessage(@Nonnull MessageWithId message) { } } diff --git a/rskj-core/src/test/java/co/rsk/net/sync/SnapSyncStateTest.java b/rskj-core/src/test/java/co/rsk/net/sync/SnapSyncStateTest.java index f12e5080483..a028c023048 100644 --- a/rskj-core/src/test/java/co/rsk/net/sync/SnapSyncStateTest.java +++ b/rskj-core/src/test/java/co/rsk/net/sync/SnapSyncStateTest.java @@ -21,29 +21,22 @@ import co.rsk.core.BlockDifficulty; import co.rsk.net.Peer; import co.rsk.net.SnapshotProcessor; -import co.rsk.net.messages.MessageType; -import co.rsk.net.messages.SnapBlocksResponseMessage; -import co.rsk.net.messages.SnapStateChunkResponseMessage; -import co.rsk.net.messages.SnapStatusResponseMessage; -import co.rsk.scoring.EventType; +import co.rsk.net.messages.*; import org.apache.commons.lang3.tuple.Pair; import org.ethereum.core.Block; -import org.ethereum.core.BlockHeader; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; import java.math.BigInteger; -import java.time.Duration; +import java.util.Collections; import java.util.List; import java.util.PriorityQueue; import java.util.Queue; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; import static org.junit.jupiter.api.Assertions.*; import static org.mockito.Mockito.*; @@ -55,9 +48,10 @@ class SnapSyncStateTest { private final SyncEventsHandler syncEventsHandler = mock(SyncEventsHandler.class); private final SnapshotPeersInformation peersInformation = mock(SnapshotPeersInformation.class); private final SnapshotProcessor snapshotProcessor = mock(SnapshotProcessor.class); + private final SnapSyncRequestManager snapRequestManager = mock(SnapSyncRequestManager.class); private final SyncMessageHandler.Listener listener = mock(SyncMessageHandler.Listener.class); - private final SnapSyncState underTest = new SnapSyncState(syncEventsHandler, snapshotProcessor, syncConfiguration, listener); + private final SnapSyncState underTest = new SnapSyncState(syncEventsHandler, snapshotProcessor, snapRequestManager, syncConfiguration, listener); @BeforeEach void setUp() { @@ -96,19 +90,6 @@ void givenOnEnterWasCalledTwice_thenSyncingStartsOnlyOnce() { verify(snapshotProcessor, times(1)).startSyncing(underTest); } - @Test - void givenTickIsCalledBeforeTimeout_thenTimerIsUpdated_andNoTimeoutHappens() { - //given - Duration elapsedTime = Duration.ofMillis(10); - underTest.timeElapsed = Duration.ZERO; - // when - underTest.tick(elapsedTime); - //then - assertThat(underTest.timeElapsed, equalTo(elapsedTime)); - verify(syncEventsHandler, never()).stopSyncing(); - verify(syncEventsHandler, never()).onErrorSyncing(any(), any(), any(), any()); - } - @Test void givenFinishIsCalled_thenSyncEventHandlerStopsSync() { //given-when @@ -122,9 +103,10 @@ void givenFinishIsCalled_thenSyncEventHandlerStopsSync() { void givenOnSnapStatusIsCalled_thenJobIsAddedAndRun() throws InterruptedException { //given Peer peer = mock(Peer.class); - SnapStatusResponseMessage msg = mock(SnapStatusResponseMessage.class); + SnapStatusResponseMessage msg = new SnapStatusResponseMessage(1, Collections.emptyList(), Collections.emptyList(), 1); CountDownLatch latch = new CountDownLatch(1); doCountDownOnQueueEmpty(listener, latch); + doReturn(true).when(snapRequestManager).processResponse(any()); underTest.onEnter(); //when @@ -144,9 +126,10 @@ void givenOnSnapStatusIsCalled_thenJobIsAddedAndRun() throws InterruptedExceptio void givenOnSnapBlocksIsCalled_thenJobIsAddedAndRun() throws InterruptedException { //given Peer peer = mock(Peer.class); - SnapBlocksResponseMessage msg = mock(SnapBlocksResponseMessage.class); + SnapBlocksResponseMessage msg = new SnapBlocksResponseMessage(1, Collections.emptyList(), Collections.emptyList()); CountDownLatch latch = new CountDownLatch(1); doCountDownOnQueueEmpty(listener, latch); + doReturn(true).when(snapRequestManager).processResponse(any()); underTest.onEnter(); //when @@ -166,10 +149,10 @@ void givenOnSnapBlocksIsCalled_thenJobIsAddedAndRun() throws InterruptedExceptio void givenNewBlockHeadersIsCalled_thenJobIsAddedAndRun() throws InterruptedException { //given Peer peer = mock(Peer.class); - //noinspection unchecked - List msg = mock(List.class); + BlockHeadersResponseMessage msg = new BlockHeadersResponseMessage(1, Collections.emptyList()); CountDownLatch latch = new CountDownLatch(1); doCountDownOnQueueEmpty(listener, latch); + doReturn(true).when(snapRequestManager).processResponse(any()); underTest.onEnter(); //when @@ -189,9 +172,10 @@ void givenNewBlockHeadersIsCalled_thenJobIsAddedAndRun() throws InterruptedExcep void givenOnSnapStateChunkIsCalled_thenJobIsAddedAndRun() throws InterruptedException { //given Peer peer = mock(Peer.class); - SnapStateChunkResponseMessage msg = mock(SnapStateChunkResponseMessage.class); + SnapStateChunkResponseMessage msg = new SnapStateChunkResponseMessage(1, new byte[0], 1, 1, 1, true); CountDownLatch latch = new CountDownLatch(1); doCountDownOnQueueEmpty(listener, latch); + doReturn(true).when(snapRequestManager).processResponse(any()); underTest.onEnter(); //when @@ -207,20 +191,6 @@ void givenOnSnapStateChunkIsCalled_thenJobIsAddedAndRun() throws InterruptedExce assertEquals(msg.getMessageType(), jobArg.getValue().getMsgType()); } - @Test - void givenOnMessageTimeOut_thenShouldFail() throws InterruptedException { - //given - Peer peer = mock(Peer.class); - underTest.setLastBlock(mock(Block.class), mock(BlockDifficulty.class), peer); - underTest.setRunning(); - - //when - underTest.onMessageTimeOut(); - - //then - verify(syncEventsHandler, times(1)).onErrorSyncing(eq(peer), eq(EventType.TIMEOUT_MESSAGE), any()); - } - @Test void testSetAndGetLastBlock() { Block mockBlock = mock(Block.class); From 9a57059b7fa30885799a71a1ddcab9d34ae3586f Mon Sep 17 00:00:00 2001 From: Volodymyr Kravets Date: Mon, 23 Dec 2024 11:20:04 +0200 Subject: [PATCH 10/10] feat(snap): fix (explosive) state chunks requesting --- rskj-core/src/main/java/co/rsk/net/SnapshotProcessor.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/rskj-core/src/main/java/co/rsk/net/SnapshotProcessor.java b/rskj-core/src/main/java/co/rsk/net/SnapshotProcessor.java index d9381759558..6471779a737 100644 --- a/rskj-core/src/main/java/co/rsk/net/SnapshotProcessor.java +++ b/rskj-core/src/main/java/co/rsk/net/SnapshotProcessor.java @@ -674,9 +674,7 @@ private void processOrderedStateChunkResponse(SnapSyncState state, Peer peer, Sn state.getAllNodes().addAll(nodes); state.setStateSize(state.getStateSize().add(BigInteger.valueOf(trieElements.size()))); state.setStateChunkSize(state.getStateChunkSize().add(BigInteger.valueOf(message.getChunkOfTrieKeyValue().length))); - if (!message.isComplete()) { - executeNextChunkRequestTask(state, peer); - } else { + if (message.isComplete()) { if (!this.checkHistoricalHeaders || blocksVerified(state)) { completeSyncing(state); } else {