From 4b800165870e2761e7721cc8100c40a20aaaeab8 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Tue, 17 Sep 2024 13:22:06 +1000 Subject: [PATCH 001/106] 7311: Add PeerTask system for use in future PRs Signed-off-by: Matilda Clerke --- .../InvalidPeerTaskResponseException.java | 26 ++ .../peertask/NoAvailablePeerException.java | 17 ++ .../eth/manager/peertask/PeerManager.java | 64 +++++ .../eth/manager/peertask/PeerTask.java | 63 ++++ .../manager/peertask/PeerTaskBehavior.java | 20 ++ .../manager/peertask/PeerTaskExecutor.java | 157 ++++++++++ .../PeerTaskExecutorResponseCode.java | 24 ++ .../peertask/PeerTaskExecutorResult.java | 35 +++ .../peertask/PeerTaskRequestSender.java | 55 ++++ .../eth/manager/peertask/PeerManagerTest.java | 80 ++++++ .../peertask/PeerTaskExecutorTest.java | 268 ++++++++++++++++++ .../peertask/PeerTaskRequestSenderTest.java | 77 +++++ 12 files changed, 886 insertions(+) create mode 100644 ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/InvalidPeerTaskResponseException.java create mode 100644 ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/NoAvailablePeerException.java create mode 100644 ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerManager.java create mode 100644 ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTask.java create mode 100644 ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskBehavior.java create mode 100644 ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java create mode 100644 ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorResponseCode.java create mode 100644 ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorResult.java create mode 100644 ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskRequestSender.java create mode 100644 ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerManagerTest.java create mode 100644 ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java create mode 100644 ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskRequestSenderTest.java diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/InvalidPeerTaskResponseException.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/InvalidPeerTaskResponseException.java new file mode 100644 index 00000000000..824c0860d70 --- /dev/null +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/InvalidPeerTaskResponseException.java @@ -0,0 +1,26 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.manager.peertask; + +public class InvalidPeerTaskResponseException extends Exception { + + public InvalidPeerTaskResponseException() { + super(); + } + + public InvalidPeerTaskResponseException(final Throwable cause) { + super(cause); + } +} diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/NoAvailablePeerException.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/NoAvailablePeerException.java new file mode 100644 index 00000000000..40c600d6fb3 --- /dev/null +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/NoAvailablePeerException.java @@ -0,0 +1,17 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.manager.peertask; + +public class NoAvailablePeerException extends Exception {} diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerManager.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerManager.java new file mode 100644 index 00000000000..fc5bc691b72 --- /dev/null +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerManager.java @@ -0,0 +1,64 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.manager.peertask; + +import org.hyperledger.besu.ethereum.eth.manager.EthPeer; +import org.hyperledger.besu.ethereum.p2p.peers.PeerId; + +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.function.Predicate; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** "Manages" the EthPeers for the PeerTaskExecutor */ +public class PeerManager { + private static final Logger LOG = LoggerFactory.getLogger(PeerManager.class); + + // use a synchronized map to ensure the map is never modified by multiple threads at once + private final Map ethPeersByPeerId = + Collections.synchronizedMap(new HashMap<>()); + + /** + * Gets the highest reputation peer matching the supplies filter + * + * @param filter a filter to match prospective peers with + * @return the highest reputation peer matching the supplies filter + * @throws NoAvailablePeerException If there are no suitable peers + */ + public EthPeer getPeer(final Predicate filter) throws NoAvailablePeerException { + LOG.trace("Getting peer from pool of {} peers", ethPeersByPeerId.size()); + return ethPeersByPeerId.values().stream() + .filter(filter) + .max(Comparator.naturalOrder()) + .orElseThrow(NoAvailablePeerException::new); + } + + public Optional getPeerByPeerId(final PeerId peerId) { + return Optional.ofNullable(ethPeersByPeerId.get(peerId)); + } + + public void addPeer(final EthPeer ethPeer) { + ethPeersByPeerId.put(ethPeer.getConnection().getPeer(), ethPeer); + } + + public void removePeer(final PeerId peerId) { + ethPeersByPeerId.remove(peerId); + } +} diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTask.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTask.java new file mode 100644 index 00000000000..244908c9216 --- /dev/null +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTask.java @@ -0,0 +1,63 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.manager.peertask; + +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; + +import java.util.Collection; + +/** + * Represents a task to be executed on an EthPeer by the PeerTaskExecutor + * + * @param The type of the result of this PeerTask + */ +public interface PeerTask { + /** + * Returns the SubProtocol used for this PeerTask + * + * @return the SubProtocol used for this PeerTask + */ + String getSubProtocol(); + + /** + * Gets the minimum required block number for a peer to have to successfully execute this task + * + * @return the minimum required block number for a peer to have to successfully execute this task + */ + long getRequiredBlockNumber(); + + /** + * Gets the request data to send to the EthPeer + * + * @return the request data to send to the EthPeer + */ + MessageData getRequestMessage(); + + /** + * Parses the MessageData response from the EthPeer + * + * @param messageData the response MessageData to be parsed + * @return a T built from the response MessageData + * @throws InvalidPeerTaskResponseException if the response messageData is invalid + */ + T parseResponse(MessageData messageData) throws InvalidPeerTaskResponseException; + + /** + * Gets the Collection of behaviors this task is expected to exhibit in the PeetTaskExecutor + * + * @return the Collection of behaviors this task is expected to exhibit in the PeetTaskExecutor + */ + Collection getPeerTaskBehaviors(); +} diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskBehavior.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskBehavior.java new file mode 100644 index 00000000000..fba9000a741 --- /dev/null +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskBehavior.java @@ -0,0 +1,20 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.manager.peertask; + +public enum PeerTaskBehavior { + RETRY_WITH_SAME_PEER, + RETRY_WITH_OTHER_PEERS +} diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java new file mode 100644 index 00000000000..fabf88d05d9 --- /dev/null +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java @@ -0,0 +1,157 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.manager.peertask; + +import org.hyperledger.besu.ethereum.eth.manager.EthPeer; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; +import org.hyperledger.besu.ethereum.p2p.rlpx.connections.PeerConnection; +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; +import org.hyperledger.besu.metrics.BesuMetricCategory; +import org.hyperledger.besu.plugin.services.MetricsSystem; +import org.hyperledger.besu.plugin.services.metrics.LabelledMetric; +import org.hyperledger.besu.plugin.services.metrics.OperationTimer; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeoutException; +import java.util.function.Supplier; + +/** Manages the execution of PeerTasks, respecting their PeerTaskBehavior */ +public class PeerTaskExecutor { + private static final long[] WAIT_TIME_BEFORE_RETRY = {0, 20000, 5000}; + + private final PeerManager peerManager; + private final PeerTaskRequestSender requestSender; + private final Supplier protocolSpecSupplier; + private final LabelledMetric requestTimer; + + public PeerTaskExecutor( + final PeerManager peerManager, + final PeerTaskRequestSender requestSender, + final Supplier protocolSpecSupplier, + final MetricsSystem metricsSystem) { + this.peerManager = peerManager; + this.requestSender = requestSender; + this.protocolSpecSupplier = protocolSpecSupplier; + requestTimer = + metricsSystem.createLabelledTimer( + BesuMetricCategory.PEERS, "Peer Task Executor Request Time", "", "Task Class Name"); + } + + public PeerTaskExecutorResult execute(final PeerTask peerTask) { + PeerTaskExecutorResult executorResult; + int triesRemaining = + peerTask.getPeerTaskBehaviors().contains(PeerTaskBehavior.RETRY_WITH_OTHER_PEERS) ? 3 : 1; + final Collection usedEthPeers = new ArrayList<>(); + do { + EthPeer peer; + try { + peer = + peerManager.getPeer( + (candidatePeer) -> + isPeerUnused(candidatePeer, usedEthPeers) + && (protocolSpecSupplier.get().isPoS() + || isPeerHeightHighEnough( + candidatePeer, peerTask.getRequiredBlockNumber())) + && isPeerProtocolSuitable(candidatePeer, peerTask.getSubProtocol())); + usedEthPeers.add(peer); + executorResult = executeAgainstPeer(peerTask, peer); + } catch (NoAvailablePeerException e) { + executorResult = + new PeerTaskExecutorResult<>(null, PeerTaskExecutorResponseCode.NO_PEER_AVAILABLE); + } + } while (--triesRemaining > 0 + && executorResult.getResponseCode() != PeerTaskExecutorResponseCode.SUCCESS); + + return executorResult; + } + + public CompletableFuture> executeAsync(final PeerTask peerTask) { + return CompletableFuture.supplyAsync(() -> execute(peerTask)); + } + + public PeerTaskExecutorResult executeAgainstPeer( + final PeerTask peerTask, final EthPeer peer) { + MessageData requestMessageData = peerTask.getRequestMessage(); + PeerTaskExecutorResult executorResult; + int triesRemaining = + peerTask.getPeerTaskBehaviors().contains(PeerTaskBehavior.RETRY_WITH_SAME_PEER) ? 3 : 1; + do { + try { + + MessageData responseMessageData; + try (final OperationTimer.TimingContext timingContext = + requestTimer.labels(peerTask.getClass().getSimpleName()).startTimer()) { + responseMessageData = + requestSender.sendRequest(peerTask.getSubProtocol(), requestMessageData, peer); + } + T result = peerTask.parseResponse(responseMessageData); + peer.recordUsefulResponse(); + executorResult = new PeerTaskExecutorResult<>(result, PeerTaskExecutorResponseCode.SUCCESS); + + } catch (PeerConnection.PeerNotConnected e) { + executorResult = + new PeerTaskExecutorResult<>(null, PeerTaskExecutorResponseCode.PEER_DISCONNECTED); + + } catch (InterruptedException | TimeoutException e) { + peer.recordRequestTimeout(requestMessageData.getCode()); + executorResult = new PeerTaskExecutorResult<>(null, PeerTaskExecutorResponseCode.TIMEOUT); + + } catch (InvalidPeerTaskResponseException e) { + peer.recordUselessResponse(e.getMessage()); + executorResult = + new PeerTaskExecutorResult<>(null, PeerTaskExecutorResponseCode.INVALID_RESPONSE); + + } catch (ExecutionException e) { + executorResult = + new PeerTaskExecutorResult<>(null, PeerTaskExecutorResponseCode.INTERNAL_SERVER_ERROR); + } + } while (--triesRemaining > 0 + && executorResult.getResponseCode() != PeerTaskExecutorResponseCode.SUCCESS + && executorResult.getResponseCode() != PeerTaskExecutorResponseCode.PEER_DISCONNECTED + && sleepBetweenRetries(WAIT_TIME_BEFORE_RETRY[triesRemaining])); + + return executorResult; + } + + public CompletableFuture> executeAgainstPeerAsync( + final PeerTask peerTask, final EthPeer peer) { + return CompletableFuture.supplyAsync(() -> executeAgainstPeer(peerTask, peer)); + } + + private boolean sleepBetweenRetries(final long sleepTime) { + try { + Thread.sleep(sleepTime); + return true; + } catch (InterruptedException e) { + return false; + } + } + + private static boolean isPeerUnused( + final EthPeer ethPeer, final Collection usedEthPeers) { + return !usedEthPeers.contains(ethPeer); + } + + private static boolean isPeerHeightHighEnough(final EthPeer ethPeer, final long requiredHeight) { + return ethPeer.chainState().getEstimatedHeight() >= requiredHeight; + } + + private static boolean isPeerProtocolSuitable(final EthPeer ethPeer, final String protocol) { + return ethPeer.getProtocolName().equals(protocol); + } +} diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorResponseCode.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorResponseCode.java new file mode 100644 index 00000000000..327461de15a --- /dev/null +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorResponseCode.java @@ -0,0 +1,24 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.manager.peertask; + +public enum PeerTaskExecutorResponseCode { + SUCCESS, + NO_PEER_AVAILABLE, + PEER_DISCONNECTED, + INTERNAL_SERVER_ERROR, + TIMEOUT, + INVALID_RESPONSE +} diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorResult.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorResult.java new file mode 100644 index 00000000000..f89bc67f61f --- /dev/null +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorResult.java @@ -0,0 +1,35 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.manager.peertask; + +import java.util.Optional; + +public class PeerTaskExecutorResult { + private final Optional result; + private final PeerTaskExecutorResponseCode responseCode; + + public PeerTaskExecutorResult(final T result, final PeerTaskExecutorResponseCode responseCode) { + this.result = Optional.ofNullable(result); + this.responseCode = responseCode; + } + + public Optional getResult() { + return result; + } + + public PeerTaskExecutorResponseCode getResponseCode() { + return responseCode; + } +} diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskRequestSender.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskRequestSender.java new file mode 100644 index 00000000000..9d9ceed3214 --- /dev/null +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskRequestSender.java @@ -0,0 +1,55 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.manager.peertask; + +import org.hyperledger.besu.ethereum.eth.manager.EthPeer; +import org.hyperledger.besu.ethereum.eth.manager.RequestManager.ResponseStream; +import org.hyperledger.besu.ethereum.p2p.rlpx.connections.PeerConnection; +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +public class PeerTaskRequestSender { + private static final long DEFAULT_TIMEOUT_MS = 20_000; + + private final long timeoutMs; + + public PeerTaskRequestSender() { + this.timeoutMs = DEFAULT_TIMEOUT_MS; + } + + public PeerTaskRequestSender(final long timeoutMs) { + this.timeoutMs = timeoutMs; + } + + public MessageData sendRequest( + final String subProtocol, final MessageData requestMessageData, final EthPeer ethPeer) + throws PeerConnection.PeerNotConnected, + ExecutionException, + InterruptedException, + TimeoutException { + ResponseStream responseStream = + ethPeer.send(requestMessageData, subProtocol, ethPeer.getConnection()); + final CompletableFuture responseMessageDataFuture = new CompletableFuture<>(); + responseStream.then( + (boolean streamClosed, MessageData message, EthPeer peer) -> { + responseMessageDataFuture.complete(message); + }); + return responseMessageDataFuture.get(timeoutMs, TimeUnit.MILLISECONDS); + } +} diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerManagerTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerManagerTest.java new file mode 100644 index 00000000000..54e0b88b5f1 --- /dev/null +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerManagerTest.java @@ -0,0 +1,80 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.manager.peertask; + +import org.hyperledger.besu.ethereum.eth.manager.EthPeer; +import org.hyperledger.besu.ethereum.eth.manager.MockPeerConnection; +import org.hyperledger.besu.ethereum.p2p.rlpx.connections.PeerConnection; +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.Capability; + +import java.time.Clock; +import java.util.Collections; +import java.util.Set; + +import org.apache.tuweni.bytes.Bytes; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class PeerManagerTest { + + public PeerManager peerManager; + + @BeforeEach + public void beforeTest() { + peerManager = new PeerManager(); + } + + @Test + public void testGetPeer() throws NoAvailablePeerException { + EthPeer protocol1With5ReputationPeer = + createTestPeer(Set.of(Capability.create("capability1", 1)), "protocol1", 5); + peerManager.addPeer(protocol1With5ReputationPeer); + EthPeer protocol1With4ReputationPeer = + createTestPeer(Set.of(Capability.create("capability1", 1)), "protocol1", 4); + peerManager.addPeer(protocol1With4ReputationPeer); + EthPeer protocol2With50ReputationPeer = + createTestPeer(Set.of(Capability.create("capability1", 1)), "protocol2", 50); + peerManager.addPeer(protocol2With50ReputationPeer); + EthPeer protocol2With4ReputationPeer = + createTestPeer(Set.of(Capability.create("capability1", 1)), "protocol2", 4); + peerManager.addPeer(protocol2With4ReputationPeer); + + EthPeer result = peerManager.getPeer((p) -> p.getProtocolName().equals("protocol1")); + + Assertions.assertSame(protocol1With5ReputationPeer, result); + } + + private EthPeer createTestPeer( + final Set connectionCapabilities, + final String protocolName, + final int reputationAdjustment) { + PeerConnection peerConnection = new MockPeerConnection(connectionCapabilities); + EthPeer peer = + new EthPeer( + peerConnection, + protocolName, + null, + Collections.emptyList(), + 1, + Clock.systemUTC(), + Collections.emptyList(), + Bytes.EMPTY); + for (int i = 0; i < reputationAdjustment; i++) { + peer.getReputation().recordUsefulResponse(); + } + return peer; + } +} diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java new file mode 100644 index 00000000000..029cb19e10b --- /dev/null +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java @@ -0,0 +1,268 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.manager.peertask; + +import org.hyperledger.besu.ethereum.eth.manager.EthPeer; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; +import org.hyperledger.besu.ethereum.p2p.rlpx.connections.PeerConnection; +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; +import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; + +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeoutException; +import java.util.function.Predicate; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; + +public class PeerTaskExecutorTest { + private @Mock PeerManager peerManager; + private @Mock PeerTaskRequestSender requestSender; + private @Mock ProtocolSpec protocolSpec; + private @Mock PeerTask peerTask; + private @Mock MessageData requestMessageData; + private @Mock MessageData responseMessageData; + private @Mock EthPeer ethPeer; + private AutoCloseable mockCloser; + + private PeerTaskExecutor peerTaskExecutor; + + @BeforeEach + public void beforeTest() { + mockCloser = MockitoAnnotations.openMocks(this); + peerTaskExecutor = + new PeerTaskExecutor( + peerManager, requestSender, () -> protocolSpec, new NoOpMetricsSystem()); + } + + @AfterEach + public void afterTest() throws Exception { + mockCloser.close(); + } + + @Test + public void testExecuteAgainstPeerWithNoPeerTaskBehaviorsAndSuccessfulFlow() + throws PeerConnection.PeerNotConnected, + ExecutionException, + InterruptedException, + TimeoutException, + InvalidPeerTaskResponseException { + String subprotocol = "subprotocol"; + Object responseObject = new Object(); + + Mockito.when(peerTask.getRequestMessage()).thenReturn(requestMessageData); + Mockito.when(peerTask.getPeerTaskBehaviors()).thenReturn(Collections.emptyList()); + Mockito.when(peerTask.getSubProtocol()).thenReturn(subprotocol); + Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, ethPeer)) + .thenReturn(responseMessageData); + Mockito.when(peerTask.parseResponse(responseMessageData)).thenReturn(responseObject); + + PeerTaskExecutorResult result = peerTaskExecutor.executeAgainstPeer(peerTask, ethPeer); + + Mockito.verify(ethPeer).recordUsefulResponse(); + + Assertions.assertNotNull(result); + Assertions.assertTrue(result.getResult().isPresent()); + Assertions.assertSame(responseObject, result.getResult().get()); + Assertions.assertEquals(PeerTaskExecutorResponseCode.SUCCESS, result.getResponseCode()); + } + + @Test + public void testExecuteAgainstPeerWithRetryBehaviorsAndSuccessfulFlowAfterFirstFailure() + throws PeerConnection.PeerNotConnected, + ExecutionException, + InterruptedException, + TimeoutException, + InvalidPeerTaskResponseException { + String subprotocol = "subprotocol"; + Object responseObject = new Object(); + int requestMessageDataCode = 123; + + Mockito.when(peerTask.getRequestMessage()).thenReturn(requestMessageData); + Mockito.when(peerTask.getPeerTaskBehaviors()) + .thenReturn(List.of(PeerTaskBehavior.RETRY_WITH_SAME_PEER)); + + Mockito.when(peerTask.getSubProtocol()).thenReturn(subprotocol); + Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, ethPeer)) + .thenThrow(new TimeoutException()) + .thenReturn(responseMessageData); + Mockito.when(requestMessageData.getCode()).thenReturn(requestMessageDataCode); + Mockito.when(peerTask.parseResponse(responseMessageData)).thenReturn(responseObject); + + PeerTaskExecutorResult result = peerTaskExecutor.executeAgainstPeer(peerTask, ethPeer); + + Mockito.verify(ethPeer).recordRequestTimeout(requestMessageDataCode); + Mockito.verify(ethPeer).recordUsefulResponse(); + + Assertions.assertNotNull(result); + Assertions.assertTrue(result.getResult().isPresent()); + Assertions.assertSame(responseObject, result.getResult().get()); + Assertions.assertEquals(PeerTaskExecutorResponseCode.SUCCESS, result.getResponseCode()); + } + + @Test + public void testExecuteAgainstPeerWithNoPeerTaskBehaviorsAndPeerNotConnected() + throws PeerConnection.PeerNotConnected, + ExecutionException, + InterruptedException, + TimeoutException, + InvalidPeerTaskResponseException { + String subprotocol = "subprotocol"; + + Mockito.when(peerTask.getRequestMessage()).thenReturn(requestMessageData); + Mockito.when(peerTask.getPeerTaskBehaviors()).thenReturn(Collections.emptyList()); + Mockito.when(peerTask.getSubProtocol()).thenReturn(subprotocol); + Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, ethPeer)) + .thenThrow(new PeerConnection.PeerNotConnected("")); + + PeerTaskExecutorResult result = peerTaskExecutor.executeAgainstPeer(peerTask, ethPeer); + + Assertions.assertNotNull(result); + Assertions.assertTrue(result.getResult().isEmpty()); + Assertions.assertEquals( + PeerTaskExecutorResponseCode.PEER_DISCONNECTED, result.getResponseCode()); + } + + @Test + public void testExecuteAgainstPeerWithNoPeerTaskBehaviorsAndTimeoutException() + throws PeerConnection.PeerNotConnected, + ExecutionException, + InterruptedException, + TimeoutException, + InvalidPeerTaskResponseException { + String subprotocol = "subprotocol"; + int requestMessageDataCode = 123; + + Mockito.when(peerTask.getRequestMessage()).thenReturn(requestMessageData); + Mockito.when(peerTask.getPeerTaskBehaviors()).thenReturn(Collections.emptyList()); + Mockito.when(peerTask.getSubProtocol()).thenReturn(subprotocol); + Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, ethPeer)) + .thenThrow(new TimeoutException()); + Mockito.when(requestMessageData.getCode()).thenReturn(requestMessageDataCode); + + PeerTaskExecutorResult result = peerTaskExecutor.executeAgainstPeer(peerTask, ethPeer); + + Mockito.verify(ethPeer).recordRequestTimeout(requestMessageDataCode); + + Assertions.assertNotNull(result); + Assertions.assertTrue(result.getResult().isEmpty()); + Assertions.assertEquals(PeerTaskExecutorResponseCode.TIMEOUT, result.getResponseCode()); + } + + @Test + public void testExecuteAgainstPeerWithNoPeerTaskBehaviorsAndInvalidResponseMessage() + throws PeerConnection.PeerNotConnected, + ExecutionException, + InterruptedException, + TimeoutException, + InvalidPeerTaskResponseException { + String subprotocol = "subprotocol"; + + Mockito.when(peerTask.getRequestMessage()).thenReturn(requestMessageData); + Mockito.when(peerTask.getPeerTaskBehaviors()).thenReturn(Collections.emptyList()); + Mockito.when(peerTask.getSubProtocol()).thenReturn(subprotocol); + Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, ethPeer)) + .thenReturn(responseMessageData); + Mockito.when(peerTask.parseResponse(responseMessageData)) + .thenThrow(new InvalidPeerTaskResponseException()); + + PeerTaskExecutorResult result = peerTaskExecutor.executeAgainstPeer(peerTask, ethPeer); + + Mockito.verify(ethPeer).recordUselessResponse(null); + + Assertions.assertNotNull(result); + Assertions.assertTrue(result.getResult().isEmpty()); + Assertions.assertEquals( + PeerTaskExecutorResponseCode.INVALID_RESPONSE, result.getResponseCode()); + } + + @Test + @SuppressWarnings("unchecked") + public void testExecuteWithNoPeerTaskBehaviorsAndSuccessFlow() + throws PeerConnection.PeerNotConnected, + ExecutionException, + InterruptedException, + TimeoutException, + InvalidPeerTaskResponseException, + NoAvailablePeerException { + String subprotocol = "subprotocol"; + Object responseObject = new Object(); + + Mockito.when(peerManager.getPeer(Mockito.any(Predicate.class))).thenReturn(ethPeer); + + Mockito.when(peerTask.getRequestMessage()).thenReturn(requestMessageData); + Mockito.when(peerTask.getPeerTaskBehaviors()).thenReturn(Collections.emptyList()); + Mockito.when(peerTask.getSubProtocol()).thenReturn(subprotocol); + Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, ethPeer)) + .thenReturn(responseMessageData); + Mockito.when(peerTask.parseResponse(responseMessageData)).thenReturn(responseObject); + + PeerTaskExecutorResult result = peerTaskExecutor.executeAgainstPeer(peerTask, ethPeer); + + Mockito.verify(ethPeer).recordUsefulResponse(); + + Assertions.assertNotNull(result); + Assertions.assertTrue(result.getResult().isPresent()); + Assertions.assertSame(responseObject, result.getResult().get()); + Assertions.assertEquals(PeerTaskExecutorResponseCode.SUCCESS, result.getResponseCode()); + } + + @Test + @SuppressWarnings("unchecked") + public void testExecuteWithPeerSwitchingAndSuccessFlow() + throws PeerConnection.PeerNotConnected, + ExecutionException, + InterruptedException, + TimeoutException, + InvalidPeerTaskResponseException, + NoAvailablePeerException { + String subprotocol = "subprotocol"; + Object responseObject = new Object(); + int requestMessageDataCode = 123; + EthPeer peer2 = Mockito.mock(EthPeer.class); + + Mockito.when(peerManager.getPeer(Mockito.any(Predicate.class))) + .thenReturn(ethPeer) + .thenReturn(peer2); + + Mockito.when(peerTask.getRequestMessage()).thenReturn(requestMessageData); + Mockito.when(peerTask.getPeerTaskBehaviors()) + .thenReturn(List.of(PeerTaskBehavior.RETRY_WITH_OTHER_PEERS)); + Mockito.when(peerTask.getSubProtocol()).thenReturn(subprotocol); + Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, ethPeer)) + .thenThrow(new TimeoutException()); + Mockito.when(requestMessageData.getCode()).thenReturn(requestMessageDataCode); + Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, peer2)) + .thenReturn(responseMessageData); + Mockito.when(peerTask.parseResponse(responseMessageData)).thenReturn(responseObject); + + PeerTaskExecutorResult result = peerTaskExecutor.execute(peerTask); + + Mockito.verify(ethPeer).recordRequestTimeout(requestMessageDataCode); + Mockito.verify(peer2).recordUsefulResponse(); + + Assertions.assertNotNull(result); + Assertions.assertTrue(result.getResult().isPresent()); + Assertions.assertSame(responseObject, result.getResult().get()); + Assertions.assertEquals(PeerTaskExecutorResponseCode.SUCCESS, result.getResponseCode()); + } +} diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskRequestSenderTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskRequestSenderTest.java new file mode 100644 index 00000000000..9e161031f51 --- /dev/null +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskRequestSenderTest.java @@ -0,0 +1,77 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.manager.peertask; + +import org.hyperledger.besu.ethereum.eth.manager.EthPeer; +import org.hyperledger.besu.ethereum.eth.manager.RequestManager; +import org.hyperledger.besu.ethereum.p2p.rlpx.connections.PeerConnection; +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeoutException; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; + +public class PeerTaskRequestSenderTest { + + private PeerTaskRequestSender peerTaskRequestSender; + + @BeforeEach + public void beforeTest() { + peerTaskRequestSender = new PeerTaskRequestSender(); + } + + @Test + public void testSendRequest() + throws PeerConnection.PeerNotConnected, + ExecutionException, + InterruptedException, + TimeoutException { + String subprotocol = "subprotocol"; + MessageData requestMessageData = Mockito.mock(MessageData.class); + MessageData responseMessageData = Mockito.mock(MessageData.class); + EthPeer peer = Mockito.mock(EthPeer.class); + PeerConnection peerConnection = Mockito.mock(PeerConnection.class); + RequestManager.ResponseStream responseStream = + Mockito.mock(RequestManager.ResponseStream.class); + + Mockito.when(peer.getConnection()).thenReturn(peerConnection); + Mockito.when(peer.send(requestMessageData, subprotocol, peerConnection)) + .thenReturn(responseStream); + + CompletableFuture actualResponseMessageDataFuture = + CompletableFuture.supplyAsync( + () -> { + try { + return peerTaskRequestSender.sendRequest(subprotocol, requestMessageData, peer); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + Thread.sleep(500); + ArgumentCaptor responseCallbackArgumentCaptor = + ArgumentCaptor.forClass(RequestManager.ResponseCallback.class); + Mockito.verify(responseStream).then(responseCallbackArgumentCaptor.capture()); + RequestManager.ResponseCallback responseCallback = responseCallbackArgumentCaptor.getValue(); + responseCallback.exec(false, responseMessageData, peer); + + Assertions.assertSame(responseMessageData, actualResponseMessageDataFuture.get()); + } +} From a8d5a9f340f017ce70dad533006880faedd1d9ab Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Tue, 17 Sep 2024 13:25:07 +1000 Subject: [PATCH 002/106] 7311: Clean up some warnings Signed-off-by: Matilda Clerke --- .../ethereum/eth/manager/peertask/PeerTaskExecutorTest.java | 6 ++---- .../eth/manager/peertask/PeerTaskRequestSenderTest.java | 6 +----- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java index 029cb19e10b..2c015425f1d 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java @@ -124,8 +124,7 @@ public void testExecuteAgainstPeerWithNoPeerTaskBehaviorsAndPeerNotConnected() throws PeerConnection.PeerNotConnected, ExecutionException, InterruptedException, - TimeoutException, - InvalidPeerTaskResponseException { + TimeoutException { String subprotocol = "subprotocol"; Mockito.when(peerTask.getRequestMessage()).thenReturn(requestMessageData); @@ -147,8 +146,7 @@ public void testExecuteAgainstPeerWithNoPeerTaskBehaviorsAndTimeoutException() throws PeerConnection.PeerNotConnected, ExecutionException, InterruptedException, - TimeoutException, - InvalidPeerTaskResponseException { + TimeoutException { String subprotocol = "subprotocol"; int requestMessageDataCode = 123; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskRequestSenderTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskRequestSenderTest.java index 9e161031f51..8bc52604db7 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskRequestSenderTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskRequestSenderTest.java @@ -21,7 +21,6 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeoutException; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -40,10 +39,7 @@ public void beforeTest() { @Test public void testSendRequest() - throws PeerConnection.PeerNotConnected, - ExecutionException, - InterruptedException, - TimeoutException { + throws PeerConnection.PeerNotConnected, ExecutionException, InterruptedException { String subprotocol = "subprotocol"; MessageData requestMessageData = Mockito.mock(MessageData.class); MessageData responseMessageData = Mockito.mock(MessageData.class); From 4c64dbe1cd678efabcda798ae8bebe43e844a5d7 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Wed, 18 Sep 2024 08:15:35 +1000 Subject: [PATCH 003/106] 7311: Add feature toggle for enabling use of the peertask system where available Signed-off-by: Matilda Clerke --- .../org/hyperledger/besu/cli/BesuCommand.java | 10 ++++ .../cli/custom/PeerTaskFeatureToggle.java | 53 +++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 besu/src/main/java/org/hyperledger/besu/cli/custom/PeerTaskFeatureToggle.java diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index ecfc0eaadb2..a6e77127248 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -40,6 +40,7 @@ import org.hyperledger.besu.cli.converter.PercentageConverter; import org.hyperledger.besu.cli.converter.SubnetInfoConverter; import org.hyperledger.besu.cli.custom.JsonRPCAllowlistHostsProperty; +import org.hyperledger.besu.cli.custom.PeerTaskFeatureToggle; import org.hyperledger.besu.cli.error.BesuExecutionExceptionHandler; import org.hyperledger.besu.cli.error.BesuParameterExceptionHandler; import org.hyperledger.besu.cli.options.MiningOptions; @@ -842,6 +843,12 @@ static class PrivacyOptionGroup { description = "Specifies the number of last blocks to cache (default: ${DEFAULT-VALUE})") private final Integer numberOfblocksToCache = 0; + @Option( + names = {"--peertask-system-enabled"}, + description = + "Temporary feature toggle to enable using the new peertask system (default: ${DEFAULT-VALUE})") + private final Boolean isPeerTaskSystemEnabled = false; + @Mixin private P2PTLSConfigOptions p2pTLSConfigOptions; // Plugins Configuration Option Group @@ -1784,6 +1791,9 @@ private void configure() throws Exception { instantiateSignatureAlgorithmFactory(); + PeerTaskFeatureToggle.initialize(isPeerTaskSystemEnabled); + logger.info("PeerTask feature toggle is {}", PeerTaskFeatureToggle.usePeerTaskSystem() ? "enabled" : "disabled"); + logger.info(generateConfigurationOverview()); logger.info("Security Module: {}", securityModuleName); } diff --git a/besu/src/main/java/org/hyperledger/besu/cli/custom/PeerTaskFeatureToggle.java b/besu/src/main/java/org/hyperledger/besu/cli/custom/PeerTaskFeatureToggle.java new file mode 100644 index 00000000000..0d929f006ac --- /dev/null +++ b/besu/src/main/java/org/hyperledger/besu/cli/custom/PeerTaskFeatureToggle.java @@ -0,0 +1,53 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.cli.custom; + +/** + * Temporary class to allow easy access to the PeerTask feature toggle. This class can be removed + * once we've properly tested the PeerTask system and want to enable it permanently + */ +public class PeerTaskFeatureToggle { + private static Boolean USE_PEER_TASK_SYSTEM = null; + + /** + * Initialize the PeerTaskFeatureToggle with the supplied usePeerTaskSystem Boolean. + * PeerTaskFeatureToggle may only be initialized once! + * + * @param usePeerTaskSystem Boolean indicating whether or not the PeerTask system should be used + * @throws IllegalStateException if PeerTaskFeatureToggle has already been initialized + */ + public static void initialize(final Boolean usePeerTaskSystem) { + if (USE_PEER_TASK_SYSTEM != null) { + throw new IllegalStateException( + "PeerTaskFeatureToggle has already been initialized, and cannot be initialized again"); + } + USE_PEER_TASK_SYSTEM = usePeerTaskSystem; + } + + /** + * Indicates whether or not to use the PeerTask system. PeerTaskFeatureToggle must have been + * initialized before this method can be called! + * + * @return whether or not to use the PeerTask system + * @throws IllegalStateException if the PeerTaskFeatureToggle has not been initialized + */ + public static Boolean usePeerTaskSystem() { + if (USE_PEER_TASK_SYSTEM == null) { + throw new IllegalStateException( + "PeerTaskFeatureToggle has not been initialized, but getUsePeerTaskSystem() has been called"); + } + return USE_PEER_TASK_SYSTEM; + } +} From 7a94fe2d6d5666def85c6db4f1d3d4d2385e4df8 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Wed, 18 Sep 2024 08:32:54 +1000 Subject: [PATCH 004/106] 7311: Remove log used for testing, apply spotless Signed-off-by: Matilda Clerke --- besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java | 1 - 1 file changed, 1 deletion(-) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index a6e77127248..cee8a49c882 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -1792,7 +1792,6 @@ private void configure() throws Exception { instantiateSignatureAlgorithmFactory(); PeerTaskFeatureToggle.initialize(isPeerTaskSystemEnabled); - logger.info("PeerTask feature toggle is {}", PeerTaskFeatureToggle.usePeerTaskSystem() ? "enabled" : "disabled"); logger.info(generateConfigurationOverview()); logger.info("Security Module: {}", securityModuleName); From ace5dd1918836cb2b881d171a570ba9e2916be35 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Wed, 18 Sep 2024 08:52:20 +1000 Subject: [PATCH 005/106] 7311: Add private constructor to PeerTaskFeatureToggle to prevent instantiation Signed-off-by: Matilda Clerke --- .../org/hyperledger/besu/cli/custom/PeerTaskFeatureToggle.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/custom/PeerTaskFeatureToggle.java b/besu/src/main/java/org/hyperledger/besu/cli/custom/PeerTaskFeatureToggle.java index 0d929f006ac..a817e84e3d4 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/custom/PeerTaskFeatureToggle.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/custom/PeerTaskFeatureToggle.java @@ -50,4 +50,6 @@ public static Boolean usePeerTaskSystem() { } return USE_PEER_TASK_SYSTEM; } + + private PeerTaskFeatureToggle() {} } From 52d440afbe3c5c3b785831495672ec7ef912a381 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Wed, 18 Sep 2024 09:16:14 +1000 Subject: [PATCH 006/106] 7311: Switch to logging a warning instead of throwing an exception when initializing PeerTaskFeatureToggle multiple times Signed-off-by: Matilda Clerke --- .../besu/cli/custom/PeerTaskFeatureToggle.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/custom/PeerTaskFeatureToggle.java b/besu/src/main/java/org/hyperledger/besu/cli/custom/PeerTaskFeatureToggle.java index a817e84e3d4..a6fc636fb5f 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/custom/PeerTaskFeatureToggle.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/custom/PeerTaskFeatureToggle.java @@ -14,11 +14,15 @@ */ package org.hyperledger.besu.cli.custom; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * Temporary class to allow easy access to the PeerTask feature toggle. This class can be removed * once we've properly tested the PeerTask system and want to enable it permanently */ public class PeerTaskFeatureToggle { + private static final Logger LOGGER = LoggerFactory.getLogger(PeerTaskFeatureToggle.class); private static Boolean USE_PEER_TASK_SYSTEM = null; /** @@ -30,10 +34,10 @@ public class PeerTaskFeatureToggle { */ public static void initialize(final Boolean usePeerTaskSystem) { if (USE_PEER_TASK_SYSTEM != null) { - throw new IllegalStateException( - "PeerTaskFeatureToggle has already been initialized, and cannot be initialized again"); + LOGGER.warn("PeerTaskFeatureToggle has already been initialized, and cannot be initialized again"); + } else { + USE_PEER_TASK_SYSTEM = usePeerTaskSystem; } - USE_PEER_TASK_SYSTEM = usePeerTaskSystem; } /** From f392a0feda02546a8deb48f7158b7003f29fb498 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Wed, 18 Sep 2024 09:16:34 +1000 Subject: [PATCH 007/106] 7311: Update javadoc to match previous commit Signed-off-by: Matilda Clerke --- .../org/hyperledger/besu/cli/custom/PeerTaskFeatureToggle.java | 1 - 1 file changed, 1 deletion(-) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/custom/PeerTaskFeatureToggle.java b/besu/src/main/java/org/hyperledger/besu/cli/custom/PeerTaskFeatureToggle.java index a6fc636fb5f..19e66a92081 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/custom/PeerTaskFeatureToggle.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/custom/PeerTaskFeatureToggle.java @@ -30,7 +30,6 @@ public class PeerTaskFeatureToggle { * PeerTaskFeatureToggle may only be initialized once! * * @param usePeerTaskSystem Boolean indicating whether or not the PeerTask system should be used - * @throws IllegalStateException if PeerTaskFeatureToggle has already been initialized */ public static void initialize(final Boolean usePeerTaskSystem) { if (USE_PEER_TASK_SYSTEM != null) { From 2fb2690e0109a9bec8126d8305be04ceb4536dbc Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Wed, 18 Sep 2024 09:22:47 +1000 Subject: [PATCH 008/106] 7311: spotless Signed-off-by: Matilda Clerke --- .../org/hyperledger/besu/cli/custom/PeerTaskFeatureToggle.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/custom/PeerTaskFeatureToggle.java b/besu/src/main/java/org/hyperledger/besu/cli/custom/PeerTaskFeatureToggle.java index 19e66a92081..b1035cfd071 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/custom/PeerTaskFeatureToggle.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/custom/PeerTaskFeatureToggle.java @@ -33,7 +33,8 @@ public class PeerTaskFeatureToggle { */ public static void initialize(final Boolean usePeerTaskSystem) { if (USE_PEER_TASK_SYSTEM != null) { - LOGGER.warn("PeerTaskFeatureToggle has already been initialized, and cannot be initialized again"); + LOGGER.warn( + "PeerTaskFeatureToggle has already been initialized, and cannot be initialized again"); } else { USE_PEER_TASK_SYSTEM = usePeerTaskSystem; } From f14aaebef6ebf404753cc2c9561aa5422d85f2df Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Wed, 18 Sep 2024 10:24:10 +1000 Subject: [PATCH 009/106] 7311: Fix broken BesuCommandTest Signed-off-by: Matilda Clerke --- besu/src/test/resources/everything_config.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/besu/src/test/resources/everything_config.toml b/besu/src/test/resources/everything_config.toml index e3d7a3f28d3..73a17754623 100644 --- a/besu/src/test/resources/everything_config.toml +++ b/besu/src/test/resources/everything_config.toml @@ -226,6 +226,7 @@ Xsecp256k1-native-enabled=false Xaltbn128-native-enabled=false Xsnapsync-server-enabled=true Xbonsai-full-flat-db-enabled=true +peertask-system-enabled=false # compatibility flags compatibility-eth64-forkid-enabled=false From 1ba51845e2dffdfb0811b468fb1392c947d1ff41 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Wed, 18 Sep 2024 11:35:20 +1000 Subject: [PATCH 010/106] 7311: add class Signed-off-by: Matilda Clerke --- .../task/GetReceiptsFromPeerTask.java | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTask.java diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTask.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTask.java new file mode 100644 index 00000000000..f99fb3e8a27 --- /dev/null +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTask.java @@ -0,0 +1,103 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.manager.peertask.task; + +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.TransactionReceipt; +import org.hyperledger.besu.ethereum.eth.EthProtocol; +import org.hyperledger.besu.ethereum.eth.manager.peertask.InvalidPeerTaskResponseException; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTask; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskBehavior; +import org.hyperledger.besu.ethereum.eth.messages.GetReceiptsMessage; +import org.hyperledger.besu.ethereum.eth.messages.ReceiptsMessage; +import org.hyperledger.besu.ethereum.mainnet.BodyValidation; +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class GetReceiptsFromPeerTask + implements PeerTask>> { + + private final Collection blockHeaders; + private final Map> headersByReceiptsRoot = new HashMap<>(); + + public GetReceiptsFromPeerTask(final Collection blockHeaders) { + this.blockHeaders = blockHeaders; + blockHeaders.forEach( + header -> + headersByReceiptsRoot + .computeIfAbsent(header.getReceiptsRoot(), key -> new ArrayList<>()) + .add(header)); + } + + @Override + public String getSubProtocol() { + return EthProtocol.NAME; + } + + @Override + public long getRequiredBlockNumber() { + return blockHeaders.stream() + .mapToLong(BlockHeader::getNumber) + .max() + .orElse(BlockHeader.GENESIS_BLOCK_NUMBER); + } + + @Override + public MessageData getRequestMessage() { + // Since we have to match up the data by receipt root, we only need to request receipts + // for one of the headers with each unique receipt root. + final List blockHashes = + headersByReceiptsRoot.values().stream() + .map(headers -> headers.getFirst().getHash()) + .toList(); + return GetReceiptsMessage.create(blockHashes); + } + + @Override + public Map> parseResponse(final MessageData messageData) + throws InvalidPeerTaskResponseException { + if (messageData == null) { + throw new InvalidPeerTaskResponseException(); + } + final ReceiptsMessage receiptsMessage = ReceiptsMessage.readFrom(messageData); + final List> receiptsByBlock = receiptsMessage.receipts(); + if (receiptsByBlock.isEmpty() || receiptsByBlock.size() > blockHeaders.size()) { + throw new InvalidPeerTaskResponseException(); + } + + final Map> receiptsByHeader = new HashMap<>(); + for (final List receiptsInBlock : receiptsByBlock) { + final List blockHeaders = + headersByReceiptsRoot.get(BodyValidation.receiptsRoot(receiptsInBlock)); + if (blockHeaders == null) { + // Contains receipts that we didn't request, so mustn't be the response we're looking for. + throw new InvalidPeerTaskResponseException(); + } + blockHeaders.forEach(header -> receiptsByHeader.put(header, receiptsInBlock)); + } + return receiptsByHeader; + } + + @Override + public Collection getPeerTaskBehaviors() { + return List.of(PeerTaskBehavior.RETRY_WITH_OTHER_PEERS, PeerTaskBehavior.RETRY_WITH_SAME_PEER); + } +} \ No newline at end of file From f2500dd4d8f7af04d481ddfb61f1b038d02dc989 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Wed, 18 Sep 2024 11:40:59 +1000 Subject: [PATCH 011/106] 7311: Move PeerTaskFeatureToggle to more appropriate location Signed-off-by: Matilda Clerke --- besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java | 2 +- .../ethereum/eth/manager/peertask}/PeerTaskFeatureToggle.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename {besu/src/main/java/org/hyperledger/besu/cli/custom => ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask}/PeerTaskFeatureToggle.java (97%) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index cee8a49c882..b37bb791467 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -40,7 +40,6 @@ import org.hyperledger.besu.cli.converter.PercentageConverter; import org.hyperledger.besu.cli.converter.SubnetInfoConverter; import org.hyperledger.besu.cli.custom.JsonRPCAllowlistHostsProperty; -import org.hyperledger.besu.cli.custom.PeerTaskFeatureToggle; import org.hyperledger.besu.cli.error.BesuExecutionExceptionHandler; import org.hyperledger.besu.cli.error.BesuParameterExceptionHandler; import org.hyperledger.besu.cli.options.MiningOptions; @@ -118,6 +117,7 @@ import org.hyperledger.besu.ethereum.core.PrivacyParameters; import org.hyperledger.besu.ethereum.core.VersionMetadata; import org.hyperledger.besu.ethereum.core.plugins.PluginConfiguration; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskFeatureToggle; import org.hyperledger.besu.ethereum.eth.sync.SyncMode; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; import org.hyperledger.besu.ethereum.eth.transactions.ImmutableTransactionPoolConfiguration; diff --git a/besu/src/main/java/org/hyperledger/besu/cli/custom/PeerTaskFeatureToggle.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskFeatureToggle.java similarity index 97% rename from besu/src/main/java/org/hyperledger/besu/cli/custom/PeerTaskFeatureToggle.java rename to ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskFeatureToggle.java index b1035cfd071..24cc63b84c4 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/custom/PeerTaskFeatureToggle.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskFeatureToggle.java @@ -12,7 +12,7 @@ * * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.cli.custom; +package org.hyperledger.besu.ethereum.eth.manager.peertask; import org.slf4j.Logger; import org.slf4j.LoggerFactory; From 33b810b40d3a194e5be1f9e3689c106e37eff503 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Wed, 18 Sep 2024 11:43:36 +1000 Subject: [PATCH 012/106] 7311: add X prefix to peertask-system-enabled Signed-off-by: Matilda Clerke --- besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java | 2 +- besu/src/test/resources/everything_config.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index b37bb791467..ba1eb4b0a27 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -844,7 +844,7 @@ static class PrivacyOptionGroup { private final Integer numberOfblocksToCache = 0; @Option( - names = {"--peertask-system-enabled"}, + names = {"--Xpeertask-system-enabled"}, description = "Temporary feature toggle to enable using the new peertask system (default: ${DEFAULT-VALUE})") private final Boolean isPeerTaskSystemEnabled = false; diff --git a/besu/src/test/resources/everything_config.toml b/besu/src/test/resources/everything_config.toml index 73a17754623..d28152b47cd 100644 --- a/besu/src/test/resources/everything_config.toml +++ b/besu/src/test/resources/everything_config.toml @@ -226,7 +226,7 @@ Xsecp256k1-native-enabled=false Xaltbn128-native-enabled=false Xsnapsync-server-enabled=true Xbonsai-full-flat-db-enabled=true -peertask-system-enabled=false +Xpeertask-system-enabled=false # compatibility flags compatibility-eth64-forkid-enabled=false From 15b6bdf712f32601ae37adc984a5c53f629fa523 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Wed, 18 Sep 2024 12:39:54 +1000 Subject: [PATCH 013/106] 7311: Move --Xpeertask-system-enabled out of BesuCommand and make hidden Signed-off-by: Matilda Clerke --- .../java/org/hyperledger/besu/cli/BesuCommand.java | 8 -------- .../besu/cli/options/stable/P2PTLSConfigOptions.java | 10 ++++++++++ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index ba1eb4b0a27..e103f1e68dd 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -843,12 +843,6 @@ static class PrivacyOptionGroup { description = "Specifies the number of last blocks to cache (default: ${DEFAULT-VALUE})") private final Integer numberOfblocksToCache = 0; - @Option( - names = {"--Xpeertask-system-enabled"}, - description = - "Temporary feature toggle to enable using the new peertask system (default: ${DEFAULT-VALUE})") - private final Boolean isPeerTaskSystemEnabled = false; - @Mixin private P2PTLSConfigOptions p2pTLSConfigOptions; // Plugins Configuration Option Group @@ -1791,8 +1785,6 @@ private void configure() throws Exception { instantiateSignatureAlgorithmFactory(); - PeerTaskFeatureToggle.initialize(isPeerTaskSystemEnabled); - logger.info(generateConfigurationOverview()); logger.info("Security Module: {}", securityModuleName); } diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/P2PTLSConfigOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/P2PTLSConfigOptions.java index c3f8c56219f..440d65b5a2f 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/P2PTLSConfigOptions.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/P2PTLSConfigOptions.java @@ -20,6 +20,7 @@ import org.hyperledger.besu.cli.util.CommandLineUtils; import org.hyperledger.besu.ethereum.api.tls.FileBasedPasswordProvider; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskFeatureToggle; import org.hyperledger.besu.ethereum.p2p.rlpx.connections.netty.TLSConfiguration; import java.nio.file.Path; @@ -103,6 +104,13 @@ public class P2PTLSConfigOptions { "Whether to send a SNI header in the TLS ClientHello message (default: ${DEFAULT-VALUE})") private final Boolean p2pTlsClientHelloSniHeaderEnabled = false; + @Option( + names = {"--Xpeertask-system-enabled"}, + hidden = true, + description = + "Temporary feature toggle to enable using the new peertask system (default: ${DEFAULT-VALUE})") + private final Boolean isPeerTaskSystemEnabled = false; + /** Default constructor. */ P2PTLSConfigOptions() {} @@ -128,6 +136,8 @@ public Optional p2pTLSConfiguration(final CommandLine commandL "File containing password to unlock keystore is required when p2p TLS is enabled"); } + PeerTaskFeatureToggle.initialize(isPeerTaskSystemEnabled); + return Optional.of( TLSConfiguration.Builder.tlsConfiguration() .withKeyStoreType(p2pTLSKeyStoreType) From e3fbc6ca48bea501bf48dbb05543d3289e014e93 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Wed, 18 Sep 2024 12:44:20 +1000 Subject: [PATCH 014/106] 7311: spotless Signed-off-by: Matilda Clerke --- .../main/java/org/hyperledger/besu/cli/BesuCommand.java | 1 - .../besu/cli/options/stable/P2PTLSConfigOptions.java | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index e103f1e68dd..ecfc0eaadb2 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -117,7 +117,6 @@ import org.hyperledger.besu.ethereum.core.PrivacyParameters; import org.hyperledger.besu.ethereum.core.VersionMetadata; import org.hyperledger.besu.ethereum.core.plugins.PluginConfiguration; -import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskFeatureToggle; import org.hyperledger.besu.ethereum.eth.sync.SyncMode; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; import org.hyperledger.besu.ethereum.eth.transactions.ImmutableTransactionPoolConfiguration; diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/P2PTLSConfigOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/P2PTLSConfigOptions.java index 440d65b5a2f..a48b177bba3 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/P2PTLSConfigOptions.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/P2PTLSConfigOptions.java @@ -105,10 +105,10 @@ public class P2PTLSConfigOptions { private final Boolean p2pTlsClientHelloSniHeaderEnabled = false; @Option( - names = {"--Xpeertask-system-enabled"}, - hidden = true, - description = - "Temporary feature toggle to enable using the new peertask system (default: ${DEFAULT-VALUE})") + names = {"--Xpeertask-system-enabled"}, + hidden = true, + description = + "Temporary feature toggle to enable using the new peertask system (default: ${DEFAULT-VALUE})") private final Boolean isPeerTaskSystemEnabled = false; /** Default constructor. */ From b2a45c201e3b899127fc13b0858956cea39a230e Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Thu, 19 Sep 2024 14:11:54 +1000 Subject: [PATCH 015/106] 7311: Add GetReceiptsFromPeerTask Signed-off-by: Matilda Clerke --- .../controller/BesuControllerBuilder.java | 20 ++- ...onsensusScheduleBesuControllerBuilder.java | 7 +- .../MergeBesuControllerBuilder.java | 7 +- .../TransitionBesuControllerBuilder.java | 10 +- .../besu/ethereum/mainnet/BodyValidator.java | 89 ++++++++++ .../eth/manager/EthProtocolManager.java | 15 +- .../task/GetReceiptsFromPeerTask.java | 122 +++++++------- .../eth/sync/DefaultSynchronizer.java | 5 + .../CheckpointDownloadBlockStep.java | 62 +++++-- .../CheckpointDownloaderFactory.java | 4 + .../checkpointsync/CheckpointSyncActions.java | 4 + .../CheckpointSyncChainDownloader.java | 10 +- ...CheckpointSyncDownloadPipelineFactory.java | 14 +- .../sync/fastsync/DownloadReceiptsStep.java | 31 +++- .../eth/sync/fastsync/FastSyncActions.java | 5 + .../fastsync/FastSyncChainDownloader.java | 10 +- .../FastSyncDownloadPipelineFactory.java | 6 +- .../worldstate/FastDownloaderFactory.java | 3 + .../sync/snapsync/SnapDownloaderFactory.java | 3 + .../eth/manager/EthProtocolManagerTest.java | 4 +- .../manager/EthProtocolManagerTestUtil.java | 7 +- .../PeerTaskFeatureToggleTestHelper.java | 34 ++++ .../task/GetReceiptsFromPeerTaskTest.java | 152 ++++++++++++++++++ .../CheckPointSyncChainDownloaderTest.java | 152 +++++++++++++++++- .../fastsync/DownloadReceiptsStepTest.java | 50 +++++- .../fastsync/FastDownloaderFactoryTest.java | 7 + .../sync/fastsync/FastSyncActionsTest.java | 2 + .../fastsync/FastSyncChainDownloaderTest.java | 2 + .../ethereum/eth/transactions/TestNode.java | 4 +- .../TransactionPoolFactoryTest.java | 4 +- 30 files changed, 741 insertions(+), 104 deletions(-) create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/BodyValidator.java create mode 100644 ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskFeatureToggleTestHelper.java create mode 100644 ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTaskTest.java diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java index 6bb3fb117c1..bd10301a677 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java @@ -55,6 +55,9 @@ import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; import org.hyperledger.besu.ethereum.eth.manager.MergePeerFilter; import org.hyperledger.besu.ethereum.eth.manager.MonitoredExecutors; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerManager; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutor; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskRequestSender; import org.hyperledger.besu.ethereum.eth.manager.snap.SnapProtocolManager; import org.hyperledger.besu.ethereum.eth.peervalidation.CheckpointBlocksPeerValidator; import org.hyperledger.besu.ethereum.eth.peervalidation.ClassicForkPeerValidator; @@ -652,6 +655,11 @@ public BesuController build() { } final EthContext ethContext = new EthContext(ethPeers, ethMessages, snapMessages, scheduler); + final PeerManager peerManager = new PeerManager(); + ethPeers.streamAllPeers().forEach(peerManager::addPeer); + final PeerTaskExecutor peerTaskExecutor = + new PeerTaskExecutor( + peerManager, new PeerTaskRequestSender(), currentProtocolSpecSupplier, metricsSystem); final boolean fullSyncDisabled = !SyncMode.isFullSync(syncConfig.getSyncMode()); final SyncState syncState = new SyncState(blockchain, ethPeers, fullSyncDisabled, checkpoint); @@ -691,7 +699,8 @@ public BesuController build() { scheduler, peerValidators, Optional.empty(), - forkIdManager); + forkIdManager, + peerManager); final PivotBlockSelector pivotBlockSelector = createPivotSelector( @@ -703,6 +712,7 @@ public BesuController build() { worldStateStorageCoordinator, protocolContext, ethContext, + peerTaskExecutor, syncState, ethProtocolManager, pivotBlockSelector); @@ -835,6 +845,7 @@ protected DefaultSynchronizer createSynchronizer( final WorldStateStorageCoordinator worldStateStorageCoordinator, final ProtocolContext protocolContext, final EthContext ethContext, + final PeerTaskExecutor peerTaskExecutor, final SyncState syncState, final EthProtocolManager ethProtocolManager, final PivotBlockSelector pivotBlockSelector) { @@ -846,6 +857,7 @@ protected DefaultSynchronizer createSynchronizer( worldStateStorageCoordinator, ethProtocolManager.getBlockBroadcaster(), ethContext, + peerTaskExecutor, syncState, dataDirectory, storageProvider, @@ -1035,7 +1047,8 @@ protected EthProtocolManager createEthProtocolManager( final EthScheduler scheduler, final List peerValidators, final Optional mergePeerFilter, - final ForkIdManager forkIdManager) { + final ForkIdManager forkIdManager, + final PeerManager peerManager) { return new EthProtocolManager( protocolContext.getBlockchain(), networkId, @@ -1049,7 +1062,8 @@ protected EthProtocolManager createEthProtocolManager( mergePeerFilter, synchronizerConfiguration, scheduler, - forkIdManager); + forkIdManager, + peerManager); } /** diff --git a/besu/src/main/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilder.java index 0d0f8d2fd87..897f2e3acec 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilder.java @@ -42,6 +42,7 @@ import org.hyperledger.besu.ethereum.eth.manager.EthProtocolManager; import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; import org.hyperledger.besu.ethereum.eth.manager.MergePeerFilter; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerManager; import org.hyperledger.besu.ethereum.eth.manager.snap.SnapProtocolManager; import org.hyperledger.besu.ethereum.eth.peervalidation.PeerValidator; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; @@ -243,7 +244,8 @@ protected EthProtocolManager createEthProtocolManager( final EthScheduler scheduler, final List peerValidators, final Optional mergePeerFilter, - final ForkIdManager forkIdManager) { + final ForkIdManager forkIdManager, + final PeerManager peerManager) { return besuControllerBuilderSchedule .get(0L) .createEthProtocolManager( @@ -257,7 +259,8 @@ protected EthProtocolManager createEthProtocolManager( scheduler, peerValidators, mergePeerFilter, - forkIdManager); + forkIdManager, + peerManager); } @Override diff --git a/besu/src/main/java/org/hyperledger/besu/controller/MergeBesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/MergeBesuControllerBuilder.java index f5fc75959e1..f860f399da6 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/MergeBesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/MergeBesuControllerBuilder.java @@ -34,6 +34,7 @@ import org.hyperledger.besu.ethereum.eth.manager.EthProtocolManager; import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; import org.hyperledger.besu.ethereum.eth.manager.MergePeerFilter; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerManager; import org.hyperledger.besu.ethereum.eth.peervalidation.PeerValidator; import org.hyperledger.besu.ethereum.eth.peervalidation.RequiredBlocksPeerValidator; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; @@ -99,7 +100,8 @@ protected EthProtocolManager createEthProtocolManager( final EthScheduler scheduler, final List peerValidators, final Optional mergePeerFilter, - final ForkIdManager forkIdManager) { + final ForkIdManager forkIdManager, + final PeerManager peerManager) { var mergeContext = protocolContext.getConsensusContext(MergeContext.class); @@ -129,7 +131,8 @@ protected EthProtocolManager createEthProtocolManager( scheduler, peerValidators, filterToUse, - forkIdManager); + forkIdManager, + peerManager); return ethProtocolManager; } diff --git a/besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java index ee2611d9c87..9b71e0bdc95 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java @@ -40,6 +40,8 @@ import org.hyperledger.besu.ethereum.eth.manager.EthProtocolManager; import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; import org.hyperledger.besu.ethereum.eth.manager.MergePeerFilter; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerManager; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutor; import org.hyperledger.besu.ethereum.eth.peervalidation.PeerValidator; import org.hyperledger.besu.ethereum.eth.sync.DefaultSynchronizer; import org.hyperledger.besu.ethereum.eth.sync.PivotBlockSelector; @@ -163,7 +165,8 @@ protected EthProtocolManager createEthProtocolManager( final EthScheduler scheduler, final List peerValidators, final Optional mergePeerFilter, - final ForkIdManager forkIdManager) { + final ForkIdManager forkIdManager, + final PeerManager peerManager) { return mergeBesuControllerBuilder.createEthProtocolManager( protocolContext, synchronizerConfiguration, @@ -175,7 +178,8 @@ protected EthProtocolManager createEthProtocolManager( scheduler, peerValidators, mergePeerFilter, - forkIdManager); + forkIdManager, + peerManager); } @Override @@ -225,6 +229,7 @@ protected DefaultSynchronizer createSynchronizer( final WorldStateStorageCoordinator worldStateStorageCoordinator, final ProtocolContext protocolContext, final EthContext ethContext, + final PeerTaskExecutor peerTaskExecutor, final SyncState syncState, final EthProtocolManager ethProtocolManager, final PivotBlockSelector pivotBlockSelector) { @@ -235,6 +240,7 @@ protected DefaultSynchronizer createSynchronizer( worldStateStorageCoordinator, protocolContext, ethContext, + peerTaskExecutor, syncState, ethProtocolManager, pivotBlockSelector); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/BodyValidator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/BodyValidator.java new file mode 100644 index 00000000000..6c78337281e --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/BodyValidator.java @@ -0,0 +1,89 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.mainnet; + +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.Request; +import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.ethereum.core.TransactionReceipt; +import org.hyperledger.besu.ethereum.core.Withdrawal; +import org.hyperledger.besu.evm.log.LogsBloomFilter; + +import java.util.List; + +/** A utility class for body validation tasks. Implemented utilising BodyValidation */ +public class BodyValidator { + + /** + * Generates the transaction root for a list of transactions + * + * @param transactions the transactions + * @return the transaction root + */ + public Hash transactionsRoot(final List transactions) { + return BodyValidation.transactionsRoot(transactions); + } + + /** + * Generates the withdrawals root for a list of withdrawals + * + * @param withdrawals the transactions + * @return the transaction root + */ + public Hash withdrawalsRoot(final List withdrawals) { + return BodyValidation.withdrawalsRoot(withdrawals); + } + + /** + * Generates the requests root for a list of requests + * + * @param requests list of request + * @return the requests root + */ + public Hash requestsRoot(final List requests) { + return BodyValidation.requestsRoot(requests); + } + + /** + * Generates the receipt root for a list of receipts + * + * @param receipts the receipts + * @return the receipt root + */ + public Hash receiptsRoot(final List receipts) { + return BodyValidation.receiptsRoot(receipts); + } + + /** + * Generates the ommers hash for a list of ommer block headers + * + * @param ommers the ommer block headers + * @return the ommers hash + */ + public Hash ommersHash(final List ommers) { + return BodyValidation.ommersHash(ommers); + } + + /** + * Generates the logs bloom filter for a list of transaction receipts + * + * @param receipts the transaction receipts + * @return the logs bloom filter + */ + public LogsBloomFilter logsBloom(final List receipts) { + return BodyValidation.logsBloom(receipts); + } +} diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManager.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManager.java index 58318f9611e..d69910d59cf 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManager.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManager.java @@ -23,6 +23,7 @@ import org.hyperledger.besu.ethereum.core.Difficulty; import org.hyperledger.besu.ethereum.eth.EthProtocol; import org.hyperledger.besu.ethereum.eth.EthProtocolConfiguration; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerManager; import org.hyperledger.besu.ethereum.eth.messages.EthPV62; import org.hyperledger.besu.ethereum.eth.messages.StatusMessage; import org.hyperledger.besu.ethereum.eth.peervalidation.PeerValidator; @@ -69,6 +70,7 @@ public class EthProtocolManager implements ProtocolManager, MinedBlockObserver { private final Hash genesisHash; private final ForkIdManager forkIdManager; + private final PeerManager peerManager; private final BigInteger networkId; private final EthPeers ethPeers; private final EthMessages ethMessages; @@ -92,7 +94,8 @@ public EthProtocolManager( final Optional mergePeerFilter, final SynchronizerConfiguration synchronizerConfiguration, final EthScheduler scheduler, - final ForkIdManager forkIdManager) { + final ForkIdManager forkIdManager, + final PeerManager peerManager) { this.networkId = networkId; this.peerValidators = peerValidators; this.scheduler = scheduler; @@ -102,6 +105,7 @@ public EthProtocolManager( this.genesisHash = blockchain.getBlockHashByNumber(0L).orElse(Hash.ZERO); this.forkIdManager = forkIdManager; + this.peerManager = peerManager; this.ethPeers = ethPeers; this.ethMessages = ethMessages; @@ -140,7 +144,8 @@ public EthProtocolManager( final List peerValidators, final Optional mergePeerFilter, final SynchronizerConfiguration synchronizerConfiguration, - final EthScheduler scheduler) { + final EthScheduler scheduler, + final PeerManager peerManager) { this( blockchain, networkId, @@ -158,7 +163,8 @@ public EthProtocolManager( blockchain, Collections.emptyList(), Collections.emptyList(), - ethereumWireProtocolConfiguration.isLegacyEth64ForkIdEnabled())); + ethereumWireProtocolConfiguration.isLegacyEth64ForkIdEnabled()), + peerManager); } public EthContext ethContext() { @@ -337,7 +343,7 @@ public void processMessage(final Capability cap, final Message message) { public void handleNewConnection(final PeerConnection connection) { ethPeers.registerNewConnection(connection, peerValidators); final EthPeer peer = ethPeers.peer(connection); - + peerManager.addPeer(peer); final Capability cap = connection.capability(getSupportedProtocol()); final ForkId latestForkId = cap.getVersion() >= 64 ? forkIdManager.getForkIdForChainHead() : null; @@ -369,6 +375,7 @@ public void handleDisconnect( final DisconnectReason reason, final boolean initiatedByPeer) { final boolean wasActiveConnection = ethPeers.registerDisconnect(connection); + peerManager.removePeer(connection.getPeer()); LOG.atDebug() .setMessage("Disconnect - active Connection? {} - {} - {} - {} {} - {} peers left") .addArgument(wasActiveConnection) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTask.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTask.java index f99fb3e8a27..f4760de8847 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTask.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTask.java @@ -23,7 +23,7 @@ import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskBehavior; import org.hyperledger.besu.ethereum.eth.messages.GetReceiptsMessage; import org.hyperledger.besu.ethereum.eth.messages.ReceiptsMessage; -import org.hyperledger.besu.ethereum.mainnet.BodyValidation; +import org.hyperledger.besu.ethereum.mainnet.BodyValidator; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; import java.util.ArrayList; @@ -33,71 +33,75 @@ import java.util.Map; public class GetReceiptsFromPeerTask - implements PeerTask>> { + implements PeerTask>> { - private final Collection blockHeaders; - private final Map> headersByReceiptsRoot = new HashMap<>(); + private final Collection blockHeaders; + private final BodyValidator bodyValidator; + private final Map> headersByReceiptsRoot = new HashMap<>(); - public GetReceiptsFromPeerTask(final Collection blockHeaders) { - this.blockHeaders = blockHeaders; - blockHeaders.forEach( - header -> - headersByReceiptsRoot - .computeIfAbsent(header.getReceiptsRoot(), key -> new ArrayList<>()) - .add(header)); - } + public GetReceiptsFromPeerTask( + final Collection blockHeaders, final BodyValidator bodyValidator) { + this.blockHeaders = blockHeaders; + this.bodyValidator = bodyValidator; - @Override - public String getSubProtocol() { - return EthProtocol.NAME; - } + blockHeaders.forEach( + header -> + headersByReceiptsRoot + .computeIfAbsent(header.getReceiptsRoot(), key -> new ArrayList<>()) + .add(header)); + } - @Override - public long getRequiredBlockNumber() { - return blockHeaders.stream() - .mapToLong(BlockHeader::getNumber) - .max() - .orElse(BlockHeader.GENESIS_BLOCK_NUMBER); - } + @Override + public String getSubProtocol() { + return EthProtocol.NAME; + } - @Override - public MessageData getRequestMessage() { - // Since we have to match up the data by receipt root, we only need to request receipts - // for one of the headers with each unique receipt root. - final List blockHashes = - headersByReceiptsRoot.values().stream() - .map(headers -> headers.getFirst().getHash()) - .toList(); - return GetReceiptsMessage.create(blockHashes); - } + @Override + public long getRequiredBlockNumber() { + return blockHeaders.stream() + .mapToLong(BlockHeader::getNumber) + .max() + .orElse(BlockHeader.GENESIS_BLOCK_NUMBER); + } - @Override - public Map> parseResponse(final MessageData messageData) - throws InvalidPeerTaskResponseException { - if (messageData == null) { - throw new InvalidPeerTaskResponseException(); - } - final ReceiptsMessage receiptsMessage = ReceiptsMessage.readFrom(messageData); - final List> receiptsByBlock = receiptsMessage.receipts(); - if (receiptsByBlock.isEmpty() || receiptsByBlock.size() > blockHeaders.size()) { - throw new InvalidPeerTaskResponseException(); - } + @Override + public MessageData getRequestMessage() { + // Since we have to match up the data by receipt root, we only need to request receipts + // for one of the headers with each unique receipt root. + final List blockHashes = + headersByReceiptsRoot.values().stream() + .map(headers -> headers.getFirst().getHash()) + .toList(); + return GetReceiptsMessage.create(blockHashes); + } - final Map> receiptsByHeader = new HashMap<>(); - for (final List receiptsInBlock : receiptsByBlock) { - final List blockHeaders = - headersByReceiptsRoot.get(BodyValidation.receiptsRoot(receiptsInBlock)); - if (blockHeaders == null) { - // Contains receipts that we didn't request, so mustn't be the response we're looking for. - throw new InvalidPeerTaskResponseException(); - } - blockHeaders.forEach(header -> receiptsByHeader.put(header, receiptsInBlock)); - } - return receiptsByHeader; + @Override + public Map> parseResponse(final MessageData messageData) + throws InvalidPeerTaskResponseException { + if (messageData == null) { + throw new InvalidPeerTaskResponseException(); + } + final ReceiptsMessage receiptsMessage = ReceiptsMessage.readFrom(messageData); + final List> receiptsByBlock = receiptsMessage.receipts(); + if (receiptsByBlock.isEmpty() || receiptsByBlock.size() > blockHeaders.size()) { + throw new InvalidPeerTaskResponseException(); } - @Override - public Collection getPeerTaskBehaviors() { - return List.of(PeerTaskBehavior.RETRY_WITH_OTHER_PEERS, PeerTaskBehavior.RETRY_WITH_SAME_PEER); + final Map> receiptsByHeader = new HashMap<>(); + for (final List receiptsInBlock : receiptsByBlock) { + final List blockHeaders = + headersByReceiptsRoot.get(bodyValidator.receiptsRoot(receiptsInBlock)); + if (blockHeaders == null) { + // Contains receipts that we didn't request, so mustn't be the response we're looking for. + throw new InvalidPeerTaskResponseException(); + } + blockHeaders.forEach(header -> receiptsByHeader.put(header, receiptsInBlock)); } -} \ No newline at end of file + return receiptsByHeader; + } + + @Override + public Collection getPeerTaskBehaviors() { + return List.of(PeerTaskBehavior.RETRY_WITH_OTHER_PEERS, PeerTaskBehavior.RETRY_WITH_SAME_PEER); + } +} diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java index b7dc2adb160..f012bbd7aea 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java @@ -22,6 +22,7 @@ import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.core.Synchronizer; import org.hyperledger.besu.ethereum.eth.manager.EthContext; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutor; import org.hyperledger.besu.ethereum.eth.sync.checkpointsync.CheckpointDownloaderFactory; import org.hyperledger.besu.ethereum.eth.sync.fastsync.FastSyncDownloader; import org.hyperledger.besu.ethereum.eth.sync.fastsync.FastSyncState; @@ -82,6 +83,7 @@ public DefaultSynchronizer( final WorldStateStorageCoordinator worldStateStorageCoordinator, final BlockBroadcaster blockBroadcaster, final EthContext ethContext, + final PeerTaskExecutor peerTaskExecutor, final SyncState syncState, final Path dataDirectory, final StorageProvider storageProvider, @@ -147,6 +149,7 @@ public DefaultSynchronizer( protocolContext, metricsSystem, ethContext, + peerTaskExecutor, worldStateStorageCoordinator, syncState, clock, @@ -163,6 +166,7 @@ public DefaultSynchronizer( protocolContext, metricsSystem, ethContext, + peerTaskExecutor, worldStateStorageCoordinator, syncState, clock, @@ -179,6 +183,7 @@ public DefaultSynchronizer( protocolContext, metricsSystem, ethContext, + peerTaskExecutor, worldStateStorageCoordinator, syncState, clock, diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointDownloadBlockStep.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointDownloadBlockStep.java index b4bdf585410..f255ec69d98 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointDownloadBlockStep.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointDownloadBlockStep.java @@ -16,17 +16,24 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.core.Block; +import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.BlockWithReceipts; import org.hyperledger.besu.ethereum.core.TransactionReceipt; import org.hyperledger.besu.ethereum.eth.manager.EthContext; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutor; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutorResponseCode; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutorResult; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskFeatureToggle; +import org.hyperledger.besu.ethereum.eth.manager.peertask.task.GetReceiptsFromPeerTask; import org.hyperledger.besu.ethereum.eth.manager.task.AbstractPeerTask.PeerTaskResult; import org.hyperledger.besu.ethereum.eth.manager.task.GetBlockFromPeerTask; -import org.hyperledger.besu.ethereum.eth.manager.task.GetReceiptsFromPeerTask; import org.hyperledger.besu.ethereum.eth.sync.fastsync.checkpoint.Checkpoint; +import org.hyperledger.besu.ethereum.mainnet.BodyValidator; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.plugin.services.MetricsSystem; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; @@ -34,16 +41,19 @@ public class CheckpointDownloadBlockStep { private final ProtocolSchedule protocolSchedule; private final EthContext ethContext; + private final PeerTaskExecutor peerTaskExecutor; private final Checkpoint checkpoint; private final MetricsSystem metricsSystem; public CheckpointDownloadBlockStep( final ProtocolSchedule protocolSchedule, final EthContext ethContext, + final PeerTaskExecutor peerTaskExecutor, final Checkpoint checkpoint, final MetricsSystem metricsSystem) { this.protocolSchedule = protocolSchedule; this.ethContext = ethContext; + this.peerTaskExecutor = peerTaskExecutor; this.checkpoint = checkpoint; this.metricsSystem = metricsSystem; } @@ -65,17 +75,43 @@ public CompletableFuture> downloadBlock(final Hash h private CompletableFuture> downloadReceipts( final PeerTaskResult peerTaskResult) { final Block block = peerTaskResult.getResult(); - final GetReceiptsFromPeerTask getReceiptsFromPeerTask = - GetReceiptsFromPeerTask.forHeaders(ethContext, List.of(block.getHeader()), metricsSystem); - return getReceiptsFromPeerTask - .run() - .thenCompose( - receiptTaskResult -> { - final Optional> transactionReceipts = - Optional.ofNullable(receiptTaskResult.getResult().get(block.getHeader())); - return CompletableFuture.completedFuture( - transactionReceipts.map(receipts -> new BlockWithReceipts(block, receipts))); - }) - .exceptionally(throwable -> Optional.empty()); + if (PeerTaskFeatureToggle.usePeerTaskSystem()) { + CompletableFuture> futureReceipts = new CompletableFuture<>(); + GetReceiptsFromPeerTask task = + new GetReceiptsFromPeerTask(List.of(block.getHeader()), new BodyValidator()); + PeerTaskExecutorResult>> executorResult = + peerTaskExecutor.execute(task); + + if (executorResult.getResponseCode() == PeerTaskExecutorResponseCode.SUCCESS) { + List transactionReceipts = + executorResult + .getResult() + .map((map) -> map.get(block.getHeader())) + .orElseThrow( + () -> + new IllegalStateException("PeerTask response code was success, but empty")); + BlockWithReceipts blockWithReceipts = new BlockWithReceipts(block, transactionReceipts); + futureReceipts.complete(Optional.of(blockWithReceipts)); + } else { + futureReceipts.complete(Optional.empty()); + } + return futureReceipts; + + } else { + final org.hyperledger.besu.ethereum.eth.manager.task.GetReceiptsFromPeerTask + getReceiptsFromPeerTask = + org.hyperledger.besu.ethereum.eth.manager.task.GetReceiptsFromPeerTask.forHeaders( + ethContext, List.of(block.getHeader()), metricsSystem); + return getReceiptsFromPeerTask + .run() + .thenCompose( + receiptTaskResult -> { + final Optional> transactionReceipts = + Optional.ofNullable(receiptTaskResult.getResult().get(block.getHeader())); + return CompletableFuture.completedFuture( + transactionReceipts.map(receipts -> new BlockWithReceipts(block, receipts))); + }) + .exceptionally(throwable -> Optional.empty()); + } } } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointDownloaderFactory.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointDownloaderFactory.java index 03df47e4407..30134d9f6c5 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointDownloaderFactory.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointDownloaderFactory.java @@ -17,6 +17,7 @@ import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.eth.manager.EthContext; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutor; import org.hyperledger.besu.ethereum.eth.sync.PivotBlockSelector; import org.hyperledger.besu.ethereum.eth.sync.SyncMode; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; @@ -61,6 +62,7 @@ public static Optional> createCheckpointDownloader( final ProtocolContext protocolContext, final MetricsSystem metricsSystem, final EthContext ethContext, + final PeerTaskExecutor peerTaskExecutor, final WorldStateStorageCoordinator worldStateStorageCoordinator, final SyncState syncState, final Clock clock, @@ -110,6 +112,7 @@ public static Optional> createCheckpointDownloader( protocolSchedule, protocolContext, ethContext, + peerTaskExecutor, syncState, pivotBlockSelector, metricsSystem); @@ -127,6 +130,7 @@ public static Optional> createCheckpointDownloader( protocolSchedule, protocolContext, ethContext, + peerTaskExecutor, syncState, pivotBlockSelector, metricsSystem); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncActions.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncActions.java index 5096b74e24f..61b997e6c53 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncActions.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncActions.java @@ -16,6 +16,7 @@ import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.eth.manager.EthContext; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutor; import org.hyperledger.besu.ethereum.eth.sync.ChainDownloader; import org.hyperledger.besu.ethereum.eth.sync.PivotBlockSelector; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; @@ -34,6 +35,7 @@ public CheckpointSyncActions( final ProtocolSchedule protocolSchedule, final ProtocolContext protocolContext, final EthContext ethContext, + final PeerTaskExecutor peerTaskExecutor, final SyncState syncState, final PivotBlockSelector pivotBlockSelector, final MetricsSystem metricsSystem) { @@ -43,6 +45,7 @@ public CheckpointSyncActions( protocolSchedule, protocolContext, ethContext, + peerTaskExecutor, syncState, pivotBlockSelector, metricsSystem); @@ -57,6 +60,7 @@ public ChainDownloader createChainDownloader( protocolSchedule, protocolContext, ethContext, + peerTaskExecutor, syncState, metricsSystem, currentState, diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncChainDownloader.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncChainDownloader.java index 5450b9e5a49..2590e4736ae 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncChainDownloader.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncChainDownloader.java @@ -16,6 +16,7 @@ import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.eth.manager.EthContext; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutor; import org.hyperledger.besu.ethereum.eth.sync.ChainDownloader; import org.hyperledger.besu.ethereum.eth.sync.PipelineChainDownloader; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; @@ -36,6 +37,7 @@ public static ChainDownloader create( final ProtocolSchedule protocolSchedule, final ProtocolContext protocolContext, final EthContext ethContext, + final PeerTaskExecutor peerTaskExecutor, final SyncState syncState, final MetricsSystem metricsSystem, final FastSyncState fastSyncState, @@ -55,7 +57,13 @@ public static ChainDownloader create( syncState, syncTargetManager, new CheckpointSyncDownloadPipelineFactory( - config, protocolSchedule, protocolContext, ethContext, fastSyncState, metricsSystem), + config, + protocolSchedule, + protocolContext, + ethContext, + peerTaskExecutor, + fastSyncState, + metricsSystem), ethContext.getScheduler(), metricsSystem, syncDurationMetrics); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncDownloadPipelineFactory.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncDownloadPipelineFactory.java index 45f3f243d8c..714dfcb4af2 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncDownloadPipelineFactory.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncDownloadPipelineFactory.java @@ -19,6 +19,7 @@ import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.eth.manager.EthContext; import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutor; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; import org.hyperledger.besu.ethereum.eth.sync.fastsync.FastSyncDownloadPipelineFactory; import org.hyperledger.besu.ethereum.eth.sync.fastsync.FastSyncState; @@ -40,9 +41,17 @@ public CheckpointSyncDownloadPipelineFactory( final ProtocolSchedule protocolSchedule, final ProtocolContext protocolContext, final EthContext ethContext, + final PeerTaskExecutor peerTaskExecutor, final FastSyncState fastSyncState, final MetricsSystem metricsSystem) { - super(syncConfig, protocolSchedule, protocolContext, ethContext, fastSyncState, metricsSystem); + super( + syncConfig, + protocolSchedule, + protocolContext, + ethContext, + peerTaskExecutor, + fastSyncState, + metricsSystem); } @Override @@ -76,7 +85,8 @@ protected Pipeline createDownloadCheckPointPipeline( checkPointSource, checkpoint, protocolContext.getBlockchain()); final CheckpointDownloadBlockStep checkPointDownloadBlockStep = - new CheckpointDownloadBlockStep(protocolSchedule, ethContext, checkpoint, metricsSystem); + new CheckpointDownloadBlockStep( + protocolSchedule, ethContext, peerTaskExecutor, checkpoint, metricsSystem); return PipelineBuilder.createPipelineFrom( "fetchCheckpoints", diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java index cd57de371dd..bd8b0a8a423 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java @@ -22,10 +22,17 @@ import org.hyperledger.besu.ethereum.core.BlockWithReceipts; import org.hyperledger.besu.ethereum.core.TransactionReceipt; import org.hyperledger.besu.ethereum.eth.manager.EthContext; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutor; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutorResponseCode; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutorResult; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskFeatureToggle; +import org.hyperledger.besu.ethereum.eth.manager.peertask.task.GetReceiptsFromPeerTask; import org.hyperledger.besu.ethereum.eth.sync.tasks.GetReceiptsForHeadersTask; +import org.hyperledger.besu.ethereum.mainnet.BodyValidator; import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.util.FutureUtils; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; @@ -34,18 +41,36 @@ public class DownloadReceiptsStep implements Function, CompletableFuture>> { private final EthContext ethContext; + private final PeerTaskExecutor peerTaskExecutor; private final MetricsSystem metricsSystem; - public DownloadReceiptsStep(final EthContext ethContext, final MetricsSystem metricsSystem) { + public DownloadReceiptsStep( + final EthContext ethContext, + final PeerTaskExecutor peerTaskExecutor, + final MetricsSystem metricsSystem) { this.ethContext = ethContext; + this.peerTaskExecutor = peerTaskExecutor; this.metricsSystem = metricsSystem; } @Override public CompletableFuture> apply(final List blocks) { final List headers = blocks.stream().map(Block::getHeader).collect(toList()); - final CompletableFuture>> getReceipts = - GetReceiptsForHeadersTask.forHeaders(ethContext, headers, metricsSystem).run(); + final CompletableFuture>> getReceipts; + if (PeerTaskFeatureToggle.usePeerTaskSystem()) { + GetReceiptsFromPeerTask getReceiptsFromPeerTask = + new GetReceiptsFromPeerTask(headers, new BodyValidator()); + PeerTaskExecutorResult>> getReceiptsResult = + peerTaskExecutor.execute(getReceiptsFromPeerTask); + if (getReceiptsResult.getResponseCode() == PeerTaskExecutorResponseCode.SUCCESS + && getReceiptsResult.getResult().isPresent()) { + getReceipts = CompletableFuture.completedFuture(getReceiptsResult.getResult().get()); + } else { + getReceipts = CompletableFuture.completedFuture(Collections.emptyMap()); + } + } else { + getReceipts = GetReceiptsForHeadersTask.forHeaders(ethContext, headers, metricsSystem).run(); + } final CompletableFuture> combineWithBlocks = getReceipts.thenApply( receiptsByHeader -> combineBlocksAndReceipts(blocks, receiptsByHeader)); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActions.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActions.java index 7f6bbae3f31..58a64bd562a 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActions.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActions.java @@ -19,6 +19,7 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.eth.manager.EthContext; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutor; import org.hyperledger.besu.ethereum.eth.manager.task.WaitForPeersTask; import org.hyperledger.besu.ethereum.eth.sync.ChainDownloader; import org.hyperledger.besu.ethereum.eth.sync.PivotBlockSelector; @@ -48,6 +49,7 @@ public class FastSyncActions { protected final ProtocolSchedule protocolSchedule; protected final ProtocolContext protocolContext; protected final EthContext ethContext; + protected final PeerTaskExecutor peerTaskExecutor; protected final SyncState syncState; protected final PivotBlockSelector pivotBlockSelector; protected final MetricsSystem metricsSystem; @@ -60,6 +62,7 @@ public FastSyncActions( final ProtocolSchedule protocolSchedule, final ProtocolContext protocolContext, final EthContext ethContext, + final PeerTaskExecutor peerTaskExecutor, final SyncState syncState, final PivotBlockSelector pivotBlockSelector, final MetricsSystem metricsSystem) { @@ -68,6 +71,7 @@ public FastSyncActions( this.protocolSchedule = protocolSchedule; this.protocolContext = protocolContext; this.ethContext = ethContext; + this.peerTaskExecutor = peerTaskExecutor; this.syncState = syncState; this.pivotBlockSelector = pivotBlockSelector; this.metricsSystem = metricsSystem; @@ -164,6 +168,7 @@ public ChainDownloader createChainDownloader( protocolSchedule, protocolContext, ethContext, + peerTaskExecutor, syncState, metricsSystem, currentState, diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloader.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloader.java index c36ff7cb482..1bf55a3811a 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloader.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloader.java @@ -16,6 +16,7 @@ import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.eth.manager.EthContext; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutor; import org.hyperledger.besu.ethereum.eth.sync.ChainDownloader; import org.hyperledger.besu.ethereum.eth.sync.PipelineChainDownloader; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; @@ -35,6 +36,7 @@ public static ChainDownloader create( final ProtocolSchedule protocolSchedule, final ProtocolContext protocolContext, final EthContext ethContext, + final PeerTaskExecutor peerTaskExecutor, final SyncState syncState, final MetricsSystem metricsSystem, final FastSyncState fastSyncState, @@ -53,7 +55,13 @@ public static ChainDownloader create( syncState, syncTargetManager, new FastSyncDownloadPipelineFactory( - config, protocolSchedule, protocolContext, ethContext, fastSyncState, metricsSystem), + config, + protocolSchedule, + protocolContext, + ethContext, + peerTaskExecutor, + fastSyncState, + metricsSystem), ethContext.getScheduler(), metricsSystem, syncDurationMetrics); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloadPipelineFactory.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloadPipelineFactory.java index 07e964426cc..63df4ee0cbe 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloadPipelineFactory.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloadPipelineFactory.java @@ -26,6 +26,7 @@ import org.hyperledger.besu.ethereum.eth.manager.EthContext; import org.hyperledger.besu.ethereum.eth.manager.EthPeer; import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutor; import org.hyperledger.besu.ethereum.eth.sync.DownloadBodiesStep; import org.hyperledger.besu.ethereum.eth.sync.DownloadHeadersStep; import org.hyperledger.besu.ethereum.eth.sync.DownloadPipelineFactory; @@ -57,6 +58,7 @@ public class FastSyncDownloadPipelineFactory implements DownloadPipelineFactory protected final ProtocolSchedule protocolSchedule; protected final ProtocolContext protocolContext; protected final EthContext ethContext; + protected final PeerTaskExecutor peerTaskExecutor; protected final FastSyncState fastSyncState; protected final MetricsSystem metricsSystem; protected final FastSyncValidationPolicy attachedValidationPolicy; @@ -68,12 +70,14 @@ public FastSyncDownloadPipelineFactory( final ProtocolSchedule protocolSchedule, final ProtocolContext protocolContext, final EthContext ethContext, + final PeerTaskExecutor peerTaskExecutor, final FastSyncState fastSyncState, final MetricsSystem metricsSystem) { this.syncConfig = syncConfig; this.protocolSchedule = protocolSchedule; this.protocolContext = protocolContext; this.ethContext = ethContext; + this.peerTaskExecutor = peerTaskExecutor; this.fastSyncState = fastSyncState; this.metricsSystem = metricsSystem; final LabelledMetric fastSyncValidationCounter = @@ -140,7 +144,7 @@ public Pipeline createDownloadPipelineForSyncTarget(final SyncT final DownloadBodiesStep downloadBodiesStep = new DownloadBodiesStep(protocolSchedule, ethContext, metricsSystem); final DownloadReceiptsStep downloadReceiptsStep = - new DownloadReceiptsStep(ethContext, metricsSystem); + new DownloadReceiptsStep(ethContext, peerTaskExecutor, metricsSystem); final ImportBlocksStep importBlockStep = new ImportBlocksStep( protocolSchedule, diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastDownloaderFactory.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastDownloaderFactory.java index 8b71a57885d..1d775cc80fd 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastDownloaderFactory.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastDownloaderFactory.java @@ -17,6 +17,7 @@ import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.eth.manager.EthContext; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutor; import org.hyperledger.besu.ethereum.eth.sync.PivotBlockSelector; import org.hyperledger.besu.ethereum.eth.sync.SyncMode; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; @@ -59,6 +60,7 @@ public static Optional> create( final ProtocolContext protocolContext, final MetricsSystem metricsSystem, final EthContext ethContext, + final PeerTaskExecutor peerTaskExecutor, final WorldStateStorageCoordinator worldStateStorageCoordinator, final SyncState syncState, final Clock clock, @@ -126,6 +128,7 @@ public static Optional> create( protocolSchedule, protocolContext, ethContext, + peerTaskExecutor, syncState, pivotBlockSelector, metricsSystem), diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapDownloaderFactory.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapDownloaderFactory.java index 5de8ceb9843..6c5ce0b04e9 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapDownloaderFactory.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapDownloaderFactory.java @@ -17,6 +17,7 @@ import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.eth.manager.EthContext; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutor; import org.hyperledger.besu.ethereum.eth.sync.PivotBlockSelector; import org.hyperledger.besu.ethereum.eth.sync.SyncMode; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; @@ -57,6 +58,7 @@ public static Optional> createSnapDownloader( final ProtocolContext protocolContext, final MetricsSystem metricsSystem, final EthContext ethContext, + final PeerTaskExecutor peerTaskExecutor, final WorldStateStorageCoordinator worldStateStorageCoordinator, final SyncState syncState, final Clock clock, @@ -121,6 +123,7 @@ public static Optional> createSnapDownloader( protocolSchedule, protocolContext, ethContext, + peerTaskExecutor, syncState, pivotBlockSelector, metricsSystem), diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManagerTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManagerTest.java index 3a3331b568f..762600bdd0e 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManagerTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManagerTest.java @@ -44,6 +44,7 @@ import org.hyperledger.besu.ethereum.eth.EthProtocolConfiguration; import org.hyperledger.besu.ethereum.eth.EthProtocolVersion; import org.hyperledger.besu.ethereum.eth.manager.MockPeerConnection.PeerSendHandler; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerManager; import org.hyperledger.besu.ethereum.eth.messages.BlockBodiesMessage; import org.hyperledger.besu.ethereum.eth.messages.BlockHeadersMessage; import org.hyperledger.besu.ethereum.eth.messages.EthPV62; @@ -1243,7 +1244,8 @@ private EthProtocolManager createEthManager( Optional.empty(), syncConfig, mock(EthScheduler.class), - mock(ForkIdManager.class))) { + mock(ForkIdManager.class), + new PeerManager())) { return ethManager; } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManagerTestUtil.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManagerTestUtil.java index 0b0bd1e3eb7..d8b5029d951 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManagerTestUtil.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManagerTestUtil.java @@ -30,6 +30,7 @@ import org.hyperledger.besu.ethereum.core.ProtocolScheduleFixture; import org.hyperledger.besu.ethereum.eth.EthProtocol; import org.hyperledger.besu.ethereum.eth.EthProtocolConfiguration; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerManager; import org.hyperledger.besu.ethereum.eth.manager.snap.SnapProtocolManager; import org.hyperledger.besu.ethereum.eth.peervalidation.PeerValidator; import org.hyperledger.besu.ethereum.eth.sync.ChainHeadTracker; @@ -117,7 +118,8 @@ public static EthProtocolManager create( mergePeerFilter, mock(SynchronizerConfiguration.class), ethScheduler, - new ForkIdManager(blockchain, Collections.emptyList(), Collections.emptyList(), false)); + new ForkIdManager(blockchain, Collections.emptyList(), Collections.emptyList(), false), + new PeerManager()); } public static EthProtocolManager create( @@ -168,7 +170,8 @@ public static EthProtocolManager create( Optional.empty(), mock(SynchronizerConfiguration.class), ethScheduler, - forkIdManager); + forkIdManager, + new PeerManager()); } public static EthProtocolManager create(final Blockchain blockchain) { diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskFeatureToggleTestHelper.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskFeatureToggleTestHelper.java new file mode 100644 index 00000000000..4410ebc40ea --- /dev/null +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskFeatureToggleTestHelper.java @@ -0,0 +1,34 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.manager.peertask; + +import java.lang.reflect.Field; + +import org.junit.platform.commons.util.ReflectionUtils; + +public class PeerTaskFeatureToggleTestHelper { + + public static void setPeerTaskFeatureToggle(final boolean usePeerTaskSystem) + throws IllegalAccessException { + Field usePeerTaskSystemField = + ReflectionUtils.findFields( + PeerTaskFeatureToggle.class, + (f) -> f.getName().equals("USE_PEER_TASK_SYSTEM"), + ReflectionUtils.HierarchyTraversalMode.TOP_DOWN) + .getFirst(); + usePeerTaskSystemField.setAccessible(true); + usePeerTaskSystemField.set(null, usePeerTaskSystem); + } +} diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTaskTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTaskTest.java new file mode 100644 index 00000000000..131026a2760 --- /dev/null +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTaskTest.java @@ -0,0 +1,152 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.manager.peertask.task; + +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.TransactionReceipt; +import org.hyperledger.besu.ethereum.eth.EthProtocol; +import org.hyperledger.besu.ethereum.eth.manager.peertask.InvalidPeerTaskResponseException; +import org.hyperledger.besu.ethereum.eth.messages.EthPV63; +import org.hyperledger.besu.ethereum.eth.messages.GetReceiptsMessage; +import org.hyperledger.besu.ethereum.eth.messages.ReceiptsMessage; +import org.hyperledger.besu.ethereum.mainnet.BodyValidator; +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import org.apache.commons.lang3.StringUtils; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +public class GetReceiptsFromPeerTaskTest { + + @Test + public void testGetSubProtocol() { + GetReceiptsFromPeerTask task = new GetReceiptsFromPeerTask(Collections.emptyList(), null); + Assertions.assertEquals(EthProtocol.NAME, task.getSubProtocol()); + } + + @Test + public void testGetRequiredBlockNumber() { + GetReceiptsFromPeerTask task = + new GetReceiptsFromPeerTask( + List.of(mockBlockHeader(1), mockBlockHeader(2), mockBlockHeader(3)), null); + Assertions.assertEquals(3, task.getRequiredBlockNumber()); + } + + @Test + public void testGetRequestMessage() { + GetReceiptsFromPeerTask task = + new GetReceiptsFromPeerTask( + List.of(mockBlockHeader(1), mockBlockHeader(2), mockBlockHeader(3)), null); + + MessageData messageData = task.getRequestMessage(); + GetReceiptsMessage getReceiptsMessage = GetReceiptsMessage.readFrom(messageData); + + Assertions.assertEquals(EthPV63.GET_RECEIPTS, getReceiptsMessage.getCode()); + Iterable hashesInMessage = getReceiptsMessage.hashes(); + List expectedHashes = + List.of( + Hash.fromHexString(StringUtils.repeat("00", 31) + "11"), + Hash.fromHexString(StringUtils.repeat("00", 31) + "21"), + Hash.fromHexString(StringUtils.repeat("00", 31) + "31")); + List actualHashes = new ArrayList<>(); + hashesInMessage.forEach(actualHashes::add); + + Assertions.assertEquals(3, actualHashes.size()); + Assertions.assertEquals( + expectedHashes.stream().sorted().toList(), actualHashes.stream().sorted().toList()); + } + + @Test + public void testParseResponseWithNullResponseMessage() { + GetReceiptsFromPeerTask task = new GetReceiptsFromPeerTask(Collections.emptyList(), null); + Assertions.assertThrows(InvalidPeerTaskResponseException.class, () -> task.parseResponse(null)); + } + + @Test + public void testParseResponseForInvalidResponse() throws InvalidPeerTaskResponseException { + GetReceiptsFromPeerTask task = + new GetReceiptsFromPeerTask( + List.of(mockBlockHeader(1), mockBlockHeader(2), mockBlockHeader(3)), null); + ReceiptsMessage receiptsMessage = + ReceiptsMessage.create( + List.of( + List.of(new TransactionReceipt(1, 123, Collections.emptyList(), Optional.empty())), + List.of(new TransactionReceipt(1, 456, Collections.emptyList(), Optional.empty())), + List.of(new TransactionReceipt(1, 789, Collections.emptyList(), Optional.empty())), + List.of( + new TransactionReceipt(1, 101112, Collections.emptyList(), Optional.empty())))); + + Assertions.assertThrows( + InvalidPeerTaskResponseException.class, () -> task.parseResponse(receiptsMessage)); + } + + @Test + public void testParseResponse() throws InvalidPeerTaskResponseException { + BodyValidator bodyValidator = Mockito.mock(BodyValidator.class); + BlockHeader blockHeader1 = mockBlockHeader(1); + BlockHeader blockHeader2 = mockBlockHeader(2); + BlockHeader blockHeader3 = mockBlockHeader(3); + + GetReceiptsFromPeerTask task = + new GetReceiptsFromPeerTask( + List.of(blockHeader1, blockHeader2, blockHeader3), bodyValidator); + + TransactionReceipt receiptForBlock1 = + new TransactionReceipt(1, 123, Collections.emptyList(), Optional.empty()); + TransactionReceipt receiptForBlock2 = + new TransactionReceipt(1, 456, Collections.emptyList(), Optional.empty()); + TransactionReceipt receiptForBlock3 = + new TransactionReceipt(1, 789, Collections.emptyList(), Optional.empty()); + ReceiptsMessage receiptsMessage = + ReceiptsMessage.create( + List.of( + List.of(receiptForBlock1), List.of(receiptForBlock2), List.of(receiptForBlock3))); + + Mockito.when(bodyValidator.receiptsRoot(List.of(receiptForBlock1))) + .thenReturn(Hash.fromHexString(StringUtils.repeat("00", 31) + "12")); + Mockito.when(bodyValidator.receiptsRoot(List.of(receiptForBlock2))) + .thenReturn(Hash.fromHexString(StringUtils.repeat("00", 31) + "22")); + Mockito.when(bodyValidator.receiptsRoot(List.of(receiptForBlock3))) + .thenReturn(Hash.fromHexString(StringUtils.repeat("00", 31) + "32")); + + Map> resultMap = task.parseResponse(receiptsMessage); + + Assertions.assertEquals(3, resultMap.size()); + Assertions.assertEquals(List.of(receiptForBlock1), resultMap.get(blockHeader1)); + Assertions.assertEquals(List.of(receiptForBlock2), resultMap.get(blockHeader2)); + Assertions.assertEquals(List.of(receiptForBlock3), resultMap.get(blockHeader3)); + } + + private BlockHeader mockBlockHeader(final long blockNumber) { + BlockHeader blockHeader = Mockito.mock(BlockHeader.class); + Mockito.when(blockHeader.getNumber()).thenReturn(blockNumber); + // second to last hex digit indicates the blockNumber, last hex digit indicates the usage of the + // hash + Mockito.when(blockHeader.getHash()) + .thenReturn(Hash.fromHexString(StringUtils.repeat("00", 31) + blockNumber + "1")); + Mockito.when(blockHeader.getReceiptsRoot()) + .thenReturn(Hash.fromHexString(StringUtils.repeat("00", 31) + blockNumber + "2")); + + return blockHeader; + } +} diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java index 43f03100a75..f6b22aa967c 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java @@ -22,13 +22,20 @@ import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; +import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.BlockchainSetupUtil; import org.hyperledger.besu.ethereum.core.Difficulty; +import org.hyperledger.besu.ethereum.core.TransactionReceipt; import org.hyperledger.besu.ethereum.eth.manager.EthContext; import org.hyperledger.besu.ethereum.eth.manager.EthProtocolManager; import org.hyperledger.besu.ethereum.eth.manager.EthProtocolManagerTestUtil; import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; import org.hyperledger.besu.ethereum.eth.manager.RespondingEthPeer; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutor; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutorResponseCode; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutorResult; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskFeatureToggleTestHelper; +import org.hyperledger.besu.ethereum.eth.manager.peertask.task.GetReceiptsFromPeerTask; import org.hyperledger.besu.ethereum.eth.sync.ChainDownloader; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; import org.hyperledger.besu.ethereum.eth.sync.fastsync.FastSyncState; @@ -44,8 +51,15 @@ import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; +import java.lang.reflect.Field; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeoutException; import java.util.stream.Stream; import org.junit.jupiter.api.AfterEach; @@ -55,12 +69,16 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.ArgumentsProvider; import org.junit.jupiter.params.provider.ArgumentsSource; +import org.junit.platform.commons.util.ReflectionUtils; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; public class CheckPointSyncChainDownloaderTest { protected ProtocolSchedule protocolSchedule; protected EthProtocolManager ethProtocolManager; protected EthContext ethContext; + private PeerTaskExecutor peerTaskExecutor; protected ProtocolContext protocolContext; private SyncState syncState; @@ -100,6 +118,7 @@ public void setup(final DataStorageFormat dataStorageFormat) { localBlockchain = localBlockchainSetup.getBlockchain(); otherBlockchainSetup = BlockchainSetupUtil.forTesting(dataStorageFormat); otherBlockchain = otherBlockchainSetup.getBlockchain(); + otherBlockchainSetup.importFirstBlocks(30); protocolSchedule = localBlockchainSetup.getProtocolSchedule(); protocolContext = localBlockchainSetup.getProtocolContext(); ethProtocolManager = @@ -123,6 +142,57 @@ public void setup(final DataStorageFormat dataStorageFormat) { ethContext.getEthPeers(), true, Optional.of(checkpoint)); + + peerTaskExecutor = mock(PeerTaskExecutor.class); + + when(peerTaskExecutor.execute(any(GetReceiptsFromPeerTask.class))) + .thenAnswer( + new Answer>>>() { + @Override + public PeerTaskExecutorResult>> answer( + final InvocationOnMock invocationOnMock) throws Throwable { + GetReceiptsFromPeerTask task = + invocationOnMock.getArgument(0, GetReceiptsFromPeerTask.class); + + return processTask(task); + } + }); + + when(peerTaskExecutor.executeAsync(any(GetReceiptsFromPeerTask.class))) + .thenAnswer( + new Answer< + CompletableFuture< + PeerTaskExecutorResult>>>>() { + @Override + public CompletableFuture< + PeerTaskExecutorResult>>> + answer(final InvocationOnMock invocationOnMock) throws Throwable { + GetReceiptsFromPeerTask task = + invocationOnMock.getArgument(0, GetReceiptsFromPeerTask.class); + + return CompletableFuture.completedFuture(processTask(task)); + } + }); + } + + @SuppressWarnings("unchecked") + private PeerTaskExecutorResult>> processTask( + final GetReceiptsFromPeerTask task) throws IllegalAccessException { + Map> getReceiptsFromPeerTaskResult = new HashMap<>(); + List fields = + ReflectionUtils.findFields( + task.getClass(), + (field) -> field.getName().equals("blockHeaders"), + ReflectionUtils.HierarchyTraversalMode.TOP_DOWN); + fields.forEach((f) -> f.setAccessible(true)); + Collection blockHeaders = (Collection) fields.getFirst().get(task); + blockHeaders.forEach( + (bh) -> + getReceiptsFromPeerTaskResult.put( + bh, otherBlockchain.getTxReceipts(bh.getHash()).get())); + + return new PeerTaskExecutorResult<>( + getReceiptsFromPeerTaskResult, PeerTaskExecutorResponseCode.SUCCESS); } @AfterEach @@ -140,6 +210,7 @@ private ChainDownloader downloader( protocolSchedule, protocolContext, ethContext, + peerTaskExecutor, syncState, new NoOpMetricsSystem(), new FastSyncState(otherBlockchain.getBlockHeader(pivotBlockNumber).get()), @@ -148,9 +219,10 @@ private ChainDownloader downloader( @ParameterizedTest @ArgumentsSource(CheckPointSyncChainDownloaderTestArguments.class) - public void shouldSyncToPivotBlockInMultipleSegments(final DataStorageFormat storageFormat) { + public void shouldSyncToPivotBlockInMultipleSegments(final DataStorageFormat storageFormat) + throws IllegalAccessException { + PeerTaskFeatureToggleTestHelper.setPeerTaskFeatureToggle(false); setup(storageFormat); - otherBlockchainSetup.importFirstBlocks(30); final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, otherBlockchain); @@ -184,9 +256,81 @@ public void shouldSyncToPivotBlockInMultipleSegments(final DataStorageFormat sto @ParameterizedTest @ArgumentsSource(CheckPointSyncChainDownloaderTestArguments.class) - public void shouldSyncToPivotBlockInSingleSegment(final DataStorageFormat storageFormat) { + public void shouldSyncToPivotBlockInSingleSegment(final DataStorageFormat storageFormat) + throws IllegalAccessException { + PeerTaskFeatureToggleTestHelper.setPeerTaskFeatureToggle(false); + setup(storageFormat); + + final RespondingEthPeer peer = + EthProtocolManagerTestUtil.createPeer(ethProtocolManager, otherBlockchain); + final RespondingEthPeer.Responder responder = + RespondingEthPeer.blockchainResponder(otherBlockchain); + + final long pivotBlockNumber = 10; + final SynchronizerConfiguration syncConfig = SynchronizerConfiguration.builder().build(); + ethContext + .getEthPeers() + .streamAvailablePeers() + .forEach( + ethPeer -> { + ethPeer.setCheckpointHeader( + otherBlockchainSetup.getBlocks().get((int) checkpoint.blockNumber()).getHeader()); + }); + final ChainDownloader downloader = downloader(syncConfig, pivotBlockNumber); + final CompletableFuture result = downloader.start(); + + peer.respondWhileOtherThreadsWork(responder, () -> !result.isDone()); + + assertThat(result).isCompleted(); + assertThat(localBlockchain.getChainHeadBlockNumber()).isEqualTo(pivotBlockNumber); + assertThat(localBlockchain.getChainHeadHeader()) + .isEqualTo(otherBlockchain.getBlockHeader(pivotBlockNumber).get()); + } + + @ParameterizedTest + @ArgumentsSource(CheckPointSyncChainDownloaderTestArguments.class) + public void shouldSyncToPivotBlockInMultipleSegmentsWithPeerTaskSystem( + final DataStorageFormat storageFormat) + throws IllegalAccessException, ExecutionException, InterruptedException, TimeoutException { + PeerTaskFeatureToggleTestHelper.setPeerTaskFeatureToggle(true); + setup(storageFormat); + + final RespondingEthPeer peer = + EthProtocolManagerTestUtil.createPeer(ethProtocolManager, otherBlockchain); + final RespondingEthPeer.Responder responder = + RespondingEthPeer.blockchainResponder(otherBlockchain); + + final SynchronizerConfiguration syncConfig = + SynchronizerConfiguration.builder() + .downloaderChainSegmentSize(5) + .downloaderHeadersRequestSize(3) + .build(); + final long pivotBlockNumber = 25; + ethContext + .getEthPeers() + .streamAvailablePeers() + .forEach( + ethPeer -> { + ethPeer.setCheckpointHeader( + otherBlockchainSetup.getBlocks().get((int) checkpoint.blockNumber()).getHeader()); + }); + final ChainDownloader downloader = downloader(syncConfig, pivotBlockNumber); + final CompletableFuture result = downloader.start(); + + peer.respondWhileOtherThreadsWork(responder, () -> !result.isDone()); + + assertThat(result).isCompleted(); + assertThat(localBlockchain.getChainHeadBlockNumber()).isEqualTo(pivotBlockNumber); + assertThat(localBlockchain.getChainHeadHeader()) + .isEqualTo(otherBlockchain.getBlockHeader(pivotBlockNumber).get()); + } + + @ParameterizedTest + @ArgumentsSource(CheckPointSyncChainDownloaderTestArguments.class) + public void shouldSyncToPivotBlockInSingleSegmentWithPeerTaskSystem( + final DataStorageFormat storageFormat) throws IllegalAccessException { + PeerTaskFeatureToggleTestHelper.setPeerTaskFeatureToggle(true); setup(storageFormat); - otherBlockchainSetup.importFirstBlocks(30); final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, otherBlockchain); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStepTest.java index c9cfeda1191..1fb86a82f78 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStepTest.java @@ -30,22 +30,32 @@ import org.hyperledger.besu.ethereum.eth.manager.EthProtocolManager; import org.hyperledger.besu.ethereum.eth.manager.EthProtocolManagerTestUtil; import org.hyperledger.besu.ethereum.eth.manager.RespondingEthPeer; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutor; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutorResponseCode; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutorResult; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskFeatureToggleTestHelper; +import org.hyperledger.besu.ethereum.eth.manager.peertask.task.GetReceiptsFromPeerTask; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPool; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.mockito.Mockito; public class DownloadReceiptsStepTest { private static ProtocolContext protocolContext; private static MutableBlockchain blockchain; + private PeerTaskExecutor peerTaskExecutor; private EthProtocolManager ethProtocolManager; private DownloadReceiptsStep downloadReceiptsStep; @@ -59,6 +69,7 @@ public static void setUpClass() { @BeforeEach public void setUp() { + peerTaskExecutor = mock(PeerTaskExecutor.class); TransactionPool transactionPool = mock(TransactionPool.class); ethProtocolManager = EthProtocolManagerTestUtil.create( @@ -69,11 +80,13 @@ public void setUp() { transactionPool, EthProtocolConfiguration.defaultConfig()); downloadReceiptsStep = - new DownloadReceiptsStep(ethProtocolManager.ethContext(), new NoOpMetricsSystem()); + new DownloadReceiptsStep( + ethProtocolManager.ethContext(), peerTaskExecutor, new NoOpMetricsSystem()); } @Test - public void shouldDownloadReceiptsForBlocks() { + public void shouldDownloadReceiptsForBlocks() throws IllegalAccessException { + PeerTaskFeatureToggleTestHelper.setPeerTaskFeatureToggle(false); final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 1000); final List blocks = asList(block(1), block(2), block(3), block(4)); @@ -90,6 +103,32 @@ public void shouldDownloadReceiptsForBlocks() { blockWithReceipts(4))); } + @Test + public void shouldDownloadReceiptsForBlocksUsingPeerTaskSystem() + throws IllegalAccessException, ExecutionException, InterruptedException { + PeerTaskFeatureToggleTestHelper.setPeerTaskFeatureToggle(true); + + final List blocks = asList(mockBlock(), mockBlock(), mockBlock(), mockBlock()); + Map> receiptsMap = new HashMap<>(); + blocks.forEach( + (b) -> receiptsMap.put(b.getHeader(), List.of(Mockito.mock(TransactionReceipt.class)))); + PeerTaskExecutorResult>> peerTaskResult = + new PeerTaskExecutorResult<>(receiptsMap, PeerTaskExecutorResponseCode.SUCCESS); + Mockito.when(peerTaskExecutor.execute(Mockito.any(GetReceiptsFromPeerTask.class))) + .thenReturn(peerTaskResult); + + final CompletableFuture> result = downloadReceiptsStep.apply(blocks); + + assertThat(result.get().get(0).getBlock()).isEqualTo(blocks.get(0)); + assertThat(result.get().get(0).getReceipts().size()).isEqualTo(1); + assertThat(result.get().get(1).getBlock()).isEqualTo(blocks.get(1)); + assertThat(result.get().get(1).getReceipts().size()).isEqualTo(1); + assertThat(result.get().get(2).getBlock()).isEqualTo(blocks.get(2)); + assertThat(result.get().get(2).getReceipts().size()).isEqualTo(1); + assertThat(result.get().get(3).getBlock()).isEqualTo(blocks.get(3)); + assertThat(result.get().get(3).getReceipts().size()).isEqualTo(1); + } + private Block block(final long number) { final BlockHeader header = blockchain.getBlockHeader(number).get(); return new Block(header, blockchain.getBlockBody(header.getHash()).get()); @@ -100,4 +139,11 @@ private BlockWithReceipts blockWithReceipts(final long number) { final List receipts = blockchain.getTxReceipts(block.getHash()).get(); return new BlockWithReceipts(block, receipts); } + + private Block mockBlock() { + final Block block = Mockito.mock(Block.class); + final BlockHeader blockHeader = Mockito.mock(BlockHeader.class); + Mockito.when(block.getHeader()).thenAnswer((invocationOnMock) -> blockHeader); + return block; + } } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastDownloaderFactoryTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastDownloaderFactoryTest.java index 37ca5be2e99..bc493ebd036 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastDownloaderFactoryTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastDownloaderFactoryTest.java @@ -25,6 +25,7 @@ import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; import org.hyperledger.besu.ethereum.eth.manager.EthContext; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutor; import org.hyperledger.besu.ethereum.eth.sync.PivotBlockSelector; import org.hyperledger.besu.ethereum.eth.sync.SyncMode; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; @@ -71,6 +72,7 @@ public class FastDownloaderFactoryTest { @Mock private ProtocolContext protocolContext; @Mock private MetricsSystem metricsSystem; @Mock private EthContext ethContext; + @Mock private PeerTaskExecutor peerTaskExecutor; @Mock private SyncState syncState; @Mock private Clock clock; @Mock private Path dataDirectory; @@ -114,6 +116,7 @@ public void shouldThrowIfSyncModeChangedWhileFastSyncIncomplete( protocolContext, metricsSystem, ethContext, + peerTaskExecutor, worldStateStorageCoordinator, syncState, clock, @@ -139,6 +142,7 @@ public void shouldNotThrowIfSyncModeChangedWhileFastSyncComplete( protocolContext, metricsSystem, ethContext, + peerTaskExecutor, worldStateStorageCoordinator, syncState, clock, @@ -167,6 +171,7 @@ public void shouldNotThrowWhenFastSyncModeRequested(final DataStorageFormat data protocolContext, metricsSystem, ethContext, + peerTaskExecutor, worldStateStorageCoordinator, syncState, clock, @@ -202,6 +207,7 @@ public void shouldClearWorldStateDuringFastSyncWhenStateQueDirectoryExists( protocolContext, metricsSystem, ethContext, + peerTaskExecutor, worldStateStorageCoordinator, syncState, clock, @@ -239,6 +245,7 @@ public void shouldCrashWhenStateQueueIsNotDirectory(final DataStorageFormat data protocolContext, metricsSystem, ethContext, + peerTaskExecutor, worldStateStorageCoordinator, syncState, clock, diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActionsTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActionsTest.java index 68caf2182c0..6d194cb8e02 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActionsTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActionsTest.java @@ -34,6 +34,7 @@ import org.hyperledger.besu.ethereum.eth.manager.EthProtocolManager; import org.hyperledger.besu.ethereum.eth.manager.EthProtocolManagerTestUtil; import org.hyperledger.besu.ethereum.eth.manager.RespondingEthPeer; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutor; import org.hyperledger.besu.ethereum.eth.peervalidation.PeerValidator; import org.hyperledger.besu.ethereum.eth.sync.PivotBlockSelector; import org.hyperledger.besu.ethereum.eth.sync.SyncMode; @@ -536,6 +537,7 @@ private FastSyncActions createFastSyncActions( protocolSchedule, protocolContext, ethContext, + new PeerTaskExecutor(null, null, null, new NoOpMetricsSystem()), new SyncState(blockchain, ethContext.getEthPeers(), true, Optional.empty()), pivotBlockSelector, new NoOpMetricsSystem()); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloaderTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloaderTest.java index 34014246d28..da82034eaa6 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloaderTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloaderTest.java @@ -29,6 +29,7 @@ import org.hyperledger.besu.ethereum.eth.manager.EthProtocolManagerTestUtil; import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; import org.hyperledger.besu.ethereum.eth.manager.RespondingEthPeer; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutor; import org.hyperledger.besu.ethereum.eth.messages.EthPV62; import org.hyperledger.besu.ethereum.eth.messages.GetBlockHeadersMessage; import org.hyperledger.besu.ethereum.eth.sync.ChainDownloader; @@ -110,6 +111,7 @@ private ChainDownloader downloader( protocolSchedule, protocolContext, ethContext, + new PeerTaskExecutor(null, null, null, new NoOpMetricsSystem()), syncState, new NoOpMetricsSystem(), new FastSyncState(otherBlockchain.getBlockHeader(pivotBlockNumber).get()), diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TestNode.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TestNode.java index c679183b0ff..7f1a3423118 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TestNode.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TestNode.java @@ -44,6 +44,7 @@ import org.hyperledger.besu.ethereum.eth.manager.EthPeers; import org.hyperledger.besu.ethereum.eth.manager.EthProtocolManager; import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerManager; import org.hyperledger.besu.ethereum.eth.sync.ChainHeadTracker; import org.hyperledger.besu.ethereum.eth.sync.SyncMode; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; @@ -196,7 +197,8 @@ public boolean isMessagePermitted(final EnodeURL destinationEnode, final int cod Collections.emptyList(), Optional.empty(), syncConfig, - scheduler); + scheduler, + new PeerManager()); final NetworkRunner networkRunner = NetworkRunner.builder() diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolFactoryTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolFactoryTest.java index 5742637c3e8..c9ef635c0f9 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolFactoryTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolFactoryTest.java @@ -45,6 +45,7 @@ import org.hyperledger.besu.ethereum.eth.manager.EthPeers; import org.hyperledger.besu.ethereum.eth.manager.EthProtocolManager; import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerManager; import org.hyperledger.besu.ethereum.eth.sync.SyncMode; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; @@ -318,7 +319,8 @@ private void setupInitialSyncPhase(final SyncState syncState) { Optional.empty(), mock(SynchronizerConfiguration.class), mock(EthScheduler.class), - mock(ForkIdManager.class)); + mock(ForkIdManager.class), + new PeerManager()); } @Test From 513b74f6d41c5854619b92be88ebd62480c5b8cf Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Thu, 19 Sep 2024 14:41:36 +1000 Subject: [PATCH 016/106] 7311: Move isPeerTaskSystemEnabled to SynchronizerOptions Signed-off-by: Matilda Clerke --- .../java/org/hyperledger/besu/cli/BesuCommand.java | 3 +++ .../besu/cli/options/stable/P2PTLSConfigOptions.java | 10 ---------- .../cli/options/unstable/SynchronizerOptions.java | 11 +++++++++++ 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index ecfc0eaadb2..ef3bb6d6f8d 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -117,6 +117,7 @@ import org.hyperledger.besu.ethereum.core.PrivacyParameters; import org.hyperledger.besu.ethereum.core.VersionMetadata; import org.hyperledger.besu.ethereum.core.plugins.PluginConfiguration; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskFeatureToggle; import org.hyperledger.besu.ethereum.eth.sync.SyncMode; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; import org.hyperledger.besu.ethereum.eth.transactions.ImmutableTransactionPoolConfiguration; @@ -1784,6 +1785,8 @@ private void configure() throws Exception { instantiateSignatureAlgorithmFactory(); + PeerTaskFeatureToggle.initialize(unstableSynchronizerOptions.isPeerTaskSystemEnabled()); + logger.info(generateConfigurationOverview()); logger.info("Security Module: {}", securityModuleName); } diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/P2PTLSConfigOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/P2PTLSConfigOptions.java index a48b177bba3..c3f8c56219f 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/P2PTLSConfigOptions.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/P2PTLSConfigOptions.java @@ -20,7 +20,6 @@ import org.hyperledger.besu.cli.util.CommandLineUtils; import org.hyperledger.besu.ethereum.api.tls.FileBasedPasswordProvider; -import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskFeatureToggle; import org.hyperledger.besu.ethereum.p2p.rlpx.connections.netty.TLSConfiguration; import java.nio.file.Path; @@ -104,13 +103,6 @@ public class P2PTLSConfigOptions { "Whether to send a SNI header in the TLS ClientHello message (default: ${DEFAULT-VALUE})") private final Boolean p2pTlsClientHelloSniHeaderEnabled = false; - @Option( - names = {"--Xpeertask-system-enabled"}, - hidden = true, - description = - "Temporary feature toggle to enable using the new peertask system (default: ${DEFAULT-VALUE})") - private final Boolean isPeerTaskSystemEnabled = false; - /** Default constructor. */ P2PTLSConfigOptions() {} @@ -136,8 +128,6 @@ public Optional p2pTLSConfiguration(final CommandLine commandL "File containing password to unlock keystore is required when p2p TLS is enabled"); } - PeerTaskFeatureToggle.initialize(isPeerTaskSystemEnabled); - return Optional.of( TLSConfiguration.Builder.tlsConfiguration() .withKeyStoreType(p2pTLSKeyStoreType) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/SynchronizerOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/SynchronizerOptions.java index 95bbe0a2b1f..6d293be8d0e 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/SynchronizerOptions.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/SynchronizerOptions.java @@ -314,6 +314,13 @@ public void parseBlockPropagationRange(final String arg) { description = "Snap sync enabled for BFT chains (default: ${DEFAULT-VALUE})") private Boolean snapsyncBftEnabled = SnapSyncConfiguration.DEFAULT_SNAP_SYNC_BFT_ENABLED; + @CommandLine.Option( + names = {"--Xpeertask-system-enabled"}, + hidden = true, + description = + "Temporary feature toggle to enable using the new peertask system (default: ${DEFAULT-VALUE})") + private final Boolean isPeerTaskSystemEnabled = false; + private SynchronizerOptions() {} /** @@ -334,6 +341,10 @@ public boolean isSnapSyncBftEnabled() { return snapsyncBftEnabled; } + public boolean isPeerTaskSystemEnabled() { + return isPeerTaskSystemEnabled; + } + /** * Create synchronizer options. * From 2364ed5a52a94c14fbcac5354f8190ad291ab8d9 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Thu, 19 Sep 2024 14:54:18 +1000 Subject: [PATCH 017/106] 7311: Fix javadoc issue Signed-off-by: Matilda Clerke --- .../besu/cli/options/unstable/SynchronizerOptions.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/SynchronizerOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/SynchronizerOptions.java index 6d293be8d0e..4121f32d214 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/SynchronizerOptions.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/SynchronizerOptions.java @@ -341,6 +341,11 @@ public boolean isSnapSyncBftEnabled() { return snapsyncBftEnabled; } + /** + * Flag to indicate whether the peer task system should be used where available + * + * @return true if the peer task system should be used where available + */ public boolean isPeerTaskSystemEnabled() { return isPeerTaskSystemEnabled; } From 03f6495a545c5720f9d7d5a185c25b55e5242cd5 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Thu, 19 Sep 2024 16:26:18 +1000 Subject: [PATCH 018/106] 7311: Fix javadoc issue Signed-off-by: Matilda Clerke --- .../org/hyperledger/besu/controller/BesuControllerBuilder.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java index f6897fef354..cc9b3eeb18c 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java @@ -835,6 +835,7 @@ private TrieLogPruner createTrieLogPruner( * @param worldStateStorageCoordinator the world state storage * @param protocolContext the protocol context * @param ethContext the eth context + * @param peerTaskExecutor the PeerTaskExecutor * @param syncState the sync state * @param ethProtocolManager the eth protocol manager * @param pivotBlockSelector the pivot block selector @@ -1034,6 +1035,7 @@ protected String getSupportedProtocol() { * @param peerValidators the peer validators * @param mergePeerFilter the merge peer filter * @param forkIdManager the fork id manager + * @param peerManager the PeerManager * @return the eth protocol manager */ protected EthProtocolManager createEthProtocolManager( From 97e5918d67b40a24a7904e7ca62254450ab273ca Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Fri, 20 Sep 2024 10:00:56 +1000 Subject: [PATCH 019/106] 7311: Move PeerTaskFeatureToggleTestHelper to TestUtil and fix RunnerTest Signed-off-by: Matilda Clerke --- .../java/org/hyperledger/besu/RunnerTest.java | 39 ++++++++++++++++++- .../CheckPointSyncChainDownloaderTest.java | 2 +- .../fastsync/DownloadReceiptsStepTest.java | 2 +- .../PeerTaskFeatureToggleTestHelper.java | 3 +- 4 files changed, 41 insertions(+), 5 deletions(-) rename {ethereum/eth/src/test/java/org/hyperledger/besu => testutil/src/main/java/org/hyperledger}/ethereum/eth/manager/peertask/PeerTaskFeatureToggleTestHelper.java (90%) diff --git a/besu/src/test/java/org/hyperledger/besu/RunnerTest.java b/besu/src/test/java/org/hyperledger/besu/RunnerTest.java index 5fefc7ef224..c7c9fc1c98c 100644 --- a/besu/src/test/java/org/hyperledger/besu/RunnerTest.java +++ b/besu/src/test/java/org/hyperledger/besu/RunnerTest.java @@ -49,6 +49,7 @@ import org.hyperledger.besu.ethereum.core.MiningParameters; import org.hyperledger.besu.ethereum.core.PrivacyParameters; import org.hyperledger.besu.ethereum.eth.EthProtocolConfiguration; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskFeatureToggle; import org.hyperledger.besu.ethereum.eth.sync.SyncMode; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; @@ -104,6 +105,7 @@ import okhttp3.Response; import org.apache.tuweni.units.bigints.UInt256; import org.awaitility.Awaitility; +import org.hyperledger.ethereum.eth.manager.peertask.PeerTaskFeatureToggleTestHelper; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -119,7 +121,7 @@ public final class RunnerTest { private Vertx vertx; @BeforeEach - public void initVertx() { + public void initVertx() throws IllegalAccessException { vertx = Vertx.vertx(); } @@ -131,7 +133,18 @@ public void stopVertx() { @TempDir private Path temp; @Test - public void getFixedNodes() { + public void getFixedNodes() throws IllegalAccessException { + PeerTaskFeatureToggleTestHelper.setPeerTaskFeatureToggle(false); + doGetFixedNodes(); + } + + @Test + public void getFixedNodesUsingPeerTaskSystem() throws IllegalAccessException { + PeerTaskFeatureToggleTestHelper.setPeerTaskFeatureToggle(true); + doGetFixedNodes(); + } + + private void doGetFixedNodes() { final EnodeURL staticNode = EnodeURLImpl.fromString( "enode://8f4b88336cc40ef2516d8b27df812e007fb2384a61e93635f1899051311344f3dcdbb49a4fe49a79f66d2f589a9f282e8cc4f1d7381e8ef7e4fcc6b0db578c77@127.0.0.1:30301"); @@ -150,6 +163,17 @@ public void getFixedNodes() { @Test public void fullSyncFromGenesis() throws Exception { + PeerTaskFeatureToggleTestHelper.setPeerTaskFeatureToggle(false); + doFullSyncFromGenesis(); + } + + @Test + public void fullSyncFromGenesisUsingPeerTaskSystem() throws Exception { + PeerTaskFeatureToggleTestHelper.setPeerTaskFeatureToggle(true); + doFullSyncFromGenesis(); + } + + private void doFullSyncFromGenesis() throws Exception { // set merge flag to false, otherwise this test can fail if a merge test runs first MergeConfigOptions.setMergeEnabled(false); @@ -158,6 +182,17 @@ public void fullSyncFromGenesis() throws Exception { @Test public void fastSyncFromGenesis() throws Exception { + PeerTaskFeatureToggleTestHelper.setPeerTaskFeatureToggle(false); + doFastSyncFromGenesis(); + } + + @Test + public void fastSyncFromGenesisUsingPeerTaskSystem() throws Exception { + PeerTaskFeatureToggleTestHelper.setPeerTaskFeatureToggle(true); + doFastSyncFromGenesis(); + } + + private void doFastSyncFromGenesis() throws Exception { // set merge flag to false, otherwise this test can fail if a merge test runs first MergeConfigOptions.setMergeEnabled(false); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java index f6b22aa967c..a1f66217c3a 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java @@ -34,7 +34,7 @@ import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutor; import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutorResponseCode; import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutorResult; -import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskFeatureToggleTestHelper; +import org.hyperledger.ethereum.eth.manager.peertask.PeerTaskFeatureToggleTestHelper; import org.hyperledger.besu.ethereum.eth.manager.peertask.task.GetReceiptsFromPeerTask; import org.hyperledger.besu.ethereum.eth.sync.ChainDownloader; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStepTest.java index 1fb86a82f78..cfaeb722db6 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStepTest.java @@ -33,7 +33,7 @@ import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutor; import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutorResponseCode; import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutorResult; -import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskFeatureToggleTestHelper; +import org.hyperledger.ethereum.eth.manager.peertask.PeerTaskFeatureToggleTestHelper; import org.hyperledger.besu.ethereum.eth.manager.peertask.task.GetReceiptsFromPeerTask; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPool; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskFeatureToggleTestHelper.java b/testutil/src/main/java/org/hyperledger/ethereum/eth/manager/peertask/PeerTaskFeatureToggleTestHelper.java similarity index 90% rename from ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskFeatureToggleTestHelper.java rename to testutil/src/main/java/org/hyperledger/ethereum/eth/manager/peertask/PeerTaskFeatureToggleTestHelper.java index 4410ebc40ea..02f791309fd 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskFeatureToggleTestHelper.java +++ b/testutil/src/main/java/org/hyperledger/ethereum/eth/manager/peertask/PeerTaskFeatureToggleTestHelper.java @@ -12,10 +12,11 @@ * * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.eth.manager.peertask; +package org.hyperledger.ethereum.eth.manager.peertask; import java.lang.reflect.Field; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskFeatureToggle; import org.junit.platform.commons.util.ReflectionUtils; public class PeerTaskFeatureToggleTestHelper { From e0f736d5cf62aeb114d625b710cc4dd6a9097205 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Fri, 20 Sep 2024 10:06:06 +1000 Subject: [PATCH 020/106] 7311: spotless Signed-off-by: Matilda Clerke --- besu/src/test/java/org/hyperledger/besu/RunnerTest.java | 3 +-- .../sync/checkpointsync/CheckPointSyncChainDownloaderTest.java | 2 +- .../ethereum/eth/sync/fastsync/DownloadReceiptsStepTest.java | 2 +- .../eth/manager/peertask/PeerTaskFeatureToggleTestHelper.java | 3 ++- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/besu/src/test/java/org/hyperledger/besu/RunnerTest.java b/besu/src/test/java/org/hyperledger/besu/RunnerTest.java index c7c9fc1c98c..566331196a0 100644 --- a/besu/src/test/java/org/hyperledger/besu/RunnerTest.java +++ b/besu/src/test/java/org/hyperledger/besu/RunnerTest.java @@ -49,7 +49,6 @@ import org.hyperledger.besu.ethereum.core.MiningParameters; import org.hyperledger.besu.ethereum.core.PrivacyParameters; import org.hyperledger.besu.ethereum.eth.EthProtocolConfiguration; -import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskFeatureToggle; import org.hyperledger.besu.ethereum.eth.sync.SyncMode; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; @@ -76,6 +75,7 @@ import org.hyperledger.besu.services.PermissioningServiceImpl; import org.hyperledger.besu.services.RpcEndpointServiceImpl; import org.hyperledger.besu.testutil.TestClock; +import org.hyperledger.ethereum.eth.manager.peertask.PeerTaskFeatureToggleTestHelper; import java.io.IOException; import java.math.BigInteger; @@ -105,7 +105,6 @@ import okhttp3.Response; import org.apache.tuweni.units.bigints.UInt256; import org.awaitility.Awaitility; -import org.hyperledger.ethereum.eth.manager.peertask.PeerTaskFeatureToggleTestHelper; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java index a1f66217c3a..d1a63e50c2c 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java @@ -34,7 +34,6 @@ import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutor; import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutorResponseCode; import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutorResult; -import org.hyperledger.ethereum.eth.manager.peertask.PeerTaskFeatureToggleTestHelper; import org.hyperledger.besu.ethereum.eth.manager.peertask.task.GetReceiptsFromPeerTask; import org.hyperledger.besu.ethereum.eth.sync.ChainDownloader; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; @@ -50,6 +49,7 @@ import org.hyperledger.besu.metrics.SyncDurationMetrics; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; +import org.hyperledger.ethereum.eth.manager.peertask.PeerTaskFeatureToggleTestHelper; import java.lang.reflect.Field; import java.util.Collection; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStepTest.java index cfaeb722db6..1051578415d 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStepTest.java @@ -33,11 +33,11 @@ import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutor; import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutorResponseCode; import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutorResult; -import org.hyperledger.ethereum.eth.manager.peertask.PeerTaskFeatureToggleTestHelper; import org.hyperledger.besu.ethereum.eth.manager.peertask.task.GetReceiptsFromPeerTask; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPool; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; +import org.hyperledger.ethereum.eth.manager.peertask.PeerTaskFeatureToggleTestHelper; import java.util.HashMap; import java.util.List; diff --git a/testutil/src/main/java/org/hyperledger/ethereum/eth/manager/peertask/PeerTaskFeatureToggleTestHelper.java b/testutil/src/main/java/org/hyperledger/ethereum/eth/manager/peertask/PeerTaskFeatureToggleTestHelper.java index 02f791309fd..44a4e37b2eb 100644 --- a/testutil/src/main/java/org/hyperledger/ethereum/eth/manager/peertask/PeerTaskFeatureToggleTestHelper.java +++ b/testutil/src/main/java/org/hyperledger/ethereum/eth/manager/peertask/PeerTaskFeatureToggleTestHelper.java @@ -14,9 +14,10 @@ */ package org.hyperledger.ethereum.eth.manager.peertask; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskFeatureToggle; + import java.lang.reflect.Field; -import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskFeatureToggle; import org.junit.platform.commons.util.ReflectionUtils; public class PeerTaskFeatureToggleTestHelper { From 6e734f95a8c11dd054f94200dc3e6141462c451c Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Fri, 20 Sep 2024 11:32:30 +1000 Subject: [PATCH 021/106] 7311: Remove PeerTaskFeatureToggle in favor of including isPeerTaskSystemEnabled in SynchronizerConfiguration Signed-off-by: Matilda Clerke --- .../org/hyperledger/besu/cli/BesuCommand.java | 3 - .../options/unstable/SynchronizerOptions.java | 2 +- .../peertask/PeerTaskFeatureToggle.java | 59 ------------------- .../eth/sync/SynchronizerConfiguration.java | 18 +++++- 4 files changed, 17 insertions(+), 65 deletions(-) delete mode 100644 ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskFeatureToggle.java diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index 8d2bba300b3..79879538bd5 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -119,7 +119,6 @@ import org.hyperledger.besu.ethereum.core.PrivacyParameters; import org.hyperledger.besu.ethereum.core.VersionMetadata; import org.hyperledger.besu.ethereum.core.plugins.PluginConfiguration; -import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskFeatureToggle; import org.hyperledger.besu.ethereum.eth.sync.SyncMode; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; import org.hyperledger.besu.ethereum.eth.transactions.ImmutableTransactionPoolConfiguration; @@ -1925,8 +1924,6 @@ private void configure() throws Exception { instantiateSignatureAlgorithmFactory(); - PeerTaskFeatureToggle.initialize(unstableSynchronizerOptions.isPeerTaskSystemEnabled()); - logger.info(generateConfigurationOverview()); logger.info("Security Module: {}", securityModuleName); } diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/SynchronizerOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/SynchronizerOptions.java index 4121f32d214..816d9df00a1 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/SynchronizerOptions.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/SynchronizerOptions.java @@ -436,7 +436,7 @@ public SynchronizerConfiguration.Builder toDomainObject() { .isSnapSyncBftEnabled(snapsyncBftEnabled) .build()); builder.checkpointPostMergeEnabled(checkpointPostMergeSyncEnabled); - + builder.isPeerTaskSystemEnabled(isPeerTaskSystemEnabled); return builder; } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskFeatureToggle.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskFeatureToggle.java deleted file mode 100644 index 24cc63b84c4..00000000000 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskFeatureToggle.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright contributors to Hyperledger Besu. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.besu.ethereum.eth.manager.peertask; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Temporary class to allow easy access to the PeerTask feature toggle. This class can be removed - * once we've properly tested the PeerTask system and want to enable it permanently - */ -public class PeerTaskFeatureToggle { - private static final Logger LOGGER = LoggerFactory.getLogger(PeerTaskFeatureToggle.class); - private static Boolean USE_PEER_TASK_SYSTEM = null; - - /** - * Initialize the PeerTaskFeatureToggle with the supplied usePeerTaskSystem Boolean. - * PeerTaskFeatureToggle may only be initialized once! - * - * @param usePeerTaskSystem Boolean indicating whether or not the PeerTask system should be used - */ - public static void initialize(final Boolean usePeerTaskSystem) { - if (USE_PEER_TASK_SYSTEM != null) { - LOGGER.warn( - "PeerTaskFeatureToggle has already been initialized, and cannot be initialized again"); - } else { - USE_PEER_TASK_SYSTEM = usePeerTaskSystem; - } - } - - /** - * Indicates whether or not to use the PeerTask system. PeerTaskFeatureToggle must have been - * initialized before this method can be called! - * - * @return whether or not to use the PeerTask system - * @throws IllegalStateException if the PeerTaskFeatureToggle has not been initialized - */ - public static Boolean usePeerTaskSystem() { - if (USE_PEER_TASK_SYSTEM == null) { - throw new IllegalStateException( - "PeerTaskFeatureToggle has not been initialized, but getUsePeerTaskSystem() has been called"); - } - return USE_PEER_TASK_SYSTEM; - } - - private PeerTaskFeatureToggle() {} -} diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/SynchronizerConfiguration.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/SynchronizerConfiguration.java index d46da85dc48..d72f76c213c 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/SynchronizerConfiguration.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/SynchronizerConfiguration.java @@ -85,6 +85,7 @@ public class SynchronizerConfiguration { private final int maxTrailingPeers; private final long worldStateMinMillisBeforeStalling; private final long propagationManagerGetBlockTimeoutMillis; + private final boolean isPeerTaskSystemEnabled; private SynchronizerConfiguration( final int syncPivotDistance, @@ -108,7 +109,8 @@ private SynchronizerConfiguration( final int computationParallelism, final int maxTrailingPeers, final long propagationManagerGetBlockTimeoutMillis, - final boolean checkpointPostMergeEnabled) { + final boolean checkpointPostMergeEnabled, + final boolean isPeerTaskSystemEnabled) { this.syncPivotDistance = syncPivotDistance; this.fastSyncFullValidationRate = fastSyncFullValidationRate; this.syncMinimumPeerCount = syncMinimumPeerCount; @@ -131,6 +133,7 @@ private SynchronizerConfiguration( this.maxTrailingPeers = maxTrailingPeers; this.propagationManagerGetBlockTimeoutMillis = propagationManagerGetBlockTimeoutMillis; this.checkpointPostMergeEnabled = checkpointPostMergeEnabled; + this.isPeerTaskSystemEnabled = isPeerTaskSystemEnabled; } public static Builder builder() { @@ -256,6 +259,10 @@ public long getPropagationManagerGetBlockTimeoutMillis() { return propagationManagerGetBlockTimeoutMillis; } + public boolean isPeerTaskSystemEnabled() { + return isPeerTaskSystemEnabled; + } + public static class Builder { private SyncMode syncMode = SyncMode.FULL; private int syncMinimumPeerCount = DEFAULT_SYNC_MINIMUM_PEERS; @@ -280,6 +287,7 @@ public static class Builder { DEFAULT_WORLD_STATE_MAX_REQUESTS_WITHOUT_PROGRESS; private long worldStateMinMillisBeforeStalling = DEFAULT_WORLD_STATE_MIN_MILLIS_BEFORE_STALLING; private int worldStateTaskCacheSize = DEFAULT_WORLD_STATE_TASK_CACHE_SIZE; + private boolean isPeerTaskSystemEnabled = false; private long propagationManagerGetBlockTimeoutMillis = DEFAULT_PROPAGATION_MANAGER_GET_BLOCK_TIMEOUT_MILLIS; @@ -406,6 +414,11 @@ public Builder checkpointPostMergeEnabled(final boolean checkpointPostMergeEnabl return this; } + public Builder isPeerTaskSystemEnabled(final boolean isPeerTaskSystemEnabled) { + this.isPeerTaskSystemEnabled = isPeerTaskSystemEnabled; + return this; + } + public SynchronizerConfiguration build() { return new SynchronizerConfiguration( syncPivotDistance, @@ -429,7 +442,8 @@ public SynchronizerConfiguration build() { computationParallelism, maxTrailingPeers, propagationManagerGetBlockTimeoutMillis, - checkpointPostMergeEnabled); + checkpointPostMergeEnabled, + isPeerTaskSystemEnabled); } } } From fc9b3f2517d5834ce744058d4bb4270fa9966a16 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Fri, 20 Sep 2024 14:26:10 +1000 Subject: [PATCH 022/106] 7311: Adjust to the removal of PeerTaskFeatureToggle and use SynchronizerConfiguration to get the toggle instead Signed-off-by: Matilda Clerke --- .../java/org/hyperledger/besu/RunnerTest.java | 48 +++++++------------ .../CheckpointDownloadBlockStep.java | 7 ++- ...CheckpointSyncDownloadPipelineFactory.java | 2 +- .../sync/fastsync/DownloadReceiptsStep.java | 7 ++- .../FastSyncDownloadPipelineFactory.java | 2 +- .../CheckPointSyncChainDownloaderTest.java | 13 +++-- .../fastsync/DownloadReceiptsStepTest.java | 20 +++++--- .../PeerTaskFeatureToggleTestHelper.java | 36 -------------- 8 files changed, 49 insertions(+), 86 deletions(-) delete mode 100644 testutil/src/main/java/org/hyperledger/ethereum/eth/manager/peertask/PeerTaskFeatureToggleTestHelper.java diff --git a/besu/src/test/java/org/hyperledger/besu/RunnerTest.java b/besu/src/test/java/org/hyperledger/besu/RunnerTest.java index 566331196a0..5fbd75f0b33 100644 --- a/besu/src/test/java/org/hyperledger/besu/RunnerTest.java +++ b/besu/src/test/java/org/hyperledger/besu/RunnerTest.java @@ -75,7 +75,6 @@ import org.hyperledger.besu.services.PermissioningServiceImpl; import org.hyperledger.besu.services.RpcEndpointServiceImpl; import org.hyperledger.besu.testutil.TestClock; -import org.hyperledger.ethereum.eth.manager.peertask.PeerTaskFeatureToggleTestHelper; import java.io.IOException; import java.math.BigInteger; @@ -133,17 +132,6 @@ public void stopVertx() { @Test public void getFixedNodes() throws IllegalAccessException { - PeerTaskFeatureToggleTestHelper.setPeerTaskFeatureToggle(false); - doGetFixedNodes(); - } - - @Test - public void getFixedNodesUsingPeerTaskSystem() throws IllegalAccessException { - PeerTaskFeatureToggleTestHelper.setPeerTaskFeatureToggle(true); - doGetFixedNodes(); - } - - private void doGetFixedNodes() { final EnodeURL staticNode = EnodeURLImpl.fromString( "enode://8f4b88336cc40ef2516d8b27df812e007fb2384a61e93635f1899051311344f3dcdbb49a4fe49a79f66d2f589a9f282e8cc4f1d7381e8ef7e4fcc6b0db578c77@127.0.0.1:30301"); @@ -162,43 +150,40 @@ private void doGetFixedNodes() { @Test public void fullSyncFromGenesis() throws Exception { - PeerTaskFeatureToggleTestHelper.setPeerTaskFeatureToggle(false); - doFullSyncFromGenesis(); + // set merge flag to false, otherwise this test can fail if a merge test runs first + MergeConfigOptions.setMergeEnabled(false); + + syncFromGenesis(SyncMode.FULL, getFastSyncGenesis(), false); } @Test public void fullSyncFromGenesisUsingPeerTaskSystem() throws Exception { - PeerTaskFeatureToggleTestHelper.setPeerTaskFeatureToggle(true); - doFullSyncFromGenesis(); - } - - private void doFullSyncFromGenesis() throws Exception { // set merge flag to false, otherwise this test can fail if a merge test runs first MergeConfigOptions.setMergeEnabled(false); - syncFromGenesis(SyncMode.FULL, getFastSyncGenesis()); + syncFromGenesis(SyncMode.FULL, getFastSyncGenesis(), true); } @Test public void fastSyncFromGenesis() throws Exception { - PeerTaskFeatureToggleTestHelper.setPeerTaskFeatureToggle(false); - doFastSyncFromGenesis(); + // set merge flag to false, otherwise this test can fail if a merge test runs first + MergeConfigOptions.setMergeEnabled(false); + + syncFromGenesis(SyncMode.FAST, getFastSyncGenesis(), false); } @Test public void fastSyncFromGenesisUsingPeerTaskSystem() throws Exception { - PeerTaskFeatureToggleTestHelper.setPeerTaskFeatureToggle(true); - doFastSyncFromGenesis(); - } - - private void doFastSyncFromGenesis() throws Exception { // set merge flag to false, otherwise this test can fail if a merge test runs first MergeConfigOptions.setMergeEnabled(false); - syncFromGenesis(SyncMode.FAST, getFastSyncGenesis()); + syncFromGenesis(SyncMode.FAST, getFastSyncGenesis(), true); } - private void syncFromGenesis(final SyncMode mode, final GenesisConfigFile genesisConfig) + private void syncFromGenesis( + final SyncMode mode, + final GenesisConfigFile genesisConfig, + final boolean isPeerTaskSystemEnabled) throws Exception { final Path dataDirAhead = Files.createTempDirectory(temp, "db-ahead"); final Path dbAhead = dataDirAhead.resolve("database"); @@ -206,7 +191,10 @@ private void syncFromGenesis(final SyncMode mode, final GenesisConfigFile genesi final NodeKey aheadDbNodeKey = NodeKeyUtils.createFrom(KeyPairUtil.loadKeyPair(dataDirAhead)); final NodeKey behindDbNodeKey = NodeKeyUtils.generate(); final SynchronizerConfiguration syncConfigAhead = - SynchronizerConfiguration.builder().syncMode(SyncMode.FULL).build(); + SynchronizerConfiguration.builder() + .syncMode(SyncMode.FULL) + .isPeerTaskSystemEnabled(isPeerTaskSystemEnabled) + .build(); final ObservableMetricsSystem noOpMetricsSystem = new NoOpMetricsSystem(); final var miningParameters = MiningParameters.newDefault(); final var dataStorageConfiguration = DataStorageConfiguration.DEFAULT_FOREST_CONFIG; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointDownloadBlockStep.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointDownloadBlockStep.java index f255ec69d98..9b742af9f77 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointDownloadBlockStep.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointDownloadBlockStep.java @@ -23,10 +23,10 @@ import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutor; import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutorResponseCode; import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutorResult; -import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskFeatureToggle; import org.hyperledger.besu.ethereum.eth.manager.peertask.task.GetReceiptsFromPeerTask; import org.hyperledger.besu.ethereum.eth.manager.task.AbstractPeerTask.PeerTaskResult; import org.hyperledger.besu.ethereum.eth.manager.task.GetBlockFromPeerTask; +import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; import org.hyperledger.besu.ethereum.eth.sync.fastsync.checkpoint.Checkpoint; import org.hyperledger.besu.ethereum.mainnet.BodyValidator; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; @@ -43,6 +43,7 @@ public class CheckpointDownloadBlockStep { private final EthContext ethContext; private final PeerTaskExecutor peerTaskExecutor; private final Checkpoint checkpoint; + private final SynchronizerConfiguration synchronizerConfiguration; private final MetricsSystem metricsSystem; public CheckpointDownloadBlockStep( @@ -50,11 +51,13 @@ public CheckpointDownloadBlockStep( final EthContext ethContext, final PeerTaskExecutor peerTaskExecutor, final Checkpoint checkpoint, + final SynchronizerConfiguration synchronizerConfiguration, final MetricsSystem metricsSystem) { this.protocolSchedule = protocolSchedule; this.ethContext = ethContext; this.peerTaskExecutor = peerTaskExecutor; this.checkpoint = checkpoint; + this.synchronizerConfiguration = synchronizerConfiguration; this.metricsSystem = metricsSystem; } @@ -75,7 +78,7 @@ public CompletableFuture> downloadBlock(final Hash h private CompletableFuture> downloadReceipts( final PeerTaskResult peerTaskResult) { final Block block = peerTaskResult.getResult(); - if (PeerTaskFeatureToggle.usePeerTaskSystem()) { + if (synchronizerConfiguration.isPeerTaskSystemEnabled()) { CompletableFuture> futureReceipts = new CompletableFuture<>(); GetReceiptsFromPeerTask task = new GetReceiptsFromPeerTask(List.of(block.getHeader()), new BodyValidator()); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncDownloadPipelineFactory.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncDownloadPipelineFactory.java index 714dfcb4af2..0be10869861 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncDownloadPipelineFactory.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncDownloadPipelineFactory.java @@ -86,7 +86,7 @@ protected Pipeline createDownloadCheckPointPipeline( final CheckpointDownloadBlockStep checkPointDownloadBlockStep = new CheckpointDownloadBlockStep( - protocolSchedule, ethContext, peerTaskExecutor, checkpoint, metricsSystem); + protocolSchedule, ethContext, peerTaskExecutor, checkpoint, syncConfig, metricsSystem); return PipelineBuilder.createPipelineFrom( "fetchCheckpoints", diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java index bd8b0a8a423..a9ef69a86d6 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java @@ -25,8 +25,8 @@ import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutor; import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutorResponseCode; import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutorResult; -import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskFeatureToggle; import org.hyperledger.besu.ethereum.eth.manager.peertask.task.GetReceiptsFromPeerTask; +import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; import org.hyperledger.besu.ethereum.eth.sync.tasks.GetReceiptsForHeadersTask; import org.hyperledger.besu.ethereum.mainnet.BodyValidator; import org.hyperledger.besu.plugin.services.MetricsSystem; @@ -42,14 +42,17 @@ public class DownloadReceiptsStep implements Function, CompletableFuture>> { private final EthContext ethContext; private final PeerTaskExecutor peerTaskExecutor; + private final SynchronizerConfiguration synchronizerConfiguration; private final MetricsSystem metricsSystem; public DownloadReceiptsStep( final EthContext ethContext, final PeerTaskExecutor peerTaskExecutor, + final SynchronizerConfiguration synchronizerConfiguration, final MetricsSystem metricsSystem) { this.ethContext = ethContext; this.peerTaskExecutor = peerTaskExecutor; + this.synchronizerConfiguration = synchronizerConfiguration; this.metricsSystem = metricsSystem; } @@ -57,7 +60,7 @@ public DownloadReceiptsStep( public CompletableFuture> apply(final List blocks) { final List headers = blocks.stream().map(Block::getHeader).collect(toList()); final CompletableFuture>> getReceipts; - if (PeerTaskFeatureToggle.usePeerTaskSystem()) { + if (synchronizerConfiguration.isPeerTaskSystemEnabled()) { GetReceiptsFromPeerTask getReceiptsFromPeerTask = new GetReceiptsFromPeerTask(headers, new BodyValidator()); PeerTaskExecutorResult>> getReceiptsResult = diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloadPipelineFactory.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloadPipelineFactory.java index 63df4ee0cbe..dc56891cef0 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloadPipelineFactory.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloadPipelineFactory.java @@ -144,7 +144,7 @@ public Pipeline createDownloadPipelineForSyncTarget(final SyncT final DownloadBodiesStep downloadBodiesStep = new DownloadBodiesStep(protocolSchedule, ethContext, metricsSystem); final DownloadReceiptsStep downloadReceiptsStep = - new DownloadReceiptsStep(ethContext, peerTaskExecutor, metricsSystem); + new DownloadReceiptsStep(ethContext, peerTaskExecutor, syncConfig, metricsSystem); final ImportBlocksStep importBlockStep = new ImportBlocksStep( protocolSchedule, diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java index d1a63e50c2c..68e1023fbc7 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java @@ -49,7 +49,6 @@ import org.hyperledger.besu.metrics.SyncDurationMetrics; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; -import org.hyperledger.ethereum.eth.manager.peertask.PeerTaskFeatureToggleTestHelper; import java.lang.reflect.Field; import java.util.Collection; @@ -221,7 +220,6 @@ private ChainDownloader downloader( @ArgumentsSource(CheckPointSyncChainDownloaderTestArguments.class) public void shouldSyncToPivotBlockInMultipleSegments(final DataStorageFormat storageFormat) throws IllegalAccessException { - PeerTaskFeatureToggleTestHelper.setPeerTaskFeatureToggle(false); setup(storageFormat); final RespondingEthPeer peer = @@ -233,6 +231,7 @@ public void shouldSyncToPivotBlockInMultipleSegments(final DataStorageFormat sto SynchronizerConfiguration.builder() .downloaderChainSegmentSize(5) .downloaderHeadersRequestSize(3) + .isPeerTaskSystemEnabled(false) .build(); final long pivotBlockNumber = 25; ethContext @@ -258,7 +257,6 @@ public void shouldSyncToPivotBlockInMultipleSegments(final DataStorageFormat sto @ArgumentsSource(CheckPointSyncChainDownloaderTestArguments.class) public void shouldSyncToPivotBlockInSingleSegment(final DataStorageFormat storageFormat) throws IllegalAccessException { - PeerTaskFeatureToggleTestHelper.setPeerTaskFeatureToggle(false); setup(storageFormat); final RespondingEthPeer peer = @@ -267,7 +265,8 @@ public void shouldSyncToPivotBlockInSingleSegment(final DataStorageFormat storag RespondingEthPeer.blockchainResponder(otherBlockchain); final long pivotBlockNumber = 10; - final SynchronizerConfiguration syncConfig = SynchronizerConfiguration.builder().build(); + final SynchronizerConfiguration syncConfig = + SynchronizerConfiguration.builder().isPeerTaskSystemEnabled(false).build(); ethContext .getEthPeers() .streamAvailablePeers() @@ -292,7 +291,6 @@ public void shouldSyncToPivotBlockInSingleSegment(final DataStorageFormat storag public void shouldSyncToPivotBlockInMultipleSegmentsWithPeerTaskSystem( final DataStorageFormat storageFormat) throws IllegalAccessException, ExecutionException, InterruptedException, TimeoutException { - PeerTaskFeatureToggleTestHelper.setPeerTaskFeatureToggle(true); setup(storageFormat); final RespondingEthPeer peer = @@ -304,6 +302,7 @@ public void shouldSyncToPivotBlockInMultipleSegmentsWithPeerTaskSystem( SynchronizerConfiguration.builder() .downloaderChainSegmentSize(5) .downloaderHeadersRequestSize(3) + .isPeerTaskSystemEnabled(true) .build(); final long pivotBlockNumber = 25; ethContext @@ -329,7 +328,6 @@ public void shouldSyncToPivotBlockInMultipleSegmentsWithPeerTaskSystem( @ArgumentsSource(CheckPointSyncChainDownloaderTestArguments.class) public void shouldSyncToPivotBlockInSingleSegmentWithPeerTaskSystem( final DataStorageFormat storageFormat) throws IllegalAccessException { - PeerTaskFeatureToggleTestHelper.setPeerTaskFeatureToggle(true); setup(storageFormat); final RespondingEthPeer peer = @@ -338,7 +336,8 @@ public void shouldSyncToPivotBlockInSingleSegmentWithPeerTaskSystem( RespondingEthPeer.blockchainResponder(otherBlockchain); final long pivotBlockNumber = 10; - final SynchronizerConfiguration syncConfig = SynchronizerConfiguration.builder().build(); + final SynchronizerConfiguration syncConfig = + SynchronizerConfiguration.builder().isPeerTaskSystemEnabled(true).build(); ethContext .getEthPeers() .streamAvailablePeers() diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStepTest.java index 1051578415d..c6f40151609 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStepTest.java @@ -34,10 +34,10 @@ import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutorResponseCode; import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutorResult; import org.hyperledger.besu.ethereum.eth.manager.peertask.task.GetReceiptsFromPeerTask; +import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPool; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; -import org.hyperledger.ethereum.eth.manager.peertask.PeerTaskFeatureToggleTestHelper; import java.util.HashMap; import java.util.List; @@ -57,7 +57,6 @@ public class DownloadReceiptsStepTest { private PeerTaskExecutor peerTaskExecutor; private EthProtocolManager ethProtocolManager; - private DownloadReceiptsStep downloadReceiptsStep; @BeforeAll public static void setUpClass() { @@ -79,14 +78,16 @@ public void setUp() { protocolContext.getWorldStateArchive(), transactionPool, EthProtocolConfiguration.defaultConfig()); - downloadReceiptsStep = - new DownloadReceiptsStep( - ethProtocolManager.ethContext(), peerTaskExecutor, new NoOpMetricsSystem()); } @Test public void shouldDownloadReceiptsForBlocks() throws IllegalAccessException { - PeerTaskFeatureToggleTestHelper.setPeerTaskFeatureToggle(false); + DownloadReceiptsStep downloadReceiptsStep = + new DownloadReceiptsStep( + ethProtocolManager.ethContext(), + peerTaskExecutor, + SynchronizerConfiguration.builder().isPeerTaskSystemEnabled(false).build(), + new NoOpMetricsSystem()); final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 1000); final List blocks = asList(block(1), block(2), block(3), block(4)); @@ -106,7 +107,12 @@ public void shouldDownloadReceiptsForBlocks() throws IllegalAccessException { @Test public void shouldDownloadReceiptsForBlocksUsingPeerTaskSystem() throws IllegalAccessException, ExecutionException, InterruptedException { - PeerTaskFeatureToggleTestHelper.setPeerTaskFeatureToggle(true); + DownloadReceiptsStep downloadReceiptsStep = + new DownloadReceiptsStep( + ethProtocolManager.ethContext(), + peerTaskExecutor, + SynchronizerConfiguration.builder().isPeerTaskSystemEnabled(true).build(), + new NoOpMetricsSystem()); final List blocks = asList(mockBlock(), mockBlock(), mockBlock(), mockBlock()); Map> receiptsMap = new HashMap<>(); diff --git a/testutil/src/main/java/org/hyperledger/ethereum/eth/manager/peertask/PeerTaskFeatureToggleTestHelper.java b/testutil/src/main/java/org/hyperledger/ethereum/eth/manager/peertask/PeerTaskFeatureToggleTestHelper.java deleted file mode 100644 index 44a4e37b2eb..00000000000 --- a/testutil/src/main/java/org/hyperledger/ethereum/eth/manager/peertask/PeerTaskFeatureToggleTestHelper.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright contributors to Hyperledger Besu. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.ethereum.eth.manager.peertask; - -import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskFeatureToggle; - -import java.lang.reflect.Field; - -import org.junit.platform.commons.util.ReflectionUtils; - -public class PeerTaskFeatureToggleTestHelper { - - public static void setPeerTaskFeatureToggle(final boolean usePeerTaskSystem) - throws IllegalAccessException { - Field usePeerTaskSystemField = - ReflectionUtils.findFields( - PeerTaskFeatureToggle.class, - (f) -> f.getName().equals("USE_PEER_TASK_SYSTEM"), - ReflectionUtils.HierarchyTraversalMode.TOP_DOWN) - .getFirst(); - usePeerTaskSystemField.setAccessible(true); - usePeerTaskSystemField.set(null, usePeerTaskSystem); - } -} From 08c66fd916e7e0ddc8d3a6870416503eafb4fc0f Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Fri, 20 Sep 2024 14:49:41 +1000 Subject: [PATCH 023/106] 7311: Reduce timeout in PeerTaskRequestSender to 5s Signed-off-by: Matilda Clerke --- .../ethereum/eth/manager/peertask/PeerTaskRequestSender.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskRequestSender.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskRequestSender.java index 9d9ceed3214..77ff5e7251d 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskRequestSender.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskRequestSender.java @@ -25,7 +25,7 @@ import java.util.concurrent.TimeoutException; public class PeerTaskRequestSender { - private static final long DEFAULT_TIMEOUT_MS = 20_000; + private static final long DEFAULT_TIMEOUT_MS = 5_000; private final long timeoutMs; From 049cae271c69b2b0572215a0babf5b73393cf3f5 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Fri, 20 Sep 2024 15:13:09 +1000 Subject: [PATCH 024/106] 7311: Refactor PeerManager to be an interface Signed-off-by: Matilda Clerke --- .../manager/peertask/DefaultPeerManager.java | 64 +++++++++++++++++++ .../eth/manager/peertask/PeerManager.java | 51 +++++++-------- ...rTest.java => DefaultPeerManagerTest.java} | 6 +- 3 files changed, 89 insertions(+), 32 deletions(-) create mode 100644 ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerManager.java rename ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/{PeerManagerTest.java => DefaultPeerManagerTest.java} (95%) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerManager.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerManager.java new file mode 100644 index 00000000000..9424ee0b48f --- /dev/null +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerManager.java @@ -0,0 +1,64 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.manager.peertask; + +import org.hyperledger.besu.ethereum.eth.manager.EthPeer; +import org.hyperledger.besu.ethereum.p2p.peers.PeerId; + +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.function.Predicate; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This is a simple PeerManager implementation that can be used the default implementation in most + * situations + */ +public class DefaultPeerManager implements PeerManager { + private static final Logger LOG = LoggerFactory.getLogger(DefaultPeerManager.class); + + // use a synchronized map to ensure the map is never modified by multiple threads at once + private final Map ethPeersByPeerId = + Collections.synchronizedMap(new HashMap<>()); + + @Override + public EthPeer getPeer(final Predicate filter) throws NoAvailablePeerException { + LOG.trace("Getting peer from pool of {} peers", ethPeersByPeerId.size()); + return ethPeersByPeerId.values().stream() + .filter(filter) + .max(Comparator.naturalOrder()) + .orElseThrow(NoAvailablePeerException::new); + } + + @Override + public Optional getPeerByPeerId(final PeerId peerId) { + return Optional.ofNullable(ethPeersByPeerId.get(peerId)); + } + + @Override + public void addPeer(final EthPeer ethPeer) { + ethPeersByPeerId.put(ethPeer.getConnection().getPeer(), ethPeer); + } + + @Override + public void removePeer(final PeerId peerId) { + ethPeersByPeerId.remove(peerId); + } +} diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerManager.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerManager.java index fc5bc691b72..14323474a14 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerManager.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerManager.java @@ -17,23 +17,11 @@ import org.hyperledger.besu.ethereum.eth.manager.EthPeer; import org.hyperledger.besu.ethereum.p2p.peers.PeerId; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.Map; import java.util.Optional; import java.util.function.Predicate; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - /** "Manages" the EthPeers for the PeerTaskExecutor */ -public class PeerManager { - private static final Logger LOG = LoggerFactory.getLogger(PeerManager.class); - - // use a synchronized map to ensure the map is never modified by multiple threads at once - private final Map ethPeersByPeerId = - Collections.synchronizedMap(new HashMap<>()); +public interface PeerManager { /** * Gets the highest reputation peer matching the supplies filter @@ -42,23 +30,28 @@ public class PeerManager { * @return the highest reputation peer matching the supplies filter * @throws NoAvailablePeerException If there are no suitable peers */ - public EthPeer getPeer(final Predicate filter) throws NoAvailablePeerException { - LOG.trace("Getting peer from pool of {} peers", ethPeersByPeerId.size()); - return ethPeersByPeerId.values().stream() - .filter(filter) - .max(Comparator.naturalOrder()) - .orElseThrow(NoAvailablePeerException::new); - } + EthPeer getPeer(final Predicate filter) throws NoAvailablePeerException; - public Optional getPeerByPeerId(final PeerId peerId) { - return Optional.ofNullable(ethPeersByPeerId.get(peerId)); - } + /** + * Attempts to get the EthPeer identified by peerId + * + * @param peerId the peerId of the desired EthPeer + * @return An Optional\ containing the EthPeer identified by peerId if present in the + * PeerManager, or empty otherwise + */ + Optional getPeerByPeerId(final PeerId peerId); - public void addPeer(final EthPeer ethPeer) { - ethPeersByPeerId.put(ethPeer.getConnection().getPeer(), ethPeer); - } + /** + * Add the supplied EthPeer to the PeerManager + * + * @param ethPeer the EthPeer to be added to the PeerManager + */ + void addPeer(final EthPeer ethPeer); - public void removePeer(final PeerId peerId) { - ethPeersByPeerId.remove(peerId); - } + /** + * Remove the EthPeer identified by peerId from the PeerManager + * + * @param peerId the PeerId of the EthPeer to be removed from the PeerManager + */ + void removePeer(final PeerId peerId); } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerManagerTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerManagerTest.java similarity index 95% rename from ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerManagerTest.java rename to ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerManagerTest.java index 54e0b88b5f1..5aa04f8f9a8 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerManagerTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerManagerTest.java @@ -28,13 +28,13 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -public class PeerManagerTest { +public class DefaultPeerManagerTest { - public PeerManager peerManager; + public DefaultPeerManager peerManager; @BeforeEach public void beforeTest() { - peerManager = new PeerManager(); + peerManager = new DefaultPeerManager(); } @Test From 5c3a61ae77c6dd8c4acc3c91152df2218e7bd2f1 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Fri, 20 Sep 2024 15:34:03 +1000 Subject: [PATCH 025/106] 7311: Fix up compile errors after merge Signed-off-by: Matilda Clerke --- .../hyperledger/besu/controller/BesuControllerBuilder.java | 3 ++- .../besu/ethereum/eth/manager/EthProtocolManagerTest.java | 4 ++-- .../ethereum/eth/manager/EthProtocolManagerTestUtil.java | 6 +++--- .../besu/ethereum/eth/transactions/TestNode.java | 4 ++-- .../eth/transactions/TransactionPoolFactoryTest.java | 4 ++-- 5 files changed, 11 insertions(+), 10 deletions(-) diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java index cc9b3eeb18c..7fff62db991 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java @@ -55,6 +55,7 @@ import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; import org.hyperledger.besu.ethereum.eth.manager.MergePeerFilter; import org.hyperledger.besu.ethereum.eth.manager.MonitoredExecutors; +import org.hyperledger.besu.ethereum.eth.manager.peertask.DefaultPeerManager; import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerManager; import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutor; import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskRequestSender; @@ -655,7 +656,7 @@ public BesuController build() { } final EthContext ethContext = new EthContext(ethPeers, ethMessages, snapMessages, scheduler); - final PeerManager peerManager = new PeerManager(); + final PeerManager peerManager = new DefaultPeerManager(); ethPeers.streamAllPeers().forEach(peerManager::addPeer); final PeerTaskExecutor peerTaskExecutor = new PeerTaskExecutor( diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManagerTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManagerTest.java index 762600bdd0e..31f66e31d22 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManagerTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManagerTest.java @@ -44,7 +44,7 @@ import org.hyperledger.besu.ethereum.eth.EthProtocolConfiguration; import org.hyperledger.besu.ethereum.eth.EthProtocolVersion; import org.hyperledger.besu.ethereum.eth.manager.MockPeerConnection.PeerSendHandler; -import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerManager; +import org.hyperledger.besu.ethereum.eth.manager.peertask.DefaultPeerManager; import org.hyperledger.besu.ethereum.eth.messages.BlockBodiesMessage; import org.hyperledger.besu.ethereum.eth.messages.BlockHeadersMessage; import org.hyperledger.besu.ethereum.eth.messages.EthPV62; @@ -1245,7 +1245,7 @@ private EthProtocolManager createEthManager( syncConfig, mock(EthScheduler.class), mock(ForkIdManager.class), - new PeerManager())) { + new DefaultPeerManager())) { return ethManager; } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManagerTestUtil.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManagerTestUtil.java index d8b5029d951..b71ffa91331 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManagerTestUtil.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManagerTestUtil.java @@ -30,7 +30,7 @@ import org.hyperledger.besu.ethereum.core.ProtocolScheduleFixture; import org.hyperledger.besu.ethereum.eth.EthProtocol; import org.hyperledger.besu.ethereum.eth.EthProtocolConfiguration; -import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerManager; +import org.hyperledger.besu.ethereum.eth.manager.peertask.DefaultPeerManager; import org.hyperledger.besu.ethereum.eth.manager.snap.SnapProtocolManager; import org.hyperledger.besu.ethereum.eth.peervalidation.PeerValidator; import org.hyperledger.besu.ethereum.eth.sync.ChainHeadTracker; @@ -119,7 +119,7 @@ public static EthProtocolManager create( mock(SynchronizerConfiguration.class), ethScheduler, new ForkIdManager(blockchain, Collections.emptyList(), Collections.emptyList(), false), - new PeerManager()); + new DefaultPeerManager()); } public static EthProtocolManager create( @@ -171,7 +171,7 @@ public static EthProtocolManager create( mock(SynchronizerConfiguration.class), ethScheduler, forkIdManager, - new PeerManager()); + new DefaultPeerManager()); } public static EthProtocolManager create(final Blockchain blockchain) { diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TestNode.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TestNode.java index 7f1a3423118..f8671b675fa 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TestNode.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TestNode.java @@ -44,7 +44,7 @@ import org.hyperledger.besu.ethereum.eth.manager.EthPeers; import org.hyperledger.besu.ethereum.eth.manager.EthProtocolManager; import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; -import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerManager; +import org.hyperledger.besu.ethereum.eth.manager.peertask.DefaultPeerManager; import org.hyperledger.besu.ethereum.eth.sync.ChainHeadTracker; import org.hyperledger.besu.ethereum.eth.sync.SyncMode; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; @@ -198,7 +198,7 @@ public boolean isMessagePermitted(final EnodeURL destinationEnode, final int cod Optional.empty(), syncConfig, scheduler, - new PeerManager()); + new DefaultPeerManager()); final NetworkRunner networkRunner = NetworkRunner.builder() diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolFactoryTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolFactoryTest.java index c9ef635c0f9..f85abc4e963 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolFactoryTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolFactoryTest.java @@ -45,7 +45,7 @@ import org.hyperledger.besu.ethereum.eth.manager.EthPeers; import org.hyperledger.besu.ethereum.eth.manager.EthProtocolManager; import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; -import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerManager; +import org.hyperledger.besu.ethereum.eth.manager.peertask.DefaultPeerManager; import org.hyperledger.besu.ethereum.eth.sync.SyncMode; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; @@ -320,7 +320,7 @@ private void setupInitialSyncPhase(final SyncState syncState) { mock(SynchronizerConfiguration.class), mock(EthScheduler.class), mock(ForkIdManager.class), - new PeerManager()); + new DefaultPeerManager()); } @Test From 84488980f542a6abc520d4995fa0c59a5fa824fa Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Fri, 20 Sep 2024 16:04:33 +1000 Subject: [PATCH 026/106] 7311: Fix MetricsAcceptanceTest Signed-off-by: Matilda Clerke --- .../besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java index fabf88d05d9..b0573755fd8 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java @@ -49,7 +49,7 @@ public PeerTaskExecutor( this.protocolSpecSupplier = protocolSpecSupplier; requestTimer = metricsSystem.createLabelledTimer( - BesuMetricCategory.PEERS, "Peer Task Executor Request Time", "", "Task Class Name"); + BesuMetricCategory.PEERS, "PeerTaskExecutor:RequestTime", "Time taken to send a request", "className"); } public PeerTaskExecutorResult execute(final PeerTask peerTask) { From ab21100042cfc92ebd358b30c597d2eb68eb0030 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Fri, 20 Sep 2024 16:20:29 +1000 Subject: [PATCH 027/106] 7311: Fix MetricsAcceptanceTest Signed-off-by: Matilda Clerke --- .../besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java index b0573755fd8..da9969686c3 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java @@ -49,7 +49,10 @@ public PeerTaskExecutor( this.protocolSpecSupplier = protocolSpecSupplier; requestTimer = metricsSystem.createLabelledTimer( - BesuMetricCategory.PEERS, "PeerTaskExecutor:RequestTime", "Time taken to send a request", "className"); + BesuMetricCategory.PEERS, + "PeerTaskExecutor:RequestTime", + "Time taken to send a request", + "className"); } public PeerTaskExecutorResult execute(final PeerTask peerTask) { From f077206c66e339c8afd12ee8c46dc2a3b8da3fa4 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Wed, 25 Sep 2024 15:38:15 +1000 Subject: [PATCH 028/106] 7311: Fix DownloadReceiptsStep when using peer task system Signed-off-by: Matilda Clerke --- .../CheckpointDownloadBlockStep.java | 4 ++ .../sync/fastsync/DownloadReceiptsStep.java | 63 +++++++++++++------ .../fastsync/DownloadReceiptsStepTest.java | 10 ++- 3 files changed, 55 insertions(+), 22 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointDownloadBlockStep.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointDownloadBlockStep.java index 9b742af9f77..603d4448a67 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointDownloadBlockStep.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointDownloadBlockStep.java @@ -93,6 +93,10 @@ private CompletableFuture> downloadReceipts( .orElseThrow( () -> new IllegalStateException("PeerTask response code was success, but empty")); + if (block.getBody().getTransactions().size() != transactionReceipts.size()) { + throw new IllegalStateException( + "PeerTask response code was success, but incorrect number of receipts returned"); + } BlockWithReceipts blockWithReceipts = new BlockWithReceipts(block, transactionReceipts); futureReceipts.complete(Optional.of(blockWithReceipts)); } else { diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java index a9ef69a86d6..5e0a44cee28 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java @@ -30,9 +30,8 @@ import org.hyperledger.besu.ethereum.eth.sync.tasks.GetReceiptsForHeadersTask; import org.hyperledger.besu.ethereum.mainnet.BodyValidator; import org.hyperledger.besu.plugin.services.MetricsSystem; -import org.hyperledger.besu.util.FutureUtils; -import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; @@ -59,26 +58,41 @@ public DownloadReceiptsStep( @Override public CompletableFuture> apply(final List blocks) { final List headers = blocks.stream().map(Block::getHeader).collect(toList()); - final CompletableFuture>> getReceipts; + final Map> getReceipts; if (synchronizerConfiguration.isPeerTaskSystemEnabled()) { - GetReceiptsFromPeerTask getReceiptsFromPeerTask = - new GetReceiptsFromPeerTask(headers, new BodyValidator()); - PeerTaskExecutorResult>> getReceiptsResult = - peerTaskExecutor.execute(getReceiptsFromPeerTask); - if (getReceiptsResult.getResponseCode() == PeerTaskExecutorResponseCode.SUCCESS - && getReceiptsResult.getResult().isPresent()) { - getReceipts = CompletableFuture.completedFuture(getReceiptsResult.getResult().get()); - } else { - getReceipts = CompletableFuture.completedFuture(Collections.emptyMap()); - } + getReceipts = new HashMap>(); + do { + GetReceiptsFromPeerTask getReceiptsFromPeerTask = + new GetReceiptsFromPeerTask(headers, new BodyValidator()); + PeerTaskExecutorResult>> getReceiptsResult = + peerTaskExecutor.execute(getReceiptsFromPeerTask); + if (getReceiptsResult.getResponseCode() == PeerTaskExecutorResponseCode.SUCCESS + && getReceiptsResult.getResult().isPresent()) { + Map> receiptsResult = + getReceiptsResult.getResult().get(); + receiptsResult + .keySet() + .forEach( + (bh) -> + getReceipts.merge( + bh, + receiptsResult.get(bh), + (initialReceipts, newReceipts) -> { + throw new IllegalStateException( + "Unexpectedly got receipts for block header already populated!"); + })); + } + // remove all the headers we found receipts for + headers.removeAll(getReceipts.keySet()); + // repeat until all headers have receipts + } while (!headers.isEmpty()); + return CompletableFuture.completedFuture(combineBlocksAndReceipts(blocks, getReceipts)); + } else { - getReceipts = GetReceiptsForHeadersTask.forHeaders(ethContext, headers, metricsSystem).run(); + return GetReceiptsForHeadersTask.forHeaders(ethContext, headers, metricsSystem) + .run() + .thenApply((receipts) -> combineBlocksAndReceipts(blocks, receipts)); } - final CompletableFuture> combineWithBlocks = - getReceipts.thenApply( - receiptsByHeader -> combineBlocksAndReceipts(blocks, receiptsByHeader)); - FutureUtils.propagateCancellation(combineWithBlocks, getReceipts); - return combineWithBlocks; } private List combineBlocksAndReceipts( @@ -88,8 +102,17 @@ private List combineBlocksAndReceipts( block -> { final List receipts = receiptsByHeader.getOrDefault(block.getHeader(), emptyList()); + if (block.getBody().getTransactions().size() != receipts.size()) { + throw new IllegalStateException( + "PeerTask response code was success, but incorrect number of receipts returned. Header hash: " + + block.getHeader().getHash() + + ", Transactions: " + + block.getBody().getTransactions().size() + + ", receipts: " + + receipts.size()); + } return new BlockWithReceipts(block, receipts); }) - .collect(toList()); + .toList(); } } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStepTest.java index c6f40151609..4270251e6c8 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStepTest.java @@ -21,10 +21,12 @@ import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; import org.hyperledger.besu.ethereum.core.Block; +import org.hyperledger.besu.ethereum.core.BlockBody; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.BlockWithReceipts; import org.hyperledger.besu.ethereum.core.BlockchainSetupUtil; import org.hyperledger.besu.ethereum.core.ProtocolScheduleFixture; +import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.core.TransactionReceipt; import org.hyperledger.besu.ethereum.eth.EthProtocolConfiguration; import org.hyperledger.besu.ethereum.eth.manager.EthProtocolManager; @@ -81,7 +83,7 @@ public void setUp() { } @Test - public void shouldDownloadReceiptsForBlocks() throws IllegalAccessException { + public void shouldDownloadReceiptsForBlocks() { DownloadReceiptsStep downloadReceiptsStep = new DownloadReceiptsStep( ethProtocolManager.ethContext(), @@ -106,7 +108,7 @@ public void shouldDownloadReceiptsForBlocks() throws IllegalAccessException { @Test public void shouldDownloadReceiptsForBlocksUsingPeerTaskSystem() - throws IllegalAccessException, ExecutionException, InterruptedException { + throws ExecutionException, InterruptedException { DownloadReceiptsStep downloadReceiptsStep = new DownloadReceiptsStep( ethProtocolManager.ethContext(), @@ -150,6 +152,10 @@ private Block mockBlock() { final Block block = Mockito.mock(Block.class); final BlockHeader blockHeader = Mockito.mock(BlockHeader.class); Mockito.when(block.getHeader()).thenAnswer((invocationOnMock) -> blockHeader); + final BlockBody blockBody = Mockito.mock(BlockBody.class); + Mockito.when(block.getBody()).thenAnswer((invocationOnMock) -> blockBody); + Mockito.when(blockBody.getTransactions()) + .thenAnswer((invocationOnMock) -> List.of(Mockito.mock(Transaction.class))); return block; } } From ad86ae6e711eecd8a0499ea4992a9e5a086e3d9c Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Wed, 25 Sep 2024 16:23:15 +1000 Subject: [PATCH 029/106] 7311: Rename PeerManager to PeerSelector Signed-off-by: Matilda Clerke --- ...PeerManager.java => DefaultPeerSelector.java} | 6 +++--- .../{PeerManager.java => PeerSelector.java} | 16 ++++++++-------- .../eth/manager/peertask/PeerTaskExecutor.java | 8 ++++---- ...gerTest.java => DefaultPeerSelectorTest.java} | 16 ++++++++-------- .../manager/peertask/PeerTaskExecutorTest.java | 8 ++++---- 5 files changed, 27 insertions(+), 27 deletions(-) rename ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/{DefaultPeerManager.java => DefaultPeerSelector.java} (91%) rename ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/{PeerManager.java => PeerSelector.java} (80%) rename ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/{DefaultPeerManagerTest.java => DefaultPeerSelectorTest.java} (85%) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerManager.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelector.java similarity index 91% rename from ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerManager.java rename to ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelector.java index 9424ee0b48f..f3999631732 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerManager.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelector.java @@ -28,11 +28,11 @@ import org.slf4j.LoggerFactory; /** - * This is a simple PeerManager implementation that can be used the default implementation in most + * This is a simple PeerSelector implementation that can be used the default implementation in most * situations */ -public class DefaultPeerManager implements PeerManager { - private static final Logger LOG = LoggerFactory.getLogger(DefaultPeerManager.class); +public class DefaultPeerSelector implements PeerSelector { + private static final Logger LOG = LoggerFactory.getLogger(DefaultPeerSelector.class); // use a synchronized map to ensure the map is never modified by multiple threads at once private final Map ethPeersByPeerId = diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerManager.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerSelector.java similarity index 80% rename from ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerManager.java rename to ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerSelector.java index 14323474a14..73af26ec4c3 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerManager.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerSelector.java @@ -20,11 +20,11 @@ import java.util.Optional; import java.util.function.Predicate; -/** "Manages" the EthPeers for the PeerTaskExecutor */ -public interface PeerManager { +/** Selects the EthPeers for the PeerTaskExecutor */ +public interface PeerSelector { /** - * Gets the highest reputation peer matching the supplies filter + * Gets the highest reputation peer matching the supplied filter * * @param filter a filter to match prospective peers with * @return the highest reputation peer matching the supplies filter @@ -37,21 +37,21 @@ public interface PeerManager { * * @param peerId the peerId of the desired EthPeer * @return An Optional\ containing the EthPeer identified by peerId if present in the - * PeerManager, or empty otherwise + * PeerSelector, or empty otherwise */ Optional getPeerByPeerId(final PeerId peerId); /** - * Add the supplied EthPeer to the PeerManager + * Add the supplied EthPeer to the PeerSelector * - * @param ethPeer the EthPeer to be added to the PeerManager + * @param ethPeer the EthPeer to be added to the PeerSelector */ void addPeer(final EthPeer ethPeer); /** - * Remove the EthPeer identified by peerId from the PeerManager + * Remove the EthPeer identified by peerId from the PeerSelector * - * @param peerId the PeerId of the EthPeer to be removed from the PeerManager + * @param peerId the PeerId of the EthPeer to be removed from the PeerSelector */ void removePeer(final PeerId peerId); } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java index fabf88d05d9..ad56987efc6 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java @@ -34,17 +34,17 @@ public class PeerTaskExecutor { private static final long[] WAIT_TIME_BEFORE_RETRY = {0, 20000, 5000}; - private final PeerManager peerManager; + private final PeerSelector peerSelector; private final PeerTaskRequestSender requestSender; private final Supplier protocolSpecSupplier; private final LabelledMetric requestTimer; public PeerTaskExecutor( - final PeerManager peerManager, + final PeerSelector peerSelector, final PeerTaskRequestSender requestSender, final Supplier protocolSpecSupplier, final MetricsSystem metricsSystem) { - this.peerManager = peerManager; + this.peerSelector = peerSelector; this.requestSender = requestSender; this.protocolSpecSupplier = protocolSpecSupplier; requestTimer = @@ -61,7 +61,7 @@ public PeerTaskExecutorResult execute(final PeerTask peerTask) { EthPeer peer; try { peer = - peerManager.getPeer( + peerSelector.getPeer( (candidatePeer) -> isPeerUnused(candidatePeer, usedEthPeers) && (protocolSpecSupplier.get().isPoS() diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerManagerTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelectorTest.java similarity index 85% rename from ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerManagerTest.java rename to ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelectorTest.java index 5aa04f8f9a8..f3b0b27033c 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerManagerTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelectorTest.java @@ -28,31 +28,31 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -public class DefaultPeerManagerTest { +public class DefaultPeerSelectorTest { - public DefaultPeerManager peerManager; + public DefaultPeerSelector peerSelector; @BeforeEach public void beforeTest() { - peerManager = new DefaultPeerManager(); + peerSelector = new DefaultPeerSelector(); } @Test public void testGetPeer() throws NoAvailablePeerException { EthPeer protocol1With5ReputationPeer = createTestPeer(Set.of(Capability.create("capability1", 1)), "protocol1", 5); - peerManager.addPeer(protocol1With5ReputationPeer); + peerSelector.addPeer(protocol1With5ReputationPeer); EthPeer protocol1With4ReputationPeer = createTestPeer(Set.of(Capability.create("capability1", 1)), "protocol1", 4); - peerManager.addPeer(protocol1With4ReputationPeer); + peerSelector.addPeer(protocol1With4ReputationPeer); EthPeer protocol2With50ReputationPeer = createTestPeer(Set.of(Capability.create("capability1", 1)), "protocol2", 50); - peerManager.addPeer(protocol2With50ReputationPeer); + peerSelector.addPeer(protocol2With50ReputationPeer); EthPeer protocol2With4ReputationPeer = createTestPeer(Set.of(Capability.create("capability1", 1)), "protocol2", 4); - peerManager.addPeer(protocol2With4ReputationPeer); + peerSelector.addPeer(protocol2With4ReputationPeer); - EthPeer result = peerManager.getPeer((p) -> p.getProtocolName().equals("protocol1")); + EthPeer result = peerSelector.getPeer((p) -> p.getProtocolName().equals("protocol1")); Assertions.assertSame(protocol1With5ReputationPeer, result); } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java index 2c015425f1d..930f4325b6b 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java @@ -35,7 +35,7 @@ import org.mockito.MockitoAnnotations; public class PeerTaskExecutorTest { - private @Mock PeerManager peerManager; + private @Mock PeerSelector peerSelector; private @Mock PeerTaskRequestSender requestSender; private @Mock ProtocolSpec protocolSpec; private @Mock PeerTask peerTask; @@ -51,7 +51,7 @@ public void beforeTest() { mockCloser = MockitoAnnotations.openMocks(this); peerTaskExecutor = new PeerTaskExecutor( - peerManager, requestSender, () -> protocolSpec, new NoOpMetricsSystem()); + peerSelector, requestSender, () -> protocolSpec, new NoOpMetricsSystem()); } @AfterEach @@ -205,7 +205,7 @@ public void testExecuteWithNoPeerTaskBehaviorsAndSuccessFlow() String subprotocol = "subprotocol"; Object responseObject = new Object(); - Mockito.when(peerManager.getPeer(Mockito.any(Predicate.class))).thenReturn(ethPeer); + Mockito.when(peerSelector.getPeer(Mockito.any(Predicate.class))).thenReturn(ethPeer); Mockito.when(peerTask.getRequestMessage()).thenReturn(requestMessageData); Mockito.when(peerTask.getPeerTaskBehaviors()).thenReturn(Collections.emptyList()); @@ -238,7 +238,7 @@ public void testExecuteWithPeerSwitchingAndSuccessFlow() int requestMessageDataCode = 123; EthPeer peer2 = Mockito.mock(EthPeer.class); - Mockito.when(peerManager.getPeer(Mockito.any(Predicate.class))) + Mockito.when(peerSelector.getPeer(Mockito.any(Predicate.class))) .thenReturn(ethPeer) .thenReturn(peer2); From 38f04ab5a1be4d130b3423703cec2779040a36ee Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Wed, 25 Sep 2024 16:33:04 +1000 Subject: [PATCH 030/106] 7311: Reword PeerSelector javadoc to avoid implementation details Signed-off-by: Matilda Clerke --- .../ethereum/eth/manager/peertask/DefaultPeerSelector.java | 7 +++++++ .../besu/ethereum/eth/manager/peertask/PeerSelector.java | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelector.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelector.java index f3999631732..5d32a37bc85 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelector.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelector.java @@ -38,6 +38,13 @@ public class DefaultPeerSelector implements PeerSelector { private final Map ethPeersByPeerId = Collections.synchronizedMap(new HashMap<>()); + /** + * Gets the highest reputation peer matching the supplied filter + * + * @param filter a filter to match prospective peers with + * @return the highest reputation peer matching the supplies filter + * @throws NoAvailablePeerException If there are no suitable peers + */ @Override public EthPeer getPeer(final Predicate filter) throws NoAvailablePeerException { LOG.trace("Getting peer from pool of {} peers", ethPeersByPeerId.size()); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerSelector.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerSelector.java index 73af26ec4c3..3f5589f93b2 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerSelector.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerSelector.java @@ -24,10 +24,10 @@ public interface PeerSelector { /** - * Gets the highest reputation peer matching the supplied filter + * Gets a peer matching the supplied filter * * @param filter a filter to match prospective peers with - * @return the highest reputation peer matching the supplies filter + * @return a peer matching the supplied filter * @throws NoAvailablePeerException If there are no suitable peers */ EthPeer getPeer(final Predicate filter) throws NoAvailablePeerException; From 6de3fb366a8b920bd1a830b43ff6a6af96e53315 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Wed, 25 Sep 2024 16:36:51 +1000 Subject: [PATCH 031/106] 7311: Use ConcurrentHashMap in DefaultPeerSelector Signed-off-by: Matilda Clerke --- .../ethereum/eth/manager/peertask/DefaultPeerSelector.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelector.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelector.java index 5d32a37bc85..37a6fd1895c 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelector.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelector.java @@ -22,6 +22,7 @@ import java.util.HashMap; import java.util.Map; import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; import java.util.function.Predicate; import org.slf4j.Logger; @@ -34,9 +35,7 @@ public class DefaultPeerSelector implements PeerSelector { private static final Logger LOG = LoggerFactory.getLogger(DefaultPeerSelector.class); - // use a synchronized map to ensure the map is never modified by multiple threads at once - private final Map ethPeersByPeerId = - Collections.synchronizedMap(new HashMap<>()); + private final Map ethPeersByPeerId = new ConcurrentHashMap<>(); /** * Gets the highest reputation peer matching the supplied filter From da9cd438aed5f84954ebe66c640c7575d8cbc021 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Wed, 25 Sep 2024 16:40:10 +1000 Subject: [PATCH 032/106] 7311: Reword trace log in DefaultPeerSelector Signed-off-by: Matilda Clerke --- .../besu/ethereum/eth/manager/peertask/DefaultPeerSelector.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelector.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelector.java index 37a6fd1895c..be6eed7fee7 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelector.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelector.java @@ -46,7 +46,7 @@ public class DefaultPeerSelector implements PeerSelector { */ @Override public EthPeer getPeer(final Predicate filter) throws NoAvailablePeerException { - LOG.trace("Getting peer from pool of {} peers", ethPeersByPeerId.size()); + LOG.trace("Finding peer from pool of {} peers", ethPeersByPeerId.size()); return ethPeersByPeerId.values().stream() .filter(filter) .max(Comparator.naturalOrder()) From ce7d24582c1a443050003525abf92c8d80521581 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Wed, 25 Sep 2024 16:40:28 +1000 Subject: [PATCH 033/106] 7311: Remove unused imports Signed-off-by: Matilda Clerke --- .../besu/ethereum/eth/manager/peertask/DefaultPeerSelector.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelector.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelector.java index be6eed7fee7..c334773d168 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelector.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelector.java @@ -17,9 +17,7 @@ import org.hyperledger.besu.ethereum.eth.manager.EthPeer; import org.hyperledger.besu.ethereum.p2p.peers.PeerId; -import java.util.Collections; import java.util.Comparator; -import java.util.HashMap; import java.util.Map; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; From c9eb22e614ca5ca2a15d149042a0806b129fedde Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Wed, 25 Sep 2024 16:51:37 +1000 Subject: [PATCH 034/106] 7311: Use a 1 second delay between retries in PeerTaskExecutor to match old implementation Signed-off-by: Matilda Clerke --- .../ethereum/eth/manager/peertask/PeerTaskExecutor.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java index ad56987efc6..894568b8b72 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java @@ -32,7 +32,6 @@ /** Manages the execution of PeerTasks, respecting their PeerTaskBehavior */ public class PeerTaskExecutor { - private static final long[] WAIT_TIME_BEFORE_RETRY = {0, 20000, 5000}; private final PeerSelector peerSelector; private final PeerTaskRequestSender requestSender; @@ -123,7 +122,7 @@ public PeerTaskExecutorResult executeAgainstPeer( } while (--triesRemaining > 0 && executorResult.getResponseCode() != PeerTaskExecutorResponseCode.SUCCESS && executorResult.getResponseCode() != PeerTaskExecutorResponseCode.PEER_DISCONNECTED - && sleepBetweenRetries(WAIT_TIME_BEFORE_RETRY[triesRemaining])); + && sleepBetweenRetries()); return executorResult; } @@ -133,9 +132,10 @@ public CompletableFuture> executeAgainstPeerAsync( return CompletableFuture.supplyAsync(() -> executeAgainstPeer(peerTask, peer)); } - private boolean sleepBetweenRetries(final long sleepTime) { + private boolean sleepBetweenRetries() { try { - Thread.sleep(sleepTime); + //sleep for 1 second to match implemented wait between retries in AbstractRetryingPeerTask + Thread.sleep(1000); return true; } catch (InterruptedException e) { return false; From e2fda731928109306b85e0bd7ff5f84672d99e58 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Wed, 25 Sep 2024 16:54:15 +1000 Subject: [PATCH 035/106] 7311: Add testGetPeerButNoPeerMatchesFilter to DefaultPeerSelectorTest Signed-off-by: Matilda Clerke --- .../peertask/DefaultPeerSelectorTest.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelectorTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelectorTest.java index f3b0b27033c..bc92da6fb00 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelectorTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelectorTest.java @@ -57,6 +57,24 @@ public void testGetPeer() throws NoAvailablePeerException { Assertions.assertSame(protocol1With5ReputationPeer, result); } + @Test + public void testGetPeerButNoPeerMatchesFilter() throws NoAvailablePeerException { + EthPeer protocol1With5ReputationPeer = + createTestPeer(Set.of(Capability.create("capability1", 1)), "protocol1", 5); + peerSelector.addPeer(protocol1With5ReputationPeer); + EthPeer protocol1With4ReputationPeer = + createTestPeer(Set.of(Capability.create("capability1", 1)), "protocol1", 4); + peerSelector.addPeer(protocol1With4ReputationPeer); + EthPeer protocol2With50ReputationPeer = + createTestPeer(Set.of(Capability.create("capability1", 1)), "protocol2", 50); + peerSelector.addPeer(protocol2With50ReputationPeer); + EthPeer protocol2With4ReputationPeer = + createTestPeer(Set.of(Capability.create("capability1", 1)), "protocol2", 4); + peerSelector.addPeer(protocol2With4ReputationPeer); + + Assertions.assertThrows(NoAvailablePeerException.class, () -> peerSelector.getPeer((p) -> p.getProtocolName().equals("fake protocol"))); + } + private EthPeer createTestPeer( final Set connectionCapabilities, final String protocolName, From 608feceff4c7105209f8003e8e0eeab0d807b2e0 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Wed, 25 Sep 2024 16:54:54 +1000 Subject: [PATCH 036/106] 7311: Add testGetPeerButNoPeerMatchesFilter to DefaultPeerSelectorTest Signed-off-by: Matilda Clerke --- .../ethereum/eth/manager/peertask/DefaultPeerSelectorTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelectorTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelectorTest.java index bc92da6fb00..16d53f474bc 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelectorTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelectorTest.java @@ -58,7 +58,7 @@ public void testGetPeer() throws NoAvailablePeerException { } @Test - public void testGetPeerButNoPeerMatchesFilter() throws NoAvailablePeerException { + public void testGetPeerButNoPeerMatchesFilter() { EthPeer protocol1With5ReputationPeer = createTestPeer(Set.of(Capability.create("capability1", 1)), "protocol1", 5); peerSelector.addPeer(protocol1With5ReputationPeer); From 2d0780055baeba3922e4b00c173c85c43f843b7b Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Wed, 25 Sep 2024 16:59:02 +1000 Subject: [PATCH 037/106] 7311: spotless Signed-off-by: Matilda Clerke --- .../eth/manager/peertask/PeerTaskExecutor.java | 2 +- .../manager/peertask/DefaultPeerSelectorTest.java | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java index 894568b8b72..15833989268 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java @@ -134,7 +134,7 @@ public CompletableFuture> executeAgainstPeerAsync( private boolean sleepBetweenRetries() { try { - //sleep for 1 second to match implemented wait between retries in AbstractRetryingPeerTask + // sleep for 1 second to match implemented wait between retries in AbstractRetryingPeerTask Thread.sleep(1000); return true; } catch (InterruptedException e) { diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelectorTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelectorTest.java index 16d53f474bc..8913b80a457 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelectorTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelectorTest.java @@ -60,19 +60,21 @@ public void testGetPeer() throws NoAvailablePeerException { @Test public void testGetPeerButNoPeerMatchesFilter() { EthPeer protocol1With5ReputationPeer = - createTestPeer(Set.of(Capability.create("capability1", 1)), "protocol1", 5); + createTestPeer(Set.of(Capability.create("capability1", 1)), "protocol1", 5); peerSelector.addPeer(protocol1With5ReputationPeer); EthPeer protocol1With4ReputationPeer = - createTestPeer(Set.of(Capability.create("capability1", 1)), "protocol1", 4); + createTestPeer(Set.of(Capability.create("capability1", 1)), "protocol1", 4); peerSelector.addPeer(protocol1With4ReputationPeer); EthPeer protocol2With50ReputationPeer = - createTestPeer(Set.of(Capability.create("capability1", 1)), "protocol2", 50); + createTestPeer(Set.of(Capability.create("capability1", 1)), "protocol2", 50); peerSelector.addPeer(protocol2With50ReputationPeer); EthPeer protocol2With4ReputationPeer = - createTestPeer(Set.of(Capability.create("capability1", 1)), "protocol2", 4); + createTestPeer(Set.of(Capability.create("capability1", 1)), "protocol2", 4); peerSelector.addPeer(protocol2With4ReputationPeer); - Assertions.assertThrows(NoAvailablePeerException.class, () -> peerSelector.getPeer((p) -> p.getProtocolName().equals("fake protocol"))); + Assertions.assertThrows( + NoAvailablePeerException.class, + () -> peerSelector.getPeer((p) -> p.getProtocolName().equals("fake protocol"))); } private EthPeer createTestPeer( From ad262979632ac98a57bccd95092f1660a27cb372 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Fri, 20 Sep 2024 16:04:33 +1000 Subject: [PATCH 038/106] 7311: Fix MetricsAcceptanceTest Signed-off-by: Matilda Clerke --- .../besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java index 15833989268..30703496dbc 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java @@ -48,7 +48,7 @@ public PeerTaskExecutor( this.protocolSpecSupplier = protocolSpecSupplier; requestTimer = metricsSystem.createLabelledTimer( - BesuMetricCategory.PEERS, "Peer Task Executor Request Time", "", "Task Class Name"); + BesuMetricCategory.PEERS, "PeerTaskExecutor:RequestTime", "Time taken to send a request", "className"); } public PeerTaskExecutorResult execute(final PeerTask peerTask) { From 96c803030c3ae3308ef8c593603a0ede6b0eb2ef Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Fri, 20 Sep 2024 16:20:29 +1000 Subject: [PATCH 039/106] 7311: Fix MetricsAcceptanceTest Signed-off-by: Matilda Clerke --- .../besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java index 30703496dbc..bd6445226b6 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java @@ -48,7 +48,10 @@ public PeerTaskExecutor( this.protocolSpecSupplier = protocolSpecSupplier; requestTimer = metricsSystem.createLabelledTimer( - BesuMetricCategory.PEERS, "PeerTaskExecutor:RequestTime", "Time taken to send a request", "className"); + BesuMetricCategory.PEERS, + "PeerTaskExecutor:RequestTime", + "Time taken to send a request", + "className"); } public PeerTaskExecutorResult execute(final PeerTask peerTask) { From b0f2ed024f59ce6b5846457a90a0beab3a6f2f67 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Thu, 26 Sep 2024 14:42:59 +1000 Subject: [PATCH 040/106] 7311: Modify PeerTaskExecutor metric to include response time from peer Signed-off-by: Matilda Clerke --- .../eth/manager/peertask/PeerTaskExecutor.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java index bd6445226b6..76d5eb38394 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java @@ -50,7 +50,7 @@ public PeerTaskExecutor( metricsSystem.createLabelledTimer( BesuMetricCategory.PEERS, "PeerTaskExecutor:RequestTime", - "Time taken to send a request", + "Time taken to send a request and receive a response", "className"); } @@ -95,13 +95,14 @@ public PeerTaskExecutorResult executeAgainstPeer( do { try { - MessageData responseMessageData; + T result ; try (final OperationTimer.TimingContext timingContext = requestTimer.labels(peerTask.getClass().getSimpleName()).startTimer()) { - responseMessageData = - requestSender.sendRequest(peerTask.getSubProtocol(), requestMessageData, peer); + MessageData responseMessageData = + requestSender.sendRequest(peerTask.getSubProtocol(), requestMessageData, peer); + + result = peerTask.parseResponse(responseMessageData); } - T result = peerTask.parseResponse(responseMessageData); peer.recordUsefulResponse(); executorResult = new PeerTaskExecutorResult<>(result, PeerTaskExecutorResponseCode.SUCCESS); From 598b519c08992266ea19a8332af8ff5adae7a02b Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Thu, 26 Sep 2024 15:30:42 +1000 Subject: [PATCH 041/106] 7311: Use SubProtocol instead of subprotocol name string in PeerTask Signed-off-by: Matilda Clerke --- .../ethereum/eth/manager/peertask/PeerTask.java | 3 ++- .../eth/manager/peertask/PeerTaskExecutor.java | 9 +++++---- .../manager/peertask/PeerTaskRequestSender.java | 5 +++-- .../manager/peertask/PeerTaskExecutorTest.java | 17 ++++++++++------- .../peertask/PeerTaskRequestSenderTest.java | 6 ++++-- 5 files changed, 24 insertions(+), 16 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTask.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTask.java index 244908c9216..7ae78754b22 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTask.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTask.java @@ -15,6 +15,7 @@ package org.hyperledger.besu.ethereum.eth.manager.peertask; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.SubProtocol; import java.util.Collection; @@ -29,7 +30,7 @@ public interface PeerTask { * * @return the SubProtocol used for this PeerTask */ - String getSubProtocol(); + SubProtocol getSubProtocol(); /** * Gets the minimum required block number for a peer to have to successfully execute this task diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java index 76d5eb38394..bab1a1f0bdd 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java @@ -18,6 +18,7 @@ import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; import org.hyperledger.besu.ethereum.p2p.rlpx.connections.PeerConnection; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.SubProtocol; import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.plugin.services.metrics.LabelledMetric; @@ -95,11 +96,11 @@ public PeerTaskExecutorResult executeAgainstPeer( do { try { - T result ; + T result; try (final OperationTimer.TimingContext timingContext = requestTimer.labels(peerTask.getClass().getSimpleName()).startTimer()) { MessageData responseMessageData = - requestSender.sendRequest(peerTask.getSubProtocol(), requestMessageData, peer); + requestSender.sendRequest(peerTask.getSubProtocol(), requestMessageData, peer); result = peerTask.parseResponse(responseMessageData); } @@ -155,7 +156,7 @@ private static boolean isPeerHeightHighEnough(final EthPeer ethPeer, final long return ethPeer.chainState().getEstimatedHeight() >= requiredHeight; } - private static boolean isPeerProtocolSuitable(final EthPeer ethPeer, final String protocol) { - return ethPeer.getProtocolName().equals(protocol); + private static boolean isPeerProtocolSuitable(final EthPeer ethPeer, final SubProtocol protocol) { + return ethPeer.getProtocolName().equals(protocol.getName()); } } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskRequestSender.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskRequestSender.java index 77ff5e7251d..7a597eca8e8 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskRequestSender.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskRequestSender.java @@ -18,6 +18,7 @@ import org.hyperledger.besu.ethereum.eth.manager.RequestManager.ResponseStream; import org.hyperledger.besu.ethereum.p2p.rlpx.connections.PeerConnection; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.SubProtocol; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; @@ -38,13 +39,13 @@ public PeerTaskRequestSender(final long timeoutMs) { } public MessageData sendRequest( - final String subProtocol, final MessageData requestMessageData, final EthPeer ethPeer) + final SubProtocol subProtocol, final MessageData requestMessageData, final EthPeer ethPeer) throws PeerConnection.PeerNotConnected, ExecutionException, InterruptedException, TimeoutException { ResponseStream responseStream = - ethPeer.send(requestMessageData, subProtocol, ethPeer.getConnection()); + ethPeer.send(requestMessageData, subProtocol.getName(), ethPeer.getConnection()); final CompletableFuture responseMessageDataFuture = new CompletableFuture<>(); responseStream.then( (boolean streamClosed, MessageData message, EthPeer peer) -> { diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java index 930f4325b6b..413b68f77c5 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java @@ -18,6 +18,7 @@ import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; import org.hyperledger.besu.ethereum.p2p.rlpx.connections.PeerConnection; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.SubProtocol; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import java.util.Collections; @@ -39,6 +40,7 @@ public class PeerTaskExecutorTest { private @Mock PeerTaskRequestSender requestSender; private @Mock ProtocolSpec protocolSpec; private @Mock PeerTask peerTask; + private @Mock SubProtocol subprotocol; private @Mock MessageData requestMessageData; private @Mock MessageData responseMessageData; private @Mock EthPeer ethPeer; @@ -66,12 +68,13 @@ public void testExecuteAgainstPeerWithNoPeerTaskBehaviorsAndSuccessfulFlow() InterruptedException, TimeoutException, InvalidPeerTaskResponseException { - String subprotocol = "subprotocol"; + Object responseObject = new Object(); Mockito.when(peerTask.getRequestMessage()).thenReturn(requestMessageData); Mockito.when(peerTask.getPeerTaskBehaviors()).thenReturn(Collections.emptyList()); Mockito.when(peerTask.getSubProtocol()).thenReturn(subprotocol); + Mockito.when(subprotocol.getName()).thenReturn("subprotocol"); Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, ethPeer)) .thenReturn(responseMessageData); Mockito.when(peerTask.parseResponse(responseMessageData)).thenReturn(responseObject); @@ -93,7 +96,6 @@ public void testExecuteAgainstPeerWithRetryBehaviorsAndSuccessfulFlowAfterFirstF InterruptedException, TimeoutException, InvalidPeerTaskResponseException { - String subprotocol = "subprotocol"; Object responseObject = new Object(); int requestMessageDataCode = 123; @@ -102,6 +104,7 @@ public void testExecuteAgainstPeerWithRetryBehaviorsAndSuccessfulFlowAfterFirstF .thenReturn(List.of(PeerTaskBehavior.RETRY_WITH_SAME_PEER)); Mockito.when(peerTask.getSubProtocol()).thenReturn(subprotocol); + Mockito.when(subprotocol.getName()).thenReturn("subprotocol"); Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, ethPeer)) .thenThrow(new TimeoutException()) .thenReturn(responseMessageData); @@ -125,11 +128,11 @@ public void testExecuteAgainstPeerWithNoPeerTaskBehaviorsAndPeerNotConnected() ExecutionException, InterruptedException, TimeoutException { - String subprotocol = "subprotocol"; Mockito.when(peerTask.getRequestMessage()).thenReturn(requestMessageData); Mockito.when(peerTask.getPeerTaskBehaviors()).thenReturn(Collections.emptyList()); Mockito.when(peerTask.getSubProtocol()).thenReturn(subprotocol); + Mockito.when(subprotocol.getName()).thenReturn("subprotocol"); Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, ethPeer)) .thenThrow(new PeerConnection.PeerNotConnected("")); @@ -147,12 +150,12 @@ public void testExecuteAgainstPeerWithNoPeerTaskBehaviorsAndTimeoutException() ExecutionException, InterruptedException, TimeoutException { - String subprotocol = "subprotocol"; int requestMessageDataCode = 123; Mockito.when(peerTask.getRequestMessage()).thenReturn(requestMessageData); Mockito.when(peerTask.getPeerTaskBehaviors()).thenReturn(Collections.emptyList()); Mockito.when(peerTask.getSubProtocol()).thenReturn(subprotocol); + Mockito.when(subprotocol.getName()).thenReturn("subprotocol"); Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, ethPeer)) .thenThrow(new TimeoutException()); Mockito.when(requestMessageData.getCode()).thenReturn(requestMessageDataCode); @@ -173,11 +176,11 @@ public void testExecuteAgainstPeerWithNoPeerTaskBehaviorsAndInvalidResponseMessa InterruptedException, TimeoutException, InvalidPeerTaskResponseException { - String subprotocol = "subprotocol"; Mockito.when(peerTask.getRequestMessage()).thenReturn(requestMessageData); Mockito.when(peerTask.getPeerTaskBehaviors()).thenReturn(Collections.emptyList()); Mockito.when(peerTask.getSubProtocol()).thenReturn(subprotocol); + Mockito.when(subprotocol.getName()).thenReturn("subprotocol"); Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, ethPeer)) .thenReturn(responseMessageData); Mockito.when(peerTask.parseResponse(responseMessageData)) @@ -202,7 +205,6 @@ public void testExecuteWithNoPeerTaskBehaviorsAndSuccessFlow() TimeoutException, InvalidPeerTaskResponseException, NoAvailablePeerException { - String subprotocol = "subprotocol"; Object responseObject = new Object(); Mockito.when(peerSelector.getPeer(Mockito.any(Predicate.class))).thenReturn(ethPeer); @@ -210,6 +212,7 @@ public void testExecuteWithNoPeerTaskBehaviorsAndSuccessFlow() Mockito.when(peerTask.getRequestMessage()).thenReturn(requestMessageData); Mockito.when(peerTask.getPeerTaskBehaviors()).thenReturn(Collections.emptyList()); Mockito.when(peerTask.getSubProtocol()).thenReturn(subprotocol); + Mockito.when(subprotocol.getName()).thenReturn("subprotocol"); Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, ethPeer)) .thenReturn(responseMessageData); Mockito.when(peerTask.parseResponse(responseMessageData)).thenReturn(responseObject); @@ -233,7 +236,6 @@ public void testExecuteWithPeerSwitchingAndSuccessFlow() TimeoutException, InvalidPeerTaskResponseException, NoAvailablePeerException { - String subprotocol = "subprotocol"; Object responseObject = new Object(); int requestMessageDataCode = 123; EthPeer peer2 = Mockito.mock(EthPeer.class); @@ -246,6 +248,7 @@ public void testExecuteWithPeerSwitchingAndSuccessFlow() Mockito.when(peerTask.getPeerTaskBehaviors()) .thenReturn(List.of(PeerTaskBehavior.RETRY_WITH_OTHER_PEERS)); Mockito.when(peerTask.getSubProtocol()).thenReturn(subprotocol); + Mockito.when(subprotocol.getName()).thenReturn("subprotocol"); Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, ethPeer)) .thenThrow(new TimeoutException()); Mockito.when(requestMessageData.getCode()).thenReturn(requestMessageDataCode); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskRequestSenderTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskRequestSenderTest.java index 8bc52604db7..4041fb63037 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskRequestSenderTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskRequestSenderTest.java @@ -18,6 +18,7 @@ import org.hyperledger.besu.ethereum.eth.manager.RequestManager; import org.hyperledger.besu.ethereum.p2p.rlpx.connections.PeerConnection; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.SubProtocol; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; @@ -40,7 +41,7 @@ public void beforeTest() { @Test public void testSendRequest() throws PeerConnection.PeerNotConnected, ExecutionException, InterruptedException { - String subprotocol = "subprotocol"; + SubProtocol subprotocol = Mockito.mock(SubProtocol.class); MessageData requestMessageData = Mockito.mock(MessageData.class); MessageData responseMessageData = Mockito.mock(MessageData.class); EthPeer peer = Mockito.mock(EthPeer.class); @@ -49,7 +50,8 @@ public void testSendRequest() Mockito.mock(RequestManager.ResponseStream.class); Mockito.when(peer.getConnection()).thenReturn(peerConnection); - Mockito.when(peer.send(requestMessageData, subprotocol, peerConnection)) + Mockito.when(subprotocol.getName()).thenReturn("subprotocol"); + Mockito.when(peer.send(requestMessageData, "subprotocol", peerConnection)) .thenReturn(responseStream); CompletableFuture actualResponseMessageDataFuture = From bc25b16dcd81f1cbbf94c4ee0486d639f182485c Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Thu, 26 Sep 2024 15:46:58 +1000 Subject: [PATCH 042/106] 7311: rename timing context to ignored to prevent intellij warnings Signed-off-by: Matilda Clerke --- .../besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java index bab1a1f0bdd..fa375d0b626 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java @@ -97,7 +97,7 @@ public PeerTaskExecutorResult executeAgainstPeer( try { T result; - try (final OperationTimer.TimingContext timingContext = + try (final OperationTimer.TimingContext ignored = requestTimer.labels(peerTask.getClass().getSimpleName()).startTimer()) { MessageData responseMessageData = requestSender.sendRequest(peerTask.getSubProtocol(), requestMessageData, peer); From e31bb7003754a8d06d4f4304004ba04514b7b298 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Thu, 26 Sep 2024 16:10:05 +1000 Subject: [PATCH 043/106] 7311: Use constants for number of retries Signed-off-by: Matilda Clerke --- .../eth/manager/peertask/PeerTaskExecutor.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java index fa375d0b626..6fdb89f8c6c 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java @@ -34,6 +34,9 @@ /** Manages the execution of PeerTasks, respecting their PeerTaskBehavior */ public class PeerTaskExecutor { + public static final int RETRIES_WITH_SAME_PEER = 3; + public static final int RETRIES_WITH_OTHER_PEER = 3; + public static final int NO_RETRIES = 1; private final PeerSelector peerSelector; private final PeerTaskRequestSender requestSender; private final Supplier protocolSpecSupplier; @@ -58,7 +61,9 @@ public PeerTaskExecutor( public PeerTaskExecutorResult execute(final PeerTask peerTask) { PeerTaskExecutorResult executorResult; int triesRemaining = - peerTask.getPeerTaskBehaviors().contains(PeerTaskBehavior.RETRY_WITH_OTHER_PEERS) ? 3 : 1; + peerTask.getPeerTaskBehaviors().contains(PeerTaskBehavior.RETRY_WITH_OTHER_PEERS) + ? RETRIES_WITH_OTHER_PEER + : NO_RETRIES; final Collection usedEthPeers = new ArrayList<>(); do { EthPeer peer; @@ -92,7 +97,9 @@ public PeerTaskExecutorResult executeAgainstPeer( MessageData requestMessageData = peerTask.getRequestMessage(); PeerTaskExecutorResult executorResult; int triesRemaining = - peerTask.getPeerTaskBehaviors().contains(PeerTaskBehavior.RETRY_WITH_SAME_PEER) ? 3 : 1; + peerTask.getPeerTaskBehaviors().contains(PeerTaskBehavior.RETRY_WITH_SAME_PEER) + ? RETRIES_WITH_SAME_PEER + : NO_RETRIES; do { try { From 41923d3c40ec303a2125241e13d31aec3101dafb Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Thu, 26 Sep 2024 16:37:28 +1000 Subject: [PATCH 044/106] 7311: Convert PeerTaskExecutorResult to a record Signed-off-by: Matilda Clerke --- .../manager/peertask/PeerTaskExecutor.java | 19 +++++----- .../peertask/PeerTaskExecutorResult.java | 22 +++--------- .../peertask/PeerTaskExecutorTest.java | 36 +++++++++---------- 3 files changed, 33 insertions(+), 44 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java index 6fdb89f8c6c..3187149a0eb 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java @@ -26,6 +26,7 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeoutException; @@ -80,10 +81,10 @@ public PeerTaskExecutorResult execute(final PeerTask peerTask) { executorResult = executeAgainstPeer(peerTask, peer); } catch (NoAvailablePeerException e) { executorResult = - new PeerTaskExecutorResult<>(null, PeerTaskExecutorResponseCode.NO_PEER_AVAILABLE); + new PeerTaskExecutorResult<>(Optional.empty(), PeerTaskExecutorResponseCode.NO_PEER_AVAILABLE); } } while (--triesRemaining > 0 - && executorResult.getResponseCode() != PeerTaskExecutorResponseCode.SUCCESS); + && executorResult.responseCode() != PeerTaskExecutorResponseCode.SUCCESS); return executorResult; } @@ -112,28 +113,28 @@ public PeerTaskExecutorResult executeAgainstPeer( result = peerTask.parseResponse(responseMessageData); } peer.recordUsefulResponse(); - executorResult = new PeerTaskExecutorResult<>(result, PeerTaskExecutorResponseCode.SUCCESS); + executorResult = new PeerTaskExecutorResult<>(Optional.ofNullable(result), PeerTaskExecutorResponseCode.SUCCESS); } catch (PeerConnection.PeerNotConnected e) { executorResult = - new PeerTaskExecutorResult<>(null, PeerTaskExecutorResponseCode.PEER_DISCONNECTED); + new PeerTaskExecutorResult<>(Optional.empty(), PeerTaskExecutorResponseCode.PEER_DISCONNECTED); } catch (InterruptedException | TimeoutException e) { peer.recordRequestTimeout(requestMessageData.getCode()); - executorResult = new PeerTaskExecutorResult<>(null, PeerTaskExecutorResponseCode.TIMEOUT); + executorResult = new PeerTaskExecutorResult<>(Optional.empty(), PeerTaskExecutorResponseCode.TIMEOUT); } catch (InvalidPeerTaskResponseException e) { peer.recordUselessResponse(e.getMessage()); executorResult = - new PeerTaskExecutorResult<>(null, PeerTaskExecutorResponseCode.INVALID_RESPONSE); + new PeerTaskExecutorResult<>(Optional.empty(), PeerTaskExecutorResponseCode.INVALID_RESPONSE); } catch (ExecutionException e) { executorResult = - new PeerTaskExecutorResult<>(null, PeerTaskExecutorResponseCode.INTERNAL_SERVER_ERROR); + new PeerTaskExecutorResult<>(Optional.empty(), PeerTaskExecutorResponseCode.INTERNAL_SERVER_ERROR); } } while (--triesRemaining > 0 - && executorResult.getResponseCode() != PeerTaskExecutorResponseCode.SUCCESS - && executorResult.getResponseCode() != PeerTaskExecutorResponseCode.PEER_DISCONNECTED + && executorResult.responseCode() != PeerTaskExecutorResponseCode.SUCCESS + && executorResult.responseCode() != PeerTaskExecutorResponseCode.PEER_DISCONNECTED && sleepBetweenRetries()); return executorResult; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorResult.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorResult.java index f89bc67f61f..ef528cd2ae5 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorResult.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorResult.java @@ -16,20 +16,8 @@ import java.util.Optional; -public class PeerTaskExecutorResult { - private final Optional result; - private final PeerTaskExecutorResponseCode responseCode; - - public PeerTaskExecutorResult(final T result, final PeerTaskExecutorResponseCode responseCode) { - this.result = Optional.ofNullable(result); - this.responseCode = responseCode; - } - - public Optional getResult() { - return result; - } - - public PeerTaskExecutorResponseCode getResponseCode() { - return responseCode; - } -} +public record PeerTaskExecutorResult ( + Optional result, + PeerTaskExecutorResponseCode responseCode +) +{} diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java index 413b68f77c5..297ce08b05b 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java @@ -84,9 +84,9 @@ public void testExecuteAgainstPeerWithNoPeerTaskBehaviorsAndSuccessfulFlow() Mockito.verify(ethPeer).recordUsefulResponse(); Assertions.assertNotNull(result); - Assertions.assertTrue(result.getResult().isPresent()); - Assertions.assertSame(responseObject, result.getResult().get()); - Assertions.assertEquals(PeerTaskExecutorResponseCode.SUCCESS, result.getResponseCode()); + Assertions.assertTrue(result.result().isPresent()); + Assertions.assertSame(responseObject, result.result().get()); + Assertions.assertEquals(PeerTaskExecutorResponseCode.SUCCESS, result.responseCode()); } @Test @@ -117,9 +117,9 @@ public void testExecuteAgainstPeerWithRetryBehaviorsAndSuccessfulFlowAfterFirstF Mockito.verify(ethPeer).recordUsefulResponse(); Assertions.assertNotNull(result); - Assertions.assertTrue(result.getResult().isPresent()); - Assertions.assertSame(responseObject, result.getResult().get()); - Assertions.assertEquals(PeerTaskExecutorResponseCode.SUCCESS, result.getResponseCode()); + Assertions.assertTrue(result.result().isPresent()); + Assertions.assertSame(responseObject, result.result().get()); + Assertions.assertEquals(PeerTaskExecutorResponseCode.SUCCESS, result.responseCode()); } @Test @@ -139,9 +139,9 @@ public void testExecuteAgainstPeerWithNoPeerTaskBehaviorsAndPeerNotConnected() PeerTaskExecutorResult result = peerTaskExecutor.executeAgainstPeer(peerTask, ethPeer); Assertions.assertNotNull(result); - Assertions.assertTrue(result.getResult().isEmpty()); + Assertions.assertTrue(result.result().isEmpty()); Assertions.assertEquals( - PeerTaskExecutorResponseCode.PEER_DISCONNECTED, result.getResponseCode()); + PeerTaskExecutorResponseCode.PEER_DISCONNECTED, result.responseCode()); } @Test @@ -165,8 +165,8 @@ public void testExecuteAgainstPeerWithNoPeerTaskBehaviorsAndTimeoutException() Mockito.verify(ethPeer).recordRequestTimeout(requestMessageDataCode); Assertions.assertNotNull(result); - Assertions.assertTrue(result.getResult().isEmpty()); - Assertions.assertEquals(PeerTaskExecutorResponseCode.TIMEOUT, result.getResponseCode()); + Assertions.assertTrue(result.result().isEmpty()); + Assertions.assertEquals(PeerTaskExecutorResponseCode.TIMEOUT, result.responseCode()); } @Test @@ -191,9 +191,9 @@ public void testExecuteAgainstPeerWithNoPeerTaskBehaviorsAndInvalidResponseMessa Mockito.verify(ethPeer).recordUselessResponse(null); Assertions.assertNotNull(result); - Assertions.assertTrue(result.getResult().isEmpty()); + Assertions.assertTrue(result.result().isEmpty()); Assertions.assertEquals( - PeerTaskExecutorResponseCode.INVALID_RESPONSE, result.getResponseCode()); + PeerTaskExecutorResponseCode.INVALID_RESPONSE, result.responseCode()); } @Test @@ -222,9 +222,9 @@ public void testExecuteWithNoPeerTaskBehaviorsAndSuccessFlow() Mockito.verify(ethPeer).recordUsefulResponse(); Assertions.assertNotNull(result); - Assertions.assertTrue(result.getResult().isPresent()); - Assertions.assertSame(responseObject, result.getResult().get()); - Assertions.assertEquals(PeerTaskExecutorResponseCode.SUCCESS, result.getResponseCode()); + Assertions.assertTrue(result.result().isPresent()); + Assertions.assertSame(responseObject, result.result().get()); + Assertions.assertEquals(PeerTaskExecutorResponseCode.SUCCESS, result.responseCode()); } @Test @@ -262,8 +262,8 @@ public void testExecuteWithPeerSwitchingAndSuccessFlow() Mockito.verify(peer2).recordUsefulResponse(); Assertions.assertNotNull(result); - Assertions.assertTrue(result.getResult().isPresent()); - Assertions.assertSame(responseObject, result.getResult().get()); - Assertions.assertEquals(PeerTaskExecutorResponseCode.SUCCESS, result.getResponseCode()); + Assertions.assertTrue(result.result().isPresent()); + Assertions.assertSame(responseObject, result.result().get()); + Assertions.assertEquals(PeerTaskExecutorResponseCode.SUCCESS, result.responseCode()); } } From 720f94ef50d4418ee500f69f9865617a0e42a7eb Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Mon, 30 Sep 2024 08:13:56 +1000 Subject: [PATCH 045/106] 7311: Rename PeerTaskBehavior to PeerTaskRetryBehavior Signed-off-by: Matilda Clerke --- .../besu/ethereum/eth/manager/peertask/PeerTask.java | 2 +- .../ethereum/eth/manager/peertask/PeerTaskExecutor.java | 6 +++--- .../{PeerTaskBehavior.java => PeerTaskRetryBehavior.java} | 2 +- .../ethereum/eth/manager/peertask/PeerTaskExecutorTest.java | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) rename ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/{PeerTaskBehavior.java => PeerTaskRetryBehavior.java} (95%) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTask.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTask.java index 7ae78754b22..36bd03531bd 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTask.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTask.java @@ -60,5 +60,5 @@ public interface PeerTask { * * @return the Collection of behaviors this task is expected to exhibit in the PeetTaskExecutor */ - Collection getPeerTaskBehaviors(); + Collection getPeerTaskBehaviors(); } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java index 3187149a0eb..caef72ba76e 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java @@ -32,7 +32,7 @@ import java.util.concurrent.TimeoutException; import java.util.function.Supplier; -/** Manages the execution of PeerTasks, respecting their PeerTaskBehavior */ +/** Manages the execution of PeerTasks, respecting their PeerTaskRetryBehavior */ public class PeerTaskExecutor { public static final int RETRIES_WITH_SAME_PEER = 3; @@ -62,7 +62,7 @@ public PeerTaskExecutor( public PeerTaskExecutorResult execute(final PeerTask peerTask) { PeerTaskExecutorResult executorResult; int triesRemaining = - peerTask.getPeerTaskBehaviors().contains(PeerTaskBehavior.RETRY_WITH_OTHER_PEERS) + peerTask.getPeerTaskBehaviors().contains(PeerTaskRetryBehavior.RETRY_WITH_OTHER_PEERS) ? RETRIES_WITH_OTHER_PEER : NO_RETRIES; final Collection usedEthPeers = new ArrayList<>(); @@ -98,7 +98,7 @@ public PeerTaskExecutorResult executeAgainstPeer( MessageData requestMessageData = peerTask.getRequestMessage(); PeerTaskExecutorResult executorResult; int triesRemaining = - peerTask.getPeerTaskBehaviors().contains(PeerTaskBehavior.RETRY_WITH_SAME_PEER) + peerTask.getPeerTaskBehaviors().contains(PeerTaskRetryBehavior.RETRY_WITH_SAME_PEER) ? RETRIES_WITH_SAME_PEER : NO_RETRIES; do { diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskBehavior.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskRetryBehavior.java similarity index 95% rename from ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskBehavior.java rename to ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskRetryBehavior.java index fba9000a741..53e2def6612 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskBehavior.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskRetryBehavior.java @@ -14,7 +14,7 @@ */ package org.hyperledger.besu.ethereum.eth.manager.peertask; -public enum PeerTaskBehavior { +public enum PeerTaskRetryBehavior { RETRY_WITH_SAME_PEER, RETRY_WITH_OTHER_PEERS } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java index 297ce08b05b..0077c904188 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java @@ -101,7 +101,7 @@ public void testExecuteAgainstPeerWithRetryBehaviorsAndSuccessfulFlowAfterFirstF Mockito.when(peerTask.getRequestMessage()).thenReturn(requestMessageData); Mockito.when(peerTask.getPeerTaskBehaviors()) - .thenReturn(List.of(PeerTaskBehavior.RETRY_WITH_SAME_PEER)); + .thenReturn(List.of(PeerTaskRetryBehavior.RETRY_WITH_SAME_PEER)); Mockito.when(peerTask.getSubProtocol()).thenReturn(subprotocol); Mockito.when(subprotocol.getName()).thenReturn("subprotocol"); @@ -246,7 +246,7 @@ public void testExecuteWithPeerSwitchingAndSuccessFlow() Mockito.when(peerTask.getRequestMessage()).thenReturn(requestMessageData); Mockito.when(peerTask.getPeerTaskBehaviors()) - .thenReturn(List.of(PeerTaskBehavior.RETRY_WITH_OTHER_PEERS)); + .thenReturn(List.of(PeerTaskRetryBehavior.RETRY_WITH_OTHER_PEERS)); Mockito.when(peerTask.getSubProtocol()).thenReturn(subprotocol); Mockito.when(subprotocol.getName()).thenReturn("subprotocol"); Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, ethPeer)) From 7d845b327b42b4f7b37716b199052111e0309116 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Mon, 30 Sep 2024 15:57:23 +1000 Subject: [PATCH 046/106] 7311: Move peer selection logic to PeerSelector Signed-off-by: Matilda Clerke --- .../manager/peertask/DefaultPeerSelector.java | 38 ++++- .../eth/manager/peertask/PeerSelector.java | 23 ++- .../manager/peertask/PeerTaskExecutor.java | 49 +++--- .../peertask/PeerTaskExecutorResult.java | 7 +- .../peertask/DefaultPeerSelectorTest.java | 139 +++++++++++------- .../peertask/PeerTaskExecutorTest.java | 22 +-- 6 files changed, 168 insertions(+), 110 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelector.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelector.java index c334773d168..41d2e9b700b 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelector.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelector.java @@ -15,13 +15,17 @@ package org.hyperledger.besu.ethereum.eth.manager.peertask; import org.hyperledger.besu.ethereum.eth.manager.EthPeer; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; import org.hyperledger.besu.ethereum.p2p.peers.PeerId; +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.SubProtocol; +import java.util.Collection; import java.util.Comparator; import java.util.Map; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.function.Predicate; +import java.util.function.Supplier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,8 +37,13 @@ public class DefaultPeerSelector implements PeerSelector { private static final Logger LOG = LoggerFactory.getLogger(DefaultPeerSelector.class); + private final Supplier protocolSpecSupplier; private final Map ethPeersByPeerId = new ConcurrentHashMap<>(); + public DefaultPeerSelector(final Supplier protocolSpecSupplier) { + this.protocolSpecSupplier = protocolSpecSupplier; + } + /** * Gets the highest reputation peer matching the supplied filter * @@ -42,8 +51,7 @@ public class DefaultPeerSelector implements PeerSelector { * @return the highest reputation peer matching the supplies filter * @throws NoAvailablePeerException If there are no suitable peers */ - @Override - public EthPeer getPeer(final Predicate filter) throws NoAvailablePeerException { + private EthPeer getPeer(final Predicate filter) throws NoAvailablePeerException { LOG.trace("Finding peer from pool of {} peers", ethPeersByPeerId.size()); return ethPeersByPeerId.values().stream() .filter(filter) @@ -51,6 +59,20 @@ public EthPeer getPeer(final Predicate filter) throws NoAvailablePeerEx .orElseThrow(NoAvailablePeerException::new); } + @Override + public EthPeer getPeer( + final Collection usedEthPeers, + final long requiredPeerHeight, + final SubProtocol requiredSubProtocol) + throws NoAvailablePeerException { + return getPeer( + (candidatePeer) -> + isPeerUnused(candidatePeer, usedEthPeers) + && (protocolSpecSupplier.get().isPoS() + || isPeerHeightHighEnough(candidatePeer, requiredPeerHeight)) + && isPeerProtocolSuitable(candidatePeer, requiredSubProtocol)); + } + @Override public Optional getPeerByPeerId(final PeerId peerId) { return Optional.ofNullable(ethPeersByPeerId.get(peerId)); @@ -65,4 +87,16 @@ public void addPeer(final EthPeer ethPeer) { public void removePeer(final PeerId peerId) { ethPeersByPeerId.remove(peerId); } + + private boolean isPeerUnused(final EthPeer ethPeer, final Collection usedEthPeers) { + return !usedEthPeers.contains(ethPeer); + } + + private boolean isPeerHeightHighEnough(final EthPeer ethPeer, final long requiredHeight) { + return ethPeer.chainState().getEstimatedHeight() >= requiredHeight; + } + + private boolean isPeerProtocolSuitable(final EthPeer ethPeer, final SubProtocol protocol) { + return ethPeer.getProtocolName().equals(protocol.getName()); + } } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerSelector.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerSelector.java index 3f5589f93b2..93d98a193b1 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerSelector.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerSelector.java @@ -16,21 +16,28 @@ import org.hyperledger.besu.ethereum.eth.manager.EthPeer; import org.hyperledger.besu.ethereum.p2p.peers.PeerId; +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.SubProtocol; +import java.util.Collection; import java.util.Optional; -import java.util.function.Predicate; /** Selects the EthPeers for the PeerTaskExecutor */ public interface PeerSelector { /** - * Gets a peer matching the supplied filter + * Gets a peer with the requiredPeerHeight (if not PoS), and with the requiredSubProtocol, and + * which is not in the supplied collection of usedEthPeers * - * @param filter a filter to match prospective peers with - * @return a peer matching the supplied filter + * @param usedEthPeers a collection of EthPeers to be excluded from selection because they have + * already been used + * @param requiredPeerHeight the minimum peer height required of the selected peer + * @param requiredSubProtocol the SubProtocol required of the peer + * @return a peer matching the supplied conditions * @throws NoAvailablePeerException If there are no suitable peers */ - EthPeer getPeer(final Predicate filter) throws NoAvailablePeerException; + EthPeer getPeer( + Collection usedEthPeers, long requiredPeerHeight, SubProtocol requiredSubProtocol) + throws NoAvailablePeerException; /** * Attempts to get the EthPeer identified by peerId @@ -39,19 +46,19 @@ public interface PeerSelector { * @return An Optional\ containing the EthPeer identified by peerId if present in the * PeerSelector, or empty otherwise */ - Optional getPeerByPeerId(final PeerId peerId); + Optional getPeerByPeerId(PeerId peerId); /** * Add the supplied EthPeer to the PeerSelector * * @param ethPeer the EthPeer to be added to the PeerSelector */ - void addPeer(final EthPeer ethPeer); + void addPeer(EthPeer ethPeer); /** * Remove the EthPeer identified by peerId from the PeerSelector * * @param peerId the PeerId of the EthPeer to be removed from the PeerSelector */ - void removePeer(final PeerId peerId); + void removePeer(PeerId peerId); } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java index caef72ba76e..5a71a8f2608 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java @@ -18,14 +18,13 @@ import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; import org.hyperledger.besu.ethereum.p2p.rlpx.connections.PeerConnection; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; -import org.hyperledger.besu.ethereum.p2p.rlpx.wire.SubProtocol; import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.plugin.services.metrics.LabelledMetric; import org.hyperledger.besu.plugin.services.metrics.OperationTimer; -import java.util.ArrayList; import java.util.Collection; +import java.util.HashSet; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; @@ -40,17 +39,15 @@ public class PeerTaskExecutor { public static final int NO_RETRIES = 1; private final PeerSelector peerSelector; private final PeerTaskRequestSender requestSender; - private final Supplier protocolSpecSupplier; + private final LabelledMetric requestTimer; public PeerTaskExecutor( final PeerSelector peerSelector, final PeerTaskRequestSender requestSender, - final Supplier protocolSpecSupplier, final MetricsSystem metricsSystem) { this.peerSelector = peerSelector; this.requestSender = requestSender; - this.protocolSpecSupplier = protocolSpecSupplier; requestTimer = metricsSystem.createLabelledTimer( BesuMetricCategory.PEERS, @@ -65,23 +62,19 @@ public PeerTaskExecutorResult execute(final PeerTask peerTask) { peerTask.getPeerTaskBehaviors().contains(PeerTaskRetryBehavior.RETRY_WITH_OTHER_PEERS) ? RETRIES_WITH_OTHER_PEER : NO_RETRIES; - final Collection usedEthPeers = new ArrayList<>(); + final Collection usedEthPeers = new HashSet<>(); do { EthPeer peer; try { peer = peerSelector.getPeer( - (candidatePeer) -> - isPeerUnused(candidatePeer, usedEthPeers) - && (protocolSpecSupplier.get().isPoS() - || isPeerHeightHighEnough( - candidatePeer, peerTask.getRequiredBlockNumber())) - && isPeerProtocolSuitable(candidatePeer, peerTask.getSubProtocol())); + usedEthPeers, peerTask.getRequiredBlockNumber(), peerTask.getSubProtocol()); usedEthPeers.add(peer); executorResult = executeAgainstPeer(peerTask, peer); } catch (NoAvailablePeerException e) { executorResult = - new PeerTaskExecutorResult<>(Optional.empty(), PeerTaskExecutorResponseCode.NO_PEER_AVAILABLE); + new PeerTaskExecutorResult<>( + Optional.empty(), PeerTaskExecutorResponseCode.NO_PEER_AVAILABLE); } } while (--triesRemaining > 0 && executorResult.responseCode() != PeerTaskExecutorResponseCode.SUCCESS); @@ -103,7 +96,6 @@ public PeerTaskExecutorResult executeAgainstPeer( : NO_RETRIES; do { try { - T result; try (final OperationTimer.TimingContext ignored = requestTimer.labels(peerTask.getClass().getSimpleName()).startTimer()) { @@ -113,24 +105,30 @@ public PeerTaskExecutorResult executeAgainstPeer( result = peerTask.parseResponse(responseMessageData); } peer.recordUsefulResponse(); - executorResult = new PeerTaskExecutorResult<>(Optional.ofNullable(result), PeerTaskExecutorResponseCode.SUCCESS); + executorResult = + new PeerTaskExecutorResult<>( + Optional.ofNullable(result), PeerTaskExecutorResponseCode.SUCCESS); } catch (PeerConnection.PeerNotConnected e) { executorResult = - new PeerTaskExecutorResult<>(Optional.empty(), PeerTaskExecutorResponseCode.PEER_DISCONNECTED); + new PeerTaskExecutorResult<>( + Optional.empty(), PeerTaskExecutorResponseCode.PEER_DISCONNECTED); } catch (InterruptedException | TimeoutException e) { peer.recordRequestTimeout(requestMessageData.getCode()); - executorResult = new PeerTaskExecutorResult<>(Optional.empty(), PeerTaskExecutorResponseCode.TIMEOUT); + executorResult = + new PeerTaskExecutorResult<>(Optional.empty(), PeerTaskExecutorResponseCode.TIMEOUT); } catch (InvalidPeerTaskResponseException e) { peer.recordUselessResponse(e.getMessage()); executorResult = - new PeerTaskExecutorResult<>(Optional.empty(), PeerTaskExecutorResponseCode.INVALID_RESPONSE); + new PeerTaskExecutorResult<>( + Optional.empty(), PeerTaskExecutorResponseCode.INVALID_RESPONSE); } catch (ExecutionException e) { executorResult = - new PeerTaskExecutorResult<>(Optional.empty(), PeerTaskExecutorResponseCode.INTERNAL_SERVER_ERROR); + new PeerTaskExecutorResult<>( + Optional.empty(), PeerTaskExecutorResponseCode.INTERNAL_SERVER_ERROR); } } while (--triesRemaining > 0 && executorResult.responseCode() != PeerTaskExecutorResponseCode.SUCCESS @@ -154,17 +152,4 @@ private boolean sleepBetweenRetries() { return false; } } - - private static boolean isPeerUnused( - final EthPeer ethPeer, final Collection usedEthPeers) { - return !usedEthPeers.contains(ethPeer); - } - - private static boolean isPeerHeightHighEnough(final EthPeer ethPeer, final long requiredHeight) { - return ethPeer.chainState().getEstimatedHeight() >= requiredHeight; - } - - private static boolean isPeerProtocolSuitable(final EthPeer ethPeer, final SubProtocol protocol) { - return ethPeer.getProtocolName().equals(protocol.getName()); - } } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorResult.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorResult.java index ef528cd2ae5..86dec85c295 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorResult.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorResult.java @@ -16,8 +16,5 @@ import java.util.Optional; -public record PeerTaskExecutorResult ( - Optional result, - PeerTaskExecutorResponseCode responseCode -) -{} +public record PeerTaskExecutorResult( + Optional result, PeerTaskExecutorResponseCode responseCode) {} diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelectorTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelectorTest.java index 8913b80a457..add2b1e612c 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelectorTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelectorTest.java @@ -14,19 +14,23 @@ */ package org.hyperledger.besu.ethereum.eth.manager.peertask; +import org.hyperledger.besu.ethereum.eth.EthProtocol; +import org.hyperledger.besu.ethereum.eth.SnapProtocol; +import org.hyperledger.besu.ethereum.eth.manager.ChainState; import org.hyperledger.besu.ethereum.eth.manager.EthPeer; -import org.hyperledger.besu.ethereum.eth.manager.MockPeerConnection; +import org.hyperledger.besu.ethereum.eth.manager.PeerReputation; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; +import org.hyperledger.besu.ethereum.p2p.peers.Peer; import org.hyperledger.besu.ethereum.p2p.rlpx.connections.PeerConnection; -import org.hyperledger.besu.ethereum.p2p.rlpx.wire.Capability; +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.SubProtocol; -import java.time.Clock; -import java.util.Collections; +import java.util.HashSet; import java.util.Set; -import org.apache.tuweni.bytes.Bytes; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.mockito.Mockito; public class DefaultPeerSelectorTest { @@ -34,67 +38,98 @@ public class DefaultPeerSelectorTest { @BeforeEach public void beforeTest() { - peerSelector = new DefaultPeerSelector(); + ProtocolSpec protocolSpec = Mockito.mock(ProtocolSpec.class); + Mockito.when(protocolSpec.isPoS()).thenReturn(false); + peerSelector = new DefaultPeerSelector(() -> protocolSpec); } @Test public void testGetPeer() throws NoAvailablePeerException { - EthPeer protocol1With5ReputationPeer = - createTestPeer(Set.of(Capability.create("capability1", 1)), "protocol1", 5); - peerSelector.addPeer(protocol1With5ReputationPeer); - EthPeer protocol1With4ReputationPeer = - createTestPeer(Set.of(Capability.create("capability1", 1)), "protocol1", 4); - peerSelector.addPeer(protocol1With4ReputationPeer); - EthPeer protocol2With50ReputationPeer = - createTestPeer(Set.of(Capability.create("capability1", 1)), "protocol2", 50); - peerSelector.addPeer(protocol2With50ReputationPeer); - EthPeer protocol2With4ReputationPeer = - createTestPeer(Set.of(Capability.create("capability1", 1)), "protocol2", 4); - peerSelector.addPeer(protocol2With4ReputationPeer); - - EthPeer result = peerSelector.getPeer((p) -> p.getProtocolName().equals("protocol1")); - - Assertions.assertSame(protocol1With5ReputationPeer, result); + EthPeer expectedPeer = createTestPeer(10, EthProtocol.get(), 5); + peerSelector.addPeer(expectedPeer); + EthPeer excludedForLowChainHeightPeer = createTestPeer(5, EthProtocol.get(), 50); + peerSelector.addPeer(excludedForLowChainHeightPeer); + EthPeer excludedForWrongProtocolPeer = createTestPeer(10, SnapProtocol.get(), 50); + peerSelector.addPeer(excludedForWrongProtocolPeer); + EthPeer excludedForLowReputationPeer = createTestPeer(10, EthProtocol.get(), 1); + peerSelector.addPeer(excludedForLowReputationPeer); + EthPeer excludedForBeingAlreadyUsedPeer = createTestPeer(10, EthProtocol.get(), 50); + peerSelector.addPeer(excludedForBeingAlreadyUsedPeer); + + Set usedEthPeers = new HashSet<>(); + usedEthPeers.add(excludedForBeingAlreadyUsedPeer); + + EthPeer result = peerSelector.getPeer(usedEthPeers, 10, EthProtocol.get()); + + Assertions.assertSame(expectedPeer, result); } @Test public void testGetPeerButNoPeerMatchesFilter() { - EthPeer protocol1With5ReputationPeer = - createTestPeer(Set.of(Capability.create("capability1", 1)), "protocol1", 5); - peerSelector.addPeer(protocol1With5ReputationPeer); - EthPeer protocol1With4ReputationPeer = - createTestPeer(Set.of(Capability.create("capability1", 1)), "protocol1", 4); - peerSelector.addPeer(protocol1With4ReputationPeer); - EthPeer protocol2With50ReputationPeer = - createTestPeer(Set.of(Capability.create("capability1", 1)), "protocol2", 50); - peerSelector.addPeer(protocol2With50ReputationPeer); - EthPeer protocol2With4ReputationPeer = - createTestPeer(Set.of(Capability.create("capability1", 1)), "protocol2", 4); - peerSelector.addPeer(protocol2With4ReputationPeer); + EthPeer expectedPeer = createTestPeer(10, EthProtocol.get(), 5); + peerSelector.addPeer(expectedPeer); + EthPeer excludedForLowChainHeightPeer = createTestPeer(5, EthProtocol.get(), 50); + peerSelector.addPeer(excludedForLowChainHeightPeer); + EthPeer excludedForWrongProtocolPeer = createTestPeer(10, SnapProtocol.get(), 50); + peerSelector.addPeer(excludedForWrongProtocolPeer); + EthPeer excludedForLowReputationPeer = createTestPeer(10, EthProtocol.get(), 1); + peerSelector.addPeer(excludedForLowReputationPeer); + EthPeer excludedForBeingAlreadyUsedPeer = createTestPeer(10, EthProtocol.get(), 50); + peerSelector.addPeer(excludedForBeingAlreadyUsedPeer); + + Set usedEthPeers = new HashSet<>(); + usedEthPeers.add(excludedForBeingAlreadyUsedPeer); Assertions.assertThrows( NoAvailablePeerException.class, - () -> peerSelector.getPeer((p) -> p.getProtocolName().equals("fake protocol"))); + () -> peerSelector.getPeer(usedEthPeers, 10, new MockSubProtocol())); } private EthPeer createTestPeer( - final Set connectionCapabilities, - final String protocolName, - final int reputationAdjustment) { - PeerConnection peerConnection = new MockPeerConnection(connectionCapabilities); - EthPeer peer = - new EthPeer( - peerConnection, - protocolName, - null, - Collections.emptyList(), - 1, - Clock.systemUTC(), - Collections.emptyList(), - Bytes.EMPTY); - for (int i = 0; i < reputationAdjustment; i++) { - peer.getReputation().recordUsefulResponse(); + final long chainHeight, final SubProtocol protocol, final int reputation) { + EthPeer ethPeer = Mockito.mock(EthPeer.class); + PeerConnection peerConnection = Mockito.mock(PeerConnection.class); + Peer peer = Mockito.mock(Peer.class); + ChainState chainState = Mockito.mock(ChainState.class); + PeerReputation peerReputation = Mockito.mock(PeerReputation.class); + + Mockito.when(ethPeer.getConnection()).thenReturn(peerConnection); + Mockito.when(peerConnection.getPeer()).thenReturn(peer); + Mockito.when(ethPeer.getProtocolName()).thenReturn(protocol.getName()); + Mockito.when(ethPeer.chainState()).thenReturn(chainState); + Mockito.when(chainState.getEstimatedHeight()).thenReturn(chainHeight); + Mockito.when(ethPeer.getReputation()).thenReturn(peerReputation); + Mockito.when(peerReputation.getScore()).thenReturn(reputation); + + Mockito.when(ethPeer.compareTo(Mockito.any(EthPeer.class))) + .thenAnswer( + (invocationOnMock) -> { + EthPeer otherPeer = invocationOnMock.getArgument(0, EthPeer.class); + return Integer.compare(reputation, otherPeer.getReputation().getScore()); + }); + return ethPeer; + } + + private static class MockSubProtocol implements SubProtocol { + + @Override + public String getName() { + return "Mock"; + } + + @Override + public int messageSpace(final int protocolVersion) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isValidMessageCode(final int protocolVersion, final int code) { + throw new UnsupportedOperationException(); + } + + @Override + public String messageName(final int protocolVersion, final int code) { + throw new UnsupportedOperationException(); } - return peer; } } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java index 0077c904188..a7b59145175 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java @@ -15,17 +15,16 @@ package org.hyperledger.besu.ethereum.eth.manager.peertask; import org.hyperledger.besu.ethereum.eth.manager.EthPeer; -import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; import org.hyperledger.besu.ethereum.p2p.rlpx.connections.PeerConnection; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.SubProtocol; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeoutException; -import java.util.function.Predicate; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; @@ -38,7 +37,6 @@ public class PeerTaskExecutorTest { private @Mock PeerSelector peerSelector; private @Mock PeerTaskRequestSender requestSender; - private @Mock ProtocolSpec protocolSpec; private @Mock PeerTask peerTask; private @Mock SubProtocol subprotocol; private @Mock MessageData requestMessageData; @@ -53,7 +51,7 @@ public void beforeTest() { mockCloser = MockitoAnnotations.openMocks(this); peerTaskExecutor = new PeerTaskExecutor( - peerSelector, requestSender, () -> protocolSpec, new NoOpMetricsSystem()); + peerSelector, requestSender, new NoOpMetricsSystem()); } @AfterEach @@ -140,8 +138,7 @@ public void testExecuteAgainstPeerWithNoPeerTaskBehaviorsAndPeerNotConnected() Assertions.assertNotNull(result); Assertions.assertTrue(result.result().isEmpty()); - Assertions.assertEquals( - PeerTaskExecutorResponseCode.PEER_DISCONNECTED, result.responseCode()); + Assertions.assertEquals(PeerTaskExecutorResponseCode.PEER_DISCONNECTED, result.responseCode()); } @Test @@ -192,8 +189,7 @@ public void testExecuteAgainstPeerWithNoPeerTaskBehaviorsAndInvalidResponseMessa Assertions.assertNotNull(result); Assertions.assertTrue(result.result().isEmpty()); - Assertions.assertEquals( - PeerTaskExecutorResponseCode.INVALID_RESPONSE, result.responseCode()); + Assertions.assertEquals(PeerTaskExecutorResponseCode.INVALID_RESPONSE, result.responseCode()); } @Test @@ -207,10 +203,13 @@ public void testExecuteWithNoPeerTaskBehaviorsAndSuccessFlow() NoAvailablePeerException { Object responseObject = new Object(); - Mockito.when(peerSelector.getPeer(Mockito.any(Predicate.class))).thenReturn(ethPeer); + Mockito.when( + peerSelector.getPeer(Mockito.any(Collection.class), Mockito.eq(10L), Mockito.eq(subprotocol))) + .thenReturn(ethPeer); Mockito.when(peerTask.getRequestMessage()).thenReturn(requestMessageData); Mockito.when(peerTask.getPeerTaskBehaviors()).thenReturn(Collections.emptyList()); + Mockito.when(peerTask.getRequiredBlockNumber()).thenReturn(10L); Mockito.when(peerTask.getSubProtocol()).thenReturn(subprotocol); Mockito.when(subprotocol.getName()).thenReturn("subprotocol"); Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, ethPeer)) @@ -240,15 +239,16 @@ public void testExecuteWithPeerSwitchingAndSuccessFlow() int requestMessageDataCode = 123; EthPeer peer2 = Mockito.mock(EthPeer.class); - Mockito.when(peerSelector.getPeer(Mockito.any(Predicate.class))) + Mockito.when( + peerSelector.getPeer(Mockito.any(Collection.class), Mockito.eq(10L), Mockito.eq(subprotocol))) .thenReturn(ethPeer) .thenReturn(peer2); Mockito.when(peerTask.getRequestMessage()).thenReturn(requestMessageData); Mockito.when(peerTask.getPeerTaskBehaviors()) .thenReturn(List.of(PeerTaskRetryBehavior.RETRY_WITH_OTHER_PEERS)); + Mockito.when(peerTask.getRequiredBlockNumber()).thenReturn(10L); Mockito.when(peerTask.getSubProtocol()).thenReturn(subprotocol); - Mockito.when(subprotocol.getName()).thenReturn("subprotocol"); Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, ethPeer)) .thenThrow(new TimeoutException()); Mockito.when(requestMessageData.getCode()).thenReturn(requestMessageDataCode); From 50c26f138b63203cf8254a9b46c582c91e192469 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Mon, 30 Sep 2024 16:02:30 +1000 Subject: [PATCH 047/106] 7311: spotless Signed-off-by: Matilda Clerke --- .../eth/manager/peertask/PeerTaskExecutor.java | 2 -- .../eth/manager/peertask/PeerTaskExecutorTest.java | 10 +++++----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java index 5a71a8f2608..10c882e7e5a 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java @@ -15,7 +15,6 @@ package org.hyperledger.besu.ethereum.eth.manager.peertask; import org.hyperledger.besu.ethereum.eth.manager.EthPeer; -import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; import org.hyperledger.besu.ethereum.p2p.rlpx.connections.PeerConnection; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; import org.hyperledger.besu.metrics.BesuMetricCategory; @@ -29,7 +28,6 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeoutException; -import java.util.function.Supplier; /** Manages the execution of PeerTasks, respecting their PeerTaskRetryBehavior */ public class PeerTaskExecutor { diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java index a7b59145175..15b1747bc7b 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java @@ -49,9 +49,7 @@ public class PeerTaskExecutorTest { @BeforeEach public void beforeTest() { mockCloser = MockitoAnnotations.openMocks(this); - peerTaskExecutor = - new PeerTaskExecutor( - peerSelector, requestSender, new NoOpMetricsSystem()); + peerTaskExecutor = new PeerTaskExecutor(peerSelector, requestSender, new NoOpMetricsSystem()); } @AfterEach @@ -204,7 +202,8 @@ public void testExecuteWithNoPeerTaskBehaviorsAndSuccessFlow() Object responseObject = new Object(); Mockito.when( - peerSelector.getPeer(Mockito.any(Collection.class), Mockito.eq(10L), Mockito.eq(subprotocol))) + peerSelector.getPeer( + Mockito.any(Collection.class), Mockito.eq(10L), Mockito.eq(subprotocol))) .thenReturn(ethPeer); Mockito.when(peerTask.getRequestMessage()).thenReturn(requestMessageData); @@ -240,7 +239,8 @@ public void testExecuteWithPeerSwitchingAndSuccessFlow() EthPeer peer2 = Mockito.mock(EthPeer.class); Mockito.when( - peerSelector.getPeer(Mockito.any(Collection.class), Mockito.eq(10L), Mockito.eq(subprotocol))) + peerSelector.getPeer( + Mockito.any(Collection.class), Mockito.eq(10L), Mockito.eq(subprotocol))) .thenReturn(ethPeer) .thenReturn(peer2); From 2c1446efa02e222f69df5b9424768f9d4f6596b7 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Tue, 1 Oct 2024 12:02:40 +1000 Subject: [PATCH 048/106] 7311: Fix up everything broken after merge Signed-off-by: Matilda Clerke --- .../controller/BesuControllerBuilder.java | 19 +++++++++---------- ...onsensusScheduleBesuControllerBuilder.java | 6 +++--- .../MergeBesuControllerBuilder.java | 6 +++--- .../TransitionBesuControllerBuilder.java | 6 +++--- .../eth/manager/EthProtocolManager.java | 16 ++++++++-------- .../task/GetReceiptsFromPeerTask.java | 12 +++++++----- .../CheckpointDownloadBlockStep.java | 4 ++-- .../sync/fastsync/DownloadReceiptsStep.java | 6 +++--- .../eth/manager/EthProtocolManagerTest.java | 4 ++-- .../manager/EthProtocolManagerTestUtil.java | 6 +++--- .../task/GetReceiptsFromPeerTaskTest.java | 4 ++-- .../CheckPointSyncChainDownloaderTest.java | 2 +- .../fastsync/DownloadReceiptsStepTest.java | 4 +++- .../sync/fastsync/FastSyncActionsTest.java | 2 +- .../fastsync/FastSyncChainDownloaderTest.java | 2 +- .../ethereum/eth/transactions/TestNode.java | 4 ++-- .../TransactionPoolFactoryTest.java | 4 ++-- 17 files changed, 55 insertions(+), 52 deletions(-) diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java index 7fff62db991..18646a13060 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java @@ -55,8 +55,8 @@ import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; import org.hyperledger.besu.ethereum.eth.manager.MergePeerFilter; import org.hyperledger.besu.ethereum.eth.manager.MonitoredExecutors; -import org.hyperledger.besu.ethereum.eth.manager.peertask.DefaultPeerManager; -import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerManager; +import org.hyperledger.besu.ethereum.eth.manager.peertask.DefaultPeerSelector; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerSelector; import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutor; import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskRequestSender; import org.hyperledger.besu.ethereum.eth.manager.snap.SnapProtocolManager; @@ -656,11 +656,10 @@ public BesuController build() { } final EthContext ethContext = new EthContext(ethPeers, ethMessages, snapMessages, scheduler); - final PeerManager peerManager = new DefaultPeerManager(); - ethPeers.streamAllPeers().forEach(peerManager::addPeer); + final PeerSelector peerSelector = new DefaultPeerSelector(currentProtocolSpecSupplier); + ethPeers.streamAllPeers().forEach(peerSelector::addPeer); final PeerTaskExecutor peerTaskExecutor = - new PeerTaskExecutor( - peerManager, new PeerTaskRequestSender(), currentProtocolSpecSupplier, metricsSystem); + new PeerTaskExecutor(peerSelector, new PeerTaskRequestSender(), metricsSystem); final boolean fullSyncDisabled = !SyncMode.isFullSync(syncConfig.getSyncMode()); final SyncState syncState = new SyncState(blockchain, ethPeers, fullSyncDisabled, checkpoint); @@ -701,7 +700,7 @@ public BesuController build() { peerValidators, Optional.empty(), forkIdManager, - peerManager); + peerSelector); final PivotBlockSelector pivotBlockSelector = createPivotSelector( @@ -1036,7 +1035,7 @@ protected String getSupportedProtocol() { * @param peerValidators the peer validators * @param mergePeerFilter the merge peer filter * @param forkIdManager the fork id manager - * @param peerManager the PeerManager + * @param peerSelector the PeerSelector * @return the eth protocol manager */ protected EthProtocolManager createEthProtocolManager( @@ -1051,7 +1050,7 @@ protected EthProtocolManager createEthProtocolManager( final List peerValidators, final Optional mergePeerFilter, final ForkIdManager forkIdManager, - final PeerManager peerManager) { + final PeerSelector peerSelector) { return new EthProtocolManager( protocolContext.getBlockchain(), networkId, @@ -1066,7 +1065,7 @@ protected EthProtocolManager createEthProtocolManager( synchronizerConfiguration, scheduler, forkIdManager, - peerManager); + peerSelector); } /** diff --git a/besu/src/main/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilder.java index 897f2e3acec..2e28a3179c8 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilder.java @@ -42,7 +42,7 @@ import org.hyperledger.besu.ethereum.eth.manager.EthProtocolManager; import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; import org.hyperledger.besu.ethereum.eth.manager.MergePeerFilter; -import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerManager; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerSelector; import org.hyperledger.besu.ethereum.eth.manager.snap.SnapProtocolManager; import org.hyperledger.besu.ethereum.eth.peervalidation.PeerValidator; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; @@ -245,7 +245,7 @@ protected EthProtocolManager createEthProtocolManager( final List peerValidators, final Optional mergePeerFilter, final ForkIdManager forkIdManager, - final PeerManager peerManager) { + final PeerSelector peerSelector) { return besuControllerBuilderSchedule .get(0L) .createEthProtocolManager( @@ -260,7 +260,7 @@ protected EthProtocolManager createEthProtocolManager( peerValidators, mergePeerFilter, forkIdManager, - peerManager); + peerSelector); } @Override diff --git a/besu/src/main/java/org/hyperledger/besu/controller/MergeBesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/MergeBesuControllerBuilder.java index f860f399da6..88629a7441e 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/MergeBesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/MergeBesuControllerBuilder.java @@ -34,7 +34,7 @@ import org.hyperledger.besu.ethereum.eth.manager.EthProtocolManager; import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; import org.hyperledger.besu.ethereum.eth.manager.MergePeerFilter; -import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerManager; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerSelector; import org.hyperledger.besu.ethereum.eth.peervalidation.PeerValidator; import org.hyperledger.besu.ethereum.eth.peervalidation.RequiredBlocksPeerValidator; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; @@ -101,7 +101,7 @@ protected EthProtocolManager createEthProtocolManager( final List peerValidators, final Optional mergePeerFilter, final ForkIdManager forkIdManager, - final PeerManager peerManager) { + final PeerSelector peerSelector) { var mergeContext = protocolContext.getConsensusContext(MergeContext.class); @@ -132,7 +132,7 @@ protected EthProtocolManager createEthProtocolManager( peerValidators, filterToUse, forkIdManager, - peerManager); + peerSelector); return ethProtocolManager; } diff --git a/besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java index 9b71e0bdc95..0f56d0c618a 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java @@ -40,7 +40,7 @@ import org.hyperledger.besu.ethereum.eth.manager.EthProtocolManager; import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; import org.hyperledger.besu.ethereum.eth.manager.MergePeerFilter; -import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerManager; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerSelector; import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutor; import org.hyperledger.besu.ethereum.eth.peervalidation.PeerValidator; import org.hyperledger.besu.ethereum.eth.sync.DefaultSynchronizer; @@ -166,7 +166,7 @@ protected EthProtocolManager createEthProtocolManager( final List peerValidators, final Optional mergePeerFilter, final ForkIdManager forkIdManager, - final PeerManager peerManager) { + final PeerSelector peerSelector) { return mergeBesuControllerBuilder.createEthProtocolManager( protocolContext, synchronizerConfiguration, @@ -179,7 +179,7 @@ protected EthProtocolManager createEthProtocolManager( peerValidators, mergePeerFilter, forkIdManager, - peerManager); + peerSelector); } @Override diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManager.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManager.java index d69910d59cf..97e91f6badc 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManager.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManager.java @@ -23,7 +23,7 @@ import org.hyperledger.besu.ethereum.core.Difficulty; import org.hyperledger.besu.ethereum.eth.EthProtocol; import org.hyperledger.besu.ethereum.eth.EthProtocolConfiguration; -import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerManager; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerSelector; import org.hyperledger.besu.ethereum.eth.messages.EthPV62; import org.hyperledger.besu.ethereum.eth.messages.StatusMessage; import org.hyperledger.besu.ethereum.eth.peervalidation.PeerValidator; @@ -70,7 +70,7 @@ public class EthProtocolManager implements ProtocolManager, MinedBlockObserver { private final Hash genesisHash; private final ForkIdManager forkIdManager; - private final PeerManager peerManager; + private final PeerSelector peerSelector; private final BigInteger networkId; private final EthPeers ethPeers; private final EthMessages ethMessages; @@ -95,7 +95,7 @@ public EthProtocolManager( final SynchronizerConfiguration synchronizerConfiguration, final EthScheduler scheduler, final ForkIdManager forkIdManager, - final PeerManager peerManager) { + final PeerSelector peerSelector) { this.networkId = networkId; this.peerValidators = peerValidators; this.scheduler = scheduler; @@ -105,7 +105,7 @@ public EthProtocolManager( this.genesisHash = blockchain.getBlockHashByNumber(0L).orElse(Hash.ZERO); this.forkIdManager = forkIdManager; - this.peerManager = peerManager; + this.peerSelector = peerSelector; this.ethPeers = ethPeers; this.ethMessages = ethMessages; @@ -145,7 +145,7 @@ public EthProtocolManager( final Optional mergePeerFilter, final SynchronizerConfiguration synchronizerConfiguration, final EthScheduler scheduler, - final PeerManager peerManager) { + final PeerSelector peerSelector) { this( blockchain, networkId, @@ -164,7 +164,7 @@ public EthProtocolManager( Collections.emptyList(), Collections.emptyList(), ethereumWireProtocolConfiguration.isLegacyEth64ForkIdEnabled()), - peerManager); + peerSelector); } public EthContext ethContext() { @@ -343,7 +343,7 @@ public void processMessage(final Capability cap, final Message message) { public void handleNewConnection(final PeerConnection connection) { ethPeers.registerNewConnection(connection, peerValidators); final EthPeer peer = ethPeers.peer(connection); - peerManager.addPeer(peer); + peerSelector.addPeer(peer); final Capability cap = connection.capability(getSupportedProtocol()); final ForkId latestForkId = cap.getVersion() >= 64 ? forkIdManager.getForkIdForChainHead() : null; @@ -375,7 +375,7 @@ public void handleDisconnect( final DisconnectReason reason, final boolean initiatedByPeer) { final boolean wasActiveConnection = ethPeers.registerDisconnect(connection); - peerManager.removePeer(connection.getPeer()); + peerSelector.removePeer(connection.getPeer()); LOG.atDebug() .setMessage("Disconnect - active Connection? {} - {} - {} - {} {} - {} peers left") .addArgument(wasActiveConnection) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTask.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTask.java index f4760de8847..75263f460e1 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTask.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTask.java @@ -20,11 +20,12 @@ import org.hyperledger.besu.ethereum.eth.EthProtocol; import org.hyperledger.besu.ethereum.eth.manager.peertask.InvalidPeerTaskResponseException; import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTask; -import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskBehavior; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskRetryBehavior; import org.hyperledger.besu.ethereum.eth.messages.GetReceiptsMessage; import org.hyperledger.besu.ethereum.eth.messages.ReceiptsMessage; import org.hyperledger.besu.ethereum.mainnet.BodyValidator; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.SubProtocol; import java.util.ArrayList; import java.util.Collection; @@ -52,8 +53,8 @@ public GetReceiptsFromPeerTask( } @Override - public String getSubProtocol() { - return EthProtocol.NAME; + public SubProtocol getSubProtocol() { + return EthProtocol.get(); } @Override @@ -101,7 +102,8 @@ public Map> parseResponse(final MessageDat } @Override - public Collection getPeerTaskBehaviors() { - return List.of(PeerTaskBehavior.RETRY_WITH_OTHER_PEERS, PeerTaskBehavior.RETRY_WITH_SAME_PEER); + public Collection getPeerTaskBehaviors() { + return List.of( + PeerTaskRetryBehavior.RETRY_WITH_OTHER_PEERS, PeerTaskRetryBehavior.RETRY_WITH_SAME_PEER); } } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointDownloadBlockStep.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointDownloadBlockStep.java index 603d4448a67..a0bb5ee595f 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointDownloadBlockStep.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointDownloadBlockStep.java @@ -85,10 +85,10 @@ private CompletableFuture> downloadReceipts( PeerTaskExecutorResult>> executorResult = peerTaskExecutor.execute(task); - if (executorResult.getResponseCode() == PeerTaskExecutorResponseCode.SUCCESS) { + if (executorResult.responseCode() == PeerTaskExecutorResponseCode.SUCCESS) { List transactionReceipts = executorResult - .getResult() + .result() .map((map) -> map.get(block.getHeader())) .orElseThrow( () -> diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java index 5e0a44cee28..bd7bd934ec9 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java @@ -66,10 +66,10 @@ public CompletableFuture> apply(final List blocks new GetReceiptsFromPeerTask(headers, new BodyValidator()); PeerTaskExecutorResult>> getReceiptsResult = peerTaskExecutor.execute(getReceiptsFromPeerTask); - if (getReceiptsResult.getResponseCode() == PeerTaskExecutorResponseCode.SUCCESS - && getReceiptsResult.getResult().isPresent()) { + if (getReceiptsResult.responseCode() == PeerTaskExecutorResponseCode.SUCCESS + && getReceiptsResult.result().isPresent()) { Map> receiptsResult = - getReceiptsResult.getResult().get(); + getReceiptsResult.result().get(); receiptsResult .keySet() .forEach( diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManagerTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManagerTest.java index 31f66e31d22..95f3d762220 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManagerTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManagerTest.java @@ -44,7 +44,7 @@ import org.hyperledger.besu.ethereum.eth.EthProtocolConfiguration; import org.hyperledger.besu.ethereum.eth.EthProtocolVersion; import org.hyperledger.besu.ethereum.eth.manager.MockPeerConnection.PeerSendHandler; -import org.hyperledger.besu.ethereum.eth.manager.peertask.DefaultPeerManager; +import org.hyperledger.besu.ethereum.eth.manager.peertask.DefaultPeerSelector; import org.hyperledger.besu.ethereum.eth.messages.BlockBodiesMessage; import org.hyperledger.besu.ethereum.eth.messages.BlockHeadersMessage; import org.hyperledger.besu.ethereum.eth.messages.EthPV62; @@ -1245,7 +1245,7 @@ private EthProtocolManager createEthManager( syncConfig, mock(EthScheduler.class), mock(ForkIdManager.class), - new DefaultPeerManager())) { + new DefaultPeerSelector(() -> null))) { return ethManager; } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManagerTestUtil.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManagerTestUtil.java index b71ffa91331..16199571e66 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManagerTestUtil.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManagerTestUtil.java @@ -30,7 +30,7 @@ import org.hyperledger.besu.ethereum.core.ProtocolScheduleFixture; import org.hyperledger.besu.ethereum.eth.EthProtocol; import org.hyperledger.besu.ethereum.eth.EthProtocolConfiguration; -import org.hyperledger.besu.ethereum.eth.manager.peertask.DefaultPeerManager; +import org.hyperledger.besu.ethereum.eth.manager.peertask.DefaultPeerSelector; import org.hyperledger.besu.ethereum.eth.manager.snap.SnapProtocolManager; import org.hyperledger.besu.ethereum.eth.peervalidation.PeerValidator; import org.hyperledger.besu.ethereum.eth.sync.ChainHeadTracker; @@ -119,7 +119,7 @@ public static EthProtocolManager create( mock(SynchronizerConfiguration.class), ethScheduler, new ForkIdManager(blockchain, Collections.emptyList(), Collections.emptyList(), false), - new DefaultPeerManager()); + new DefaultPeerSelector(() -> null)); } public static EthProtocolManager create( @@ -171,7 +171,7 @@ public static EthProtocolManager create( mock(SynchronizerConfiguration.class), ethScheduler, forkIdManager, - new DefaultPeerManager()); + new DefaultPeerSelector(() -> null)); } public static EthProtocolManager create(final Blockchain blockchain) { diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTaskTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTaskTest.java index 131026a2760..f8f1c88e1ef 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTaskTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTaskTest.java @@ -41,7 +41,7 @@ public class GetReceiptsFromPeerTaskTest { @Test public void testGetSubProtocol() { GetReceiptsFromPeerTask task = new GetReceiptsFromPeerTask(Collections.emptyList(), null); - Assertions.assertEquals(EthProtocol.NAME, task.getSubProtocol()); + Assertions.assertEquals(EthProtocol.get(), task.getSubProtocol()); } @Test @@ -83,7 +83,7 @@ public void testParseResponseWithNullResponseMessage() { } @Test - public void testParseResponseForInvalidResponse() throws InvalidPeerTaskResponseException { + public void testParseResponseForInvalidResponse() { GetReceiptsFromPeerTask task = new GetReceiptsFromPeerTask( List.of(mockBlockHeader(1), mockBlockHeader(2), mockBlockHeader(3)), null); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java index 68e1023fbc7..c2a99306670 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java @@ -191,7 +191,7 @@ private PeerTaskExecutorResult>> proce bh, otherBlockchain.getTxReceipts(bh.getHash()).get())); return new PeerTaskExecutorResult<>( - getReceiptsFromPeerTaskResult, PeerTaskExecutorResponseCode.SUCCESS); + Optional.of(getReceiptsFromPeerTaskResult), PeerTaskExecutorResponseCode.SUCCESS); } @AfterEach diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStepTest.java index 4270251e6c8..554e09c3e91 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStepTest.java @@ -44,6 +44,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; @@ -121,7 +122,8 @@ public void shouldDownloadReceiptsForBlocksUsingPeerTaskSystem() blocks.forEach( (b) -> receiptsMap.put(b.getHeader(), List.of(Mockito.mock(TransactionReceipt.class)))); PeerTaskExecutorResult>> peerTaskResult = - new PeerTaskExecutorResult<>(receiptsMap, PeerTaskExecutorResponseCode.SUCCESS); + new PeerTaskExecutorResult<>( + Optional.of(receiptsMap), PeerTaskExecutorResponseCode.SUCCESS); Mockito.when(peerTaskExecutor.execute(Mockito.any(GetReceiptsFromPeerTask.class))) .thenReturn(peerTaskResult); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActionsTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActionsTest.java index 6d194cb8e02..7af807c1c7b 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActionsTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActionsTest.java @@ -537,7 +537,7 @@ private FastSyncActions createFastSyncActions( protocolSchedule, protocolContext, ethContext, - new PeerTaskExecutor(null, null, null, new NoOpMetricsSystem()), + new PeerTaskExecutor(null, null, new NoOpMetricsSystem()), new SyncState(blockchain, ethContext.getEthPeers(), true, Optional.empty()), pivotBlockSelector, new NoOpMetricsSystem()); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloaderTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloaderTest.java index da82034eaa6..0e5b5ec2c7c 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloaderTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloaderTest.java @@ -111,7 +111,7 @@ private ChainDownloader downloader( protocolSchedule, protocolContext, ethContext, - new PeerTaskExecutor(null, null, null, new NoOpMetricsSystem()), + new PeerTaskExecutor(null, null, new NoOpMetricsSystem()), syncState, new NoOpMetricsSystem(), new FastSyncState(otherBlockchain.getBlockHeader(pivotBlockNumber).get()), diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TestNode.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TestNode.java index f8671b675fa..c41e326facb 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TestNode.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TestNode.java @@ -44,7 +44,7 @@ import org.hyperledger.besu.ethereum.eth.manager.EthPeers; import org.hyperledger.besu.ethereum.eth.manager.EthProtocolManager; import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; -import org.hyperledger.besu.ethereum.eth.manager.peertask.DefaultPeerManager; +import org.hyperledger.besu.ethereum.eth.manager.peertask.DefaultPeerSelector; import org.hyperledger.besu.ethereum.eth.sync.ChainHeadTracker; import org.hyperledger.besu.ethereum.eth.sync.SyncMode; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; @@ -198,7 +198,7 @@ public boolean isMessagePermitted(final EnodeURL destinationEnode, final int cod Optional.empty(), syncConfig, scheduler, - new DefaultPeerManager()); + new DefaultPeerSelector(() -> null)); final NetworkRunner networkRunner = NetworkRunner.builder() diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolFactoryTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolFactoryTest.java index f85abc4e963..f4dac8a244e 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolFactoryTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolFactoryTest.java @@ -45,7 +45,7 @@ import org.hyperledger.besu.ethereum.eth.manager.EthPeers; import org.hyperledger.besu.ethereum.eth.manager.EthProtocolManager; import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; -import org.hyperledger.besu.ethereum.eth.manager.peertask.DefaultPeerManager; +import org.hyperledger.besu.ethereum.eth.manager.peertask.DefaultPeerSelector; import org.hyperledger.besu.ethereum.eth.sync.SyncMode; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; @@ -320,7 +320,7 @@ private void setupInitialSyncPhase(final SyncState syncState) { mock(SynchronizerConfiguration.class), mock(EthScheduler.class), mock(ForkIdManager.class), - new DefaultPeerManager()); + new DefaultPeerSelector(() -> null)); } @Test From 3c0c47b533701988032ce441f2ec8f1eb027d94a Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Wed, 2 Oct 2024 12:10:55 +1000 Subject: [PATCH 049/106] 7311: Attempt to improve performance of peer task system in pipeline Signed-off-by: Matilda Clerke --- .../sync/fastsync/DownloadReceiptsStep.java | 88 +++++++++++-------- 1 file changed, 50 insertions(+), 38 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java index bd7bd934ec9..6f07763d2d4 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java @@ -31,14 +31,19 @@ import org.hyperledger.besu.ethereum.mainnet.BodyValidator; import org.hyperledger.besu.plugin.services.MetricsSystem; -import java.util.HashMap; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + public class DownloadReceiptsStep implements Function, CompletableFuture>> { + private static final Logger LOG = LoggerFactory.getLogger(DownloadReceiptsStep.class); private final EthContext ethContext; private final PeerTaskExecutor peerTaskExecutor; private final SynchronizerConfiguration synchronizerConfiguration; @@ -58,35 +63,51 @@ public DownloadReceiptsStep( @Override public CompletableFuture> apply(final List blocks) { final List headers = blocks.stream().map(Block::getHeader).collect(toList()); - final Map> getReceipts; if (synchronizerConfiguration.isPeerTaskSystemEnabled()) { - getReceipts = new HashMap>(); - do { - GetReceiptsFromPeerTask getReceiptsFromPeerTask = - new GetReceiptsFromPeerTask(headers, new BodyValidator()); - PeerTaskExecutorResult>> getReceiptsResult = - peerTaskExecutor.execute(getReceiptsFromPeerTask); - if (getReceiptsResult.responseCode() == PeerTaskExecutorResponseCode.SUCCESS - && getReceiptsResult.result().isPresent()) { - Map> receiptsResult = - getReceiptsResult.result().get(); - receiptsResult - .keySet() - .forEach( - (bh) -> - getReceipts.merge( - bh, - receiptsResult.get(bh), - (initialReceipts, newReceipts) -> { - throw new IllegalStateException( - "Unexpectedly got receipts for block header already populated!"); - })); - } - // remove all the headers we found receipts for - headers.removeAll(getReceipts.keySet()); - // repeat until all headers have receipts - } while (!headers.isEmpty()); - return CompletableFuture.completedFuture(combineBlocksAndReceipts(blocks, getReceipts)); + return CompletableFuture.supplyAsync( + () -> { + final Map> getReceipts = + new ConcurrentHashMap>(); + List blockWithReceiptsList = new ArrayList<>(headers.size()); + do { + GetReceiptsFromPeerTask getReceiptsFromPeerTask = + new GetReceiptsFromPeerTask(headers, new BodyValidator()); + PeerTaskExecutorResult>> getReceiptsResult = + peerTaskExecutor.execute(getReceiptsFromPeerTask); + if (getReceiptsResult.responseCode() == PeerTaskExecutorResponseCode.SUCCESS + && getReceiptsResult.result().isPresent()) { + Map> receiptsResult = + getReceiptsResult.result().get(); + for (BlockHeader blockHeader : receiptsResult.keySet()) { + getReceipts.merge( + blockHeader, + receiptsResult.get(blockHeader), + (initialReceipts, newReceipts) -> { + throw new IllegalStateException( + "Unexpectedly got receipts for block header already populated!"); + }); + } + } + // remove all the headers we found receipts for + headers.removeAll(getReceipts.keySet()); + blockWithReceiptsList.addAll(combineBlocksAndReceipts(blocks, getReceipts)); + // repeat until all headers have receipts + } while (!headers.isEmpty()); + + // verify that all blocks have receipts + if (!blocks.stream() + .filter( + (b) -> + blockWithReceiptsList.stream() + .map((bwr) -> bwr.getBlock()) + .toList() + .contains(b)) + .toList() + .isEmpty()) { + throw new IllegalStateException("Not all blocks have been matched to receipts!"); + } + return blockWithReceiptsList; + }); } else { return GetReceiptsForHeadersTask.forHeaders(ethContext, headers, metricsSystem) @@ -102,15 +123,6 @@ private List combineBlocksAndReceipts( block -> { final List receipts = receiptsByHeader.getOrDefault(block.getHeader(), emptyList()); - if (block.getBody().getTransactions().size() != receipts.size()) { - throw new IllegalStateException( - "PeerTask response code was success, but incorrect number of receipts returned. Header hash: " - + block.getHeader().getHash() - + ", Transactions: " - + block.getBody().getTransactions().size() - + ", receipts: " - + receipts.size()); - } return new BlockWithReceipts(block, receipts); }) .toList(); From d0bd5ede1e1b08b33852edf36ae35d1a11f6ce46 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Wed, 2 Oct 2024 12:25:54 +1000 Subject: [PATCH 050/106] 7311: fix compile check Signed-off-by: Matilda Clerke --- .../besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java index 6f07763d2d4..5b13b8f2b99 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java @@ -38,12 +38,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - public class DownloadReceiptsStep implements Function, CompletableFuture>> { - private static final Logger LOG = LoggerFactory.getLogger(DownloadReceiptsStep.class); private final EthContext ethContext; private final PeerTaskExecutor peerTaskExecutor; private final SynchronizerConfiguration synchronizerConfiguration; From 1c25ac533716e084f903dde105dc6a8db4fbf3d5 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Wed, 2 Oct 2024 14:53:48 +1000 Subject: [PATCH 051/106] 7311: Fix broken workflow Signed-off-by: Matilda Clerke --- .../sync/fastsync/DownloadReceiptsStep.java | 25 +++++-------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java index 5b13b8f2b99..d1fc60b9375 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java @@ -32,10 +32,10 @@ import org.hyperledger.besu.plugin.services.MetricsSystem; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; public class DownloadReceiptsStep @@ -62,10 +62,10 @@ public CompletableFuture> apply(final List blocks if (synchronizerConfiguration.isPeerTaskSystemEnabled()) { return CompletableFuture.supplyAsync( () -> { - final Map> getReceipts = - new ConcurrentHashMap>(); List blockWithReceiptsList = new ArrayList<>(headers.size()); do { + final Map> getReceipts = + new HashMap>(); GetReceiptsFromPeerTask getReceiptsFromPeerTask = new GetReceiptsFromPeerTask(headers, new BodyValidator()); PeerTaskExecutorResult>> getReceiptsResult = @@ -75,13 +75,7 @@ public CompletableFuture> apply(final List blocks Map> receiptsResult = getReceiptsResult.result().get(); for (BlockHeader blockHeader : receiptsResult.keySet()) { - getReceipts.merge( - blockHeader, - receiptsResult.get(blockHeader), - (initialReceipts, newReceipts) -> { - throw new IllegalStateException( - "Unexpectedly got receipts for block header already populated!"); - }); + getReceipts.put(blockHeader, receiptsResult.get(blockHeader)); } } // remove all the headers we found receipts for @@ -91,15 +85,7 @@ public CompletableFuture> apply(final List blocks } while (!headers.isEmpty()); // verify that all blocks have receipts - if (!blocks.stream() - .filter( - (b) -> - blockWithReceiptsList.stream() - .map((bwr) -> bwr.getBlock()) - .toList() - .contains(b)) - .toList() - .isEmpty()) { + if (blocks.size() != blockWithReceiptsList.size()) { throw new IllegalStateException("Not all blocks have been matched to receipts!"); } return blockWithReceiptsList; @@ -115,6 +101,7 @@ public CompletableFuture> apply(final List blocks private List combineBlocksAndReceipts( final List blocks, final Map> receiptsByHeader) { return blocks.stream() + .filter((b) -> receiptsByHeader.keySet().contains(b.getHeader())) .map( block -> { final List receipts = From 2e6dfd992eeaa67cac2fe16ec68fad05839d07fe Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Wed, 2 Oct 2024 16:27:36 +1000 Subject: [PATCH 052/106] 7311: Reduce logging in JsonRpcExecutor to trace level Signed-off-by: Matilda Clerke --- .../besu/ethereum/api/jsonrpc/execution/JsonRpcExecutor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/execution/JsonRpcExecutor.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/execution/JsonRpcExecutor.java index 6fedcc70ca4..b8786eb7a5d 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/execution/JsonRpcExecutor.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/execution/JsonRpcExecutor.java @@ -104,9 +104,9 @@ public JsonRpcResponse execute( private Optional validateMethodAvailability(final JsonRpcRequest request) { final String name = request.getMethod(); - if (LOG.isDebugEnabled()) { + if (LOG.isTraceEnabled()) { final JsonArray params = JsonObject.mapFrom(request).getJsonArray("params"); - LOG.debug("JSON-RPC request -> {} {}", name, params); + LOG.trace("JSON-RPC request -> {} {}", name, params); } final JsonRpcMethod method = rpcMethods.get(name); From aca80585f459103c2b9ebc0e80c8fc0e737a8e3d Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Wed, 2 Oct 2024 16:28:18 +1000 Subject: [PATCH 053/106] 7311: More changes in DownloadReceiptsStep Signed-off-by: Matilda Clerke --- .../eth/sync/fastsync/DownloadReceiptsStep.java | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java index d1fc60b9375..407effb7e0c 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java @@ -32,7 +32,6 @@ import org.hyperledger.besu.plugin.services.MetricsSystem; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; @@ -64,23 +63,16 @@ public CompletableFuture> apply(final List blocks () -> { List blockWithReceiptsList = new ArrayList<>(headers.size()); do { - final Map> getReceipts = - new HashMap>(); GetReceiptsFromPeerTask getReceiptsFromPeerTask = new GetReceiptsFromPeerTask(headers, new BodyValidator()); PeerTaskExecutorResult>> getReceiptsResult = peerTaskExecutor.execute(getReceiptsFromPeerTask); if (getReceiptsResult.responseCode() == PeerTaskExecutorResponseCode.SUCCESS && getReceiptsResult.result().isPresent()) { - Map> receiptsResult = - getReceiptsResult.result().get(); - for (BlockHeader blockHeader : receiptsResult.keySet()) { - getReceipts.put(blockHeader, receiptsResult.get(blockHeader)); - } + // remove all the headers we found receipts for + headers.removeAll(getReceiptsResult.result().get().keySet()); + blockWithReceiptsList.addAll(combineBlocksAndReceipts(blocks, getReceiptsResult.result().get())); } - // remove all the headers we found receipts for - headers.removeAll(getReceipts.keySet()); - blockWithReceiptsList.addAll(combineBlocksAndReceipts(blocks, getReceipts)); // repeat until all headers have receipts } while (!headers.isEmpty()); @@ -101,7 +93,7 @@ public CompletableFuture> apply(final List blocks private List combineBlocksAndReceipts( final List blocks, final Map> receiptsByHeader) { return blocks.stream() - .filter((b) -> receiptsByHeader.keySet().contains(b.getHeader())) + .filter((b) -> receiptsByHeader.containsKey(b.getHeader())) .map( block -> { final List receipts = From 4d59b10c6abf10220a351c43dc69fcbbd0608dc2 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Thu, 3 Oct 2024 12:10:58 +1000 Subject: [PATCH 054/106] 7311: Rework DownloadReceiptsStep Signed-off-by: Matilda Clerke --- .../controller/BesuControllerBuilder.java | 2 +- .../manager/peertask/PeerTaskExecutor.java | 12 +++- .../sync/fastsync/DownloadReceiptsStep.java | 67 ++++++++++++++----- .../peertask/PeerTaskExecutorTest.java | 3 +- .../sync/fastsync/FastSyncActionsTest.java | 2 +- .../fastsync/FastSyncChainDownloaderTest.java | 2 +- 6 files changed, 66 insertions(+), 22 deletions(-) diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java index 18646a13060..b85ad75736d 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java @@ -659,7 +659,7 @@ public BesuController build() { final PeerSelector peerSelector = new DefaultPeerSelector(currentProtocolSpecSupplier); ethPeers.streamAllPeers().forEach(peerSelector::addPeer); final PeerTaskExecutor peerTaskExecutor = - new PeerTaskExecutor(peerSelector, new PeerTaskRequestSender(), metricsSystem); + new PeerTaskExecutor(peerSelector, new PeerTaskRequestSender(), scheduler, metricsSystem); final boolean fullSyncDisabled = !SyncMode.isFullSync(syncConfig.getSyncMode()); final SyncState syncState = new SyncState(blockchain, ethPeers, fullSyncDisabled, checkpoint); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java index 10c882e7e5a..c7a7d2079f3 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java @@ -15,6 +15,7 @@ package org.hyperledger.besu.ethereum.eth.manager.peertask; import org.hyperledger.besu.ethereum.eth.manager.EthPeer; +import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; import org.hyperledger.besu.ethereum.p2p.rlpx.connections.PeerConnection; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; import org.hyperledger.besu.metrics.BesuMetricCategory; @@ -37,15 +38,18 @@ public class PeerTaskExecutor { public static final int NO_RETRIES = 1; private final PeerSelector peerSelector; private final PeerTaskRequestSender requestSender; + private final EthScheduler ethScheduler; private final LabelledMetric requestTimer; public PeerTaskExecutor( final PeerSelector peerSelector, final PeerTaskRequestSender requestSender, + final EthScheduler ethScheduler, final MetricsSystem metricsSystem) { this.peerSelector = peerSelector; this.requestSender = requestSender; + this.ethScheduler = ethScheduler; requestTimer = metricsSystem.createLabelledTimer( BesuMetricCategory.PEERS, @@ -81,7 +85,13 @@ public PeerTaskExecutorResult execute(final PeerTask peerTask) { } public CompletableFuture> executeAsync(final PeerTask peerTask) { - return CompletableFuture.supplyAsync(() -> execute(peerTask)); + return ethScheduler.scheduleSyncWorkerTask( + () -> CompletableFuture.completedFuture(execute(peerTask))); + } + + public Collection>> executeBatchAsync( + final Collection> peerTasks) { + return peerTasks.stream().map(this::executeAsync).toList(); } public PeerTaskExecutorResult executeAgainstPeer( diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java index 407effb7e0c..c3741292a06 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java @@ -22,6 +22,7 @@ import org.hyperledger.besu.ethereum.core.BlockWithReceipts; import org.hyperledger.besu.ethereum.core.TransactionReceipt; import org.hyperledger.besu.ethereum.eth.manager.EthContext; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTask; import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutor; import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutorResponseCode; import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutorResult; @@ -32,11 +33,15 @@ import org.hyperledger.besu.plugin.services.MetricsSystem; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; +import com.google.common.collect.Lists; + public class DownloadReceiptsStep implements Function, CompletableFuture>> { private final EthContext ethContext; @@ -61,26 +66,46 @@ public CompletableFuture> apply(final List blocks if (synchronizerConfiguration.isPeerTaskSystemEnabled()) { return CompletableFuture.supplyAsync( () -> { - List blockWithReceiptsList = new ArrayList<>(headers.size()); + Map> getReceipts = new ConcurrentHashMap<>(); do { - GetReceiptsFromPeerTask getReceiptsFromPeerTask = - new GetReceiptsFromPeerTask(headers, new BodyValidator()); - PeerTaskExecutorResult>> getReceiptsResult = - peerTaskExecutor.execute(getReceiptsFromPeerTask); - if (getReceiptsResult.responseCode() == PeerTaskExecutorResponseCode.SUCCESS - && getReceiptsResult.result().isPresent()) { - // remove all the headers we found receipts for - headers.removeAll(getReceiptsResult.result().get().keySet()); - blockWithReceiptsList.addAll(combineBlocksAndReceipts(blocks, getReceiptsResult.result().get())); + List> blockHeaderSubLists = Lists.partition(headers, 20); + List>>> tasks = new ArrayList<>(); + for (List blockHeaderSubList : blockHeaderSubLists) { + tasks.add(new GetReceiptsFromPeerTask(blockHeaderSubList, new BodyValidator())); + } + Collection< + CompletableFuture< + PeerTaskExecutorResult>>>> + taskExecutions = peerTaskExecutor.executeBatchAsync(tasks); + for (CompletableFuture< + PeerTaskExecutorResult>>> + taskExecution : taskExecutions) { + taskExecution.thenAccept( + (getReceiptsResult) -> { + if (getReceiptsResult.responseCode() == PeerTaskExecutorResponseCode.SUCCESS + && getReceiptsResult.result().isPresent()) { + Map> taskResult = + getReceiptsResult.result().get(); + taskResult + .keySet() + .forEach( + (blockHeader) -> + getReceipts.merge( + blockHeader, + taskResult.get(blockHeader), + (initialReceipts, newReceipts) -> { + throw new IllegalStateException( + "Unexpectedly got receipts for block header already populated!"); + })); + } + }); } + taskExecutions.forEach(CompletableFuture::join); + // remove all the headers we found receipts for + headers.removeAll(getReceipts.keySet()); // repeat until all headers have receipts } while (!headers.isEmpty()); - - // verify that all blocks have receipts - if (blocks.size() != blockWithReceiptsList.size()) { - throw new IllegalStateException("Not all blocks have been matched to receipts!"); - } - return blockWithReceiptsList; + return combineBlocksAndReceipts(blocks, getReceipts); }); } else { @@ -93,11 +118,19 @@ public CompletableFuture> apply(final List blocks private List combineBlocksAndReceipts( final List blocks, final Map> receiptsByHeader) { return blocks.stream() - .filter((b) -> receiptsByHeader.containsKey(b.getHeader())) .map( block -> { final List receipts = receiptsByHeader.getOrDefault(block.getHeader(), emptyList()); + if (block.getBody().getTransactions().size() != receipts.size()) { + throw new IllegalStateException( + "PeerTask response code was success, but incorrect number of receipts returned. Header hash: " + + block.getHeader().getHash() + + ", Transactions: " + + block.getBody().getTransactions().size() + + ", receipts: " + + receipts.size()); + } return new BlockWithReceipts(block, receipts); }) .toList(); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java index 15b1747bc7b..9dbba7ef631 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java @@ -49,7 +49,8 @@ public class PeerTaskExecutorTest { @BeforeEach public void beforeTest() { mockCloser = MockitoAnnotations.openMocks(this); - peerTaskExecutor = new PeerTaskExecutor(peerSelector, requestSender, new NoOpMetricsSystem()); + peerTaskExecutor = + new PeerTaskExecutor(peerSelector, requestSender, null, new NoOpMetricsSystem()); } @AfterEach diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActionsTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActionsTest.java index 7af807c1c7b..6d194cb8e02 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActionsTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActionsTest.java @@ -537,7 +537,7 @@ private FastSyncActions createFastSyncActions( protocolSchedule, protocolContext, ethContext, - new PeerTaskExecutor(null, null, new NoOpMetricsSystem()), + new PeerTaskExecutor(null, null, null, new NoOpMetricsSystem()), new SyncState(blockchain, ethContext.getEthPeers(), true, Optional.empty()), pivotBlockSelector, new NoOpMetricsSystem()); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloaderTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloaderTest.java index 0e5b5ec2c7c..da82034eaa6 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloaderTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloaderTest.java @@ -111,7 +111,7 @@ private ChainDownloader downloader( protocolSchedule, protocolContext, ethContext, - new PeerTaskExecutor(null, null, new NoOpMetricsSystem()), + new PeerTaskExecutor(null, null, null, new NoOpMetricsSystem()), syncState, new NoOpMetricsSystem(), new FastSyncState(otherBlockchain.getBlockHeader(pivotBlockNumber).get()), From e63f4730c6473bad15a8e01004ae3cf5882ebc19 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Fri, 4 Oct 2024 08:55:24 +1000 Subject: [PATCH 055/106] 7311: Make changes as discussed in walkthrough meeting Remove DefaultPeerSelector, make EthPeers implement PeerSelector interface, and add PeerTask.getPeerRequirementFilter Signed-off-by: Matilda Clerke --- .../besu/ethereum/eth/manager/EthPeers.java | 17 ++- .../manager/peertask/DefaultPeerSelector.java | 102 ------------- .../eth/manager/peertask/PeerSelector.java | 30 +--- .../eth/manager/peertask/PeerTask.java | 16 ++- .../manager/peertask/PeerTaskExecutor.java | 17 ++- .../peertask/DefaultPeerSelectorTest.java | 135 ------------------ .../peertask/PeerTaskExecutorTest.java | 21 ++- 7 files changed, 50 insertions(+), 288 deletions(-) delete mode 100644 ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelector.java delete mode 100644 ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelectorTest.java diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthPeers.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthPeers.java index d1a54d4d3d3..1b413251be7 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthPeers.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthPeers.java @@ -17,6 +17,8 @@ import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.eth.SnapProtocol; import org.hyperledger.besu.ethereum.eth.manager.EthPeer.DisconnectCallback; +import org.hyperledger.besu.ethereum.eth.manager.exceptions.NoAvailablePeersException; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerSelector; import org.hyperledger.besu.ethereum.eth.peervalidation.PeerValidator; import org.hyperledger.besu.ethereum.eth.sync.ChainHeadTracker; import org.hyperledger.besu.ethereum.eth.sync.SnapServerChecker; @@ -26,6 +28,7 @@ import org.hyperledger.besu.ethereum.forkid.ForkIdManager; import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; import org.hyperledger.besu.ethereum.p2p.peers.Peer; +import org.hyperledger.besu.ethereum.p2p.peers.PeerId; import org.hyperledger.besu.ethereum.p2p.rlpx.RlpxAgent; import org.hyperledger.besu.ethereum.p2p.rlpx.connections.PeerConnection; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.messages.DisconnectMessage; @@ -61,7 +64,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class EthPeers { +public class EthPeers implements PeerSelector { private static final Logger LOG = LoggerFactory.getLogger(EthPeers.class); public static final Comparator TOTAL_DIFFICULTY = Comparator.comparing((final EthPeer p) -> p.chainState().getEstimatedTotalDifficulty()); @@ -465,6 +468,18 @@ public void setTrailingPeerRequirementsSupplier( this.trailingPeerRequirementsSupplier = tprSupplier; } + // Part of the PeerSelector interface, to be split apart later + @Override + public EthPeer getPeer(final Predicate filter) { + return streamBestPeers().filter(filter).findFirst().orElseThrow(NoAvailablePeersException::new); + } + + // Part of the PeerSelector interface, to be split apart later + @Override + public Optional getPeerByPeerId(final PeerId peerId) { + return Optional.ofNullable(activeConnections.get(peerId.getId())); + } + @FunctionalInterface public interface ConnectCallback { void onPeerConnected(EthPeer newPeer); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelector.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelector.java deleted file mode 100644 index 41d2e9b700b..00000000000 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelector.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright contributors to Hyperledger Besu. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.besu.ethereum.eth.manager.peertask; - -import org.hyperledger.besu.ethereum.eth.manager.EthPeer; -import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; -import org.hyperledger.besu.ethereum.p2p.peers.PeerId; -import org.hyperledger.besu.ethereum.p2p.rlpx.wire.SubProtocol; - -import java.util.Collection; -import java.util.Comparator; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; -import java.util.function.Predicate; -import java.util.function.Supplier; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * This is a simple PeerSelector implementation that can be used the default implementation in most - * situations - */ -public class DefaultPeerSelector implements PeerSelector { - private static final Logger LOG = LoggerFactory.getLogger(DefaultPeerSelector.class); - - private final Supplier protocolSpecSupplier; - private final Map ethPeersByPeerId = new ConcurrentHashMap<>(); - - public DefaultPeerSelector(final Supplier protocolSpecSupplier) { - this.protocolSpecSupplier = protocolSpecSupplier; - } - - /** - * Gets the highest reputation peer matching the supplied filter - * - * @param filter a filter to match prospective peers with - * @return the highest reputation peer matching the supplies filter - * @throws NoAvailablePeerException If there are no suitable peers - */ - private EthPeer getPeer(final Predicate filter) throws NoAvailablePeerException { - LOG.trace("Finding peer from pool of {} peers", ethPeersByPeerId.size()); - return ethPeersByPeerId.values().stream() - .filter(filter) - .max(Comparator.naturalOrder()) - .orElseThrow(NoAvailablePeerException::new); - } - - @Override - public EthPeer getPeer( - final Collection usedEthPeers, - final long requiredPeerHeight, - final SubProtocol requiredSubProtocol) - throws NoAvailablePeerException { - return getPeer( - (candidatePeer) -> - isPeerUnused(candidatePeer, usedEthPeers) - && (protocolSpecSupplier.get().isPoS() - || isPeerHeightHighEnough(candidatePeer, requiredPeerHeight)) - && isPeerProtocolSuitable(candidatePeer, requiredSubProtocol)); - } - - @Override - public Optional getPeerByPeerId(final PeerId peerId) { - return Optional.ofNullable(ethPeersByPeerId.get(peerId)); - } - - @Override - public void addPeer(final EthPeer ethPeer) { - ethPeersByPeerId.put(ethPeer.getConnection().getPeer(), ethPeer); - } - - @Override - public void removePeer(final PeerId peerId) { - ethPeersByPeerId.remove(peerId); - } - - private boolean isPeerUnused(final EthPeer ethPeer, final Collection usedEthPeers) { - return !usedEthPeers.contains(ethPeer); - } - - private boolean isPeerHeightHighEnough(final EthPeer ethPeer, final long requiredHeight) { - return ethPeer.chainState().getEstimatedHeight() >= requiredHeight; - } - - private boolean isPeerProtocolSuitable(final EthPeer ethPeer, final SubProtocol protocol) { - return ethPeer.getProtocolName().equals(protocol.getName()); - } -} diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerSelector.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerSelector.java index 93d98a193b1..0801f9f00ec 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerSelector.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerSelector.java @@ -16,28 +16,20 @@ import org.hyperledger.besu.ethereum.eth.manager.EthPeer; import org.hyperledger.besu.ethereum.p2p.peers.PeerId; -import org.hyperledger.besu.ethereum.p2p.rlpx.wire.SubProtocol; -import java.util.Collection; import java.util.Optional; +import java.util.function.Predicate; /** Selects the EthPeers for the PeerTaskExecutor */ public interface PeerSelector { /** - * Gets a peer with the requiredPeerHeight (if not PoS), and with the requiredSubProtocol, and - * which is not in the supplied collection of usedEthPeers + * Gets a peer matching the supplied filter * - * @param usedEthPeers a collection of EthPeers to be excluded from selection because they have - * already been used - * @param requiredPeerHeight the minimum peer height required of the selected peer - * @param requiredSubProtocol the SubProtocol required of the peer + * @param filter a Predicate\ matching desirable peers * @return a peer matching the supplied conditions - * @throws NoAvailablePeerException If there are no suitable peers */ - EthPeer getPeer( - Collection usedEthPeers, long requiredPeerHeight, SubProtocol requiredSubProtocol) - throws NoAvailablePeerException; + EthPeer getPeer(final Predicate filter); /** * Attempts to get the EthPeer identified by peerId @@ -47,18 +39,4 @@ EthPeer getPeer( * PeerSelector, or empty otherwise */ Optional getPeerByPeerId(PeerId peerId); - - /** - * Add the supplied EthPeer to the PeerSelector - * - * @param ethPeer the EthPeer to be added to the PeerSelector - */ - void addPeer(EthPeer ethPeer); - - /** - * Remove the EthPeer identified by peerId from the PeerSelector - * - * @param peerId the PeerId of the EthPeer to be removed from the PeerSelector - */ - void removePeer(PeerId peerId); } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTask.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTask.java index 36bd03531bd..1c5ee76c9ab 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTask.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTask.java @@ -14,10 +14,12 @@ */ package org.hyperledger.besu.ethereum.eth.manager.peertask; +import org.hyperledger.besu.ethereum.eth.manager.EthPeer; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.SubProtocol; import java.util.Collection; +import java.util.function.Predicate; /** * Represents a task to be executed on an EthPeer by the PeerTaskExecutor @@ -32,13 +34,6 @@ public interface PeerTask { */ SubProtocol getSubProtocol(); - /** - * Gets the minimum required block number for a peer to have to successfully execute this task - * - * @return the minimum required block number for a peer to have to successfully execute this task - */ - long getRequiredBlockNumber(); - /** * Gets the request data to send to the EthPeer * @@ -61,4 +56,11 @@ public interface PeerTask { * @return the Collection of behaviors this task is expected to exhibit in the PeetTaskExecutor */ Collection getPeerTaskBehaviors(); + + /** + * Gets a Predicate that checks if an EthPeer is suitable for this PeerTask + * + * @return a Predicate that checks if an EthPeer is suitable for this PeerTask + */ + Predicate getPeerRequirementFilter(); } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java index 10c882e7e5a..2d9a6edfce7 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java @@ -15,6 +15,8 @@ package org.hyperledger.besu.ethereum.eth.manager.peertask; import org.hyperledger.besu.ethereum.eth.manager.EthPeer; +import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; +import org.hyperledger.besu.ethereum.eth.manager.exceptions.NoAvailablePeersException; import org.hyperledger.besu.ethereum.p2p.rlpx.connections.PeerConnection; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; import org.hyperledger.besu.metrics.BesuMetricCategory; @@ -37,15 +39,18 @@ public class PeerTaskExecutor { public static final int NO_RETRIES = 1; private final PeerSelector peerSelector; private final PeerTaskRequestSender requestSender; + private final EthScheduler ethScheduler; private final LabelledMetric requestTimer; public PeerTaskExecutor( final PeerSelector peerSelector, final PeerTaskRequestSender requestSender, + final EthScheduler ethScheduler, final MetricsSystem metricsSystem) { this.peerSelector = peerSelector; this.requestSender = requestSender; + this.ethScheduler = ethScheduler; requestTimer = metricsSystem.createLabelledTimer( BesuMetricCategory.PEERS, @@ -66,10 +71,12 @@ public PeerTaskExecutorResult execute(final PeerTask peerTask) { try { peer = peerSelector.getPeer( - usedEthPeers, peerTask.getRequiredBlockNumber(), peerTask.getSubProtocol()); + (candidatePeer) -> + peerTask.getPeerRequirementFilter().test(candidatePeer) + && !usedEthPeers.contains(candidatePeer)); usedEthPeers.add(peer); executorResult = executeAgainstPeer(peerTask, peer); - } catch (NoAvailablePeerException e) { + } catch (NoAvailablePeersException e) { executorResult = new PeerTaskExecutorResult<>( Optional.empty(), PeerTaskExecutorResponseCode.NO_PEER_AVAILABLE); @@ -81,7 +88,8 @@ public PeerTaskExecutorResult execute(final PeerTask peerTask) { } public CompletableFuture> executeAsync(final PeerTask peerTask) { - return CompletableFuture.supplyAsync(() -> execute(peerTask)); + return ethScheduler.scheduleSyncWorkerTask( + () -> CompletableFuture.completedFuture(execute(peerTask))); } public PeerTaskExecutorResult executeAgainstPeer( @@ -138,7 +146,8 @@ public PeerTaskExecutorResult executeAgainstPeer( public CompletableFuture> executeAgainstPeerAsync( final PeerTask peerTask, final EthPeer peer) { - return CompletableFuture.supplyAsync(() -> executeAgainstPeer(peerTask, peer)); + return ethScheduler.scheduleSyncWorkerTask( + () -> CompletableFuture.completedFuture(executeAgainstPeer(peerTask, peer))); } private boolean sleepBetweenRetries() { diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelectorTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelectorTest.java deleted file mode 100644 index add2b1e612c..00000000000 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/DefaultPeerSelectorTest.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Copyright contributors to Hyperledger Besu. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.besu.ethereum.eth.manager.peertask; - -import org.hyperledger.besu.ethereum.eth.EthProtocol; -import org.hyperledger.besu.ethereum.eth.SnapProtocol; -import org.hyperledger.besu.ethereum.eth.manager.ChainState; -import org.hyperledger.besu.ethereum.eth.manager.EthPeer; -import org.hyperledger.besu.ethereum.eth.manager.PeerReputation; -import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; -import org.hyperledger.besu.ethereum.p2p.peers.Peer; -import org.hyperledger.besu.ethereum.p2p.rlpx.connections.PeerConnection; -import org.hyperledger.besu.ethereum.p2p.rlpx.wire.SubProtocol; - -import java.util.HashSet; -import java.util.Set; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - -public class DefaultPeerSelectorTest { - - public DefaultPeerSelector peerSelector; - - @BeforeEach - public void beforeTest() { - ProtocolSpec protocolSpec = Mockito.mock(ProtocolSpec.class); - Mockito.when(protocolSpec.isPoS()).thenReturn(false); - peerSelector = new DefaultPeerSelector(() -> protocolSpec); - } - - @Test - public void testGetPeer() throws NoAvailablePeerException { - EthPeer expectedPeer = createTestPeer(10, EthProtocol.get(), 5); - peerSelector.addPeer(expectedPeer); - EthPeer excludedForLowChainHeightPeer = createTestPeer(5, EthProtocol.get(), 50); - peerSelector.addPeer(excludedForLowChainHeightPeer); - EthPeer excludedForWrongProtocolPeer = createTestPeer(10, SnapProtocol.get(), 50); - peerSelector.addPeer(excludedForWrongProtocolPeer); - EthPeer excludedForLowReputationPeer = createTestPeer(10, EthProtocol.get(), 1); - peerSelector.addPeer(excludedForLowReputationPeer); - EthPeer excludedForBeingAlreadyUsedPeer = createTestPeer(10, EthProtocol.get(), 50); - peerSelector.addPeer(excludedForBeingAlreadyUsedPeer); - - Set usedEthPeers = new HashSet<>(); - usedEthPeers.add(excludedForBeingAlreadyUsedPeer); - - EthPeer result = peerSelector.getPeer(usedEthPeers, 10, EthProtocol.get()); - - Assertions.assertSame(expectedPeer, result); - } - - @Test - public void testGetPeerButNoPeerMatchesFilter() { - EthPeer expectedPeer = createTestPeer(10, EthProtocol.get(), 5); - peerSelector.addPeer(expectedPeer); - EthPeer excludedForLowChainHeightPeer = createTestPeer(5, EthProtocol.get(), 50); - peerSelector.addPeer(excludedForLowChainHeightPeer); - EthPeer excludedForWrongProtocolPeer = createTestPeer(10, SnapProtocol.get(), 50); - peerSelector.addPeer(excludedForWrongProtocolPeer); - EthPeer excludedForLowReputationPeer = createTestPeer(10, EthProtocol.get(), 1); - peerSelector.addPeer(excludedForLowReputationPeer); - EthPeer excludedForBeingAlreadyUsedPeer = createTestPeer(10, EthProtocol.get(), 50); - peerSelector.addPeer(excludedForBeingAlreadyUsedPeer); - - Set usedEthPeers = new HashSet<>(); - usedEthPeers.add(excludedForBeingAlreadyUsedPeer); - - Assertions.assertThrows( - NoAvailablePeerException.class, - () -> peerSelector.getPeer(usedEthPeers, 10, new MockSubProtocol())); - } - - private EthPeer createTestPeer( - final long chainHeight, final SubProtocol protocol, final int reputation) { - EthPeer ethPeer = Mockito.mock(EthPeer.class); - PeerConnection peerConnection = Mockito.mock(PeerConnection.class); - Peer peer = Mockito.mock(Peer.class); - ChainState chainState = Mockito.mock(ChainState.class); - PeerReputation peerReputation = Mockito.mock(PeerReputation.class); - - Mockito.when(ethPeer.getConnection()).thenReturn(peerConnection); - Mockito.when(peerConnection.getPeer()).thenReturn(peer); - Mockito.when(ethPeer.getProtocolName()).thenReturn(protocol.getName()); - Mockito.when(ethPeer.chainState()).thenReturn(chainState); - Mockito.when(chainState.getEstimatedHeight()).thenReturn(chainHeight); - Mockito.when(ethPeer.getReputation()).thenReturn(peerReputation); - Mockito.when(peerReputation.getScore()).thenReturn(reputation); - - Mockito.when(ethPeer.compareTo(Mockito.any(EthPeer.class))) - .thenAnswer( - (invocationOnMock) -> { - EthPeer otherPeer = invocationOnMock.getArgument(0, EthPeer.class); - return Integer.compare(reputation, otherPeer.getReputation().getScore()); - }); - return ethPeer; - } - - private static class MockSubProtocol implements SubProtocol { - - @Override - public String getName() { - return "Mock"; - } - - @Override - public int messageSpace(final int protocolVersion) { - throw new UnsupportedOperationException(); - } - - @Override - public boolean isValidMessageCode(final int protocolVersion, final int code) { - throw new UnsupportedOperationException(); - } - - @Override - public String messageName(final int protocolVersion, final int code) { - throw new UnsupportedOperationException(); - } - } -} diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java index 15b1747bc7b..98afe58b11c 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java @@ -15,16 +15,17 @@ package org.hyperledger.besu.ethereum.eth.manager.peertask; import org.hyperledger.besu.ethereum.eth.manager.EthPeer; +import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; import org.hyperledger.besu.ethereum.p2p.rlpx.connections.PeerConnection; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.SubProtocol; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; -import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeoutException; +import java.util.function.Predicate; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; @@ -37,6 +38,7 @@ public class PeerTaskExecutorTest { private @Mock PeerSelector peerSelector; private @Mock PeerTaskRequestSender requestSender; + private @Mock EthScheduler ethScheduler; private @Mock PeerTask peerTask; private @Mock SubProtocol subprotocol; private @Mock MessageData requestMessageData; @@ -49,7 +51,8 @@ public class PeerTaskExecutorTest { @BeforeEach public void beforeTest() { mockCloser = MockitoAnnotations.openMocks(this); - peerTaskExecutor = new PeerTaskExecutor(peerSelector, requestSender, new NoOpMetricsSystem()); + peerTaskExecutor = + new PeerTaskExecutor(peerSelector, requestSender, ethScheduler, new NoOpMetricsSystem()); } @AfterEach @@ -201,14 +204,10 @@ public void testExecuteWithNoPeerTaskBehaviorsAndSuccessFlow() NoAvailablePeerException { Object responseObject = new Object(); - Mockito.when( - peerSelector.getPeer( - Mockito.any(Collection.class), Mockito.eq(10L), Mockito.eq(subprotocol))) - .thenReturn(ethPeer); + Mockito.when(peerSelector.getPeer(Mockito.any(Predicate.class))).thenReturn(ethPeer); Mockito.when(peerTask.getRequestMessage()).thenReturn(requestMessageData); Mockito.when(peerTask.getPeerTaskBehaviors()).thenReturn(Collections.emptyList()); - Mockito.when(peerTask.getRequiredBlockNumber()).thenReturn(10L); Mockito.when(peerTask.getSubProtocol()).thenReturn(subprotocol); Mockito.when(subprotocol.getName()).thenReturn("subprotocol"); Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, ethPeer)) @@ -232,22 +231,18 @@ public void testExecuteWithPeerSwitchingAndSuccessFlow() ExecutionException, InterruptedException, TimeoutException, - InvalidPeerTaskResponseException, - NoAvailablePeerException { + InvalidPeerTaskResponseException { Object responseObject = new Object(); int requestMessageDataCode = 123; EthPeer peer2 = Mockito.mock(EthPeer.class); - Mockito.when( - peerSelector.getPeer( - Mockito.any(Collection.class), Mockito.eq(10L), Mockito.eq(subprotocol))) + Mockito.when(peerSelector.getPeer(Mockito.any(Predicate.class))) .thenReturn(ethPeer) .thenReturn(peer2); Mockito.when(peerTask.getRequestMessage()).thenReturn(requestMessageData); Mockito.when(peerTask.getPeerTaskBehaviors()) .thenReturn(List.of(PeerTaskRetryBehavior.RETRY_WITH_OTHER_PEERS)); - Mockito.when(peerTask.getRequiredBlockNumber()).thenReturn(10L); Mockito.when(peerTask.getSubProtocol()).thenReturn(subprotocol); Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, ethPeer)) .thenThrow(new TimeoutException()); From 07852dceed77965432342a7480e55b556b1d21ab Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Fri, 4 Oct 2024 11:21:31 +1000 Subject: [PATCH 056/106] 7311: Update after merge and make discussed changes from walkthrough discussion Signed-off-by: Matilda Clerke --- .../controller/BesuControllerBuilder.java | 16 +--- ...onsensusScheduleBesuControllerBuilder.java | 7 +- .../MergeBesuControllerBuilder.java | 7 +- .../TransitionBesuControllerBuilder.java | 7 +- .../eth/manager/EthProtocolManager.java | 14 +-- .../task/GetReceiptsFromPeerTask.java | 24 ++++-- .../sync/fastsync/DownloadReceiptsStep.java | 86 ++++++++----------- .../eth/manager/EthProtocolManagerTest.java | 4 +- .../manager/EthProtocolManagerTestUtil.java | 7 +- .../task/GetReceiptsFromPeerTaskTest.java | 39 +++++++-- .../fastsync/DownloadReceiptsStepTest.java | 4 +- .../ethereum/eth/transactions/TestNode.java | 4 +- .../TransactionPoolFactoryTest.java | 4 +- 13 files changed, 105 insertions(+), 118 deletions(-) diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java index b85ad75736d..61c6d057c80 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java @@ -55,8 +55,6 @@ import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; import org.hyperledger.besu.ethereum.eth.manager.MergePeerFilter; import org.hyperledger.besu.ethereum.eth.manager.MonitoredExecutors; -import org.hyperledger.besu.ethereum.eth.manager.peertask.DefaultPeerSelector; -import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerSelector; import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutor; import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskRequestSender; import org.hyperledger.besu.ethereum.eth.manager.snap.SnapProtocolManager; @@ -656,10 +654,8 @@ public BesuController build() { } final EthContext ethContext = new EthContext(ethPeers, ethMessages, snapMessages, scheduler); - final PeerSelector peerSelector = new DefaultPeerSelector(currentProtocolSpecSupplier); - ethPeers.streamAllPeers().forEach(peerSelector::addPeer); final PeerTaskExecutor peerTaskExecutor = - new PeerTaskExecutor(peerSelector, new PeerTaskRequestSender(), scheduler, metricsSystem); + new PeerTaskExecutor(ethPeers, new PeerTaskRequestSender(), scheduler, metricsSystem); final boolean fullSyncDisabled = !SyncMode.isFullSync(syncConfig.getSyncMode()); final SyncState syncState = new SyncState(blockchain, ethPeers, fullSyncDisabled, checkpoint); @@ -699,8 +695,7 @@ public BesuController build() { scheduler, peerValidators, Optional.empty(), - forkIdManager, - peerSelector); + forkIdManager); final PivotBlockSelector pivotBlockSelector = createPivotSelector( @@ -1035,7 +1030,6 @@ protected String getSupportedProtocol() { * @param peerValidators the peer validators * @param mergePeerFilter the merge peer filter * @param forkIdManager the fork id manager - * @param peerSelector the PeerSelector * @return the eth protocol manager */ protected EthProtocolManager createEthProtocolManager( @@ -1049,8 +1043,7 @@ protected EthProtocolManager createEthProtocolManager( final EthScheduler scheduler, final List peerValidators, final Optional mergePeerFilter, - final ForkIdManager forkIdManager, - final PeerSelector peerSelector) { + final ForkIdManager forkIdManager) { return new EthProtocolManager( protocolContext.getBlockchain(), networkId, @@ -1064,8 +1057,7 @@ protected EthProtocolManager createEthProtocolManager( mergePeerFilter, synchronizerConfiguration, scheduler, - forkIdManager, - peerSelector); + forkIdManager); } /** diff --git a/besu/src/main/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilder.java index 2e28a3179c8..0d0f8d2fd87 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilder.java @@ -42,7 +42,6 @@ import org.hyperledger.besu.ethereum.eth.manager.EthProtocolManager; import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; import org.hyperledger.besu.ethereum.eth.manager.MergePeerFilter; -import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerSelector; import org.hyperledger.besu.ethereum.eth.manager.snap.SnapProtocolManager; import org.hyperledger.besu.ethereum.eth.peervalidation.PeerValidator; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; @@ -244,8 +243,7 @@ protected EthProtocolManager createEthProtocolManager( final EthScheduler scheduler, final List peerValidators, final Optional mergePeerFilter, - final ForkIdManager forkIdManager, - final PeerSelector peerSelector) { + final ForkIdManager forkIdManager) { return besuControllerBuilderSchedule .get(0L) .createEthProtocolManager( @@ -259,8 +257,7 @@ protected EthProtocolManager createEthProtocolManager( scheduler, peerValidators, mergePeerFilter, - forkIdManager, - peerSelector); + forkIdManager); } @Override diff --git a/besu/src/main/java/org/hyperledger/besu/controller/MergeBesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/MergeBesuControllerBuilder.java index 88629a7441e..f5fc75959e1 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/MergeBesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/MergeBesuControllerBuilder.java @@ -34,7 +34,6 @@ import org.hyperledger.besu.ethereum.eth.manager.EthProtocolManager; import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; import org.hyperledger.besu.ethereum.eth.manager.MergePeerFilter; -import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerSelector; import org.hyperledger.besu.ethereum.eth.peervalidation.PeerValidator; import org.hyperledger.besu.ethereum.eth.peervalidation.RequiredBlocksPeerValidator; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; @@ -100,8 +99,7 @@ protected EthProtocolManager createEthProtocolManager( final EthScheduler scheduler, final List peerValidators, final Optional mergePeerFilter, - final ForkIdManager forkIdManager, - final PeerSelector peerSelector) { + final ForkIdManager forkIdManager) { var mergeContext = protocolContext.getConsensusContext(MergeContext.class); @@ -131,8 +129,7 @@ protected EthProtocolManager createEthProtocolManager( scheduler, peerValidators, filterToUse, - forkIdManager, - peerSelector); + forkIdManager); return ethProtocolManager; } diff --git a/besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java index 0f56d0c618a..703592f90a9 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java @@ -40,7 +40,6 @@ import org.hyperledger.besu.ethereum.eth.manager.EthProtocolManager; import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; import org.hyperledger.besu.ethereum.eth.manager.MergePeerFilter; -import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerSelector; import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutor; import org.hyperledger.besu.ethereum.eth.peervalidation.PeerValidator; import org.hyperledger.besu.ethereum.eth.sync.DefaultSynchronizer; @@ -165,8 +164,7 @@ protected EthProtocolManager createEthProtocolManager( final EthScheduler scheduler, final List peerValidators, final Optional mergePeerFilter, - final ForkIdManager forkIdManager, - final PeerSelector peerSelector) { + final ForkIdManager forkIdManager) { return mergeBesuControllerBuilder.createEthProtocolManager( protocolContext, synchronizerConfiguration, @@ -178,8 +176,7 @@ protected EthProtocolManager createEthProtocolManager( scheduler, peerValidators, mergePeerFilter, - forkIdManager, - peerSelector); + forkIdManager); } @Override diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManager.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManager.java index 97e91f6badc..30cd03c15c3 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManager.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManager.java @@ -23,7 +23,6 @@ import org.hyperledger.besu.ethereum.core.Difficulty; import org.hyperledger.besu.ethereum.eth.EthProtocol; import org.hyperledger.besu.ethereum.eth.EthProtocolConfiguration; -import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerSelector; import org.hyperledger.besu.ethereum.eth.messages.EthPV62; import org.hyperledger.besu.ethereum.eth.messages.StatusMessage; import org.hyperledger.besu.ethereum.eth.peervalidation.PeerValidator; @@ -70,7 +69,6 @@ public class EthProtocolManager implements ProtocolManager, MinedBlockObserver { private final Hash genesisHash; private final ForkIdManager forkIdManager; - private final PeerSelector peerSelector; private final BigInteger networkId; private final EthPeers ethPeers; private final EthMessages ethMessages; @@ -94,8 +92,7 @@ public EthProtocolManager( final Optional mergePeerFilter, final SynchronizerConfiguration synchronizerConfiguration, final EthScheduler scheduler, - final ForkIdManager forkIdManager, - final PeerSelector peerSelector) { + final ForkIdManager forkIdManager) { this.networkId = networkId; this.peerValidators = peerValidators; this.scheduler = scheduler; @@ -105,7 +102,6 @@ public EthProtocolManager( this.genesisHash = blockchain.getBlockHashByNumber(0L).orElse(Hash.ZERO); this.forkIdManager = forkIdManager; - this.peerSelector = peerSelector; this.ethPeers = ethPeers; this.ethMessages = ethMessages; @@ -144,8 +140,7 @@ public EthProtocolManager( final List peerValidators, final Optional mergePeerFilter, final SynchronizerConfiguration synchronizerConfiguration, - final EthScheduler scheduler, - final PeerSelector peerSelector) { + final EthScheduler scheduler) { this( blockchain, networkId, @@ -163,8 +158,7 @@ public EthProtocolManager( blockchain, Collections.emptyList(), Collections.emptyList(), - ethereumWireProtocolConfiguration.isLegacyEth64ForkIdEnabled()), - peerSelector); + ethereumWireProtocolConfiguration.isLegacyEth64ForkIdEnabled())); } public EthContext ethContext() { @@ -343,7 +337,6 @@ public void processMessage(final Capability cap, final Message message) { public void handleNewConnection(final PeerConnection connection) { ethPeers.registerNewConnection(connection, peerValidators); final EthPeer peer = ethPeers.peer(connection); - peerSelector.addPeer(peer); final Capability cap = connection.capability(getSupportedProtocol()); final ForkId latestForkId = cap.getVersion() >= 64 ? forkIdManager.getForkIdForChainHead() : null; @@ -375,7 +368,6 @@ public void handleDisconnect( final DisconnectReason reason, final boolean initiatedByPeer) { final boolean wasActiveConnection = ethPeers.registerDisconnect(connection); - peerSelector.removePeer(connection.getPeer()); LOG.atDebug() .setMessage("Disconnect - active Connection? {} - {} - {} - {} {} - {} peers left") .addArgument(wasActiveConnection) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTask.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTask.java index 75263f460e1..cb84f82033c 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTask.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTask.java @@ -18,6 +18,7 @@ import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.TransactionReceipt; import org.hyperledger.besu.ethereum.eth.EthProtocol; +import org.hyperledger.besu.ethereum.eth.manager.EthPeer; import org.hyperledger.besu.ethereum.eth.manager.peertask.InvalidPeerTaskResponseException; import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTask; import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskRetryBehavior; @@ -32,6 +33,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Predicate; public class GetReceiptsFromPeerTask implements PeerTask>> { @@ -39,6 +41,7 @@ public class GetReceiptsFromPeerTask private final Collection blockHeaders; private final BodyValidator bodyValidator; private final Map> headersByReceiptsRoot = new HashMap<>(); + private final long requiredBlockchainHeight; public GetReceiptsFromPeerTask( final Collection blockHeaders, final BodyValidator bodyValidator) { @@ -50,6 +53,12 @@ public GetReceiptsFromPeerTask( headersByReceiptsRoot .computeIfAbsent(header.getReceiptsRoot(), key -> new ArrayList<>()) .add(header)); + + requiredBlockchainHeight = + blockHeaders.stream() + .mapToLong(BlockHeader::getNumber) + .max() + .orElse(BlockHeader.GENESIS_BLOCK_NUMBER); } @Override @@ -57,14 +66,6 @@ public SubProtocol getSubProtocol() { return EthProtocol.get(); } - @Override - public long getRequiredBlockNumber() { - return blockHeaders.stream() - .mapToLong(BlockHeader::getNumber) - .max() - .orElse(BlockHeader.GENESIS_BLOCK_NUMBER); - } - @Override public MessageData getRequestMessage() { // Since we have to match up the data by receipt root, we only need to request receipts @@ -106,4 +107,11 @@ public Collection getPeerTaskBehaviors() { return List.of( PeerTaskRetryBehavior.RETRY_WITH_OTHER_PEERS, PeerTaskRetryBehavior.RETRY_WITH_SAME_PEER); } + + @Override + public Predicate getPeerRequirementFilter() { + return (ethPeer) -> + ethPeer.getProtocolName().equals(getSubProtocol().getName()) + && ethPeer.chainState().getEstimatedHeight() >= requiredBlockchainHeight; + } } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java index c3741292a06..86b2f31fefb 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java @@ -22,7 +22,6 @@ import org.hyperledger.besu.ethereum.core.BlockWithReceipts; import org.hyperledger.besu.ethereum.core.TransactionReceipt; import org.hyperledger.besu.ethereum.eth.manager.EthContext; -import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTask; import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutor; import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutorResponseCode; import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutorResult; @@ -32,18 +31,15 @@ import org.hyperledger.besu.ethereum.mainnet.BodyValidator; import org.hyperledger.besu.plugin.services.MetricsSystem; -import java.util.ArrayList; -import java.util.Collection; import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; -import com.google.common.collect.Lists; - public class DownloadReceiptsStep implements Function, CompletableFuture>> { + private final EthContext ethContext; private final PeerTaskExecutor peerTaskExecutor; private final SynchronizerConfiguration synchronizerConfiguration; @@ -64,49 +60,43 @@ public DownloadReceiptsStep( public CompletableFuture> apply(final List blocks) { final List headers = blocks.stream().map(Block::getHeader).collect(toList()); if (synchronizerConfiguration.isPeerTaskSystemEnabled()) { - return CompletableFuture.supplyAsync( - () -> { - Map> getReceipts = new ConcurrentHashMap<>(); - do { - List> blockHeaderSubLists = Lists.partition(headers, 20); - List>>> tasks = new ArrayList<>(); - for (List blockHeaderSubList : blockHeaderSubLists) { - tasks.add(new GetReceiptsFromPeerTask(blockHeaderSubList, new BodyValidator())); - } - Collection< - CompletableFuture< - PeerTaskExecutorResult>>>> - taskExecutions = peerTaskExecutor.executeBatchAsync(tasks); - for (CompletableFuture< - PeerTaskExecutorResult>>> - taskExecution : taskExecutions) { - taskExecution.thenAccept( - (getReceiptsResult) -> { - if (getReceiptsResult.responseCode() == PeerTaskExecutorResponseCode.SUCCESS - && getReceiptsResult.result().isPresent()) { - Map> taskResult = - getReceiptsResult.result().get(); - taskResult - .keySet() - .forEach( - (blockHeader) -> - getReceipts.merge( - blockHeader, - taskResult.get(blockHeader), - (initialReceipts, newReceipts) -> { - throw new IllegalStateException( - "Unexpectedly got receipts for block header already populated!"); - })); - } - }); - } - taskExecutions.forEach(CompletableFuture::join); - // remove all the headers we found receipts for - headers.removeAll(getReceipts.keySet()); - // repeat until all headers have receipts - } while (!headers.isEmpty()); - return combineBlocksAndReceipts(blocks, getReceipts); - }); + return ethContext + .getScheduler() + .scheduleSyncWorkerTask( + () -> { + Map> getReceipts = new ConcurrentHashMap<>(); + do { + GetReceiptsFromPeerTask task = + new GetReceiptsFromPeerTask(headers, new BodyValidator()); + PeerTaskExecutorResult>> + getReceiptsResult = peerTaskExecutor.execute(task); + if (getReceiptsResult.responseCode() == PeerTaskExecutorResponseCode.SUCCESS + && getReceiptsResult.result().isPresent()) { + Map> taskResult = + getReceiptsResult.result().get(); + taskResult + .keySet() + .forEach( + (blockHeader) -> + getReceipts.merge( + blockHeader, + taskResult.get(blockHeader), + (initialReceipts, newReceipts) -> { + throw new IllegalStateException( + "Unexpectedly got receipts for block header already populated!"); + })); + } else if (getReceiptsResult.responseCode() + == PeerTaskExecutorResponseCode.NO_PEER_AVAILABLE) { + throw new RuntimeException( + "No peer available, unable to complete DownloadReceiptsStep"); + } + // remove all the headers we found receipts for + headers.removeAll(getReceipts.keySet()); + // repeat until all headers have receipts + } while (!headers.isEmpty()); + return CompletableFuture.completedFuture( + combineBlocksAndReceipts(blocks, getReceipts)); + }); } else { return GetReceiptsForHeadersTask.forHeaders(ethContext, headers, metricsSystem) diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManagerTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManagerTest.java index 95f3d762220..3a3331b568f 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManagerTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManagerTest.java @@ -44,7 +44,6 @@ import org.hyperledger.besu.ethereum.eth.EthProtocolConfiguration; import org.hyperledger.besu.ethereum.eth.EthProtocolVersion; import org.hyperledger.besu.ethereum.eth.manager.MockPeerConnection.PeerSendHandler; -import org.hyperledger.besu.ethereum.eth.manager.peertask.DefaultPeerSelector; import org.hyperledger.besu.ethereum.eth.messages.BlockBodiesMessage; import org.hyperledger.besu.ethereum.eth.messages.BlockHeadersMessage; import org.hyperledger.besu.ethereum.eth.messages.EthPV62; @@ -1244,8 +1243,7 @@ private EthProtocolManager createEthManager( Optional.empty(), syncConfig, mock(EthScheduler.class), - mock(ForkIdManager.class), - new DefaultPeerSelector(() -> null))) { + mock(ForkIdManager.class))) { return ethManager; } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManagerTestUtil.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManagerTestUtil.java index 16199571e66..0b0bd1e3eb7 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManagerTestUtil.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManagerTestUtil.java @@ -30,7 +30,6 @@ import org.hyperledger.besu.ethereum.core.ProtocolScheduleFixture; import org.hyperledger.besu.ethereum.eth.EthProtocol; import org.hyperledger.besu.ethereum.eth.EthProtocolConfiguration; -import org.hyperledger.besu.ethereum.eth.manager.peertask.DefaultPeerSelector; import org.hyperledger.besu.ethereum.eth.manager.snap.SnapProtocolManager; import org.hyperledger.besu.ethereum.eth.peervalidation.PeerValidator; import org.hyperledger.besu.ethereum.eth.sync.ChainHeadTracker; @@ -118,8 +117,7 @@ public static EthProtocolManager create( mergePeerFilter, mock(SynchronizerConfiguration.class), ethScheduler, - new ForkIdManager(blockchain, Collections.emptyList(), Collections.emptyList(), false), - new DefaultPeerSelector(() -> null)); + new ForkIdManager(blockchain, Collections.emptyList(), Collections.emptyList(), false)); } public static EthProtocolManager create( @@ -170,8 +168,7 @@ public static EthProtocolManager create( Optional.empty(), mock(SynchronizerConfiguration.class), ethScheduler, - forkIdManager, - new DefaultPeerSelector(() -> null)); + forkIdManager); } public static EthProtocolManager create(final Blockchain blockchain) { diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTaskTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTaskTest.java index f8f1c88e1ef..4529d9d3966 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTaskTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTaskTest.java @@ -18,6 +18,8 @@ import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.TransactionReceipt; import org.hyperledger.besu.ethereum.eth.EthProtocol; +import org.hyperledger.besu.ethereum.eth.manager.ChainState; +import org.hyperledger.besu.ethereum.eth.manager.EthPeer; import org.hyperledger.besu.ethereum.eth.manager.peertask.InvalidPeerTaskResponseException; import org.hyperledger.besu.ethereum.eth.messages.EthPV63; import org.hyperledger.besu.ethereum.eth.messages.GetReceiptsMessage; @@ -44,14 +46,6 @@ public void testGetSubProtocol() { Assertions.assertEquals(EthProtocol.get(), task.getSubProtocol()); } - @Test - public void testGetRequiredBlockNumber() { - GetReceiptsFromPeerTask task = - new GetReceiptsFromPeerTask( - List.of(mockBlockHeader(1), mockBlockHeader(2), mockBlockHeader(3)), null); - Assertions.assertEquals(3, task.getRequiredBlockNumber()); - } - @Test public void testGetRequestMessage() { GetReceiptsFromPeerTask task = @@ -137,6 +131,24 @@ public void testParseResponse() throws InvalidPeerTaskResponseException { Assertions.assertEquals(List.of(receiptForBlock3), resultMap.get(blockHeader3)); } + @Test + public void testGetPeerRequirementFilter() { + BlockHeader blockHeader1 = mockBlockHeader(1); + BlockHeader blockHeader2 = mockBlockHeader(2); + BlockHeader blockHeader3 = mockBlockHeader(3); + + GetReceiptsFromPeerTask task = + new GetReceiptsFromPeerTask(List.of(blockHeader1, blockHeader2, blockHeader3), null); + + EthPeer failForIncorrectProtocol = mockPeer("incorrectProtocol", 5); + EthPeer failForShortChainHeight = mockPeer("incorrectProtocol", 1); + EthPeer successfulCandidate = mockPeer(EthProtocol.NAME, 5); + + Assertions.assertFalse(task.getPeerRequirementFilter().test(failForIncorrectProtocol)); + Assertions.assertFalse(task.getPeerRequirementFilter().test(failForShortChainHeight)); + Assertions.assertTrue(task.getPeerRequirementFilter().test(successfulCandidate)); + } + private BlockHeader mockBlockHeader(final long blockNumber) { BlockHeader blockHeader = Mockito.mock(BlockHeader.class); Mockito.when(blockHeader.getNumber()).thenReturn(blockNumber); @@ -149,4 +161,15 @@ private BlockHeader mockBlockHeader(final long blockNumber) { return blockHeader; } + + private EthPeer mockPeer(final String protocol, final long chainHeight) { + EthPeer ethPeer = Mockito.mock(EthPeer.class); + ChainState chainState = Mockito.mock(ChainState.class); + + Mockito.when(ethPeer.getProtocolName()).thenReturn(protocol); + Mockito.when(ethPeer.chainState()).thenReturn(chainState); + Mockito.when(chainState.getEstimatedHeight()).thenReturn(chainHeight); + + return ethPeer; + } } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStepTest.java index 554e09c3e91..c349df1c383 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStepTest.java @@ -124,8 +124,8 @@ public void shouldDownloadReceiptsForBlocksUsingPeerTaskSystem() PeerTaskExecutorResult>> peerTaskResult = new PeerTaskExecutorResult<>( Optional.of(receiptsMap), PeerTaskExecutorResponseCode.SUCCESS); - Mockito.when(peerTaskExecutor.execute(Mockito.any(GetReceiptsFromPeerTask.class))) - .thenReturn(peerTaskResult); + Mockito.when(peerTaskExecutor.executeAsync(Mockito.any(GetReceiptsFromPeerTask.class))) + .thenReturn(CompletableFuture.completedFuture(peerTaskResult)); final CompletableFuture> result = downloadReceiptsStep.apply(blocks); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TestNode.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TestNode.java index c41e326facb..c679183b0ff 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TestNode.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TestNode.java @@ -44,7 +44,6 @@ import org.hyperledger.besu.ethereum.eth.manager.EthPeers; import org.hyperledger.besu.ethereum.eth.manager.EthProtocolManager; import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; -import org.hyperledger.besu.ethereum.eth.manager.peertask.DefaultPeerSelector; import org.hyperledger.besu.ethereum.eth.sync.ChainHeadTracker; import org.hyperledger.besu.ethereum.eth.sync.SyncMode; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; @@ -197,8 +196,7 @@ public boolean isMessagePermitted(final EnodeURL destinationEnode, final int cod Collections.emptyList(), Optional.empty(), syncConfig, - scheduler, - new DefaultPeerSelector(() -> null)); + scheduler); final NetworkRunner networkRunner = NetworkRunner.builder() diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolFactoryTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolFactoryTest.java index f4dac8a244e..5742637c3e8 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolFactoryTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolFactoryTest.java @@ -45,7 +45,6 @@ import org.hyperledger.besu.ethereum.eth.manager.EthPeers; import org.hyperledger.besu.ethereum.eth.manager.EthProtocolManager; import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; -import org.hyperledger.besu.ethereum.eth.manager.peertask.DefaultPeerSelector; import org.hyperledger.besu.ethereum.eth.sync.SyncMode; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; @@ -319,8 +318,7 @@ private void setupInitialSyncPhase(final SyncState syncState) { Optional.empty(), mock(SynchronizerConfiguration.class), mock(EthScheduler.class), - mock(ForkIdManager.class), - new DefaultPeerSelector(() -> null)); + mock(ForkIdManager.class)); } @Test From c477d70f57bed6e2943095d490e2c7fbc4b3aaf4 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Fri, 4 Oct 2024 15:54:07 +1000 Subject: [PATCH 057/106] 7311: Change to regular HashMap Signed-off-by: Matilda Clerke --- .../besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java index 86b2f31fefb..c45d36ee98c 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java @@ -31,10 +31,10 @@ import org.hyperledger.besu.ethereum.mainnet.BodyValidator; import org.hyperledger.besu.plugin.services.MetricsSystem; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; public class DownloadReceiptsStep @@ -64,7 +64,7 @@ public CompletableFuture> apply(final List blocks .getScheduler() .scheduleSyncWorkerTask( () -> { - Map> getReceipts = new ConcurrentHashMap<>(); + Map> getReceipts = new HashMap<>(); do { GetReceiptsFromPeerTask task = new GetReceiptsFromPeerTask(headers, new BodyValidator()); From 6c57a7c3d9a8200aa55a4a2e16e79506ff2ff202 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Fri, 4 Oct 2024 16:20:23 +1000 Subject: [PATCH 058/106] 7311: Remove runtime exception again Signed-off-by: Matilda Clerke --- .../besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java index c45d36ee98c..4c60ac1d193 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java @@ -85,10 +85,6 @@ public CompletableFuture> apply(final List blocks throw new IllegalStateException( "Unexpectedly got receipts for block header already populated!"); })); - } else if (getReceiptsResult.responseCode() - == PeerTaskExecutorResponseCode.NO_PEER_AVAILABLE) { - throw new RuntimeException( - "No peer available, unable to complete DownloadReceiptsStep"); } // remove all the headers we found receipts for headers.removeAll(getReceipts.keySet()); From 6d2cb9567837c305c6280efb6ebde72741cd0f08 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Mon, 7 Oct 2024 08:46:08 +1100 Subject: [PATCH 059/106] 7311: Rename getPeerTaskBehavior to getPeerTaskRetryBehavior Signed-off-by: Matilda Clerke --- .../ethereum/eth/manager/peertask/PeerTask.java | 2 +- .../eth/manager/peertask/PeerTaskExecutor.java | 4 ++-- .../eth/manager/peertask/PeerTaskExecutorTest.java | 14 +++++++------- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTask.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTask.java index 1c5ee76c9ab..4436022c9ad 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTask.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTask.java @@ -55,7 +55,7 @@ public interface PeerTask { * * @return the Collection of behaviors this task is expected to exhibit in the PeetTaskExecutor */ - Collection getPeerTaskBehaviors(); + Collection getPeerTaskRetryBehaviors(); /** * Gets a Predicate that checks if an EthPeer is suitable for this PeerTask diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java index 2d9a6edfce7..d4b3f603d19 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java @@ -62,7 +62,7 @@ public PeerTaskExecutor( public PeerTaskExecutorResult execute(final PeerTask peerTask) { PeerTaskExecutorResult executorResult; int triesRemaining = - peerTask.getPeerTaskBehaviors().contains(PeerTaskRetryBehavior.RETRY_WITH_OTHER_PEERS) + peerTask.getPeerTaskRetryBehaviors().contains(PeerTaskRetryBehavior.RETRY_WITH_OTHER_PEERS) ? RETRIES_WITH_OTHER_PEER : NO_RETRIES; final Collection usedEthPeers = new HashSet<>(); @@ -97,7 +97,7 @@ public PeerTaskExecutorResult executeAgainstPeer( MessageData requestMessageData = peerTask.getRequestMessage(); PeerTaskExecutorResult executorResult; int triesRemaining = - peerTask.getPeerTaskBehaviors().contains(PeerTaskRetryBehavior.RETRY_WITH_SAME_PEER) + peerTask.getPeerTaskRetryBehaviors().contains(PeerTaskRetryBehavior.RETRY_WITH_SAME_PEER) ? RETRIES_WITH_SAME_PEER : NO_RETRIES; do { diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java index 98afe58b11c..a999aa7116e 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java @@ -71,7 +71,7 @@ public void testExecuteAgainstPeerWithNoPeerTaskBehaviorsAndSuccessfulFlow() Object responseObject = new Object(); Mockito.when(peerTask.getRequestMessage()).thenReturn(requestMessageData); - Mockito.when(peerTask.getPeerTaskBehaviors()).thenReturn(Collections.emptyList()); + Mockito.when(peerTask.getPeerTaskRetryBehaviors()).thenReturn(Collections.emptyList()); Mockito.when(peerTask.getSubProtocol()).thenReturn(subprotocol); Mockito.when(subprotocol.getName()).thenReturn("subprotocol"); Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, ethPeer)) @@ -99,7 +99,7 @@ public void testExecuteAgainstPeerWithRetryBehaviorsAndSuccessfulFlowAfterFirstF int requestMessageDataCode = 123; Mockito.when(peerTask.getRequestMessage()).thenReturn(requestMessageData); - Mockito.when(peerTask.getPeerTaskBehaviors()) + Mockito.when(peerTask.getPeerTaskRetryBehaviors()) .thenReturn(List.of(PeerTaskRetryBehavior.RETRY_WITH_SAME_PEER)); Mockito.when(peerTask.getSubProtocol()).thenReturn(subprotocol); @@ -129,7 +129,7 @@ public void testExecuteAgainstPeerWithNoPeerTaskBehaviorsAndPeerNotConnected() TimeoutException { Mockito.when(peerTask.getRequestMessage()).thenReturn(requestMessageData); - Mockito.when(peerTask.getPeerTaskBehaviors()).thenReturn(Collections.emptyList()); + Mockito.when(peerTask.getPeerTaskRetryBehaviors()).thenReturn(Collections.emptyList()); Mockito.when(peerTask.getSubProtocol()).thenReturn(subprotocol); Mockito.when(subprotocol.getName()).thenReturn("subprotocol"); Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, ethPeer)) @@ -151,7 +151,7 @@ public void testExecuteAgainstPeerWithNoPeerTaskBehaviorsAndTimeoutException() int requestMessageDataCode = 123; Mockito.when(peerTask.getRequestMessage()).thenReturn(requestMessageData); - Mockito.when(peerTask.getPeerTaskBehaviors()).thenReturn(Collections.emptyList()); + Mockito.when(peerTask.getPeerTaskRetryBehaviors()).thenReturn(Collections.emptyList()); Mockito.when(peerTask.getSubProtocol()).thenReturn(subprotocol); Mockito.when(subprotocol.getName()).thenReturn("subprotocol"); Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, ethPeer)) @@ -176,7 +176,7 @@ public void testExecuteAgainstPeerWithNoPeerTaskBehaviorsAndInvalidResponseMessa InvalidPeerTaskResponseException { Mockito.when(peerTask.getRequestMessage()).thenReturn(requestMessageData); - Mockito.when(peerTask.getPeerTaskBehaviors()).thenReturn(Collections.emptyList()); + Mockito.when(peerTask.getPeerTaskRetryBehaviors()).thenReturn(Collections.emptyList()); Mockito.when(peerTask.getSubProtocol()).thenReturn(subprotocol); Mockito.when(subprotocol.getName()).thenReturn("subprotocol"); Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, ethPeer)) @@ -207,7 +207,7 @@ public void testExecuteWithNoPeerTaskBehaviorsAndSuccessFlow() Mockito.when(peerSelector.getPeer(Mockito.any(Predicate.class))).thenReturn(ethPeer); Mockito.when(peerTask.getRequestMessage()).thenReturn(requestMessageData); - Mockito.when(peerTask.getPeerTaskBehaviors()).thenReturn(Collections.emptyList()); + Mockito.when(peerTask.getPeerTaskRetryBehaviors()).thenReturn(Collections.emptyList()); Mockito.when(peerTask.getSubProtocol()).thenReturn(subprotocol); Mockito.when(subprotocol.getName()).thenReturn("subprotocol"); Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, ethPeer)) @@ -241,7 +241,7 @@ public void testExecuteWithPeerSwitchingAndSuccessFlow() .thenReturn(peer2); Mockito.when(peerTask.getRequestMessage()).thenReturn(requestMessageData); - Mockito.when(peerTask.getPeerTaskBehaviors()) + Mockito.when(peerTask.getPeerTaskRetryBehaviors()) .thenReturn(List.of(PeerTaskRetryBehavior.RETRY_WITH_OTHER_PEERS)); Mockito.when(peerTask.getSubProtocol()).thenReturn(subprotocol); Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, ethPeer)) From d84520ab831b8e33503c44622ae422047464c7ca Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Mon, 7 Oct 2024 08:46:22 +1100 Subject: [PATCH 060/106] 7311: Rename getPeerTaskBehavior to getPeerTaskRetryBehavior Signed-off-by: Matilda Clerke --- .../ethereum/eth/manager/peertask/PeerTaskExecutorTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java index a999aa7116e..1bad6f2cc02 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java @@ -200,8 +200,7 @@ public void testExecuteWithNoPeerTaskBehaviorsAndSuccessFlow() ExecutionException, InterruptedException, TimeoutException, - InvalidPeerTaskResponseException, - NoAvailablePeerException { + InvalidPeerTaskResponseException { Object responseObject = new Object(); Mockito.when(peerSelector.getPeer(Mockito.any(Predicate.class))).thenReturn(ethPeer); From 0896e31de71514287c6bceaa7fc3ec6355b7cc31 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Mon, 7 Oct 2024 09:12:03 +1100 Subject: [PATCH 061/106] 7311: Rework PeerTaskExecutor retry system to be 0-based Signed-off-by: Matilda Clerke --- .../eth/manager/peertask/PeerTaskExecutor.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java index d4b3f603d19..1d56c61a7e9 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java @@ -34,9 +34,9 @@ /** Manages the execution of PeerTasks, respecting their PeerTaskRetryBehavior */ public class PeerTaskExecutor { - public static final int RETRIES_WITH_SAME_PEER = 3; - public static final int RETRIES_WITH_OTHER_PEER = 3; - public static final int NO_RETRIES = 1; + public static final int RETRIES_WITH_SAME_PEER = 2; + public static final int RETRIES_WITH_OTHER_PEER = 2; + public static final int NO_RETRIES = 0; private final PeerSelector peerSelector; private final PeerTaskRequestSender requestSender; private final EthScheduler ethScheduler; @@ -61,7 +61,7 @@ public PeerTaskExecutor( public PeerTaskExecutorResult execute(final PeerTask peerTask) { PeerTaskExecutorResult executorResult; - int triesRemaining = + int retriesRemaining = peerTask.getPeerTaskRetryBehaviors().contains(PeerTaskRetryBehavior.RETRY_WITH_OTHER_PEERS) ? RETRIES_WITH_OTHER_PEER : NO_RETRIES; @@ -81,7 +81,7 @@ public PeerTaskExecutorResult execute(final PeerTask peerTask) { new PeerTaskExecutorResult<>( Optional.empty(), PeerTaskExecutorResponseCode.NO_PEER_AVAILABLE); } - } while (--triesRemaining > 0 + } while (retriesRemaining-- > 0 && executorResult.responseCode() != PeerTaskExecutorResponseCode.SUCCESS); return executorResult; @@ -96,7 +96,7 @@ public PeerTaskExecutorResult executeAgainstPeer( final PeerTask peerTask, final EthPeer peer) { MessageData requestMessageData = peerTask.getRequestMessage(); PeerTaskExecutorResult executorResult; - int triesRemaining = + int retriesRemaining = peerTask.getPeerTaskRetryBehaviors().contains(PeerTaskRetryBehavior.RETRY_WITH_SAME_PEER) ? RETRIES_WITH_SAME_PEER : NO_RETRIES; @@ -136,7 +136,7 @@ public PeerTaskExecutorResult executeAgainstPeer( new PeerTaskExecutorResult<>( Optional.empty(), PeerTaskExecutorResponseCode.INTERNAL_SERVER_ERROR); } - } while (--triesRemaining > 0 + } while (retriesRemaining-- > 0 && executorResult.responseCode() != PeerTaskExecutorResponseCode.SUCCESS && executorResult.responseCode() != PeerTaskExecutorResponseCode.PEER_DISCONNECTED && sleepBetweenRetries()); From 5006b396ef724fe05d1af886f75d087d3ba8dde7 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Mon, 7 Oct 2024 09:20:21 +1100 Subject: [PATCH 062/106] 7311: Fix up compile errors after merge Signed-off-by: Matilda Clerke --- besu/src/test/java/org/hyperledger/besu/RunnerTest.java | 4 ++-- .../eth/manager/peertask/task/GetReceiptsFromPeerTask.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/besu/src/test/java/org/hyperledger/besu/RunnerTest.java b/besu/src/test/java/org/hyperledger/besu/RunnerTest.java index 4efe3d21d1c..f6387d5b3b0 100644 --- a/besu/src/test/java/org/hyperledger/besu/RunnerTest.java +++ b/besu/src/test/java/org/hyperledger/besu/RunnerTest.java @@ -159,7 +159,7 @@ public void fullSyncFromGenesis() throws Exception { @Test public void fullSyncFromGenesisUsingPeerTaskSystem() throws Exception { // set merge flag to false, otherwise this test can fail if a merge test runs first - MergeConfigOptions.setMergeEnabled(false); + MergeConfiguration.setMergeEnabled(false); syncFromGenesis(SyncMode.FULL, getFastSyncGenesis(), true); } @@ -175,7 +175,7 @@ public void fastSyncFromGenesis() throws Exception { @Test public void fastSyncFromGenesisUsingPeerTaskSystem() throws Exception { // set merge flag to false, otherwise this test can fail if a merge test runs first - MergeConfigOptions.setMergeEnabled(false); + MergeConfiguration.setMergeEnabled(false); syncFromGenesis(SyncMode.FAST, getFastSyncGenesis(), true); } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTask.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTask.java index cb84f82033c..544f73b7822 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTask.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTask.java @@ -103,7 +103,7 @@ public Map> parseResponse(final MessageDat } @Override - public Collection getPeerTaskBehaviors() { + public Collection getPeerTaskRetryBehaviors() { return List.of( PeerTaskRetryBehavior.RETRY_WITH_OTHER_PEERS, PeerTaskRetryBehavior.RETRY_WITH_SAME_PEER); } From 07f3a7e111bad52772ce2ffa77672529da5947ed Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Mon, 7 Oct 2024 10:19:39 +1100 Subject: [PATCH 063/106] 7311: Fix broken DownloadReceiptsStepTest test Signed-off-by: Matilda Clerke --- .../ethereum/eth/sync/fastsync/DownloadReceiptsStepTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStepTest.java index c349df1c383..554e09c3e91 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStepTest.java @@ -124,8 +124,8 @@ public void shouldDownloadReceiptsForBlocksUsingPeerTaskSystem() PeerTaskExecutorResult>> peerTaskResult = new PeerTaskExecutorResult<>( Optional.of(receiptsMap), PeerTaskExecutorResponseCode.SUCCESS); - Mockito.when(peerTaskExecutor.executeAsync(Mockito.any(GetReceiptsFromPeerTask.class))) - .thenReturn(CompletableFuture.completedFuture(peerTaskResult)); + Mockito.when(peerTaskExecutor.execute(Mockito.any(GetReceiptsFromPeerTask.class))) + .thenReturn(peerTaskResult); final CompletableFuture> result = downloadReceiptsStep.apply(blocks); From 493ac9152945ef8e23a15e00a625f05bef6ffc26 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Mon, 7 Oct 2024 16:48:13 +1100 Subject: [PATCH 064/106] 7311: Move GetReceipts to services worker for parallelism Signed-off-by: Matilda Clerke --- .../besu/ethereum/eth/manager/EthScheduler.java | 15 ++++++++++++++- .../eth/sync/fastsync/DownloadReceiptsStep.java | 5 ++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthScheduler.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthScheduler.java index 1e2f3eb6abb..8c90993c689 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthScheduler.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthScheduler.java @@ -145,7 +145,7 @@ public void executeServiceTask(final Runnable command) { servicesExecutor.execute(command); } - public CompletableFuture scheduleServiceTask(final Runnable task) { + public CompletableFuture scheduleServiceTask(final Runnable task) { return CompletableFuture.runAsync(task, servicesExecutor); } @@ -156,6 +156,19 @@ public CompletableFuture scheduleServiceTask(final EthTask task) { return serviceFuture; } + public CompletableFuture scheduleServiceTask(final Supplier> future) { + final CompletableFuture promise = new CompletableFuture<>(); + final Future workerFuture = servicesExecutor.submit(() -> propagateResult(future, promise)); + // If returned promise is cancelled, cancel the worker future + promise.whenComplete( + (r, t) -> { + if (t instanceof CancellationException) { + workerFuture.cancel(false); + } + }); + return promise; + } + public CompletableFuture startPipeline(final Pipeline pipeline) { final CompletableFuture pipelineFuture = pipeline.start(servicesExecutor); pendingFutures.add(pipelineFuture); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java index 4c60ac1d193..5f540b7f649 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java @@ -37,6 +37,9 @@ import java.util.concurrent.CompletableFuture; import java.util.function.Function; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + public class DownloadReceiptsStep implements Function, CompletableFuture>> { @@ -62,7 +65,7 @@ public CompletableFuture> apply(final List blocks if (synchronizerConfiguration.isPeerTaskSystemEnabled()) { return ethContext .getScheduler() - .scheduleSyncWorkerTask( + .scheduleServiceTask( () -> { Map> getReceipts = new HashMap<>(); do { From 1a301741280554327941ac8372c629a800dce9de Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Tue, 8 Oct 2024 08:53:51 +1100 Subject: [PATCH 065/106] 7311: Refactor peer task system usage in DownloadReceiptsStep to better match old system Signed-off-by: Matilda Clerke --- .../sync/fastsync/DownloadReceiptsStep.java | 65 +++++++++---------- 1 file changed, 31 insertions(+), 34 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java index 5f540b7f649..8ee7ef414ad 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java @@ -37,9 +37,6 @@ import java.util.concurrent.CompletableFuture; import java.util.function.Function; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - public class DownloadReceiptsStep implements Function, CompletableFuture>> { @@ -65,37 +62,8 @@ public CompletableFuture> apply(final List blocks if (synchronizerConfiguration.isPeerTaskSystemEnabled()) { return ethContext .getScheduler() - .scheduleServiceTask( - () -> { - Map> getReceipts = new HashMap<>(); - do { - GetReceiptsFromPeerTask task = - new GetReceiptsFromPeerTask(headers, new BodyValidator()); - PeerTaskExecutorResult>> - getReceiptsResult = peerTaskExecutor.execute(task); - if (getReceiptsResult.responseCode() == PeerTaskExecutorResponseCode.SUCCESS - && getReceiptsResult.result().isPresent()) { - Map> taskResult = - getReceiptsResult.result().get(); - taskResult - .keySet() - .forEach( - (blockHeader) -> - getReceipts.merge( - blockHeader, - taskResult.get(blockHeader), - (initialReceipts, newReceipts) -> { - throw new IllegalStateException( - "Unexpectedly got receipts for block header already populated!"); - })); - } - // remove all the headers we found receipts for - headers.removeAll(getReceipts.keySet()); - // repeat until all headers have receipts - } while (!headers.isEmpty()); - return CompletableFuture.completedFuture( - combineBlocksAndReceipts(blocks, getReceipts)); - }); + .scheduleServiceTask(() -> getReceiptsWithPeerTaskSystem(headers)) + .thenApply((receipts) -> combineBlocksAndReceipts(blocks, receipts)); } else { return GetReceiptsForHeadersTask.forHeaders(ethContext, headers, metricsSystem) @@ -104,6 +72,35 @@ public CompletableFuture> apply(final List blocks } } + private CompletableFuture>> + getReceiptsWithPeerTaskSystem(final List headers) { + Map> getReceipts = new HashMap<>(); + do { + GetReceiptsFromPeerTask task = new GetReceiptsFromPeerTask(headers, new BodyValidator()); + PeerTaskExecutorResult>> getReceiptsResult = + peerTaskExecutor.execute(task); + if (getReceiptsResult.responseCode() == PeerTaskExecutorResponseCode.SUCCESS + && getReceiptsResult.result().isPresent()) { + Map> taskResult = getReceiptsResult.result().get(); + taskResult + .keySet() + .forEach( + (blockHeader) -> + getReceipts.merge( + blockHeader, + taskResult.get(blockHeader), + (initialReceipts, newReceipts) -> { + throw new IllegalStateException( + "Unexpectedly got receipts for block header already populated!"); + })); + } + // remove all the headers we found receipts for + headers.removeAll(getReceipts.keySet()); + // repeat until all headers have receipts + } while (!headers.isEmpty()); + return CompletableFuture.completedFuture(getReceipts); + } + private List combineBlocksAndReceipts( final List blocks, final Map> receiptsByHeader) { return blocks.stream() From c047f428bf5959f542e7849f0858c9f12f2725c4 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Tue, 8 Oct 2024 12:37:22 +1100 Subject: [PATCH 066/106] 7311: Remove unused async methods in PeerTaskExecutor Signed-off-by: Matilda Clerke --- .../eth/manager/peertask/PeerTaskExecutor.java | 16 ---------------- .../manager/peertask/PeerTaskExecutorTest.java | 4 +--- 2 files changed, 1 insertion(+), 19 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java index 1d56c61a7e9..7e38d8cfa7b 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java @@ -15,7 +15,6 @@ package org.hyperledger.besu.ethereum.eth.manager.peertask; import org.hyperledger.besu.ethereum.eth.manager.EthPeer; -import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; import org.hyperledger.besu.ethereum.eth.manager.exceptions.NoAvailablePeersException; import org.hyperledger.besu.ethereum.p2p.rlpx.connections.PeerConnection; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; @@ -27,7 +26,6 @@ import java.util.Collection; import java.util.HashSet; import java.util.Optional; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeoutException; @@ -39,18 +37,15 @@ public class PeerTaskExecutor { public static final int NO_RETRIES = 0; private final PeerSelector peerSelector; private final PeerTaskRequestSender requestSender; - private final EthScheduler ethScheduler; private final LabelledMetric requestTimer; public PeerTaskExecutor( final PeerSelector peerSelector, final PeerTaskRequestSender requestSender, - final EthScheduler ethScheduler, final MetricsSystem metricsSystem) { this.peerSelector = peerSelector; this.requestSender = requestSender; - this.ethScheduler = ethScheduler; requestTimer = metricsSystem.createLabelledTimer( BesuMetricCategory.PEERS, @@ -87,11 +82,6 @@ public PeerTaskExecutorResult execute(final PeerTask peerTask) { return executorResult; } - public CompletableFuture> executeAsync(final PeerTask peerTask) { - return ethScheduler.scheduleSyncWorkerTask( - () -> CompletableFuture.completedFuture(execute(peerTask))); - } - public PeerTaskExecutorResult executeAgainstPeer( final PeerTask peerTask, final EthPeer peer) { MessageData requestMessageData = peerTask.getRequestMessage(); @@ -144,12 +134,6 @@ public PeerTaskExecutorResult executeAgainstPeer( return executorResult; } - public CompletableFuture> executeAgainstPeerAsync( - final PeerTask peerTask, final EthPeer peer) { - return ethScheduler.scheduleSyncWorkerTask( - () -> CompletableFuture.completedFuture(executeAgainstPeer(peerTask, peer))); - } - private boolean sleepBetweenRetries() { try { // sleep for 1 second to match implemented wait between retries in AbstractRetryingPeerTask diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java index 1bad6f2cc02..55878d144e9 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java @@ -15,7 +15,6 @@ package org.hyperledger.besu.ethereum.eth.manager.peertask; import org.hyperledger.besu.ethereum.eth.manager.EthPeer; -import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; import org.hyperledger.besu.ethereum.p2p.rlpx.connections.PeerConnection; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.SubProtocol; @@ -38,7 +37,6 @@ public class PeerTaskExecutorTest { private @Mock PeerSelector peerSelector; private @Mock PeerTaskRequestSender requestSender; - private @Mock EthScheduler ethScheduler; private @Mock PeerTask peerTask; private @Mock SubProtocol subprotocol; private @Mock MessageData requestMessageData; @@ -52,7 +50,7 @@ public class PeerTaskExecutorTest { public void beforeTest() { mockCloser = MockitoAnnotations.openMocks(this); peerTaskExecutor = - new PeerTaskExecutor(peerSelector, requestSender, ethScheduler, new NoOpMetricsSystem()); + new PeerTaskExecutor(peerSelector, requestSender, new NoOpMetricsSystem()); } @AfterEach From 5aa6b0be5fe883e96752edb847fec0e41bcad618 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Tue, 8 Oct 2024 14:53:40 +1100 Subject: [PATCH 067/106] 7311: Return Optional in PeerSelector.getPeer and utilise existing peer selection behavior in EthPeers Signed-off-by: Matilda Clerke --- .../besu/ethereum/eth/manager/EthPeers.java | 5 ++--- .../eth/manager/peertask/PeerSelector.java | 2 +- .../manager/peertask/PeerTaskExecutor.java | 20 +++++++++---------- .../peertask/PeerTaskExecutorTest.java | 11 +++++----- 4 files changed, 18 insertions(+), 20 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthPeers.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthPeers.java index 1b413251be7..47ceba65980 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthPeers.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthPeers.java @@ -17,7 +17,6 @@ import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.eth.SnapProtocol; import org.hyperledger.besu.ethereum.eth.manager.EthPeer.DisconnectCallback; -import org.hyperledger.besu.ethereum.eth.manager.exceptions.NoAvailablePeersException; import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerSelector; import org.hyperledger.besu.ethereum.eth.peervalidation.PeerValidator; import org.hyperledger.besu.ethereum.eth.sync.ChainHeadTracker; @@ -470,8 +469,8 @@ public void setTrailingPeerRequirementsSupplier( // Part of the PeerSelector interface, to be split apart later @Override - public EthPeer getPeer(final Predicate filter) { - return streamBestPeers().filter(filter).findFirst().orElseThrow(NoAvailablePeersException::new); + public Optional getPeer(final Predicate filter) { + return bestPeerMatchingCriteria(filter); } // Part of the PeerSelector interface, to be split apart later diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerSelector.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerSelector.java index 0801f9f00ec..8f7ab33e42d 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerSelector.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerSelector.java @@ -29,7 +29,7 @@ public interface PeerSelector { * @param filter a Predicate\ matching desirable peers * @return a peer matching the supplied conditions */ - EthPeer getPeer(final Predicate filter); + Optional getPeer(final Predicate filter); /** * Attempts to get the EthPeer identified by peerId diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java index 7e38d8cfa7b..1734c1e8765 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java @@ -15,7 +15,6 @@ package org.hyperledger.besu.ethereum.eth.manager.peertask; import org.hyperledger.besu.ethereum.eth.manager.EthPeer; -import org.hyperledger.besu.ethereum.eth.manager.exceptions.NoAvailablePeersException; import org.hyperledger.besu.ethereum.p2p.rlpx.connections.PeerConnection; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; import org.hyperledger.besu.metrics.BesuMetricCategory; @@ -62,20 +61,19 @@ public PeerTaskExecutorResult execute(final PeerTask peerTask) { : NO_RETRIES; final Collection usedEthPeers = new HashSet<>(); do { - EthPeer peer; - try { - peer = - peerSelector.getPeer( - (candidatePeer) -> - peerTask.getPeerRequirementFilter().test(candidatePeer) - && !usedEthPeers.contains(candidatePeer)); - usedEthPeers.add(peer); - executorResult = executeAgainstPeer(peerTask, peer); - } catch (NoAvailablePeersException e) { + Optional peer = + peerSelector.getPeer( + (candidatePeer) -> + peerTask.getPeerRequirementFilter().test(candidatePeer) + && !usedEthPeers.contains(candidatePeer)); + if (peer.isEmpty()) { executorResult = new PeerTaskExecutorResult<>( Optional.empty(), PeerTaskExecutorResponseCode.NO_PEER_AVAILABLE); + continue; } + usedEthPeers.add(peer.get()); + executorResult = executeAgainstPeer(peerTask, peer.get()); } while (retriesRemaining-- > 0 && executorResult.responseCode() != PeerTaskExecutorResponseCode.SUCCESS); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java index 55878d144e9..6dfd8d0e203 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java @@ -22,6 +22,7 @@ import java.util.Collections; import java.util.List; +import java.util.Optional; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeoutException; import java.util.function.Predicate; @@ -49,8 +50,7 @@ public class PeerTaskExecutorTest { @BeforeEach public void beforeTest() { mockCloser = MockitoAnnotations.openMocks(this); - peerTaskExecutor = - new PeerTaskExecutor(peerSelector, requestSender, new NoOpMetricsSystem()); + peerTaskExecutor = new PeerTaskExecutor(peerSelector, requestSender, new NoOpMetricsSystem()); } @AfterEach @@ -201,7 +201,8 @@ public void testExecuteWithNoPeerTaskBehaviorsAndSuccessFlow() InvalidPeerTaskResponseException { Object responseObject = new Object(); - Mockito.when(peerSelector.getPeer(Mockito.any(Predicate.class))).thenReturn(ethPeer); + Mockito.when(peerSelector.getPeer(Mockito.any(Predicate.class))) + .thenReturn(Optional.of(ethPeer)); Mockito.when(peerTask.getRequestMessage()).thenReturn(requestMessageData); Mockito.when(peerTask.getPeerTaskRetryBehaviors()).thenReturn(Collections.emptyList()); @@ -234,8 +235,8 @@ public void testExecuteWithPeerSwitchingAndSuccessFlow() EthPeer peer2 = Mockito.mock(EthPeer.class); Mockito.when(peerSelector.getPeer(Mockito.any(Predicate.class))) - .thenReturn(ethPeer) - .thenReturn(peer2); + .thenReturn(Optional.of(ethPeer)) + .thenReturn(Optional.of(peer2)); Mockito.when(peerTask.getRequestMessage()).thenReturn(requestMessageData); Mockito.when(peerTask.getPeerTaskRetryBehaviors()) From d6120b0b2ee03caa208b04942dfbccd10e0c925d Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Tue, 8 Oct 2024 14:57:19 +1100 Subject: [PATCH 068/106] 7311: Update after merge Signed-off-by: Matilda Clerke --- .../org/hyperledger/besu/controller/BesuControllerBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java index a15decff57b..46d7d10fc71 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java @@ -656,7 +656,7 @@ public BesuController build() { final EthContext ethContext = new EthContext(ethPeers, ethMessages, snapMessages, scheduler); final PeerTaskExecutor peerTaskExecutor = - new PeerTaskExecutor(ethPeers, new PeerTaskRequestSender(), scheduler, metricsSystem); + new PeerTaskExecutor(ethPeers, new PeerTaskRequestSender(), metricsSystem); final boolean fullSyncDisabled = !SyncMode.isFullSync(syncConfig.getSyncMode()); final SyncState syncState = new SyncState(blockchain, ethPeers, fullSyncDisabled, checkpoint); From 8becdb3ef5077216ad464f7c71ba62aacb11ba33 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Tue, 8 Oct 2024 16:05:12 +1100 Subject: [PATCH 069/106] 7311: Redo getPeer again to include hasAvailableRequestCapacity check Signed-off-by: Matilda Clerke --- .../org/hyperledger/besu/ethereum/eth/manager/EthPeers.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthPeers.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthPeers.java index 47ceba65980..d070c35ce19 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthPeers.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthPeers.java @@ -470,7 +470,7 @@ public void setTrailingPeerRequirementsSupplier( // Part of the PeerSelector interface, to be split apart later @Override public Optional getPeer(final Predicate filter) { - return bestPeerMatchingCriteria(filter); + return streamBestPeers().filter(filter).filter(EthPeer::hasAvailableRequestCapacity).findFirst(); } // Part of the PeerSelector interface, to be split apart later From 86a1f0bf5139afe24c350ab6f3fca0ff8a791870 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Wed, 9 Oct 2024 10:14:58 +1100 Subject: [PATCH 070/106] 7311: Add protocol spec supplier to GetReceiptsFromPeerTask Signed-off-by: Matilda Clerke --- .../controller/BesuControllerBuilder.java | 3 +++ .../TransitionBesuControllerBuilder.java | 4 ++++ .../besu/ethereum/eth/manager/EthPeers.java | 5 ++++- .../task/GetReceiptsFromPeerTask.java | 11 +++++++++-- .../eth/sync/DefaultSynchronizer.java | 5 +++++ .../CheckpointDownloadBlockStep.java | 8 +++++++- .../CheckpointDownloaderFactory.java | 5 +++++ .../checkpointsync/CheckpointSyncActions.java | 6 ++++++ .../CheckpointSyncChainDownloader.java | 5 +++++ ...CheckpointSyncDownloadPipelineFactory.java | 12 +++++++++++- .../sync/fastsync/DownloadReceiptsStep.java | 8 +++++++- .../eth/sync/fastsync/FastSyncActions.java | 5 +++++ .../fastsync/FastSyncChainDownloader.java | 5 +++++ .../FastSyncDownloadPipelineFactory.java | 8 +++++++- .../worldstate/FastDownloaderFactory.java | 4 ++++ .../sync/snapsync/SnapDownloaderFactory.java | 4 ++++ .../task/GetReceiptsFromPeerTaskTest.java | 19 +++++++++++++------ .../CheckPointSyncChainDownloaderTest.java | 3 ++- .../fastsync/DownloadReceiptsStepTest.java | 2 ++ .../fastsync/FastDownloaderFactoryTest.java | 5 +++++ .../sync/fastsync/FastSyncActionsTest.java | 3 ++- .../fastsync/FastSyncChainDownloaderTest.java | 3 ++- 22 files changed, 117 insertions(+), 16 deletions(-) diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java index 46d7d10fc71..d1e04d451ba 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java @@ -705,6 +705,7 @@ public BesuController build() { final DefaultSynchronizer synchronizer = createSynchronizer( protocolSchedule, + currentProtocolSpecSupplier, worldStateStorageCoordinator, protocolContext, ethContext, @@ -839,6 +840,7 @@ private TrieLogPruner createTrieLogPruner( */ protected DefaultSynchronizer createSynchronizer( final ProtocolSchedule protocolSchedule, + final Supplier currentProtocolSpecSupplier, final WorldStateStorageCoordinator worldStateStorageCoordinator, final ProtocolContext protocolContext, final EthContext ethContext, @@ -850,6 +852,7 @@ protected DefaultSynchronizer createSynchronizer( return new DefaultSynchronizer( syncConfig, protocolSchedule, + currentProtocolSpecSupplier, protocolContext, worldStateStorageCoordinator, ethProtocolManager.getBlockBroadcaster(), diff --git a/besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java index 703592f90a9..a363efed44f 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java @@ -51,6 +51,7 @@ import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; import org.hyperledger.besu.ethereum.forkid.ForkIdManager; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; @@ -66,6 +67,7 @@ import java.util.Map; import java.util.Optional; import java.util.function.Consumer; +import java.util.function.Supplier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -223,6 +225,7 @@ protected PluginServiceFactory createAdditionalPluginServices( @Override protected DefaultSynchronizer createSynchronizer( final ProtocolSchedule protocolSchedule, + final Supplier currentProtocolSpecSupplier, final WorldStateStorageCoordinator worldStateStorageCoordinator, final ProtocolContext protocolContext, final EthContext ethContext, @@ -234,6 +237,7 @@ protected DefaultSynchronizer createSynchronizer( DefaultSynchronizer sync = super.createSynchronizer( protocolSchedule, + currentProtocolSpecSupplier, worldStateStorageCoordinator, protocolContext, ethContext, diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthPeers.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthPeers.java index d070c35ce19..a721697717f 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthPeers.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthPeers.java @@ -470,7 +470,10 @@ public void setTrailingPeerRequirementsSupplier( // Part of the PeerSelector interface, to be split apart later @Override public Optional getPeer(final Predicate filter) { - return streamBestPeers().filter(filter).filter(EthPeer::hasAvailableRequestCapacity).findFirst(); + return streamBestPeers() + .filter(filter) + .filter(EthPeer::hasAvailableRequestCapacity) + .findFirst(); } // Part of the PeerSelector interface, to be split apart later diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTask.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTask.java index 544f73b7822..706e8779a73 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTask.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTask.java @@ -25,6 +25,7 @@ import org.hyperledger.besu.ethereum.eth.messages.GetReceiptsMessage; import org.hyperledger.besu.ethereum.eth.messages.ReceiptsMessage; import org.hyperledger.besu.ethereum.mainnet.BodyValidator; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.SubProtocol; @@ -34,19 +35,24 @@ import java.util.List; import java.util.Map; import java.util.function.Predicate; +import java.util.function.Supplier; public class GetReceiptsFromPeerTask implements PeerTask>> { private final Collection blockHeaders; private final BodyValidator bodyValidator; + private final Supplier currentProtocolSpecSupplier; private final Map> headersByReceiptsRoot = new HashMap<>(); private final long requiredBlockchainHeight; public GetReceiptsFromPeerTask( - final Collection blockHeaders, final BodyValidator bodyValidator) { + final Collection blockHeaders, + final BodyValidator bodyValidator, + final Supplier currentProtocolSpecSupplier) { this.blockHeaders = blockHeaders; this.bodyValidator = bodyValidator; + this.currentProtocolSpecSupplier = currentProtocolSpecSupplier; blockHeaders.forEach( header -> @@ -112,6 +118,7 @@ public Collection getPeerTaskRetryBehaviors() { public Predicate getPeerRequirementFilter() { return (ethPeer) -> ethPeer.getProtocolName().equals(getSubProtocol().getName()) - && ethPeer.chainState().getEstimatedHeight() >= requiredBlockchainHeight; + && (currentProtocolSpecSupplier.get().isPoS() + || ethPeer.chainState().getEstimatedHeight() >= requiredBlockchainHeight); } } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java index f012bbd7aea..bbaea664329 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java @@ -35,6 +35,7 @@ import org.hyperledger.besu.ethereum.eth.sync.state.PendingBlocksManager; import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiWorldStateProvider; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; @@ -79,6 +80,7 @@ public class DefaultSynchronizer implements Synchronizer, UnverifiedForkchoiceLi public DefaultSynchronizer( final SynchronizerConfiguration syncConfig, final ProtocolSchedule protocolSchedule, + final Supplier currentProtocolSpecSupplier, final ProtocolContext protocolContext, final WorldStateStorageCoordinator worldStateStorageCoordinator, final BlockBroadcaster blockBroadcaster, @@ -146,6 +148,7 @@ public DefaultSynchronizer( syncConfig, dataDirectory, protocolSchedule, + currentProtocolSpecSupplier, protocolContext, metricsSystem, ethContext, @@ -163,6 +166,7 @@ public DefaultSynchronizer( syncConfig, dataDirectory, protocolSchedule, + currentProtocolSpecSupplier, protocolContext, metricsSystem, ethContext, @@ -180,6 +184,7 @@ public DefaultSynchronizer( syncConfig, dataDirectory, protocolSchedule, + currentProtocolSpecSupplier, protocolContext, metricsSystem, ethContext, diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointDownloadBlockStep.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointDownloadBlockStep.java index a0bb5ee595f..70e9958beac 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointDownloadBlockStep.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointDownloadBlockStep.java @@ -30,16 +30,19 @@ import org.hyperledger.besu.ethereum.eth.sync.fastsync.checkpoint.Checkpoint; import org.hyperledger.besu.ethereum.mainnet.BodyValidator; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; import org.hyperledger.besu.plugin.services.MetricsSystem; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.function.Supplier; public class CheckpointDownloadBlockStep { private final ProtocolSchedule protocolSchedule; + private final Supplier currentProtocolSpecSupplier; private final EthContext ethContext; private final PeerTaskExecutor peerTaskExecutor; private final Checkpoint checkpoint; @@ -48,12 +51,14 @@ public class CheckpointDownloadBlockStep { public CheckpointDownloadBlockStep( final ProtocolSchedule protocolSchedule, + final Supplier currentProtocolSpecSupplier, final EthContext ethContext, final PeerTaskExecutor peerTaskExecutor, final Checkpoint checkpoint, final SynchronizerConfiguration synchronizerConfiguration, final MetricsSystem metricsSystem) { this.protocolSchedule = protocolSchedule; + this.currentProtocolSpecSupplier = currentProtocolSpecSupplier; this.ethContext = ethContext; this.peerTaskExecutor = peerTaskExecutor; this.checkpoint = checkpoint; @@ -81,7 +86,8 @@ private CompletableFuture> downloadReceipts( if (synchronizerConfiguration.isPeerTaskSystemEnabled()) { CompletableFuture> futureReceipts = new CompletableFuture<>(); GetReceiptsFromPeerTask task = - new GetReceiptsFromPeerTask(List.of(block.getHeader()), new BodyValidator()); + new GetReceiptsFromPeerTask( + List.of(block.getHeader()), new BodyValidator(), currentProtocolSpecSupplier); PeerTaskExecutorResult>> executorResult = peerTaskExecutor.execute(task); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointDownloaderFactory.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointDownloaderFactory.java index 30134d9f6c5..e8a6fe2804f 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointDownloaderFactory.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointDownloaderFactory.java @@ -34,6 +34,7 @@ import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloader; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; import org.hyperledger.besu.ethereum.mainnet.ScheduleBasedBlockHeaderFunctions; import org.hyperledger.besu.ethereum.trie.CompactEncoding; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; @@ -44,6 +45,7 @@ import java.nio.file.Path; import java.time.Clock; import java.util.Optional; +import java.util.function.Supplier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -59,6 +61,7 @@ public static Optional> createCheckpointDownloader( final SynchronizerConfiguration syncConfig, final Path dataDirectory, final ProtocolSchedule protocolSchedule, + final Supplier currentProtocolSpecSupplier, final ProtocolContext protocolContext, final MetricsSystem metricsSystem, final EthContext ethContext, @@ -110,6 +113,7 @@ public static Optional> createCheckpointDownloader( syncConfig, worldStateStorageCoordinator, protocolSchedule, + currentProtocolSpecSupplier, protocolContext, ethContext, peerTaskExecutor, @@ -128,6 +132,7 @@ public static Optional> createCheckpointDownloader( syncConfig, worldStateStorageCoordinator, protocolSchedule, + currentProtocolSpecSupplier, protocolContext, ethContext, peerTaskExecutor, diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncActions.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncActions.java index 61b997e6c53..ebc86836b0e 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncActions.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncActions.java @@ -24,15 +24,19 @@ import org.hyperledger.besu.ethereum.eth.sync.fastsync.FastSyncState; import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.metrics.SyncDurationMetrics; import org.hyperledger.besu.plugin.services.MetricsSystem; +import java.util.function.Supplier; + public class CheckpointSyncActions extends FastSyncActions { public CheckpointSyncActions( final SynchronizerConfiguration syncConfig, final WorldStateStorageCoordinator worldStateStorageCoordinator, final ProtocolSchedule protocolSchedule, + final Supplier currentProtocolSpecSupplier, final ProtocolContext protocolContext, final EthContext ethContext, final PeerTaskExecutor peerTaskExecutor, @@ -43,6 +47,7 @@ public CheckpointSyncActions( syncConfig, worldStateStorageCoordinator, protocolSchedule, + currentProtocolSpecSupplier, protocolContext, ethContext, peerTaskExecutor, @@ -58,6 +63,7 @@ public ChainDownloader createChainDownloader( syncConfig, worldStateStorageCoordinator, protocolSchedule, + currentProtocolSpecSupplier, protocolContext, ethContext, peerTaskExecutor, diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncChainDownloader.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncChainDownloader.java index 2590e4736ae..58fbec93e04 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncChainDownloader.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncChainDownloader.java @@ -25,16 +25,20 @@ import org.hyperledger.besu.ethereum.eth.sync.fastsync.SyncTargetManager; import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.metrics.SyncDurationMetrics; import org.hyperledger.besu.plugin.services.MetricsSystem; +import java.util.function.Supplier; + public class CheckpointSyncChainDownloader extends FastSyncChainDownloader { public static ChainDownloader create( final SynchronizerConfiguration config, final WorldStateStorageCoordinator worldStateStorageCoordinator, final ProtocolSchedule protocolSchedule, + final Supplier currentProtocolSpecSupplier, final ProtocolContext protocolContext, final EthContext ethContext, final PeerTaskExecutor peerTaskExecutor, @@ -59,6 +63,7 @@ public static ChainDownloader create( new CheckpointSyncDownloadPipelineFactory( config, protocolSchedule, + currentProtocolSpecSupplier, protocolContext, ethContext, peerTaskExecutor, diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncDownloadPipelineFactory.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncDownloadPipelineFactory.java index 0be10869861..696d5e8d0f5 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncDownloadPipelineFactory.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncDownloadPipelineFactory.java @@ -27,18 +27,21 @@ import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; import org.hyperledger.besu.ethereum.eth.sync.state.SyncTarget; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.services.pipeline.Pipeline; import org.hyperledger.besu.services.pipeline.PipelineBuilder; import java.util.concurrent.CompletionStage; +import java.util.function.Supplier; public class CheckpointSyncDownloadPipelineFactory extends FastSyncDownloadPipelineFactory { public CheckpointSyncDownloadPipelineFactory( final SynchronizerConfiguration syncConfig, final ProtocolSchedule protocolSchedule, + final Supplier currentProtocolSpecSupplier, final ProtocolContext protocolContext, final EthContext ethContext, final PeerTaskExecutor peerTaskExecutor, @@ -47,6 +50,7 @@ public CheckpointSyncDownloadPipelineFactory( super( syncConfig, protocolSchedule, + currentProtocolSpecSupplier, protocolContext, ethContext, peerTaskExecutor, @@ -86,7 +90,13 @@ protected Pipeline createDownloadCheckPointPipeline( final CheckpointDownloadBlockStep checkPointDownloadBlockStep = new CheckpointDownloadBlockStep( - protocolSchedule, ethContext, peerTaskExecutor, checkpoint, syncConfig, metricsSystem); + protocolSchedule, + currentProtocolSpecSupplier, + ethContext, + peerTaskExecutor, + checkpoint, + syncConfig, + metricsSystem); return PipelineBuilder.createPipelineFrom( "fetchCheckpoints", diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java index 8ee7ef414ad..65e6f846363 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java @@ -29,6 +29,7 @@ import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; import org.hyperledger.besu.ethereum.eth.sync.tasks.GetReceiptsForHeadersTask; import org.hyperledger.besu.ethereum.mainnet.BodyValidator; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; import org.hyperledger.besu.plugin.services.MetricsSystem; import java.util.HashMap; @@ -36,20 +37,24 @@ import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.function.Function; +import java.util.function.Supplier; public class DownloadReceiptsStep implements Function, CompletableFuture>> { + private final Supplier currentProtocolSpecSupplier; private final EthContext ethContext; private final PeerTaskExecutor peerTaskExecutor; private final SynchronizerConfiguration synchronizerConfiguration; private final MetricsSystem metricsSystem; public DownloadReceiptsStep( + final Supplier currentProtocolSpecSupplier, final EthContext ethContext, final PeerTaskExecutor peerTaskExecutor, final SynchronizerConfiguration synchronizerConfiguration, final MetricsSystem metricsSystem) { + this.currentProtocolSpecSupplier = currentProtocolSpecSupplier; this.ethContext = ethContext; this.peerTaskExecutor = peerTaskExecutor; this.synchronizerConfiguration = synchronizerConfiguration; @@ -76,7 +81,8 @@ public CompletableFuture> apply(final List blocks getReceiptsWithPeerTaskSystem(final List headers) { Map> getReceipts = new HashMap<>(); do { - GetReceiptsFromPeerTask task = new GetReceiptsFromPeerTask(headers, new BodyValidator()); + GetReceiptsFromPeerTask task = + new GetReceiptsFromPeerTask(headers, new BodyValidator(), currentProtocolSpecSupplier); PeerTaskExecutorResult>> getReceiptsResult = peerTaskExecutor.execute(task); if (getReceiptsResult.responseCode() == PeerTaskExecutorResponseCode.SUCCESS diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActions.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActions.java index 58a64bd562a..b1b31298fc9 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActions.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActions.java @@ -27,6 +27,7 @@ import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; import org.hyperledger.besu.ethereum.eth.sync.tasks.RetryingGetHeaderFromPeerByHashTask; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.metrics.SyncDurationMetrics; @@ -47,6 +48,7 @@ public class FastSyncActions { protected final SynchronizerConfiguration syncConfig; protected final WorldStateStorageCoordinator worldStateStorageCoordinator; protected final ProtocolSchedule protocolSchedule; + protected final Supplier currentProtocolSpecSupplier; protected final ProtocolContext protocolContext; protected final EthContext ethContext; protected final PeerTaskExecutor peerTaskExecutor; @@ -60,6 +62,7 @@ public FastSyncActions( final SynchronizerConfiguration syncConfig, final WorldStateStorageCoordinator worldStateStorageCoordinator, final ProtocolSchedule protocolSchedule, + final Supplier currentProtocolSpecSupplier, final ProtocolContext protocolContext, final EthContext ethContext, final PeerTaskExecutor peerTaskExecutor, @@ -69,6 +72,7 @@ public FastSyncActions( this.syncConfig = syncConfig; this.worldStateStorageCoordinator = worldStateStorageCoordinator; this.protocolSchedule = protocolSchedule; + this.currentProtocolSpecSupplier = currentProtocolSpecSupplier; this.protocolContext = protocolContext; this.ethContext = ethContext; this.peerTaskExecutor = peerTaskExecutor; @@ -166,6 +170,7 @@ public ChainDownloader createChainDownloader( syncConfig, worldStateStorageCoordinator, protocolSchedule, + currentProtocolSpecSupplier, protocolContext, ethContext, peerTaskExecutor, diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloader.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloader.java index 1bf55a3811a..206284a4947 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloader.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloader.java @@ -22,10 +22,13 @@ import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.metrics.SyncDurationMetrics; import org.hyperledger.besu.plugin.services.MetricsSystem; +import java.util.function.Supplier; + public class FastSyncChainDownloader { protected FastSyncChainDownloader() {} @@ -34,6 +37,7 @@ public static ChainDownloader create( final SynchronizerConfiguration config, final WorldStateStorageCoordinator worldStateStorageCoordinator, final ProtocolSchedule protocolSchedule, + final Supplier currentProtocolSpecSupplier, final ProtocolContext protocolContext, final EthContext ethContext, final PeerTaskExecutor peerTaskExecutor, @@ -57,6 +61,7 @@ public static ChainDownloader create( new FastSyncDownloadPipelineFactory( config, protocolSchedule, + currentProtocolSpecSupplier, protocolContext, ethContext, peerTaskExecutor, diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloadPipelineFactory.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloadPipelineFactory.java index d7c5769bdb5..b55b3720679 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloadPipelineFactory.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloadPipelineFactory.java @@ -40,6 +40,7 @@ import org.hyperledger.besu.ethereum.eth.sync.state.SyncTarget; import org.hyperledger.besu.ethereum.mainnet.BodyValidationMode; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.plugin.services.metrics.Counter; @@ -48,6 +49,7 @@ import org.hyperledger.besu.services.pipeline.PipelineBuilder; import java.util.concurrent.CompletionStage; +import java.util.function.Supplier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -57,6 +59,7 @@ public class FastSyncDownloadPipelineFactory implements DownloadPipelineFactory protected final SynchronizerConfiguration syncConfig; protected final ProtocolSchedule protocolSchedule; + protected final Supplier currentProtocolSpecSupplier; protected final ProtocolContext protocolContext; protected final EthContext ethContext; protected final PeerTaskExecutor peerTaskExecutor; @@ -69,6 +72,7 @@ public class FastSyncDownloadPipelineFactory implements DownloadPipelineFactory public FastSyncDownloadPipelineFactory( final SynchronizerConfiguration syncConfig, final ProtocolSchedule protocolSchedule, + final Supplier currentProtocolSpecSupplier, final ProtocolContext protocolContext, final EthContext ethContext, final PeerTaskExecutor peerTaskExecutor, @@ -76,6 +80,7 @@ public FastSyncDownloadPipelineFactory( final MetricsSystem metricsSystem) { this.syncConfig = syncConfig; this.protocolSchedule = protocolSchedule; + this.currentProtocolSpecSupplier = currentProtocolSpecSupplier; this.protocolContext = protocolContext; this.ethContext = ethContext; this.peerTaskExecutor = peerTaskExecutor; @@ -149,7 +154,8 @@ public Pipeline createDownloadPipelineForSyncTarget(final SyncT final DownloadBodiesStep downloadBodiesStep = new DownloadBodiesStep(protocolSchedule, ethContext, metricsSystem); final DownloadReceiptsStep downloadReceiptsStep = - new DownloadReceiptsStep(ethContext, peerTaskExecutor, syncConfig, metricsSystem); + new DownloadReceiptsStep( + currentProtocolSpecSupplier, ethContext, peerTaskExecutor, syncConfig, metricsSystem); final ImportBlocksStep importBlockStep = new ImportBlocksStep( protocolSchedule, diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastDownloaderFactory.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastDownloaderFactory.java index 1d775cc80fd..0b6fad7bb62 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastDownloaderFactory.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastDownloaderFactory.java @@ -28,6 +28,7 @@ import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloader; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; import org.hyperledger.besu.ethereum.mainnet.ScheduleBasedBlockHeaderFunctions; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.metrics.BesuMetricCategory; @@ -41,6 +42,7 @@ import java.nio.file.Path; import java.time.Clock; import java.util.Optional; +import java.util.function.Supplier; import java.util.stream.Stream; import org.slf4j.Logger; @@ -57,6 +59,7 @@ public static Optional> create( final SynchronizerConfiguration syncConfig, final Path dataDirectory, final ProtocolSchedule protocolSchedule, + final Supplier currentProtocolSpecSupplier, final ProtocolContext protocolContext, final MetricsSystem metricsSystem, final EthContext ethContext, @@ -126,6 +129,7 @@ public static Optional> create( syncConfig, worldStateStorageCoordinator, protocolSchedule, + currentProtocolSpecSupplier, protocolContext, ethContext, peerTaskExecutor, diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapDownloaderFactory.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapDownloaderFactory.java index 6c5ce0b04e9..423bb6b8790 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapDownloaderFactory.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapDownloaderFactory.java @@ -31,6 +31,7 @@ import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloader; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; import org.hyperledger.besu.ethereum.mainnet.ScheduleBasedBlockHeaderFunctions; import org.hyperledger.besu.ethereum.trie.CompactEncoding; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; @@ -41,6 +42,7 @@ import java.nio.file.Path; import java.time.Clock; import java.util.Optional; +import java.util.function.Supplier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -55,6 +57,7 @@ public static Optional> createSnapDownloader( final SynchronizerConfiguration syncConfig, final Path dataDirectory, final ProtocolSchedule protocolSchedule, + final Supplier currentProtocolSpecSupplier, final ProtocolContext protocolContext, final MetricsSystem metricsSystem, final EthContext ethContext, @@ -121,6 +124,7 @@ public static Optional> createSnapDownloader( syncConfig, worldStateStorageCoordinator, protocolSchedule, + currentProtocolSpecSupplier, protocolContext, ethContext, peerTaskExecutor, diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTaskTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTaskTest.java index 4529d9d3966..0e3e7365f47 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTaskTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTaskTest.java @@ -25,6 +25,7 @@ import org.hyperledger.besu.ethereum.eth.messages.GetReceiptsMessage; import org.hyperledger.besu.ethereum.eth.messages.ReceiptsMessage; import org.hyperledger.besu.ethereum.mainnet.BodyValidator; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; import java.util.ArrayList; @@ -42,7 +43,8 @@ public class GetReceiptsFromPeerTaskTest { @Test public void testGetSubProtocol() { - GetReceiptsFromPeerTask task = new GetReceiptsFromPeerTask(Collections.emptyList(), null); + GetReceiptsFromPeerTask task = + new GetReceiptsFromPeerTask(Collections.emptyList(), null, () -> null); Assertions.assertEquals(EthProtocol.get(), task.getSubProtocol()); } @@ -50,7 +52,7 @@ public void testGetSubProtocol() { public void testGetRequestMessage() { GetReceiptsFromPeerTask task = new GetReceiptsFromPeerTask( - List.of(mockBlockHeader(1), mockBlockHeader(2), mockBlockHeader(3)), null); + List.of(mockBlockHeader(1), mockBlockHeader(2), mockBlockHeader(3)), null, () -> null); MessageData messageData = task.getRequestMessage(); GetReceiptsMessage getReceiptsMessage = GetReceiptsMessage.readFrom(messageData); @@ -72,7 +74,8 @@ public void testGetRequestMessage() { @Test public void testParseResponseWithNullResponseMessage() { - GetReceiptsFromPeerTask task = new GetReceiptsFromPeerTask(Collections.emptyList(), null); + GetReceiptsFromPeerTask task = + new GetReceiptsFromPeerTask(Collections.emptyList(), null, () -> null); Assertions.assertThrows(InvalidPeerTaskResponseException.class, () -> task.parseResponse(null)); } @@ -80,7 +83,7 @@ public void testParseResponseWithNullResponseMessage() { public void testParseResponseForInvalidResponse() { GetReceiptsFromPeerTask task = new GetReceiptsFromPeerTask( - List.of(mockBlockHeader(1), mockBlockHeader(2), mockBlockHeader(3)), null); + List.of(mockBlockHeader(1), mockBlockHeader(2), mockBlockHeader(3)), null, () -> null); ReceiptsMessage receiptsMessage = ReceiptsMessage.create( List.of( @@ -103,7 +106,7 @@ public void testParseResponse() throws InvalidPeerTaskResponseException { GetReceiptsFromPeerTask task = new GetReceiptsFromPeerTask( - List.of(blockHeader1, blockHeader2, blockHeader3), bodyValidator); + List.of(blockHeader1, blockHeader2, blockHeader3), bodyValidator, () -> null); TransactionReceipt receiptForBlock1 = new TransactionReceipt(1, 123, Collections.emptyList(), Optional.empty()); @@ -137,8 +140,12 @@ public void testGetPeerRequirementFilter() { BlockHeader blockHeader2 = mockBlockHeader(2); BlockHeader blockHeader3 = mockBlockHeader(3); + ProtocolSpec protocolSpec = Mockito.mock(ProtocolSpec.class); + Mockito.when(protocolSpec.isPoS()).thenReturn(false); + GetReceiptsFromPeerTask task = - new GetReceiptsFromPeerTask(List.of(blockHeader1, blockHeader2, blockHeader3), null); + new GetReceiptsFromPeerTask( + List.of(blockHeader1, blockHeader2, blockHeader3), null, () -> protocolSpec); EthPeer failForIncorrectProtocol = mockPeer("incorrectProtocol", 5); EthPeer failForShortChainHeight = mockPeer("incorrectProtocol", 1); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java index c2a99306670..8458a5664f2 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java @@ -157,7 +157,7 @@ public PeerTaskExecutorResult>> answer } }); - when(peerTaskExecutor.executeAsync(any(GetReceiptsFromPeerTask.class))) + when(peerTaskExecutor.execute(any(GetReceiptsFromPeerTask.class))) .thenAnswer( new Answer< CompletableFuture< @@ -207,6 +207,7 @@ private ChainDownloader downloader( syncConfig, worldStateStorageCoordinator, protocolSchedule, + () -> null, protocolContext, ethContext, peerTaskExecutor, diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStepTest.java index 554e09c3e91..6261d397ddf 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStepTest.java @@ -87,6 +87,7 @@ public void setUp() { public void shouldDownloadReceiptsForBlocks() { DownloadReceiptsStep downloadReceiptsStep = new DownloadReceiptsStep( + () -> null, ethProtocolManager.ethContext(), peerTaskExecutor, SynchronizerConfiguration.builder().isPeerTaskSystemEnabled(false).build(), @@ -112,6 +113,7 @@ public void shouldDownloadReceiptsForBlocksUsingPeerTaskSystem() throws ExecutionException, InterruptedException { DownloadReceiptsStep downloadReceiptsStep = new DownloadReceiptsStep( + () -> null, ethProtocolManager.ethContext(), peerTaskExecutor, SynchronizerConfiguration.builder().isPeerTaskSystemEnabled(true).build(), diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastDownloaderFactoryTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastDownloaderFactoryTest.java index bc493ebd036..924780fe572 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastDownloaderFactoryTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastDownloaderFactoryTest.java @@ -113,6 +113,7 @@ public void shouldThrowIfSyncModeChangedWhileFastSyncIncomplete( syncConfig, dataDirectory, protocolSchedule, + () -> null, protocolContext, metricsSystem, ethContext, @@ -139,6 +140,7 @@ public void shouldNotThrowIfSyncModeChangedWhileFastSyncComplete( syncConfig, dataDirectory, protocolSchedule, + () -> null, protocolContext, metricsSystem, ethContext, @@ -168,6 +170,7 @@ public void shouldNotThrowWhenFastSyncModeRequested(final DataStorageFormat data syncConfig, dataDirectory, protocolSchedule, + () -> null, protocolContext, metricsSystem, ethContext, @@ -204,6 +207,7 @@ public void shouldClearWorldStateDuringFastSyncWhenStateQueDirectoryExists( syncConfig, dataDirectory, protocolSchedule, + () -> null, protocolContext, metricsSystem, ethContext, @@ -242,6 +246,7 @@ public void shouldCrashWhenStateQueueIsNotDirectory(final DataStorageFormat data syncConfig, dataDirectory, protocolSchedule, + () -> null, protocolContext, metricsSystem, ethContext, diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActionsTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActionsTest.java index 7e1ec3c83a6..0ffecd6842a 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActionsTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActionsTest.java @@ -585,9 +585,10 @@ private FastSyncActions createFastSyncActions( syncConfig, worldStateStorageCoordinator, protocolSchedule, + () -> null, protocolContext, ethContext, - new PeerTaskExecutor(null, null, null, new NoOpMetricsSystem()), + new PeerTaskExecutor(null, null, new NoOpMetricsSystem()), new SyncState(blockchain, ethContext.getEthPeers(), true, Optional.empty()), pivotBlockSelector, new NoOpMetricsSystem()); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloaderTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloaderTest.java index da82034eaa6..f0c433ebc3f 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloaderTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloaderTest.java @@ -109,9 +109,10 @@ private ChainDownloader downloader( syncConfig, worldStateStorageCoordinator, protocolSchedule, + () -> null, protocolContext, ethContext, - new PeerTaskExecutor(null, null, null, new NoOpMetricsSystem()), + new PeerTaskExecutor(null, null, new NoOpMetricsSystem()), syncState, new NoOpMetricsSystem(), new FastSyncState(otherBlockchain.getBlockHeader(pivotBlockNumber).get()), From 8186a77d70b92323442b32b0e73a17d8318e4d0c Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Wed, 9 Oct 2024 10:29:03 +1100 Subject: [PATCH 071/106] 7311: Rework getPeer again to use LEAST_TO_MOST_BUSY comparator Signed-off-by: Matilda Clerke --- .../org/hyperledger/besu/ethereum/eth/manager/EthPeers.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthPeers.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthPeers.java index d070c35ce19..09a998cf647 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthPeers.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthPeers.java @@ -470,7 +470,10 @@ public void setTrailingPeerRequirementsSupplier( // Part of the PeerSelector interface, to be split apart later @Override public Optional getPeer(final Predicate filter) { - return streamBestPeers().filter(filter).filter(EthPeer::hasAvailableRequestCapacity).findFirst(); + return streamAvailablePeers() + .filter(filter) + .filter(EthPeer::hasAvailableRequestCapacity) + .min(LEAST_TO_MOST_BUSY); } // Part of the PeerSelector interface, to be split apart later From 37b0ec26597b2192bbfa2d0adadbb7c7c7d75fbe Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Wed, 9 Oct 2024 10:30:42 +1100 Subject: [PATCH 072/106] 7311: Import PeerNotConnected class instead of using fully qualified class name Signed-off-by: Matilda Clerke --- .../besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java index 1734c1e8765..f3ddc5abbe8 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java @@ -15,7 +15,7 @@ package org.hyperledger.besu.ethereum.eth.manager.peertask; import org.hyperledger.besu.ethereum.eth.manager.EthPeer; -import org.hyperledger.besu.ethereum.p2p.rlpx.connections.PeerConnection; +import org.hyperledger.besu.ethereum.p2p.rlpx.connections.PeerConnection.PeerNotConnected; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.plugin.services.MetricsSystem; @@ -103,7 +103,7 @@ public PeerTaskExecutorResult executeAgainstPeer( new PeerTaskExecutorResult<>( Optional.ofNullable(result), PeerTaskExecutorResponseCode.SUCCESS); - } catch (PeerConnection.PeerNotConnected e) { + } catch (PeerNotConnected e) { executorResult = new PeerTaskExecutorResult<>( Optional.empty(), PeerTaskExecutorResponseCode.PEER_DISCONNECTED); From 545fd5c29dd00a7f0f3117dd864b4650a9b643ac Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Thu, 10 Oct 2024 08:41:18 +1100 Subject: [PATCH 073/106] 7311: Change to specifying retry counts in PeerTask instead of behavior enums Signed-off-by: Matilda Clerke --- .../eth/manager/peertask/PeerTask.java | 18 +++++++++++++---- .../manager/peertask/PeerTaskExecutor.java | 13 ++---------- .../peertask/PeerTaskRetryBehavior.java | 20 ------------------- .../peertask/PeerTaskExecutorTest.java | 20 +++++++++---------- 4 files changed, 25 insertions(+), 46 deletions(-) delete mode 100644 ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskRetryBehavior.java diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTask.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTask.java index 4436022c9ad..1d6464ac26d 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTask.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTask.java @@ -18,7 +18,6 @@ import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.SubProtocol; -import java.util.Collection; import java.util.function.Predicate; /** @@ -51,11 +50,22 @@ public interface PeerTask { T parseResponse(MessageData messageData) throws InvalidPeerTaskResponseException; /** - * Gets the Collection of behaviors this task is expected to exhibit in the PeetTaskExecutor + * Gets the number of times this task may be attempted against other peers * - * @return the Collection of behaviors this task is expected to exhibit in the PeetTaskExecutor + * @return the number of times this task may be attempted against other peers */ - Collection getPeerTaskRetryBehaviors(); + default int getRetriesWithOtherPeer() { + return 5; + } + + /** + * Gets the number of times this task may be attempted against the same peer + * + * @return the number of times this task may be attempted against the same peer + */ + default int getRetriesWithSamePeer() { + return 5; + } /** * Gets a Predicate that checks if an EthPeer is suitable for this PeerTask diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java index f3ddc5abbe8..7b485ee6c37 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java @@ -31,9 +31,6 @@ /** Manages the execution of PeerTasks, respecting their PeerTaskRetryBehavior */ public class PeerTaskExecutor { - public static final int RETRIES_WITH_SAME_PEER = 2; - public static final int RETRIES_WITH_OTHER_PEER = 2; - public static final int NO_RETRIES = 0; private final PeerSelector peerSelector; private final PeerTaskRequestSender requestSender; @@ -55,10 +52,7 @@ public PeerTaskExecutor( public PeerTaskExecutorResult execute(final PeerTask peerTask) { PeerTaskExecutorResult executorResult; - int retriesRemaining = - peerTask.getPeerTaskRetryBehaviors().contains(PeerTaskRetryBehavior.RETRY_WITH_OTHER_PEERS) - ? RETRIES_WITH_OTHER_PEER - : NO_RETRIES; + int retriesRemaining = peerTask.getRetriesWithOtherPeer(); final Collection usedEthPeers = new HashSet<>(); do { Optional peer = @@ -84,10 +78,7 @@ public PeerTaskExecutorResult executeAgainstPeer( final PeerTask peerTask, final EthPeer peer) { MessageData requestMessageData = peerTask.getRequestMessage(); PeerTaskExecutorResult executorResult; - int retriesRemaining = - peerTask.getPeerTaskRetryBehaviors().contains(PeerTaskRetryBehavior.RETRY_WITH_SAME_PEER) - ? RETRIES_WITH_SAME_PEER - : NO_RETRIES; + int retriesRemaining = peerTask.getRetriesWithSamePeer(); do { try { T result; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskRetryBehavior.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskRetryBehavior.java deleted file mode 100644 index 53e2def6612..00000000000 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskRetryBehavior.java +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright contributors to Hyperledger Besu. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.besu.ethereum.eth.manager.peertask; - -public enum PeerTaskRetryBehavior { - RETRY_WITH_SAME_PEER, - RETRY_WITH_OTHER_PEERS -} diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java index 6dfd8d0e203..0015c1ffbcb 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java @@ -20,8 +20,6 @@ import org.hyperledger.besu.ethereum.p2p.rlpx.wire.SubProtocol; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; -import java.util.Collections; -import java.util.List; import java.util.Optional; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeoutException; @@ -69,7 +67,7 @@ public void testExecuteAgainstPeerWithNoPeerTaskBehaviorsAndSuccessfulFlow() Object responseObject = new Object(); Mockito.when(peerTask.getRequestMessage()).thenReturn(requestMessageData); - Mockito.when(peerTask.getPeerTaskRetryBehaviors()).thenReturn(Collections.emptyList()); + Mockito.when(peerTask.getRetriesWithSamePeer()).thenReturn(0); Mockito.when(peerTask.getSubProtocol()).thenReturn(subprotocol); Mockito.when(subprotocol.getName()).thenReturn("subprotocol"); Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, ethPeer)) @@ -97,8 +95,7 @@ public void testExecuteAgainstPeerWithRetryBehaviorsAndSuccessfulFlowAfterFirstF int requestMessageDataCode = 123; Mockito.when(peerTask.getRequestMessage()).thenReturn(requestMessageData); - Mockito.when(peerTask.getPeerTaskRetryBehaviors()) - .thenReturn(List.of(PeerTaskRetryBehavior.RETRY_WITH_SAME_PEER)); + Mockito.when(peerTask.getRetriesWithSamePeer()).thenReturn(2); Mockito.when(peerTask.getSubProtocol()).thenReturn(subprotocol); Mockito.when(subprotocol.getName()).thenReturn("subprotocol"); @@ -127,7 +124,7 @@ public void testExecuteAgainstPeerWithNoPeerTaskBehaviorsAndPeerNotConnected() TimeoutException { Mockito.when(peerTask.getRequestMessage()).thenReturn(requestMessageData); - Mockito.when(peerTask.getPeerTaskRetryBehaviors()).thenReturn(Collections.emptyList()); + Mockito.when(peerTask.getRetriesWithSamePeer()).thenReturn(0); Mockito.when(peerTask.getSubProtocol()).thenReturn(subprotocol); Mockito.when(subprotocol.getName()).thenReturn("subprotocol"); Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, ethPeer)) @@ -149,7 +146,7 @@ public void testExecuteAgainstPeerWithNoPeerTaskBehaviorsAndTimeoutException() int requestMessageDataCode = 123; Mockito.when(peerTask.getRequestMessage()).thenReturn(requestMessageData); - Mockito.when(peerTask.getPeerTaskRetryBehaviors()).thenReturn(Collections.emptyList()); + Mockito.when(peerTask.getRetriesWithSamePeer()).thenReturn(0); Mockito.when(peerTask.getSubProtocol()).thenReturn(subprotocol); Mockito.when(subprotocol.getName()).thenReturn("subprotocol"); Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, ethPeer)) @@ -174,7 +171,7 @@ public void testExecuteAgainstPeerWithNoPeerTaskBehaviorsAndInvalidResponseMessa InvalidPeerTaskResponseException { Mockito.when(peerTask.getRequestMessage()).thenReturn(requestMessageData); - Mockito.when(peerTask.getPeerTaskRetryBehaviors()).thenReturn(Collections.emptyList()); + Mockito.when(peerTask.getRetriesWithSamePeer()).thenReturn(0); Mockito.when(peerTask.getSubProtocol()).thenReturn(subprotocol); Mockito.when(subprotocol.getName()).thenReturn("subprotocol"); Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, ethPeer)) @@ -205,7 +202,8 @@ public void testExecuteWithNoPeerTaskBehaviorsAndSuccessFlow() .thenReturn(Optional.of(ethPeer)); Mockito.when(peerTask.getRequestMessage()).thenReturn(requestMessageData); - Mockito.when(peerTask.getPeerTaskRetryBehaviors()).thenReturn(Collections.emptyList()); + Mockito.when(peerTask.getRetriesWithOtherPeer()).thenReturn(0); + Mockito.when(peerTask.getRetriesWithSamePeer()).thenReturn(0); Mockito.when(peerTask.getSubProtocol()).thenReturn(subprotocol); Mockito.when(subprotocol.getName()).thenReturn("subprotocol"); Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, ethPeer)) @@ -239,8 +237,8 @@ public void testExecuteWithPeerSwitchingAndSuccessFlow() .thenReturn(Optional.of(peer2)); Mockito.when(peerTask.getRequestMessage()).thenReturn(requestMessageData); - Mockito.when(peerTask.getPeerTaskRetryBehaviors()) - .thenReturn(List.of(PeerTaskRetryBehavior.RETRY_WITH_OTHER_PEERS)); + Mockito.when(peerTask.getRetriesWithOtherPeer()).thenReturn(2); + Mockito.when(peerTask.getRetriesWithSamePeer()).thenReturn(0); Mockito.when(peerTask.getSubProtocol()).thenReturn(subprotocol); Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, ethPeer)) .thenThrow(new TimeoutException()); From e9d08f3f5550b55fe26f3ef5b7100e16bd0d7cc9 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Thu, 10 Oct 2024 09:16:22 +1100 Subject: [PATCH 074/106] 7311: clean up after merge Signed-off-by: Matilda Clerke --- .../eth/manager/peertask/task/GetReceiptsFromPeerTask.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTask.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTask.java index 706e8779a73..4d95bfdc018 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTask.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTask.java @@ -21,7 +21,6 @@ import org.hyperledger.besu.ethereum.eth.manager.EthPeer; import org.hyperledger.besu.ethereum.eth.manager.peertask.InvalidPeerTaskResponseException; import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTask; -import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskRetryBehavior; import org.hyperledger.besu.ethereum.eth.messages.GetReceiptsMessage; import org.hyperledger.besu.ethereum.eth.messages.ReceiptsMessage; import org.hyperledger.besu.ethereum.mainnet.BodyValidator; @@ -108,12 +107,6 @@ public Map> parseResponse(final MessageDat return receiptsByHeader; } - @Override - public Collection getPeerTaskRetryBehaviors() { - return List.of( - PeerTaskRetryBehavior.RETRY_WITH_OTHER_PEERS, PeerTaskRetryBehavior.RETRY_WITH_SAME_PEER); - } - @Override public Predicate getPeerRequirementFilter() { return (ethPeer) -> From 20478d32dfb278bbe939b63229e1de8c4ae4eba1 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Thu, 10 Oct 2024 10:37:21 +1100 Subject: [PATCH 075/106] 7311: clean up after merge Signed-off-by: Matilda Clerke --- .../CheckPointSyncChainDownloaderTest.java | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java index 8458a5664f2..0eb518ba7e3 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java @@ -156,22 +156,6 @@ public PeerTaskExecutorResult>> answer return processTask(task); } }); - - when(peerTaskExecutor.execute(any(GetReceiptsFromPeerTask.class))) - .thenAnswer( - new Answer< - CompletableFuture< - PeerTaskExecutorResult>>>>() { - @Override - public CompletableFuture< - PeerTaskExecutorResult>>> - answer(final InvocationOnMock invocationOnMock) throws Throwable { - GetReceiptsFromPeerTask task = - invocationOnMock.getArgument(0, GetReceiptsFromPeerTask.class); - - return CompletableFuture.completedFuture(processTask(task)); - } - }); } @SuppressWarnings("unchecked") From 3ddfe715a94436803e6e7c4c29dd1c86f6d24d31 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Thu, 10 Oct 2024 13:53:05 +1100 Subject: [PATCH 076/106] 7311: Fix up javadoc Signed-off-by: Matilda Clerke --- .../org/hyperledger/besu/controller/BesuControllerBuilder.java | 1 + 1 file changed, 1 insertion(+) diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java index d1e04d451ba..f749b4fe78a 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java @@ -829,6 +829,7 @@ private TrieLogPruner createTrieLogPruner( * Create synchronizer synchronizer. * * @param protocolSchedule the protocol schedule + * @param currentProtocolSpecSupplier the protocol spec supplier * @param worldStateStorageCoordinator the world state storage * @param protocolContext the protocol context * @param ethContext the eth context From 1c268b70cd64613b57953714ea291c084696dd91 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Thu, 10 Oct 2024 15:14:49 +1100 Subject: [PATCH 077/106] 7311: Add additional metrics to PeerTaskExecutor Signed-off-by: Matilda Clerke --- .../manager/peertask/PeerTaskExecutor.java | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java index 7b485ee6c37..7c096698fba 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java @@ -19,6 +19,7 @@ import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.plugin.services.MetricsSystem; +import org.hyperledger.besu.plugin.services.metrics.Counter; import org.hyperledger.besu.plugin.services.metrics.LabelledMetric; import org.hyperledger.besu.plugin.services.metrics.OperationTimer; @@ -35,6 +36,9 @@ public class PeerTaskExecutor { private final PeerTaskRequestSender requestSender; private final LabelledMetric requestTimer; + private final LabelledMetric timeoutCounter; + private final LabelledMetric invalidResponseCounter; + private final LabelledMetric internalExceptionCounter; public PeerTaskExecutor( final PeerSelector peerSelector, @@ -48,6 +52,24 @@ public PeerTaskExecutor( "PeerTaskExecutor:RequestTime", "Time taken to send a request and receive a response", "className"); + timeoutCounter = + metricsSystem.createLabelledCounter( + BesuMetricCategory.PEERS, + "PeerTaskExecutor:TimeoutCounter", + "Counter of the number of timeouts occurred", + "className"); + invalidResponseCounter = + metricsSystem.createLabelledCounter( + BesuMetricCategory.PEERS, + "PeerTaskExecutor:InvalidResponseCounter", + "Counter of the number of invalid responses received", + "className"); + internalExceptionCounter = + metricsSystem.createLabelledCounter( + BesuMetricCategory.PEERS, + "PeerTaskExecutor:InternalExceptionCounter", + "Counter of the number of internal exceptions occurred", + "className"); } public PeerTaskExecutorResult execute(final PeerTask peerTask) { @@ -80,6 +102,7 @@ public PeerTaskExecutorResult executeAgainstPeer( PeerTaskExecutorResult executorResult; int retriesRemaining = peerTask.getRetriesWithSamePeer(); do { + try { T result; try (final OperationTimer.TimingContext ignored = @@ -101,16 +124,19 @@ public PeerTaskExecutorResult executeAgainstPeer( } catch (InterruptedException | TimeoutException e) { peer.recordRequestTimeout(requestMessageData.getCode()); + timeoutCounter.labels(peerTask.getClass().getSimpleName()).inc(); executorResult = new PeerTaskExecutorResult<>(Optional.empty(), PeerTaskExecutorResponseCode.TIMEOUT); } catch (InvalidPeerTaskResponseException e) { peer.recordUselessResponse(e.getMessage()); + invalidResponseCounter.labels(peerTask.getClass().getSimpleName()).inc(); executorResult = new PeerTaskExecutorResult<>( Optional.empty(), PeerTaskExecutorResponseCode.INVALID_RESPONSE); } catch (ExecutionException e) { + internalExceptionCounter.labels(peerTask.getClass().getSimpleName()).inc(); executorResult = new PeerTaskExecutorResult<>( Optional.empty(), PeerTaskExecutorResponseCode.INTERNAL_SERVER_ERROR); From b06f38b2607f5c479d6d15f3780c0d0f190c17e2 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Thu, 10 Oct 2024 16:14:19 +1100 Subject: [PATCH 078/106] 7311: Add Predicate to PeerTask to check for partial success Signed-off-by: Matilda Clerke --- .../eth/manager/peertask/PeerTask.java | 7 +++ .../manager/peertask/PeerTaskExecutor.java | 15 +++++-- .../PeerTaskExecutorResponseCode.java | 1 + .../peertask/PeerTaskExecutorTest.java | 43 ++++++++++++++++--- 4 files changed, 56 insertions(+), 10 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTask.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTask.java index 1d6464ac26d..57976ac2e9a 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTask.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTask.java @@ -73,4 +73,11 @@ default int getRetriesWithSamePeer() { * @return a Predicate that checks if an EthPeer is suitable for this PeerTask */ Predicate getPeerRequirementFilter(); + + /** + * Checks if the supplied result is considered a partial success + * + * @return true if the supplied result is considered a partial success + */ + boolean isPartialSuccessTest(T result); } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java index 7c096698fba..5d1883c7d80 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java @@ -112,10 +112,17 @@ public PeerTaskExecutorResult executeAgainstPeer( result = peerTask.parseResponse(responseMessageData); } - peer.recordUsefulResponse(); - executorResult = - new PeerTaskExecutorResult<>( - Optional.ofNullable(result), PeerTaskExecutorResponseCode.SUCCESS); + + if (peerTask.isPartialSuccessTest(result)) { + executorResult = + new PeerTaskExecutorResult<>( + Optional.ofNullable(result), PeerTaskExecutorResponseCode.PARTIAL_SUCCESS); + } else { + peer.recordUsefulResponse(); + executorResult = + new PeerTaskExecutorResult<>( + Optional.ofNullable(result), PeerTaskExecutorResponseCode.SUCCESS); + } } catch (PeerNotConnected e) { executorResult = diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorResponseCode.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorResponseCode.java index 327461de15a..123c3267c04 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorResponseCode.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorResponseCode.java @@ -16,6 +16,7 @@ public enum PeerTaskExecutorResponseCode { SUCCESS, + PARTIAL_SUCCESS, NO_PEER_AVAILABLE, PEER_DISCONNECTED, INTERNAL_SERVER_ERROR, diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java index 0015c1ffbcb..a4e78056e14 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java @@ -57,7 +57,7 @@ public void afterTest() throws Exception { } @Test - public void testExecuteAgainstPeerWithNoPeerTaskBehaviorsAndSuccessfulFlow() + public void testExecuteAgainstPeerWithNoRetriesAndSuccessfulFlow() throws PeerConnection.PeerNotConnected, ExecutionException, InterruptedException, @@ -73,6 +73,7 @@ public void testExecuteAgainstPeerWithNoPeerTaskBehaviorsAndSuccessfulFlow() Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, ethPeer)) .thenReturn(responseMessageData); Mockito.when(peerTask.parseResponse(responseMessageData)).thenReturn(responseObject); + Mockito.when(peerTask.isPartialSuccessTest(responseObject)).thenReturn(false); PeerTaskExecutorResult result = peerTaskExecutor.executeAgainstPeer(peerTask, ethPeer); @@ -85,7 +86,34 @@ public void testExecuteAgainstPeerWithNoPeerTaskBehaviorsAndSuccessfulFlow() } @Test - public void testExecuteAgainstPeerWithRetryBehaviorsAndSuccessfulFlowAfterFirstFailure() + public void testExecuteAgainstPeerWithNoRetriesAndPartialSuccessfulFlow() + throws PeerConnection.PeerNotConnected, + ExecutionException, + InterruptedException, + TimeoutException, + InvalidPeerTaskResponseException { + + Object responseObject = new Object(); + + Mockito.when(peerTask.getRequestMessage()).thenReturn(requestMessageData); + Mockito.when(peerTask.getRetriesWithSamePeer()).thenReturn(0); + Mockito.when(peerTask.getSubProtocol()).thenReturn(subprotocol); + Mockito.when(subprotocol.getName()).thenReturn("subprotocol"); + Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, ethPeer)) + .thenReturn(responseMessageData); + Mockito.when(peerTask.parseResponse(responseMessageData)).thenReturn(responseObject); + Mockito.when(peerTask.isPartialSuccessTest(responseObject)).thenReturn(true); + + PeerTaskExecutorResult result = peerTaskExecutor.executeAgainstPeer(peerTask, ethPeer); + + Assertions.assertNotNull(result); + Assertions.assertTrue(result.result().isPresent()); + Assertions.assertSame(responseObject, result.result().get()); + Assertions.assertEquals(PeerTaskExecutorResponseCode.PARTIAL_SUCCESS, result.responseCode()); + } + + @Test + public void testExecuteAgainstPeerWithRetriesAndSuccessfulFlowAfterFirstFailure() throws PeerConnection.PeerNotConnected, ExecutionException, InterruptedException, @@ -104,6 +132,7 @@ public void testExecuteAgainstPeerWithRetryBehaviorsAndSuccessfulFlowAfterFirstF .thenReturn(responseMessageData); Mockito.when(requestMessageData.getCode()).thenReturn(requestMessageDataCode); Mockito.when(peerTask.parseResponse(responseMessageData)).thenReturn(responseObject); + Mockito.when(peerTask.isPartialSuccessTest(responseObject)).thenReturn(false); PeerTaskExecutorResult result = peerTaskExecutor.executeAgainstPeer(peerTask, ethPeer); @@ -117,7 +146,7 @@ public void testExecuteAgainstPeerWithRetryBehaviorsAndSuccessfulFlowAfterFirstF } @Test - public void testExecuteAgainstPeerWithNoPeerTaskBehaviorsAndPeerNotConnected() + public void testExecuteAgainstPeerWithNoRetriesAndPeerNotConnected() throws PeerConnection.PeerNotConnected, ExecutionException, InterruptedException, @@ -138,7 +167,7 @@ public void testExecuteAgainstPeerWithNoPeerTaskBehaviorsAndPeerNotConnected() } @Test - public void testExecuteAgainstPeerWithNoPeerTaskBehaviorsAndTimeoutException() + public void testExecuteAgainstPeerWithNoRetriesAndTimeoutException() throws PeerConnection.PeerNotConnected, ExecutionException, InterruptedException, @@ -163,7 +192,7 @@ public void testExecuteAgainstPeerWithNoPeerTaskBehaviorsAndTimeoutException() } @Test - public void testExecuteAgainstPeerWithNoPeerTaskBehaviorsAndInvalidResponseMessage() + public void testExecuteAgainstPeerWithNoRetriesAndInvalidResponseMessage() throws PeerConnection.PeerNotConnected, ExecutionException, InterruptedException, @@ -190,7 +219,7 @@ public void testExecuteAgainstPeerWithNoPeerTaskBehaviorsAndInvalidResponseMessa @Test @SuppressWarnings("unchecked") - public void testExecuteWithNoPeerTaskBehaviorsAndSuccessFlow() + public void testExecuteWithNoRetriesAndSuccessFlow() throws PeerConnection.PeerNotConnected, ExecutionException, InterruptedException, @@ -209,6 +238,7 @@ public void testExecuteWithNoPeerTaskBehaviorsAndSuccessFlow() Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, ethPeer)) .thenReturn(responseMessageData); Mockito.when(peerTask.parseResponse(responseMessageData)).thenReturn(responseObject); + Mockito.when(peerTask.isPartialSuccessTest(responseObject)).thenReturn(false); PeerTaskExecutorResult result = peerTaskExecutor.executeAgainstPeer(peerTask, ethPeer); @@ -246,6 +276,7 @@ public void testExecuteWithPeerSwitchingAndSuccessFlow() Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, peer2)) .thenReturn(responseMessageData); Mockito.when(peerTask.parseResponse(responseMessageData)).thenReturn(responseObject); + Mockito.when(peerTask.isPartialSuccessTest(responseObject)).thenReturn(false); PeerTaskExecutorResult result = peerTaskExecutor.execute(peerTask); From 3c12d3d8c33641487265f57fecfb05ec840c3f4b Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Thu, 10 Oct 2024 16:18:58 +1100 Subject: [PATCH 079/106] 7311: Fix incorrect name on isPartialSuccessTest Signed-off-by: Matilda Clerke --- .../besu/ethereum/eth/manager/peertask/PeerTask.java | 2 +- .../eth/manager/peertask/PeerTaskExecutor.java | 2 +- .../eth/manager/peertask/PeerTaskExecutorTest.java | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTask.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTask.java index 57976ac2e9a..40b995503f8 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTask.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTask.java @@ -79,5 +79,5 @@ default int getRetriesWithSamePeer() { * * @return true if the supplied result is considered a partial success */ - boolean isPartialSuccessTest(T result); + boolean isPartialSuccess(T result); } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java index 5d1883c7d80..9b062977461 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java @@ -113,7 +113,7 @@ public PeerTaskExecutorResult executeAgainstPeer( result = peerTask.parseResponse(responseMessageData); } - if (peerTask.isPartialSuccessTest(result)) { + if (peerTask.isPartialSuccess(result)) { executorResult = new PeerTaskExecutorResult<>( Optional.ofNullable(result), PeerTaskExecutorResponseCode.PARTIAL_SUCCESS); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java index a4e78056e14..853210e685f 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java @@ -73,7 +73,7 @@ public void testExecuteAgainstPeerWithNoRetriesAndSuccessfulFlow() Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, ethPeer)) .thenReturn(responseMessageData); Mockito.when(peerTask.parseResponse(responseMessageData)).thenReturn(responseObject); - Mockito.when(peerTask.isPartialSuccessTest(responseObject)).thenReturn(false); + Mockito.when(peerTask.isPartialSuccess(responseObject)).thenReturn(false); PeerTaskExecutorResult result = peerTaskExecutor.executeAgainstPeer(peerTask, ethPeer); @@ -102,7 +102,7 @@ public void testExecuteAgainstPeerWithNoRetriesAndPartialSuccessfulFlow() Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, ethPeer)) .thenReturn(responseMessageData); Mockito.when(peerTask.parseResponse(responseMessageData)).thenReturn(responseObject); - Mockito.when(peerTask.isPartialSuccessTest(responseObject)).thenReturn(true); + Mockito.when(peerTask.isPartialSuccess(responseObject)).thenReturn(true); PeerTaskExecutorResult result = peerTaskExecutor.executeAgainstPeer(peerTask, ethPeer); @@ -132,7 +132,7 @@ public void testExecuteAgainstPeerWithRetriesAndSuccessfulFlowAfterFirstFailure( .thenReturn(responseMessageData); Mockito.when(requestMessageData.getCode()).thenReturn(requestMessageDataCode); Mockito.when(peerTask.parseResponse(responseMessageData)).thenReturn(responseObject); - Mockito.when(peerTask.isPartialSuccessTest(responseObject)).thenReturn(false); + Mockito.when(peerTask.isPartialSuccess(responseObject)).thenReturn(false); PeerTaskExecutorResult result = peerTaskExecutor.executeAgainstPeer(peerTask, ethPeer); @@ -238,7 +238,7 @@ public void testExecuteWithNoRetriesAndSuccessFlow() Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, ethPeer)) .thenReturn(responseMessageData); Mockito.when(peerTask.parseResponse(responseMessageData)).thenReturn(responseObject); - Mockito.when(peerTask.isPartialSuccessTest(responseObject)).thenReturn(false); + Mockito.when(peerTask.isPartialSuccess(responseObject)).thenReturn(false); PeerTaskExecutorResult result = peerTaskExecutor.executeAgainstPeer(peerTask, ethPeer); @@ -276,7 +276,7 @@ public void testExecuteWithPeerSwitchingAndSuccessFlow() Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, peer2)) .thenReturn(responseMessageData); Mockito.when(peerTask.parseResponse(responseMessageData)).thenReturn(responseObject); - Mockito.when(peerTask.isPartialSuccessTest(responseObject)).thenReturn(false); + Mockito.when(peerTask.isPartialSuccess(responseObject)).thenReturn(false); PeerTaskExecutorResult result = peerTaskExecutor.execute(peerTask); From 3b8b7d59380cef6569001e7fedc1220341ec0c4c Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Thu, 10 Oct 2024 16:32:16 +1100 Subject: [PATCH 080/106] 7311: Implement isPartialSuccess and add unit tests Signed-off-by: Matilda Clerke --- .../task/GetReceiptsFromPeerTask.java | 5 +++++ .../task/GetReceiptsFromPeerTaskTest.java | 20 +++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTask.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTask.java index 4d95bfdc018..802148b1155 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTask.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTask.java @@ -114,4 +114,9 @@ public Predicate getPeerRequirementFilter() { && (currentProtocolSpecSupplier.get().isPoS() || ethPeer.chainState().getEstimatedHeight() >= requiredBlockchainHeight); } + + @Override + public boolean isPartialSuccess(final Map> result) { + return result.isEmpty(); + } } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTaskTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTaskTest.java index 0e3e7365f47..a06ec3a7511 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTaskTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTaskTest.java @@ -30,6 +30,7 @@ import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -156,6 +157,25 @@ public void testGetPeerRequirementFilter() { Assertions.assertTrue(task.getPeerRequirementFilter().test(successfulCandidate)); } + @Test + public void testIsPartialSuccessForPartialSuccess() { + GetReceiptsFromPeerTask task = + new GetReceiptsFromPeerTask(Collections.emptyList(), null, () -> null); + + Assertions.assertTrue(task.isPartialSuccess(Collections.emptyMap())); + } + + @Test + public void testIsPartialSuccessForFullSuccess() { + GetReceiptsFromPeerTask task = + new GetReceiptsFromPeerTask(Collections.emptyList(), null, () -> null); + + Map> map = new HashMap<>(); + map.put(mockBlockHeader(1), null); + + Assertions.assertFalse(task.isPartialSuccess(map)); + } + private BlockHeader mockBlockHeader(final long blockNumber) { BlockHeader blockHeader = Mockito.mock(BlockHeader.class); Mockito.when(blockHeader.getNumber()).thenReturn(blockNumber); From d66dd3a4cd0ab6ec267ea87a740905ab495cafff Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Fri, 11 Oct 2024 11:10:30 +1100 Subject: [PATCH 081/106] 7311: Add partialSuccessCounter and inflightRequestGauge in PeerTaskExecutor Signed-off-by: Matilda Clerke --- .../manager/peertask/PeerTaskExecutor.java | 40 ++++++++++++++++--- .../besu/metrics/noop/NoOpMetricsSystem.java | 9 +++++ .../besu/metrics/noop/NoOpValueCollector.java | 6 +++ .../opentelemetry/OpenTelemetryGauge.java | 8 ++++ .../metrics/prometheus/PrometheusGauge.java | 18 +++++++-- .../services/metrics/LabelledGauge.java | 2 + 6 files changed, 74 insertions(+), 9 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java index 9b062977461..6fee0a05779 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java @@ -20,14 +20,18 @@ import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.plugin.services.metrics.Counter; +import org.hyperledger.besu.plugin.services.metrics.LabelledGauge; import org.hyperledger.besu.plugin.services.metrics.LabelledMetric; import org.hyperledger.besu.plugin.services.metrics.OperationTimer; import java.util.Collection; import java.util.HashSet; +import java.util.Map; import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; /** Manages the execution of PeerTasks, respecting their PeerTaskRetryBehavior */ public class PeerTaskExecutor { @@ -36,9 +40,12 @@ public class PeerTaskExecutor { private final PeerTaskRequestSender requestSender; private final LabelledMetric requestTimer; + private final LabelledMetric partialSuccessCounter; private final LabelledMetric timeoutCounter; private final LabelledMetric invalidResponseCounter; private final LabelledMetric internalExceptionCounter; + private final LabelledGauge inflightRequestGauge; + private final Map inflightRequestCountByClassName; public PeerTaskExecutor( final PeerSelector peerSelector, @@ -52,6 +59,12 @@ public PeerTaskExecutor( "PeerTaskExecutor:RequestTime", "Time taken to send a request and receive a response", "className"); + partialSuccessCounter = + metricsSystem.createLabelledCounter( + BesuMetricCategory.PEERS, + "PeerTaskExecutor:PartialSuccessCounter", + "Counter of the number of partial success occurred", + "className"); timeoutCounter = metricsSystem.createLabelledCounter( BesuMetricCategory.PEERS, @@ -70,6 +83,13 @@ public PeerTaskExecutor( "PeerTaskExecutor:InternalExceptionCounter", "Counter of the number of internal exceptions occurred", "className"); + inflightRequestGauge = + metricsSystem.createLabelledGauge( + BesuMetricCategory.PEERS, + "PeerTaskExecutor:InflightRequestGauge", + "Gauge of the number of inflight requests", + "className"); + inflightRequestCountByClassName = new ConcurrentHashMap<>(); } public PeerTaskExecutorResult execute(final PeerTask peerTask) { @@ -98,22 +118,32 @@ public PeerTaskExecutorResult execute(final PeerTask peerTask) { public PeerTaskExecutorResult executeAgainstPeer( final PeerTask peerTask, final EthPeer peer) { + String taskClassName = peerTask.getClass().getSimpleName(); + AtomicInteger inflightRequestCountForThisTaskClass = + inflightRequestCountByClassName.getOrDefault(taskClassName, new AtomicInteger(0)); + if (!inflightRequestGauge.isLabelsObserved(taskClassName)) { + inflightRequestGauge.labels(inflightRequestCountForThisTaskClass::get, taskClassName); + } MessageData requestMessageData = peerTask.getRequestMessage(); PeerTaskExecutorResult executorResult; int retriesRemaining = peerTask.getRetriesWithSamePeer(); do { - try { T result; try (final OperationTimer.TimingContext ignored = - requestTimer.labels(peerTask.getClass().getSimpleName()).startTimer()) { + requestTimer.labels(taskClassName).startTimer()) { + inflightRequestCountForThisTaskClass.incrementAndGet(); + MessageData responseMessageData = requestSender.sendRequest(peerTask.getSubProtocol(), requestMessageData, peer); result = peerTask.parseResponse(responseMessageData); + } finally { + inflightRequestCountForThisTaskClass.decrementAndGet(); } if (peerTask.isPartialSuccess(result)) { + partialSuccessCounter.labels(taskClassName).inc(); executorResult = new PeerTaskExecutorResult<>( Optional.ofNullable(result), PeerTaskExecutorResponseCode.PARTIAL_SUCCESS); @@ -131,19 +161,19 @@ public PeerTaskExecutorResult executeAgainstPeer( } catch (InterruptedException | TimeoutException e) { peer.recordRequestTimeout(requestMessageData.getCode()); - timeoutCounter.labels(peerTask.getClass().getSimpleName()).inc(); + timeoutCounter.labels(taskClassName).inc(); executorResult = new PeerTaskExecutorResult<>(Optional.empty(), PeerTaskExecutorResponseCode.TIMEOUT); } catch (InvalidPeerTaskResponseException e) { peer.recordUselessResponse(e.getMessage()); - invalidResponseCounter.labels(peerTask.getClass().getSimpleName()).inc(); + invalidResponseCounter.labels(taskClassName).inc(); executorResult = new PeerTaskExecutorResult<>( Optional.empty(), PeerTaskExecutorResponseCode.INVALID_RESPONSE); } catch (ExecutionException e) { - internalExceptionCounter.labels(peerTask.getClass().getSimpleName()).inc(); + internalExceptionCounter.labels(taskClassName).inc(); executorResult = new PeerTaskExecutorResult<>( Optional.empty(), PeerTaskExecutorResponseCode.INTERNAL_SERVER_ERROR); diff --git a/metrics/core/src/main/java/org/hyperledger/besu/metrics/noop/NoOpMetricsSystem.java b/metrics/core/src/main/java/org/hyperledger/besu/metrics/noop/NoOpMetricsSystem.java index 2d1ee26cfd1..5f876fa4d80 100644 --- a/metrics/core/src/main/java/org/hyperledger/besu/metrics/noop/NoOpMetricsSystem.java +++ b/metrics/core/src/main/java/org/hyperledger/besu/metrics/noop/NoOpMetricsSystem.java @@ -253,5 +253,14 @@ public void labels(final DoubleSupplier valueSupplier, final String... labelValu "The count of labels used must match the count of labels expected."); Preconditions.checkNotNull(valueSupplier, "No valueSupplier specified"); } + + @Override + public boolean isLabelsObserved(final String... labelValues) { + Preconditions.checkArgument( + labelValues.length == labelCount, + "The count of labels used must match the count of labels expected."); + final String labelValuesString = String.join(",", labelValues); + return labelValuesCache.contains(labelValuesString); + } } } diff --git a/metrics/core/src/main/java/org/hyperledger/besu/metrics/noop/NoOpValueCollector.java b/metrics/core/src/main/java/org/hyperledger/besu/metrics/noop/NoOpValueCollector.java index 144e7f3187c..6f36f10d2c7 100644 --- a/metrics/core/src/main/java/org/hyperledger/besu/metrics/noop/NoOpValueCollector.java +++ b/metrics/core/src/main/java/org/hyperledger/besu/metrics/noop/NoOpValueCollector.java @@ -36,4 +36,10 @@ public synchronized void labels(final DoubleSupplier valueSupplier, final String } labelValuesCreated.add(labelValuesString); } + + @Override + public boolean isLabelsObserved(final String... labelValues) { + final String labelValuesString = String.join(",", labelValues); + return labelValuesCreated.contains(labelValuesString); + } } diff --git a/metrics/core/src/main/java/org/hyperledger/besu/metrics/opentelemetry/OpenTelemetryGauge.java b/metrics/core/src/main/java/org/hyperledger/besu/metrics/opentelemetry/OpenTelemetryGauge.java index 6aea56586a8..e1d785c68ec 100644 --- a/metrics/core/src/main/java/org/hyperledger/besu/metrics/opentelemetry/OpenTelemetryGauge.java +++ b/metrics/core/src/main/java/org/hyperledger/besu/metrics/opentelemetry/OpenTelemetryGauge.java @@ -63,6 +63,14 @@ public void labels(final DoubleSupplier valueSupplier, final String... labelValu } } + @Override + public boolean isLabelsObserved(final String... labelValues) { + Preconditions.checkArgument( + labelValues.length == labelNames.size(), + "label values and label names need the same number of elements"); + return observationsMap.containsKey(getLabels(labelValues)); + } + private Attributes getLabels(final String... labelValues) { final AttributesBuilder labelsBuilder = Attributes.builder(); for (int i = 0; i < labelNames.size(); i++) { diff --git a/metrics/core/src/main/java/org/hyperledger/besu/metrics/prometheus/PrometheusGauge.java b/metrics/core/src/main/java/org/hyperledger/besu/metrics/prometheus/PrometheusGauge.java index 83000dfac32..b69e3f90626 100644 --- a/metrics/core/src/main/java/org/hyperledger/besu/metrics/prometheus/PrometheusGauge.java +++ b/metrics/core/src/main/java/org/hyperledger/besu/metrics/prometheus/PrometheusGauge.java @@ -47,10 +47,7 @@ public PrometheusGauge( @Override public synchronized void labels(final DoubleSupplier valueSupplier, final String... labelValues) { - if (labelValues.length != labelNames.size()) { - throw new IllegalArgumentException( - "Label values and label names must be the same cardinality"); - } + validateLabelsCardinality(labelValues); if (observationsMap.putIfAbsent(List.of(labelValues), valueSupplier) != null) { final String labelValuesString = String.join(",", labelValues); throw new IllegalArgumentException( @@ -58,6 +55,12 @@ public synchronized void labels(final DoubleSupplier valueSupplier, final String } } + @Override + public boolean isLabelsObserved(final String... labelValues) { + validateLabelsCardinality(labelValues); + return observationsMap.containsKey(List.of(labelValues)); + } + @Override public List collect() { final List samples = new ArrayList<>(); @@ -68,4 +71,11 @@ public List collect() { metricName, labelNames, labels, valueSupplier.getAsDouble()))); return List.of(new MetricFamilySamples(metricName, Type.GAUGE, help, samples)); } + + private void validateLabelsCardinality(final String... labelValues) { + if (labelValues.length != labelNames.size()) { + throw new IllegalArgumentException( + "Label values and label names must be the same cardinality"); + } + } } diff --git a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/metrics/LabelledGauge.java b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/metrics/LabelledGauge.java index 724e31c58de..c2f64c1113f 100644 --- a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/metrics/LabelledGauge.java +++ b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/metrics/LabelledGauge.java @@ -25,4 +25,6 @@ public interface LabelledGauge { * @param labelValues the label values */ void labels(final DoubleSupplier valueSupplier, final String... labelValues); + + boolean isLabelsObserved(final String... labelValues); } From a3f5d4ac689cf1c572fe5c2bdad18077c29555ef Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Fri, 11 Oct 2024 14:35:06 +1100 Subject: [PATCH 082/106] 7311: Also filter by whether a peer is fully validated Signed-off-by: Matilda Clerke --- .../java/org/hyperledger/besu/ethereum/eth/manager/EthPeers.java | 1 + 1 file changed, 1 insertion(+) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthPeers.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthPeers.java index 09a998cf647..d19c7dfca2e 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthPeers.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthPeers.java @@ -473,6 +473,7 @@ public Optional getPeer(final Predicate filter) { return streamAvailablePeers() .filter(filter) .filter(EthPeer::hasAvailableRequestCapacity) + .filter(EthPeer::isFullyValidated) .min(LEAST_TO_MOST_BUSY); } From 714db0a77be164b6d67b7b5f27b51714a6863022 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Fri, 11 Oct 2024 15:17:53 +1100 Subject: [PATCH 083/106] 7311: Remove unneeded throws in RunnerTest Signed-off-by: Matilda Clerke --- besu/src/test/java/org/hyperledger/besu/RunnerTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/besu/src/test/java/org/hyperledger/besu/RunnerTest.java b/besu/src/test/java/org/hyperledger/besu/RunnerTest.java index f6387d5b3b0..6324fabb743 100644 --- a/besu/src/test/java/org/hyperledger/besu/RunnerTest.java +++ b/besu/src/test/java/org/hyperledger/besu/RunnerTest.java @@ -119,7 +119,7 @@ public final class RunnerTest { private Vertx vertx; @BeforeEach - public void initVertx() throws IllegalAccessException { + public void initVertx() { vertx = Vertx.vertx(); } @@ -131,7 +131,7 @@ public void stopVertx() { @TempDir private Path temp; @Test - public void getFixedNodes() throws IllegalAccessException { + public void getFixedNodes() { final EnodeURL staticNode = EnodeURLImpl.fromString( "enode://8f4b88336cc40ef2516d8b27df812e007fb2384a61e93635f1899051311344f3dcdbb49a4fe49a79f66d2f589a9f282e8cc4f1d7381e8ef7e4fcc6b0db578c77@127.0.0.1:30301"); From 3a689800880d42f656907df019757f4eb4f398b2 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Fri, 11 Oct 2024 15:57:44 +1100 Subject: [PATCH 084/106] 7311: Fix up inflight requests gauge in PeerTaskExecutor Signed-off-by: Matilda Clerke --- .../eth/manager/peertask/PeerTaskExecutor.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java index 6fee0a05779..2653fd6f359 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java @@ -120,10 +120,13 @@ public PeerTaskExecutorResult executeAgainstPeer( final PeerTask peerTask, final EthPeer peer) { String taskClassName = peerTask.getClass().getSimpleName(); AtomicInteger inflightRequestCountForThisTaskClass = - inflightRequestCountByClassName.getOrDefault(taskClassName, new AtomicInteger(0)); - if (!inflightRequestGauge.isLabelsObserved(taskClassName)) { - inflightRequestGauge.labels(inflightRequestCountForThisTaskClass::get, taskClassName); - } + inflightRequestCountByClassName.computeIfAbsent( + taskClassName, + (k) -> { + AtomicInteger inflightRequests = new AtomicInteger(0); + inflightRequestGauge.labels(inflightRequests::get, taskClassName); + return inflightRequests; + }); MessageData requestMessageData = peerTask.getRequestMessage(); PeerTaskExecutorResult executorResult; int retriesRemaining = peerTask.getRetriesWithSamePeer(); From 74aa7a0fa46f9362e11d91cfd3fc5f4450ed746d Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Fri, 11 Oct 2024 16:08:43 +1100 Subject: [PATCH 085/106] 7311: Update plugin api hash Signed-off-by: Matilda Clerke --- plugin-api/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin-api/build.gradle b/plugin-api/build.gradle index dd2da0a15ec..0b44097b90c 100644 --- a/plugin-api/build.gradle +++ b/plugin-api/build.gradle @@ -71,7 +71,7 @@ Calculated : ${currentHash} tasks.register('checkAPIChanges', FileStateChecker) { description = "Checks that the API for the Plugin-API project does not change without deliberate thought" files = sourceSets.main.allJava.files - knownHash = '4jVaj9yW88nHbX0KmTR3dPQRvj9x8Pvh5E9Ry7KRT6w=' + knownHash = 'LBcFfNI00D9oKdnuLo2t5sXZZXfYZQSDB/t58U0PmgI=' } check.dependsOn('checkAPIChanges') From 56c1f9d1bc4f3d9516d9399b687b18911f049a95 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Fri, 11 Oct 2024 16:09:35 +1100 Subject: [PATCH 086/106] 7311: Update plugin api hash Signed-off-by: Matilda Clerke --- plugin-api/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin-api/build.gradle b/plugin-api/build.gradle index dd2da0a15ec..0b44097b90c 100644 --- a/plugin-api/build.gradle +++ b/plugin-api/build.gradle @@ -71,7 +71,7 @@ Calculated : ${currentHash} tasks.register('checkAPIChanges', FileStateChecker) { description = "Checks that the API for the Plugin-API project does not change without deliberate thought" files = sourceSets.main.allJava.files - knownHash = '4jVaj9yW88nHbX0KmTR3dPQRvj9x8Pvh5E9Ry7KRT6w=' + knownHash = 'LBcFfNI00D9oKdnuLo2t5sXZZXfYZQSDB/t58U0PmgI=' } check.dependsOn('checkAPIChanges') From e733452205e54c73fe1ce7f6522ef41e029f79ae Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Mon, 14 Oct 2024 08:47:21 +1100 Subject: [PATCH 087/106] 7311: Add javadoc to LabelledGauge.isLabelsObserved Signed-off-by: Matilda Clerke --- .../besu/plugin/services/metrics/LabelledGauge.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/metrics/LabelledGauge.java b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/metrics/LabelledGauge.java index c2f64c1113f..16a2f1e59b1 100644 --- a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/metrics/LabelledGauge.java +++ b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/metrics/LabelledGauge.java @@ -26,5 +26,12 @@ public interface LabelledGauge { */ void labels(final DoubleSupplier valueSupplier, final String... labelValues); + /** + * Checks whether the supplied labelValues are already observed by this LabelledGauge + * + * @param labelValues The labelValues to check + * @return true if the supplied labelValues are already observed by this LabelledGauge, false + * otherwise + */ boolean isLabelsObserved(final String... labelValues); } From b3a252be4337b160d35b883952b228420dbbfc55 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Mon, 14 Oct 2024 08:53:23 +1100 Subject: [PATCH 088/106] 7311: Update plugin-api hash Signed-off-by: Matilda Clerke --- plugin-api/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin-api/build.gradle b/plugin-api/build.gradle index 0b44097b90c..5d2d28d196f 100644 --- a/plugin-api/build.gradle +++ b/plugin-api/build.gradle @@ -71,7 +71,7 @@ Calculated : ${currentHash} tasks.register('checkAPIChanges', FileStateChecker) { description = "Checks that the API for the Plugin-API project does not change without deliberate thought" files = sourceSets.main.allJava.files - knownHash = 'LBcFfNI00D9oKdnuLo2t5sXZZXfYZQSDB/t58U0PmgI=' + knownHash = 'VN2JB2HPpEUDQaDvd7QcMkmmgedasVChfA8tnSf1GHU=' } check.dependsOn('checkAPIChanges') From 3c96ebaf09679c098722b9c260ce86f6a69f4288 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Mon, 14 Oct 2024 09:13:20 +1100 Subject: [PATCH 089/106] 7311: Update changelog Signed-off-by: Matilda Clerke --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c26ef6e777..91a23d61fa8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## [Unreleased] ### Breaking Changes +- Added isLabelsObserved to LabelledGauge in plugin-api. Implementing classes will need to also implement this method ### Upcoming Breaking Changes From e664a51c003d252710ab69d0bbc4cb81693804a9 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Tue, 15 Oct 2024 15:33:09 +1100 Subject: [PATCH 090/106] 7311: Handle headers with no receipts as a special case in DownloadReceiptsStep Signed-off-by: Matilda Clerke --- .../ethereum/eth/sync/fastsync/DownloadReceiptsStep.java | 5 +++++ .../ethereum/eth/sync/tasks/GetReceiptsForHeadersTask.java | 1 + .../ethereum/eth/sync/fastsync/DownloadReceiptsStepTest.java | 2 ++ 3 files changed, 8 insertions(+) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java index 65e6f846363..a7015369dbb 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStep.java @@ -17,6 +17,7 @@ import static java.util.Collections.emptyList; import static java.util.stream.Collectors.toList; +import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.core.Block; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.BlockWithReceipts; @@ -80,6 +81,10 @@ public CompletableFuture> apply(final List blocks private CompletableFuture>> getReceiptsWithPeerTaskSystem(final List headers) { Map> getReceipts = new HashMap<>(); + headers.stream() + .filter(header -> header.getReceiptsRoot().equals(Hash.EMPTY_TRIE_HASH)) + .forEach(header -> getReceipts.put(header, emptyList())); + headers.removeAll(getReceipts.keySet()); do { GetReceiptsFromPeerTask task = new GetReceiptsFromPeerTask(headers, new BodyValidator(), currentProtocolSpecSupplier); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/tasks/GetReceiptsForHeadersTask.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/tasks/GetReceiptsForHeadersTask.java index 58c4d3a7afa..3096eb1ac9f 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/tasks/GetReceiptsForHeadersTask.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/tasks/GetReceiptsForHeadersTask.java @@ -65,6 +65,7 @@ private GetReceiptsForHeadersTask( completeEmptyReceipts(headers); } + // TODO: This is only used in test code and should be removed public static GetReceiptsForHeadersTask forHeaders( final EthContext ethContext, final List headers, diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStepTest.java index 6261d397ddf..caa31bdf5c5 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStepTest.java @@ -18,6 +18,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; +import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; import org.hyperledger.besu.ethereum.core.Block; @@ -156,6 +157,7 @@ private Block mockBlock() { final Block block = Mockito.mock(Block.class); final BlockHeader blockHeader = Mockito.mock(BlockHeader.class); Mockito.when(block.getHeader()).thenAnswer((invocationOnMock) -> blockHeader); + Mockito.when(blockHeader.getReceiptsRoot()).thenReturn(Hash.fromHexStringLenient("DEADBEEF")); final BlockBody blockBody = Mockito.mock(BlockBody.class); Mockito.when(block.getBody()).thenAnswer((invocationOnMock) -> blockBody); Mockito.when(blockBody.getTransactions()) From 6800cdd2ed84a77acf08ec417f812af28d49e8c1 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Wed, 16 Oct 2024 09:35:23 +1100 Subject: [PATCH 091/106] 7311: Complete merge Signed-off-by: Matilda Clerke --- .../besu/ethereum/eth/sync/tasks/GetReceiptsForHeadersTask.java | 1 - 1 file changed, 1 deletion(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/tasks/GetReceiptsForHeadersTask.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/tasks/GetReceiptsForHeadersTask.java index 398ea4f635c..2e464834385 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/tasks/GetReceiptsForHeadersTask.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/tasks/GetReceiptsForHeadersTask.java @@ -65,7 +65,6 @@ private GetReceiptsForHeadersTask( completeEmptyReceipts(headers); } - // TODO: This is only used in test code and should be removed public static GetReceiptsForHeadersTask forHeaders( final EthContext ethContext, final List headers, From 44fd3a88705f0b3bce0b4f4c4e56d5eb3bfcf18e Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Thu, 17 Oct 2024 10:17:21 +1100 Subject: [PATCH 092/106] 7311: Use taskName instead of className for labelNames Signed-off-by: Matilda Clerke --- .../eth/manager/peertask/PeerTaskExecutor.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java index 2653fd6f359..e769fcc8e4c 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java @@ -58,37 +58,37 @@ public PeerTaskExecutor( BesuMetricCategory.PEERS, "PeerTaskExecutor:RequestTime", "Time taken to send a request and receive a response", - "className"); + "taskName"); partialSuccessCounter = metricsSystem.createLabelledCounter( BesuMetricCategory.PEERS, "PeerTaskExecutor:PartialSuccessCounter", "Counter of the number of partial success occurred", - "className"); + "taskName"); timeoutCounter = metricsSystem.createLabelledCounter( BesuMetricCategory.PEERS, "PeerTaskExecutor:TimeoutCounter", "Counter of the number of timeouts occurred", - "className"); + "taskName"); invalidResponseCounter = metricsSystem.createLabelledCounter( BesuMetricCategory.PEERS, "PeerTaskExecutor:InvalidResponseCounter", "Counter of the number of invalid responses received", - "className"); + "taskName"); internalExceptionCounter = metricsSystem.createLabelledCounter( BesuMetricCategory.PEERS, "PeerTaskExecutor:InternalExceptionCounter", "Counter of the number of internal exceptions occurred", - "className"); + "taskName"); inflightRequestGauge = metricsSystem.createLabelledGauge( BesuMetricCategory.PEERS, "PeerTaskExecutor:InflightRequestGauge", "Gauge of the number of inflight requests", - "className"); + "taskName"); inflightRequestCountByClassName = new ConcurrentHashMap<>(); } From ac1c4ed9d475f08c885cc71db42443c6fa52c540 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Thu, 17 Oct 2024 10:19:47 +1100 Subject: [PATCH 093/106] 7311: Use snake_case for metric names Signed-off-by: Matilda Clerke --- .../eth/manager/peertask/PeerTaskExecutor.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java index e769fcc8e4c..fcd53281573 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java @@ -56,37 +56,37 @@ public PeerTaskExecutor( requestTimer = metricsSystem.createLabelledTimer( BesuMetricCategory.PEERS, - "PeerTaskExecutor:RequestTime", + "request_time", "Time taken to send a request and receive a response", "taskName"); partialSuccessCounter = metricsSystem.createLabelledCounter( BesuMetricCategory.PEERS, - "PeerTaskExecutor:PartialSuccessCounter", + "partial_success_counter", "Counter of the number of partial success occurred", "taskName"); timeoutCounter = metricsSystem.createLabelledCounter( BesuMetricCategory.PEERS, - "PeerTaskExecutor:TimeoutCounter", + "timeout_counter", "Counter of the number of timeouts occurred", "taskName"); invalidResponseCounter = metricsSystem.createLabelledCounter( BesuMetricCategory.PEERS, - "PeerTaskExecutor:InvalidResponseCounter", + "invalid_response_counter", "Counter of the number of invalid responses received", "taskName"); internalExceptionCounter = metricsSystem.createLabelledCounter( BesuMetricCategory.PEERS, - "PeerTaskExecutor:InternalExceptionCounter", + "internal_exception_counter", "Counter of the number of internal exceptions occurred", "taskName"); inflightRequestGauge = metricsSystem.createLabelledGauge( BesuMetricCategory.PEERS, - "PeerTaskExecutor:InflightRequestGauge", + "inflight_request_gauge", "Gauge of the number of inflight requests", "taskName"); inflightRequestCountByClassName = new ConcurrentHashMap<>(); From 750353585f8368312d4b3743c3001f8027557c04 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Thu, 17 Oct 2024 10:21:16 +1100 Subject: [PATCH 094/106] 7311: Use _total metric name suffix Signed-off-by: Matilda Clerke --- .../ethereum/eth/manager/peertask/PeerTaskExecutor.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java index fcd53281573..06717914360 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java @@ -62,25 +62,25 @@ public PeerTaskExecutor( partialSuccessCounter = metricsSystem.createLabelledCounter( BesuMetricCategory.PEERS, - "partial_success_counter", + "partial_success_total", "Counter of the number of partial success occurred", "taskName"); timeoutCounter = metricsSystem.createLabelledCounter( BesuMetricCategory.PEERS, - "timeout_counter", + "timeout_total", "Counter of the number of timeouts occurred", "taskName"); invalidResponseCounter = metricsSystem.createLabelledCounter( BesuMetricCategory.PEERS, - "invalid_response_counter", + "invalid_response_total", "Counter of the number of invalid responses received", "taskName"); internalExceptionCounter = metricsSystem.createLabelledCounter( BesuMetricCategory.PEERS, - "internal_exception_counter", + "internal_exception_total", "Counter of the number of internal exceptions occurred", "taskName"); inflightRequestGauge = From ed2594169dfb84fe71850adf6ae74e35150ad139 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Thu, 17 Oct 2024 14:12:58 +1100 Subject: [PATCH 095/106] 7311: rework partial success handling Signed-off-by: Matilda Clerke --- .../eth/manager/peertask/PeerTask.java | 6 +++--- .../manager/peertask/PeerTaskExecutor.java | 19 +++++++------------ .../PeerTaskExecutorResponseCode.java | 1 - .../peertask/PeerTaskExecutorTest.java | 15 +++++++-------- 4 files changed, 17 insertions(+), 24 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTask.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTask.java index 40b995503f8..1243846ac3d 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTask.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTask.java @@ -75,9 +75,9 @@ default int getRetriesWithSamePeer() { Predicate getPeerRequirementFilter(); /** - * Checks if the supplied result is considered a partial success + * Checks if the supplied result is considered a success * - * @return true if the supplied result is considered a partial success + * @return true if the supplied result is considered a success */ - boolean isPartialSuccess(T result); + boolean isSuccess(T result); } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java index 06717914360..984cedccecb 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java @@ -40,7 +40,6 @@ public class PeerTaskExecutor { private final PeerTaskRequestSender requestSender; private final LabelledMetric requestTimer; - private final LabelledMetric partialSuccessCounter; private final LabelledMetric timeoutCounter; private final LabelledMetric invalidResponseCounter; private final LabelledMetric internalExceptionCounter; @@ -59,12 +58,6 @@ public PeerTaskExecutor( "request_time", "Time taken to send a request and receive a response", "taskName"); - partialSuccessCounter = - metricsSystem.createLabelledCounter( - BesuMetricCategory.PEERS, - "partial_success_total", - "Counter of the number of partial success occurred", - "taskName"); timeoutCounter = metricsSystem.createLabelledCounter( BesuMetricCategory.PEERS, @@ -145,16 +138,18 @@ public PeerTaskExecutorResult executeAgainstPeer( inflightRequestCountForThisTaskClass.decrementAndGet(); } - if (peerTask.isPartialSuccess(result)) { - partialSuccessCounter.labels(taskClassName).inc(); + if (peerTask.isSuccess(result)) { + peer.recordUsefulResponse(); executorResult = new PeerTaskExecutorResult<>( - Optional.ofNullable(result), PeerTaskExecutorResponseCode.PARTIAL_SUCCESS); + Optional.ofNullable(result), PeerTaskExecutorResponseCode.SUCCESS); } else { - peer.recordUsefulResponse(); + // At this point, the result is most likely empty. Technically, this is a valid result, so + // we don't penalise the peer, but it's also a useless result, so we return + // INVALID_RESPONSE code executorResult = new PeerTaskExecutorResult<>( - Optional.ofNullable(result), PeerTaskExecutorResponseCode.SUCCESS); + Optional.ofNullable(result), PeerTaskExecutorResponseCode.INVALID_RESPONSE); } } catch (PeerNotConnected e) { diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorResponseCode.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorResponseCode.java index 123c3267c04..327461de15a 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorResponseCode.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorResponseCode.java @@ -16,7 +16,6 @@ public enum PeerTaskExecutorResponseCode { SUCCESS, - PARTIAL_SUCCESS, NO_PEER_AVAILABLE, PEER_DISCONNECTED, INTERNAL_SERVER_ERROR, diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java index 853210e685f..4226f3a1332 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java @@ -73,7 +73,7 @@ public void testExecuteAgainstPeerWithNoRetriesAndSuccessfulFlow() Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, ethPeer)) .thenReturn(responseMessageData); Mockito.when(peerTask.parseResponse(responseMessageData)).thenReturn(responseObject); - Mockito.when(peerTask.isPartialSuccess(responseObject)).thenReturn(false); + Mockito.when(peerTask.isSuccess(responseObject)).thenReturn(true); PeerTaskExecutorResult result = peerTaskExecutor.executeAgainstPeer(peerTask, ethPeer); @@ -102,14 +102,13 @@ public void testExecuteAgainstPeerWithNoRetriesAndPartialSuccessfulFlow() Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, ethPeer)) .thenReturn(responseMessageData); Mockito.when(peerTask.parseResponse(responseMessageData)).thenReturn(responseObject); - Mockito.when(peerTask.isPartialSuccess(responseObject)).thenReturn(true); + Mockito.when(peerTask.isSuccess(responseObject)).thenReturn(false); PeerTaskExecutorResult result = peerTaskExecutor.executeAgainstPeer(peerTask, ethPeer); Assertions.assertNotNull(result); - Assertions.assertTrue(result.result().isPresent()); - Assertions.assertSame(responseObject, result.result().get()); - Assertions.assertEquals(PeerTaskExecutorResponseCode.PARTIAL_SUCCESS, result.responseCode()); + Assertions.assertTrue(result.result().isEmpty()); + Assertions.assertEquals(PeerTaskExecutorResponseCode.INVALID_RESPONSE, result.responseCode()); } @Test @@ -132,7 +131,7 @@ public void testExecuteAgainstPeerWithRetriesAndSuccessfulFlowAfterFirstFailure( .thenReturn(responseMessageData); Mockito.when(requestMessageData.getCode()).thenReturn(requestMessageDataCode); Mockito.when(peerTask.parseResponse(responseMessageData)).thenReturn(responseObject); - Mockito.when(peerTask.isPartialSuccess(responseObject)).thenReturn(false); + Mockito.when(peerTask.isSuccess(responseObject)).thenReturn(true); PeerTaskExecutorResult result = peerTaskExecutor.executeAgainstPeer(peerTask, ethPeer); @@ -238,7 +237,7 @@ public void testExecuteWithNoRetriesAndSuccessFlow() Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, ethPeer)) .thenReturn(responseMessageData); Mockito.when(peerTask.parseResponse(responseMessageData)).thenReturn(responseObject); - Mockito.when(peerTask.isPartialSuccess(responseObject)).thenReturn(false); + Mockito.when(peerTask.isSuccess(responseObject)).thenReturn(true); PeerTaskExecutorResult result = peerTaskExecutor.executeAgainstPeer(peerTask, ethPeer); @@ -276,7 +275,7 @@ public void testExecuteWithPeerSwitchingAndSuccessFlow() Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, peer2)) .thenReturn(responseMessageData); Mockito.when(peerTask.parseResponse(responseMessageData)).thenReturn(responseObject); - Mockito.when(peerTask.isPartialSuccess(responseObject)).thenReturn(false); + Mockito.when(peerTask.isSuccess(responseObject)).thenReturn(true); PeerTaskExecutorResult result = peerTaskExecutor.execute(peerTask); From c396fb5ad92644c92c0586971c0c8fe8a64e2e19 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Thu, 17 Oct 2024 14:15:56 +1100 Subject: [PATCH 096/106] 7311: Update GetReceiptsFromPeerTask with partialSuccess changes Signed-off-by: Matilda Clerke --- .../manager/peertask/task/GetReceiptsFromPeerTask.java | 4 ++-- .../peertask/task/GetReceiptsFromPeerTaskTest.java | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTask.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTask.java index 802148b1155..afa4fecf962 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTask.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTask.java @@ -116,7 +116,7 @@ public Predicate getPeerRequirementFilter() { } @Override - public boolean isPartialSuccess(final Map> result) { - return result.isEmpty(); + public boolean isSuccess(final Map> result) { + return !result.isEmpty(); } } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTaskTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTaskTest.java index a06ec3a7511..18c5d803e34 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTaskTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTaskTest.java @@ -158,22 +158,22 @@ public void testGetPeerRequirementFilter() { } @Test - public void testIsPartialSuccessForPartialSuccess() { + public void testIsSuccessForPartialSuccess() { GetReceiptsFromPeerTask task = new GetReceiptsFromPeerTask(Collections.emptyList(), null, () -> null); - Assertions.assertTrue(task.isPartialSuccess(Collections.emptyMap())); + Assertions.assertFalse(task.isSuccess(Collections.emptyMap())); } @Test - public void testIsPartialSuccessForFullSuccess() { + public void testIsSuccessForFullSuccess() { GetReceiptsFromPeerTask task = new GetReceiptsFromPeerTask(Collections.emptyList(), null, () -> null); Map> map = new HashMap<>(); map.put(mockBlockHeader(1), null); - Assertions.assertFalse(task.isPartialSuccess(map)); + Assertions.assertTrue(task.isSuccess(map)); } private BlockHeader mockBlockHeader(final long blockNumber) { From 5a796369e23eff5b9176d9940d2b7ac4e92f55f1 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Thu, 17 Oct 2024 14:52:46 +1100 Subject: [PATCH 097/106] 7311: Add default implementation to LabelledGauge.isLabelsObserved Signed-off-by: Matilda Clerke --- CHANGELOG.md | 2 +- plugin-api/build.gradle | 2 +- .../besu/plugin/services/metrics/LabelledGauge.java | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fd5c7ccb0e7..d9c8290f26d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,9 @@ # Changelog ## [Unreleased] +- Added isLabelsObserved to LabelledGauge in plugin-api. Default implementation returns false. ### Breaking Changes -- Added isLabelsObserved to LabelledGauge in plugin-api. Implementing classes will need to also implement this method ### Upcoming Breaking Changes diff --git a/plugin-api/build.gradle b/plugin-api/build.gradle index 5d2d28d196f..91fb45239dc 100644 --- a/plugin-api/build.gradle +++ b/plugin-api/build.gradle @@ -71,7 +71,7 @@ Calculated : ${currentHash} tasks.register('checkAPIChanges', FileStateChecker) { description = "Checks that the API for the Plugin-API project does not change without deliberate thought" files = sourceSets.main.allJava.files - knownHash = 'VN2JB2HPpEUDQaDvd7QcMkmmgedasVChfA8tnSf1GHU=' + knownHash = 'WRdnBaP05fItpWHYSFz/vBBlRWL3sLGqzR3tzd+pOkA=' } check.dependsOn('checkAPIChanges') diff --git a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/metrics/LabelledGauge.java b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/metrics/LabelledGauge.java index 16a2f1e59b1..5357c6505ae 100644 --- a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/metrics/LabelledGauge.java +++ b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/metrics/LabelledGauge.java @@ -33,5 +33,7 @@ public interface LabelledGauge { * @return true if the supplied labelValues are already observed by this LabelledGauge, false * otherwise */ - boolean isLabelsObserved(final String... labelValues); + default boolean isLabelsObserved(final String... labelValues) { + return false; + } } From b075a917df8030372754af6c0130d422ecefaace Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Fri, 18 Oct 2024 10:29:53 +1100 Subject: [PATCH 098/106] 7311: Fix broken unit test Signed-off-by: Matilda Clerke --- .../ethereum/eth/manager/peertask/PeerTaskExecutorTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java index 4226f3a1332..0262e276da2 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java @@ -107,7 +107,7 @@ public void testExecuteAgainstPeerWithNoRetriesAndPartialSuccessfulFlow() PeerTaskExecutorResult result = peerTaskExecutor.executeAgainstPeer(peerTask, ethPeer); Assertions.assertNotNull(result); - Assertions.assertTrue(result.result().isEmpty()); + Assertions.assertTrue(result.result().isPresent()); Assertions.assertEquals(PeerTaskExecutorResponseCode.INVALID_RESPONSE, result.responseCode()); } From c2205c47688e056a39d861743c5f8183294682d9 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Thu, 24 Oct 2024 09:53:30 +1100 Subject: [PATCH 099/106] 7311: Rename parseResponse to processResponse Signed-off-by: Matilda Clerke --- .../besu/ethereum/eth/manager/peertask/PeerTask.java | 4 ++-- .../eth/manager/peertask/PeerTaskExecutor.java | 2 +- .../peertask/task/GetReceiptsFromPeerTask.java | 2 +- .../eth/manager/peertask/PeerTaskExecutorTest.java | 12 ++++++------ .../peertask/task/GetReceiptsFromPeerTaskTest.java | 7 ++++--- 5 files changed, 14 insertions(+), 13 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTask.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTask.java index 1243846ac3d..fed671d38d2 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTask.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTask.java @@ -41,13 +41,13 @@ public interface PeerTask { MessageData getRequestMessage(); /** - * Parses the MessageData response from the EthPeer + * Parses and processes the MessageData response from the EthPeer * * @param messageData the response MessageData to be parsed * @return a T built from the response MessageData * @throws InvalidPeerTaskResponseException if the response messageData is invalid */ - T parseResponse(MessageData messageData) throws InvalidPeerTaskResponseException; + T processResponse(MessageData messageData) throws InvalidPeerTaskResponseException; /** * Gets the number of times this task may be attempted against other peers diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java index 984cedccecb..a2ae0455263 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutor.java @@ -133,7 +133,7 @@ public PeerTaskExecutorResult executeAgainstPeer( MessageData responseMessageData = requestSender.sendRequest(peerTask.getSubProtocol(), requestMessageData, peer); - result = peerTask.parseResponse(responseMessageData); + result = peerTask.processResponse(responseMessageData); } finally { inflightRequestCountForThisTaskClass.decrementAndGet(); } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTask.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTask.java index afa4fecf962..1ee0ed3980c 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTask.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTask.java @@ -83,7 +83,7 @@ public MessageData getRequestMessage() { } @Override - public Map> parseResponse(final MessageData messageData) + public Map> processResponse(final MessageData messageData) throws InvalidPeerTaskResponseException { if (messageData == null) { throw new InvalidPeerTaskResponseException(); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java index 0262e276da2..9639de154d7 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/PeerTaskExecutorTest.java @@ -72,7 +72,7 @@ public void testExecuteAgainstPeerWithNoRetriesAndSuccessfulFlow() Mockito.when(subprotocol.getName()).thenReturn("subprotocol"); Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, ethPeer)) .thenReturn(responseMessageData); - Mockito.when(peerTask.parseResponse(responseMessageData)).thenReturn(responseObject); + Mockito.when(peerTask.processResponse(responseMessageData)).thenReturn(responseObject); Mockito.when(peerTask.isSuccess(responseObject)).thenReturn(true); PeerTaskExecutorResult result = peerTaskExecutor.executeAgainstPeer(peerTask, ethPeer); @@ -101,7 +101,7 @@ public void testExecuteAgainstPeerWithNoRetriesAndPartialSuccessfulFlow() Mockito.when(subprotocol.getName()).thenReturn("subprotocol"); Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, ethPeer)) .thenReturn(responseMessageData); - Mockito.when(peerTask.parseResponse(responseMessageData)).thenReturn(responseObject); + Mockito.when(peerTask.processResponse(responseMessageData)).thenReturn(responseObject); Mockito.when(peerTask.isSuccess(responseObject)).thenReturn(false); PeerTaskExecutorResult result = peerTaskExecutor.executeAgainstPeer(peerTask, ethPeer); @@ -130,7 +130,7 @@ public void testExecuteAgainstPeerWithRetriesAndSuccessfulFlowAfterFirstFailure( .thenThrow(new TimeoutException()) .thenReturn(responseMessageData); Mockito.when(requestMessageData.getCode()).thenReturn(requestMessageDataCode); - Mockito.when(peerTask.parseResponse(responseMessageData)).thenReturn(responseObject); + Mockito.when(peerTask.processResponse(responseMessageData)).thenReturn(responseObject); Mockito.when(peerTask.isSuccess(responseObject)).thenReturn(true); PeerTaskExecutorResult result = peerTaskExecutor.executeAgainstPeer(peerTask, ethPeer); @@ -204,7 +204,7 @@ public void testExecuteAgainstPeerWithNoRetriesAndInvalidResponseMessage() Mockito.when(subprotocol.getName()).thenReturn("subprotocol"); Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, ethPeer)) .thenReturn(responseMessageData); - Mockito.when(peerTask.parseResponse(responseMessageData)) + Mockito.when(peerTask.processResponse(responseMessageData)) .thenThrow(new InvalidPeerTaskResponseException()); PeerTaskExecutorResult result = peerTaskExecutor.executeAgainstPeer(peerTask, ethPeer); @@ -236,7 +236,7 @@ public void testExecuteWithNoRetriesAndSuccessFlow() Mockito.when(subprotocol.getName()).thenReturn("subprotocol"); Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, ethPeer)) .thenReturn(responseMessageData); - Mockito.when(peerTask.parseResponse(responseMessageData)).thenReturn(responseObject); + Mockito.when(peerTask.processResponse(responseMessageData)).thenReturn(responseObject); Mockito.when(peerTask.isSuccess(responseObject)).thenReturn(true); PeerTaskExecutorResult result = peerTaskExecutor.executeAgainstPeer(peerTask, ethPeer); @@ -274,7 +274,7 @@ public void testExecuteWithPeerSwitchingAndSuccessFlow() Mockito.when(requestMessageData.getCode()).thenReturn(requestMessageDataCode); Mockito.when(requestSender.sendRequest(subprotocol, requestMessageData, peer2)) .thenReturn(responseMessageData); - Mockito.when(peerTask.parseResponse(responseMessageData)).thenReturn(responseObject); + Mockito.when(peerTask.processResponse(responseMessageData)).thenReturn(responseObject); Mockito.when(peerTask.isSuccess(responseObject)).thenReturn(true); PeerTaskExecutorResult result = peerTaskExecutor.execute(peerTask); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTaskTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTaskTest.java index 18c5d803e34..86c3f604806 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTaskTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTaskTest.java @@ -77,7 +77,8 @@ public void testGetRequestMessage() { public void testParseResponseWithNullResponseMessage() { GetReceiptsFromPeerTask task = new GetReceiptsFromPeerTask(Collections.emptyList(), null, () -> null); - Assertions.assertThrows(InvalidPeerTaskResponseException.class, () -> task.parseResponse(null)); + Assertions.assertThrows( + InvalidPeerTaskResponseException.class, () -> task.processResponse(null)); } @Test @@ -95,7 +96,7 @@ public void testParseResponseForInvalidResponse() { new TransactionReceipt(1, 101112, Collections.emptyList(), Optional.empty())))); Assertions.assertThrows( - InvalidPeerTaskResponseException.class, () -> task.parseResponse(receiptsMessage)); + InvalidPeerTaskResponseException.class, () -> task.processResponse(receiptsMessage)); } @Test @@ -127,7 +128,7 @@ public void testParseResponse() throws InvalidPeerTaskResponseException { Mockito.when(bodyValidator.receiptsRoot(List.of(receiptForBlock3))) .thenReturn(Hash.fromHexString(StringUtils.repeat("00", 31) + "32")); - Map> resultMap = task.parseResponse(receiptsMessage); + Map> resultMap = task.processResponse(receiptsMessage); Assertions.assertEquals(3, resultMap.size()); Assertions.assertEquals(List.of(receiptForBlock1), resultMap.get(blockHeader1)); From dc1f3bdf9fca6a672aeb685411f68d3c4187d59b Mon Sep 17 00:00:00 2001 From: "stefan.pingel@consensys.net" Date: Thu, 24 Oct 2024 15:13:13 +1000 Subject: [PATCH 100/106] add possibility to use the new peer task system when downloading the bodies Signed-off-by: stefan.pingel@consensys.net --- .../peertask/task/GetBodiesFromPeerTask.java | 154 +++++++++++ .../eth/sync/DefaultSynchronizer.java | 1 + .../ethereum/eth/sync/DownloadBodiesStep.java | 28 +- .../FastSyncDownloadPipelineFactory.java | 3 +- .../fullsync/FullSyncChainDownloader.java | 5 +- .../FullSyncDownloadPipelineFactory.java | 10 +- .../eth/sync/fullsync/FullSyncDownloader.java | 5 +- .../tasks/CompleteBlocksWithPeerTask.java | 133 ++++++++++ .../task/GetBodiesFromPeerTaskTest.java | 199 ++++++++++++++ .../FullSyncChainDownloaderForkTest.java | 5 +- .../fullsync/FullSyncChainDownloaderTest.java | 5 +- ...DownloaderTotalTerminalDifficultyTest.java | 5 +- .../sync/fullsync/FullSyncDownloaderTest.java | 1 + .../tasks/CompleteBlocksWithPeerTaskTest.java | 251 ++++++++++++++++++ 14 files changed, 795 insertions(+), 10 deletions(-) create mode 100644 ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetBodiesFromPeerTask.java create mode 100644 ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/tasks/CompleteBlocksWithPeerTask.java create mode 100644 ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetBodiesFromPeerTaskTest.java create mode 100644 ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/tasks/CompleteBlocksWithPeerTaskTest.java diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetBodiesFromPeerTask.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetBodiesFromPeerTask.java new file mode 100644 index 00000000000..3688cab107d --- /dev/null +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetBodiesFromPeerTask.java @@ -0,0 +1,154 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.manager.peertask.task; + +import org.hyperledger.besu.ethereum.core.Block; +import org.hyperledger.besu.ethereum.core.BlockBody; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.eth.EthProtocol; +import org.hyperledger.besu.ethereum.eth.manager.EthPeer; +import org.hyperledger.besu.ethereum.eth.manager.peertask.InvalidPeerTaskResponseException; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTask; +import org.hyperledger.besu.ethereum.eth.messages.BlockBodiesMessage; +import org.hyperledger.besu.ethereum.eth.messages.GetBlockBodiesMessage; +import org.hyperledger.besu.ethereum.mainnet.BodyValidation; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.SubProtocol; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Predicate; +import java.util.function.Supplier; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class GetBodiesFromPeerTask implements PeerTask> { + + private static final Logger LOG = LoggerFactory.getLogger(GetBodiesFromPeerTask.class); + + private final List blockHeaders; + private final Supplier currentProtocolSpecSupplier; + private final ProtocolSchedule protocolSchedule; + + private final long requiredBlockchainHeight; + private final List blocks = new ArrayList<>(); + + public GetBodiesFromPeerTask( + final List blockHeaders, + final Supplier currentProtocolSpecSupplier, + final ProtocolSchedule protocolSchedule) { + + this.blockHeaders = blockHeaders; + this.currentProtocolSpecSupplier = currentProtocolSpecSupplier; + this.protocolSchedule = protocolSchedule; + + this.requiredBlockchainHeight = + blockHeaders.stream() + .mapToLong(BlockHeader::getNumber) + .max() + .orElse(BlockHeader.GENESIS_BLOCK_NUMBER); + } + + @Override + public SubProtocol getSubProtocol() { + return EthProtocol.get(); + } + + @Override + public MessageData getRequestMessage() { + return GetBlockBodiesMessage.create( + blockHeaders.stream().map(BlockHeader::getBlockHash).toList()); + } + + @Override + public List parseResponse(final MessageData messageData) + throws InvalidPeerTaskResponseException { + // Blocks returned by this method are in the same order as the headers, but might not be + // complete + if (messageData == null) { + throw new InvalidPeerTaskResponseException(); + } + final BlockBodiesMessage blocksMessage = BlockBodiesMessage.readFrom(messageData); + final List blockBodies = blocksMessage.bodies(protocolSchedule); + if (blockBodies.isEmpty() || blockBodies.size() > blockHeaders.size()) { + throw new InvalidPeerTaskResponseException(); + } + + for (int i = 0; i < blockBodies.size(); i++) { + final BlockBody blockBody = blockBodies.get(i); + final BlockHeader blockHeader = blockHeaders.get(i); + if (!blockBodyMatchesBlockHeader(blockBody, blockHeader)) { + LOG.atDebug().setMessage("Received block body does not match block header").log(); + throw new InvalidPeerTaskResponseException(); + } + + blocks.add(new Block(blockHeader, blockBody)); + } + return blocks; + } + + private boolean blockBodyMatchesBlockHeader( + final BlockBody blockBody, final BlockHeader blockHeader) { + // this method validates that the block body matches the block header by calculating the roots + // of the block body + // and comparing them to the roots in the block header + if (!BodyValidation.transactionsRoot(blockBody.getTransactions()) + .equals(blockHeader.getTransactionsRoot())) { + return false; + } + if (!BodyValidation.ommersHash(blockBody.getOmmers()).equals(blockHeader.getOmmersHash())) { + return false; + } + if (blockBody.getWithdrawals().isPresent()) { + if (blockHeader.getWithdrawalsRoot().isEmpty()) { + return false; + } + if (!BodyValidation.withdrawalsRoot(blockBody.getWithdrawals().get()) + .equals(blockHeader.getWithdrawalsRoot().get())) { + return false; + } + } else if (blockHeader.getWithdrawalsRoot().isPresent()) { + return false; + } + if (blockBody.getRequests().isPresent()) { + if (blockHeader.getRequestsRoot().isEmpty()) { + return false; + } + if (!BodyValidation.requestsRoot(blockBody.getRequests().get()) + .equals(blockHeader.getRequestsRoot().get())) { + return false; + } + } else if (blockHeader.getRequestsRoot().isPresent()) { + return false; + } + return true; + } + + @Override + public Predicate getPeerRequirementFilter() { + return (ethPeer) -> + ethPeer.getProtocolName().equals(getSubProtocol().getName()) + && (currentProtocolSpecSupplier.get().isPoS() + || ethPeer.chainState().getEstimatedHeight() >= requiredBlockchainHeight); + } + + @Override + public boolean isSuccess(final List result) { + return !result.isEmpty(); + } +} diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java index bbaea664329..4fcd541f384 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java @@ -138,6 +138,7 @@ public DefaultSynchronizer( syncState, metricsSystem, terminationCondition, + peerTaskExecutor, syncDurationMetrics)); if (SyncMode.FAST.equals(syncConfig.getSyncMode())) { diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DownloadBodiesStep.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DownloadBodiesStep.java index 26fc7b544ce..3da93d87b62 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DownloadBodiesStep.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DownloadBodiesStep.java @@ -17,7 +17,9 @@ import org.hyperledger.besu.ethereum.core.Block; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.eth.manager.EthContext; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutor; import org.hyperledger.besu.ethereum.eth.sync.tasks.CompleteBlocksTask; +import org.hyperledger.besu.ethereum.eth.sync.tasks.CompleteBlocksWithPeerTask; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.plugin.services.MetricsSystem; @@ -31,19 +33,41 @@ public class DownloadBodiesStep private final ProtocolSchedule protocolSchedule; private final EthContext ethContext; private final MetricsSystem metricsSystem; + private final SynchronizerConfiguration synchronizerConfiguration; + private final PeerTaskExecutor peerTaskExecutor; public DownloadBodiesStep( final ProtocolSchedule protocolSchedule, final EthContext ethContext, + final PeerTaskExecutor peerTaskExecutor, + final SynchronizerConfiguration synchronizerConfiguration, final MetricsSystem metricsSystem) { this.protocolSchedule = protocolSchedule; this.ethContext = ethContext; + this.peerTaskExecutor = peerTaskExecutor; + this.synchronizerConfiguration = synchronizerConfiguration; this.metricsSystem = metricsSystem; } @Override public CompletableFuture> apply(final List blockHeaders) { - return CompleteBlocksTask.forHeaders(protocolSchedule, ethContext, blockHeaders, metricsSystem) - .run(); + if (synchronizerConfiguration.isPeerTaskSystemEnabled()) { + return ethContext + .getScheduler() + .scheduleServiceTask(() -> getBodiesWithPeerTaskSystem(blockHeaders)); + } else { + return CompleteBlocksTask.forHeaders( + protocolSchedule, ethContext, blockHeaders, metricsSystem) + .run(); + } + } + + private CompletableFuture> getBodiesWithPeerTaskSystem( + final List headers) { + + final CompleteBlocksWithPeerTask completeBlocksWithPeerTask = + new CompleteBlocksWithPeerTask(protocolSchedule, headers, peerTaskExecutor); + final List blocks = completeBlocksWithPeerTask.getBlocks(); + return CompletableFuture.completedFuture(blocks); } } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloadPipelineFactory.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloadPipelineFactory.java index b55b3720679..83bcd8238a0 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloadPipelineFactory.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloadPipelineFactory.java @@ -152,7 +152,8 @@ public Pipeline createDownloadPipelineForSyncTarget(final SyncT final RangeHeadersValidationStep validateHeadersJoinUpStep = new RangeHeadersValidationStep(protocolSchedule, protocolContext, detachedValidationPolicy); final DownloadBodiesStep downloadBodiesStep = - new DownloadBodiesStep(protocolSchedule, ethContext, metricsSystem); + new DownloadBodiesStep( + protocolSchedule, ethContext, peerTaskExecutor, syncConfig, metricsSystem); final DownloadReceiptsStep downloadReceiptsStep = new DownloadReceiptsStep( currentProtocolSpecSupplier, ethContext, peerTaskExecutor, syncConfig, metricsSystem); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncChainDownloader.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncChainDownloader.java index 3a0f6edb086..7faacedfb2a 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncChainDownloader.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncChainDownloader.java @@ -16,6 +16,7 @@ import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.eth.manager.EthContext; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutor; import org.hyperledger.besu.ethereum.eth.sync.ChainDownloader; import org.hyperledger.besu.ethereum.eth.sync.PipelineChainDownloader; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; @@ -35,7 +36,8 @@ public static ChainDownloader create( final SyncState syncState, final MetricsSystem metricsSystem, final SyncTerminationCondition terminationCondition, - final SyncDurationMetrics syncDurationMetrics) { + final SyncDurationMetrics syncDurationMetrics, + final PeerTaskExecutor peerTaskExecutor) { final FullSyncTargetManager syncTargetManager = new FullSyncTargetManager( @@ -54,6 +56,7 @@ public static ChainDownloader create( protocolSchedule, protocolContext, ethContext, + peerTaskExecutor, metricsSystem, terminationCondition), ethContext.getScheduler(), diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncDownloadPipelineFactory.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncDownloadPipelineFactory.java index b822236b8f0..bde94331d52 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncDownloadPipelineFactory.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncDownloadPipelineFactory.java @@ -19,6 +19,7 @@ import org.hyperledger.besu.ethereum.eth.manager.EthContext; import org.hyperledger.besu.ethereum.eth.manager.EthPeer; import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutor; import org.hyperledger.besu.ethereum.eth.sync.DownloadBodiesStep; import org.hyperledger.besu.ethereum.eth.sync.DownloadHeadersStep; import org.hyperledger.besu.ethereum.eth.sync.DownloadPipelineFactory; @@ -53,21 +54,25 @@ public class FullSyncDownloadPipelineFactory implements DownloadPipelineFactory () -> HeaderValidationMode.DETACHED_ONLY; private final BetterSyncTargetEvaluator betterSyncTargetEvaluator; private final SyncTerminationCondition fullSyncTerminationCondition; + private final PeerTaskExecutor peerTaskExecutor; public FullSyncDownloadPipelineFactory( final SynchronizerConfiguration syncConfig, final ProtocolSchedule protocolSchedule, final ProtocolContext protocolContext, final EthContext ethContext, + final PeerTaskExecutor peerTaskExecutor, final MetricsSystem metricsSystem, final SyncTerminationCondition syncTerminationCondition) { this.syncConfig = syncConfig; this.protocolSchedule = protocolSchedule; this.protocolContext = protocolContext; this.ethContext = ethContext; + this.peerTaskExecutor = peerTaskExecutor; this.metricsSystem = metricsSystem; this.fullSyncTerminationCondition = syncTerminationCondition; - betterSyncTargetEvaluator = new BetterSyncTargetEvaluator(syncConfig, ethContext.getEthPeers()); + this.betterSyncTargetEvaluator = + new BetterSyncTargetEvaluator(syncConfig, ethContext.getEthPeers()); } @Override @@ -104,7 +109,8 @@ public Pipeline createDownloadPipelineForSyncTarget(final SyncTarget target) final RangeHeadersValidationStep validateHeadersJoinUpStep = new RangeHeadersValidationStep(protocolSchedule, protocolContext, detachedValidationPolicy); final DownloadBodiesStep downloadBodiesStep = - new DownloadBodiesStep(protocolSchedule, ethContext, metricsSystem); + new DownloadBodiesStep( + protocolSchedule, ethContext, peerTaskExecutor, syncConfig, metricsSystem); final ExtractTxSignaturesStep extractTxSignaturesStep = new ExtractTxSignaturesStep(); final FullImportBlockStep importBlockStep = new FullImportBlockStep( diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncDownloader.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncDownloader.java index 8f1aca792c3..97bf38c95ec 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncDownloader.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncDownloader.java @@ -16,6 +16,7 @@ import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.eth.manager.EthContext; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutor; import org.hyperledger.besu.ethereum.eth.sync.ChainDownloader; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; import org.hyperledger.besu.ethereum.eth.sync.TrailingPeerRequirements; @@ -45,6 +46,7 @@ public FullSyncDownloader( final SyncState syncState, final MetricsSystem metricsSystem, final SyncTerminationCondition terminationCondition, + final PeerTaskExecutor peerTaskExecutor, final SyncDurationMetrics syncDurationMetrics) { this.syncConfig = syncConfig; this.protocolContext = protocolContext; @@ -59,7 +61,8 @@ public FullSyncDownloader( syncState, metricsSystem, terminationCondition, - syncDurationMetrics); + syncDurationMetrics, + peerTaskExecutor); } public CompletableFuture start() { diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/tasks/CompleteBlocksWithPeerTask.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/tasks/CompleteBlocksWithPeerTask.java new file mode 100644 index 00000000000..665cae8ab5a --- /dev/null +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/tasks/CompleteBlocksWithPeerTask.java @@ -0,0 +1,133 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.sync.tasks; + +import static com.google.common.base.Preconditions.checkArgument; + +import org.hyperledger.besu.ethereum.core.Block; +import org.hyperledger.besu.ethereum.core.BlockBody; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutor; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutorResponseCode; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutorResult; +import org.hyperledger.besu.ethereum.eth.manager.peertask.task.GetBodiesFromPeerTask; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Given a set of headers, "completes" them by repeatedly requesting additional data (bodies) needed + * to create the blocks that correspond to the supplied headers. + */ +public class CompleteBlocksWithPeerTask { + private static final Logger LOG = LoggerFactory.getLogger(CompleteBlocksWithPeerTask.class); + + private final ProtocolSchedule protocolSchedule; + private final List headersToGet = new ArrayList<>(); + private final PeerTaskExecutor peerTaskExecutor; + + private final Block[] result; + private final int resultSize; + private int nextIndex = 0; + private int remainingBlocks = 0; + + public CompleteBlocksWithPeerTask( + final ProtocolSchedule protocolSchedule, + final List headers, + final PeerTaskExecutor peerTaskExecutor) { + checkArgument(!headers.isEmpty(), "Must supply a non-empty headers list"); + this.protocolSchedule = protocolSchedule; + this.peerTaskExecutor = peerTaskExecutor; + + resultSize = headers.size(); + result = new Block[resultSize]; + remainingBlocks = resultSize; + + for (int i = 0; i < resultSize; i++) { + final BlockHeader header = headers.get(i); + if (BlockHeader.hasEmptyBlock(header)) { + final Block emptyBlock = + new Block(header, createEmptyBodyBasedOnProtocolSchedule(protocolSchedule, header)); + result[i] = emptyBlock; + remainingBlocks--; + } else { + headersToGet.add(header); + } + } + this.nextIndex = findNextIndex(0); + } + + private BlockBody createEmptyBodyBasedOnProtocolSchedule( + final ProtocolSchedule protocolSchedule, final BlockHeader header) { + return new BlockBody( + Collections.emptyList(), + Collections.emptyList(), + isWithdrawalsEnabled(protocolSchedule, header) + ? Optional.of(Collections.emptyList()) + : Optional.empty(), + Optional.empty()); + } + + private boolean isWithdrawalsEnabled( + final ProtocolSchedule protocolSchedule, final BlockHeader header) { + return protocolSchedule.getByBlockHeader(header).getWithdrawalsProcessor().isPresent(); + } + + public List getBlocks() { + while (remainingBlocks > 0) { + LOG.atDebug() + .setMessage("Requesting {} bodies from peer") + .addArgument(headersToGet.size()) + .log(); + final GetBodiesFromPeerTask task = + new GetBodiesFromPeerTask( + headersToGet, + () -> protocolSchedule.getByBlockHeader(headersToGet.getLast()), + protocolSchedule); + final PeerTaskExecutorResult> executionResult = peerTaskExecutor.execute(task); + if (executionResult.responseCode() == PeerTaskExecutorResponseCode.SUCCESS + && executionResult.result().isPresent()) { + final List blockList = executionResult.result().get(); + LOG.atDebug() + .setMessage("Received {} bodies out of {} from peer") + .addArgument(blockList.size()) + .addArgument(headersToGet.size()) + .log(); + blockList.forEach( + block -> { + remainingBlocks--; + result[nextIndex] = block; + nextIndex = findNextIndex(nextIndex + 1); + }); + } + } + return List.of(result); + } + + private int findNextIndex(final int startIndex) { + for (int i = startIndex; i < resultSize; i++) { + if (result[i] == null) { + return i; + } + } + return -1; // This only happens when we have finished processing all headers + } +} diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetBodiesFromPeerTaskTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetBodiesFromPeerTaskTest.java new file mode 100644 index 00000000000..463ea97908e --- /dev/null +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetBodiesFromPeerTaskTest.java @@ -0,0 +1,199 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.manager.peertask.task; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.core.Block; +import org.hyperledger.besu.ethereum.core.BlockBody; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.ethereum.core.encoding.EncodingContext; +import org.hyperledger.besu.ethereum.core.encoding.TransactionDecoder; +import org.hyperledger.besu.ethereum.eth.EthProtocol; +import org.hyperledger.besu.ethereum.eth.manager.ChainState; +import org.hyperledger.besu.ethereum.eth.manager.EthPeer; +import org.hyperledger.besu.ethereum.eth.manager.peertask.InvalidPeerTaskResponseException; +import org.hyperledger.besu.ethereum.eth.messages.BlockBodiesMessage; +import org.hyperledger.besu.ethereum.eth.messages.EthPV62; +import org.hyperledger.besu.ethereum.eth.messages.GetBlockBodiesMessage; +import org.hyperledger.besu.ethereum.mainnet.BodyValidation; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +import org.apache.commons.lang3.StringUtils; +import org.apache.tuweni.bytes.Bytes; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +public class GetBodiesFromPeerTaskTest { + + private static final String FRONTIER_TX_RLP = + "0xf901fc8032830138808080b901ae60056013565b6101918061001d6000396000f35b3360008190555056006001600060e060020a6000350480630a874df61461003a57806341c0e1b514610058578063a02b161e14610066578063dbbdf0831461007757005b610045600435610149565b80600160a060020a031660005260206000f35b610060610161565b60006000f35b6100716004356100d4565b60006000f35b61008560043560243561008b565b60006000f35b600054600160a060020a031632600160a060020a031614156100ac576100b1565b6100d0565b8060018360005260205260406000208190555081600060005260206000a15b5050565b600054600160a060020a031633600160a060020a031614158015610118575033600160a060020a0316600182600052602052604060002054600160a060020a031614155b61012157610126565b610146565b600060018260005260205260406000208190555080600060005260206000a15b50565b60006001826000526020526040600020549050919050565b600054600160a060020a031633600160a060020a0316146101815761018f565b600054600160a060020a0316ff5b561ca0c5689ed1ad124753d54576dfb4b571465a41900a1dff4058d8adf16f752013d0a01221cbd70ec28c94a3b55ec771bcbc70778d6ee0b51ca7ea9514594c861b1884"; + + private static final Transaction TX = + TransactionDecoder.decodeRLP( + new BytesValueRLPInput(Bytes.fromHexString(FRONTIER_TX_RLP), false), + EncodingContext.BLOCK_BODY); + public static final List TRANSACTION_LIST = List.of(TX); + public static final BlockBody BLOCK_BODY = + new BlockBody(TRANSACTION_LIST, Collections.emptyList(), Optional.empty(), Optional.empty()); + + @Test + public void testGetSubProtocol() { + GetBodiesFromPeerTask task = new GetBodiesFromPeerTask(Collections.emptyList(), null, null); + Assertions.assertEquals(EthProtocol.get(), task.getSubProtocol()); + } + + @Test + public void testGetRequestMessage() { + GetBodiesFromPeerTask task = + new GetBodiesFromPeerTask( + List.of(mockBlockHeader(1), mockBlockHeader(2), mockBlockHeader(3)), null, null); + + MessageData messageData = task.getRequestMessage(); + GetBlockBodiesMessage getBlockBodiesMessage = GetBlockBodiesMessage.readFrom(messageData); + + Assertions.assertEquals(EthPV62.GET_BLOCK_BODIES, getBlockBodiesMessage.getCode()); + Iterable hashesInMessage = getBlockBodiesMessage.hashes(); + List expectedHashes = + List.of( + Hash.fromHexString(StringUtils.repeat("00", 31) + "11"), + Hash.fromHexString(StringUtils.repeat("00", 31) + "21"), + Hash.fromHexString(StringUtils.repeat("00", 31) + "31")); + List actualHashes = new ArrayList<>(); + hashesInMessage.forEach(actualHashes::add); + + Assertions.assertEquals(3, actualHashes.size()); + Assertions.assertEquals( + expectedHashes.stream().sorted().toList(), actualHashes.stream().sorted().toList()); + } + + @Test + public void testParseResponseWithNullResponseMessage() { + GetBodiesFromPeerTask task = new GetBodiesFromPeerTask(Collections.emptyList(), null, null); + Assertions.assertThrows(InvalidPeerTaskResponseException.class, () -> task.parseResponse(null)); + } + + @Test + public void testParseResponseForInvalidResponse() { + GetBodiesFromPeerTask task = new GetBodiesFromPeerTask(List.of(mockBlockHeader(1)), null, null); + // body does not match header + BlockBodiesMessage bodiesMessage = BlockBodiesMessage.create(List.of(BLOCK_BODY)); + + Assertions.assertThrows( + InvalidPeerTaskResponseException.class, () -> task.parseResponse(bodiesMessage)); + } + + @Test + public void testParseResponse() throws InvalidPeerTaskResponseException { + final BlockHeader nonEmptyBlockHeaderMock = + getNonEmptyBlockHeaderMock(BodyValidation.transactionsRoot(TRANSACTION_LIST).toString()); + + GetBodiesFromPeerTask task = + new GetBodiesFromPeerTask(List.of(nonEmptyBlockHeaderMock), null, null); + + final BlockBodiesMessage blockBodiesMessage = BlockBodiesMessage.create(List.of(BLOCK_BODY)); + + List result = task.parseResponse(blockBodiesMessage); + + assertThat(result.size()).isEqualTo(1); + assertThat(result.getFirst().getBody().getTransactions()).isEqualTo(TRANSACTION_LIST); + } + + @Test + public void testGetPeerRequirementFilter() { + BlockHeader blockHeader1 = mockBlockHeader(1); + BlockHeader blockHeader2 = mockBlockHeader(2); + BlockHeader blockHeader3 = mockBlockHeader(3); + + ProtocolSpec protocolSpec = Mockito.mock(ProtocolSpec.class); + Mockito.when(protocolSpec.isPoS()).thenReturn(false); + + GetBodiesFromPeerTask task = + new GetBodiesFromPeerTask( + List.of(blockHeader1, blockHeader2, blockHeader3), () -> protocolSpec, null); + + EthPeer failForIncorrectProtocol = mockPeer("incorrectProtocol", 5); + EthPeer failForShortChainHeight = mockPeer("incorrectProtocol", 1); + EthPeer successfulCandidate = mockPeer(EthProtocol.NAME, 5); + + Assertions.assertFalse(task.getPeerRequirementFilter().test(failForIncorrectProtocol)); + Assertions.assertFalse(task.getPeerRequirementFilter().test(failForShortChainHeight)); + Assertions.assertTrue(task.getPeerRequirementFilter().test(successfulCandidate)); + } + + @Test + public void testIsSuccessForPartialSuccess() { + GetBodiesFromPeerTask task = new GetBodiesFromPeerTask(Collections.emptyList(), null, null); + + Assertions.assertFalse(task.isSuccess(Collections.emptyList())); + } + + @Test + public void testIsSuccessForFullSuccess() { + GetBodiesFromPeerTask task = new GetBodiesFromPeerTask(Collections.emptyList(), null, null); + + final List blockHeaders = List.of(mock(Block.class)); + + Assertions.assertTrue(task.isSuccess(blockHeaders)); + } + + private BlockHeader mockBlockHeader(final long blockNumber) { + BlockHeader blockHeader = Mockito.mock(BlockHeader.class); + Mockito.when(blockHeader.getNumber()).thenReturn(blockNumber); + // second to last hex digit indicates the blockNumber, last hex digit indicates the usage of the + // hash + Mockito.when(blockHeader.getHash()) + .thenReturn(Hash.fromHexString(StringUtils.repeat("00", 31) + blockNumber + "1")); + Mockito.when(blockHeader.getBlockHash()) + .thenReturn(Hash.fromHexString(StringUtils.repeat("00", 31) + blockNumber + "1")); + Mockito.when(blockHeader.getReceiptsRoot()) + .thenReturn(Hash.fromHexString(StringUtils.repeat("00", 31) + blockNumber + "2")); + + return blockHeader; + } + + private static BlockHeader getNonEmptyBlockHeaderMock(final String transactionsRootHexString) { + final BlockHeader blockHeader = mock(BlockHeader.class); + when(blockHeader.getTransactionsRoot()) + .thenReturn(Hash.fromHexStringLenient(transactionsRootHexString)); + when(blockHeader.getOmmersHash()).thenReturn(Hash.EMPTY_LIST_HASH); + when(blockHeader.getWithdrawalsRoot()).thenReturn(Optional.empty()); + when(blockHeader.getRequestsRoot()).thenReturn(Optional.empty()); + return blockHeader; + } + + private EthPeer mockPeer(final String protocol, final long chainHeight) { + EthPeer ethPeer = Mockito.mock(EthPeer.class); + ChainState chainState = Mockito.mock(ChainState.class); + + Mockito.when(ethPeer.getProtocolName()).thenReturn(protocol); + Mockito.when(ethPeer.chainState()).thenReturn(chainState); + Mockito.when(chainState.getEstimatedHeight()).thenReturn(chainHeight); + + return ethPeer; + } +} diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncChainDownloaderForkTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncChainDownloaderForkTest.java index d7b5970098e..c2252a117e4 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncChainDownloaderForkTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncChainDownloaderForkTest.java @@ -15,6 +15,7 @@ package org.hyperledger.besu.ethereum.eth.sync.fullsync; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.chain.Blockchain; @@ -27,6 +28,7 @@ import org.hyperledger.besu.ethereum.eth.manager.EthProtocolManagerTestUtil; import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; import org.hyperledger.besu.ethereum.eth.manager.RespondingEthPeer; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutor; import org.hyperledger.besu.ethereum.eth.sync.ChainDownloader; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; @@ -91,7 +93,8 @@ private ChainDownloader downloader(final SynchronizerConfiguration syncConfig) { syncState, metricsSystem, SyncTerminationCondition.never(), - SyncDurationMetrics.NO_OP_SYNC_DURATION_METRICS); + SyncDurationMetrics.NO_OP_SYNC_DURATION_METRICS, + mock(PeerTaskExecutor.class)); } private ChainDownloader downloader() { diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncChainDownloaderTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncChainDownloaderTest.java index ac7f0fb8257..17ce3db15a1 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncChainDownloaderTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncChainDownloaderTest.java @@ -17,6 +17,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; import static org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider.createInMemoryBlockchain; +import static org.mockito.Mockito.mock; import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.chain.Blockchain; @@ -34,6 +35,7 @@ import org.hyperledger.besu.ethereum.eth.manager.EthProtocolManagerTestUtil; import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; import org.hyperledger.besu.ethereum.eth.manager.RespondingEthPeer; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutor; import org.hyperledger.besu.ethereum.eth.messages.EthPV62; import org.hyperledger.besu.ethereum.eth.messages.GetBlockHeadersMessage; import org.hyperledger.besu.ethereum.eth.sync.ChainDownloader; @@ -123,7 +125,8 @@ private ChainDownloader downloader(final SynchronizerConfiguration syncConfig) { syncState, metricsSystem, SyncTerminationCondition.never(), - SyncDurationMetrics.NO_OP_SYNC_DURATION_METRICS); + SyncDurationMetrics.NO_OP_SYNC_DURATION_METRICS, + mock(PeerTaskExecutor.class)); } private ChainDownloader downloader() { diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncChainDownloaderTotalTerminalDifficultyTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncChainDownloaderTotalTerminalDifficultyTest.java index 311ccf5de30..ddcdca4dd95 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncChainDownloaderTotalTerminalDifficultyTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncChainDownloaderTotalTerminalDifficultyTest.java @@ -15,6 +15,7 @@ package org.hyperledger.besu.ethereum.eth.sync.fullsync; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.chain.Blockchain; @@ -27,6 +28,7 @@ import org.hyperledger.besu.ethereum.eth.manager.EthProtocolManagerTestUtil; import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; import org.hyperledger.besu.ethereum.eth.manager.RespondingEthPeer; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutor; import org.hyperledger.besu.ethereum.eth.sync.ChainDownloader; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; @@ -109,7 +111,8 @@ private ChainDownloader downloader( syncState, metricsSystem, terminalCondition, - SyncDurationMetrics.NO_OP_SYNC_DURATION_METRICS); + SyncDurationMetrics.NO_OP_SYNC_DURATION_METRICS, + mock(PeerTaskExecutor.class)); } private SynchronizerConfiguration.Builder syncConfigBuilder() { diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncDownloaderTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncDownloaderTest.java index 63e41f6d25c..c270d3fae69 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncDownloaderTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncDownloaderTest.java @@ -98,6 +98,7 @@ private FullSyncDownloader downloader(final SynchronizerConfiguration syncConfig syncState, metricsSystem, SyncTerminationCondition.never(), + null, SyncDurationMetrics.NO_OP_SYNC_DURATION_METRICS); } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/tasks/CompleteBlocksWithPeerTaskTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/tasks/CompleteBlocksWithPeerTaskTest.java new file mode 100644 index 00000000000..ed6a451df28 --- /dev/null +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/tasks/CompleteBlocksWithPeerTaskTest.java @@ -0,0 +1,251 @@ +/* + * Copyright contributors to Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.sync.tasks; + +import static java.util.Arrays.asList; +import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; +import static org.assertj.core.api.AssertionsForInterfaceTypes.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.core.Block; +import org.hyperledger.besu.ethereum.core.BlockBody; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutor; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutorResponseCode; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutorResult; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; +import org.hyperledger.besu.ethereum.mainnet.WithdrawalsProcessor; + +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +public class CompleteBlocksWithPeerTaskTest { + + @BeforeAll + public static void setUp() {} + + @Test + public void shouldFailWhenEmptyHeaders() { + assertThatThrownBy(() -> new CompleteBlocksWithPeerTask(null, Collections.emptyList(), null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Must supply a non-empty headers list"); + } + + @Test + public void shouldReturnEmptyBlock() { + final ProtocolSchedule protocolSchedule = getProtocolScheduleMock(); + final BlockHeader blockHeader = getEmptyBlockHeaderMock(); + final PeerTaskExecutor peerTaskExecutor = mock(PeerTaskExecutor.class); + + CompleteBlocksWithPeerTask completeBlocksWithPeerTask = + new CompleteBlocksWithPeerTask(protocolSchedule, List.of(blockHeader), peerTaskExecutor); + assertThat(completeBlocksWithPeerTask.getBlocks()).isNotEmpty(); + assertThat(completeBlocksWithPeerTask.getBlocks().size()).isEqualTo(1); + assertThat(BlockHeader.hasEmptyBlock(completeBlocksWithPeerTask.getBlocks().get(0).getHeader())) + .isTrue(); + + verify(peerTaskExecutor, Mockito.never()).execute(any()); + } + + @Test + public void shouldCreateWithdrawalsAwareEmptyBlock_whenWithdrawalsAreEnabled() { + final ProtocolSchedule mockProtocolSchedule = Mockito.mock(ProtocolSchedule.class); + final ProtocolSpec mockParisSpec = Mockito.mock(ProtocolSpec.class); + final ProtocolSpec mockShanghaiSpec = Mockito.mock(ProtocolSpec.class); + final WithdrawalsProcessor mockWithdrawalsProcessor = Mockito.mock(WithdrawalsProcessor.class); + + final BlockHeader header1 = + new BlockHeaderTestFixture().number(1).withdrawalsRoot(null).buildHeader(); + final BlockHeader header2 = + new BlockHeaderTestFixture().number(2).withdrawalsRoot(Hash.EMPTY_TRIE_HASH).buildHeader(); + + when(mockProtocolSchedule.getByBlockHeader((eq(header1)))).thenReturn(mockParisSpec); + when(mockParisSpec.getWithdrawalsProcessor()).thenReturn(Optional.empty()); + when(mockProtocolSchedule.getByBlockHeader((eq(header2)))).thenReturn(mockShanghaiSpec); + when(mockShanghaiSpec.getWithdrawalsProcessor()) + .thenReturn(Optional.of(mockWithdrawalsProcessor)); + + final List expectedBlocks = getExpectedBlocks(header1, header2); + + final PeerTaskExecutor peerTaskExecutor = mock(PeerTaskExecutor.class); + when(peerTaskExecutor.execute(any())) + .thenReturn( + new PeerTaskExecutorResult<>( + Optional.of(expectedBlocks), PeerTaskExecutorResponseCode.SUCCESS)); + + final CompleteBlocksWithPeerTask task = + new CompleteBlocksWithPeerTask( + mockProtocolSchedule, asList(header1, header2), peerTaskExecutor); + final List blocks = task.getBlocks(); + + assertThat(blocks).isEqualTo(expectedBlocks); + } + + @Test + public void shouldReturnNonEmptyBlock() { + final Block block = mock(Block.class); + final ProtocolSchedule protocolSchedule = getProtocolScheduleMock(); + final PeerTaskExecutor peerTaskExecutor = mock(PeerTaskExecutor.class); + final BlockHeader nonEmptyBlockHeaderMock = getNonEmptyBlockHeaderMock("0x01", "0x02"); + when(peerTaskExecutor.execute(any())) + .thenReturn( + new PeerTaskExecutorResult<>( + Optional.of(List.of(block)), PeerTaskExecutorResponseCode.SUCCESS)); + + CompleteBlocksWithPeerTask completeBlocksWithPeerTask = + new CompleteBlocksWithPeerTask( + protocolSchedule, List.of(nonEmptyBlockHeaderMock), peerTaskExecutor); + + assertThat(completeBlocksWithPeerTask.getBlocks()).isNotEmpty(); + assertThat(completeBlocksWithPeerTask.getBlocks().size()).isEqualTo(1); + assertThat(completeBlocksWithPeerTask.getBlocks().get(0)).isEqualTo(block); + } + + @Test + public void shouldReturnBlocksInRightOrderWhenEmptyAndNonEmptyBlocksRequested() { + final Block block1 = mock(Block.class); + final Block block3 = mock(Block.class); + final BlockHeader emptyBlockHeaderMock = getEmptyBlockHeaderMock(); + final BlockHeader nonEmptyBlockHeaderMock1 = getNonEmptyBlockHeaderMock("0x01", "0x02"); + final BlockHeader nonEmptyBlockHeaderMock3 = getNonEmptyBlockHeaderMock("0x03", "0x04"); + + final ProtocolSchedule protocolSchedule = getProtocolScheduleMock(); + final PeerTaskExecutor peerTaskExecutor = mock(PeerTaskExecutor.class); + when(peerTaskExecutor.execute(any())) + .thenReturn( + new PeerTaskExecutorResult<>( + Optional.of(List.of(block1, block3)), PeerTaskExecutorResponseCode.SUCCESS)); + + CompleteBlocksWithPeerTask completeBlocksWithPeerTask = + new CompleteBlocksWithPeerTask( + protocolSchedule, + List.of( + nonEmptyBlockHeaderMock1, + emptyBlockHeaderMock, + nonEmptyBlockHeaderMock3, + emptyBlockHeaderMock), + peerTaskExecutor); + + assertThat(completeBlocksWithPeerTask.getBlocks()).isNotEmpty(); + assertThat(completeBlocksWithPeerTask.getBlocks().size()).isEqualTo(4); + assertThat(completeBlocksWithPeerTask.getBlocks().get(0)).isEqualTo(block1); + assertThat(BlockHeader.hasEmptyBlock(completeBlocksWithPeerTask.getBlocks().get(1).getHeader())) + .isTrue(); + assertThat(completeBlocksWithPeerTask.getBlocks().get(2)).isEqualTo(block3); + assertThat(BlockHeader.hasEmptyBlock(completeBlocksWithPeerTask.getBlocks().get(3).getHeader())) + .isTrue(); + } + + @Test + public void shouldRequestMoreBodiesUntilFinished() { + final Block block1 = mock(Block.class); + final Block block3 = mock(Block.class); + final BlockHeader emptyBlockHeaderMock = getEmptyBlockHeaderMock(); + final BlockHeader nonEmptyBlockHeaderMock1 = getNonEmptyBlockHeaderMock("0x01", "0x02"); + final BlockHeader nonEmptyBlockHeaderMock3 = getNonEmptyBlockHeaderMock("0x03", "0x04"); + + final ProtocolSchedule protocolSchedule = getProtocolScheduleMock(); + final PeerTaskExecutor peerTaskExecutor = mock(PeerTaskExecutor.class); + when(peerTaskExecutor.execute(any())) + .thenReturn( + new PeerTaskExecutorResult<>( + Optional.of(List.of(block1)), PeerTaskExecutorResponseCode.SUCCESS)) + .thenReturn( + new PeerTaskExecutorResult<>( + Optional.of(List.of(block3)), PeerTaskExecutorResponseCode.SUCCESS)); + + CompleteBlocksWithPeerTask completeBlocksWithPeerTask = + new CompleteBlocksWithPeerTask( + protocolSchedule, + List.of( + nonEmptyBlockHeaderMock1, + emptyBlockHeaderMock, + nonEmptyBlockHeaderMock3, + emptyBlockHeaderMock), + peerTaskExecutor); + + assertThat(completeBlocksWithPeerTask.getBlocks()).isNotEmpty(); + assertThat(completeBlocksWithPeerTask.getBlocks().size()).isEqualTo(4); + assertThat(completeBlocksWithPeerTask.getBlocks().get(0)).isEqualTo(block1); + assertThat(BlockHeader.hasEmptyBlock(completeBlocksWithPeerTask.getBlocks().get(1).getHeader())) + .isTrue(); + assertThat(completeBlocksWithPeerTask.getBlocks().get(2)).isEqualTo(block3); + assertThat(BlockHeader.hasEmptyBlock(completeBlocksWithPeerTask.getBlocks().get(3).getHeader())) + .isTrue(); + } + + private static ProtocolSchedule getProtocolScheduleMock() { + final ProtocolSchedule protocolSchedule = mock(ProtocolSchedule.class); + final ProtocolSpec protocolSpec = mock(ProtocolSpec.class); + final Optional optional = Optional.of(mock(WithdrawalsProcessor.class)); + when(protocolSpec.getWithdrawalsProcessor()).thenReturn(optional); + when(protocolSchedule.getByBlockHeader(any())).thenReturn(protocolSpec); + return protocolSchedule; + } + + private static BlockHeader getEmptyBlockHeaderMock() { + final BlockHeader blockHeader = mock(BlockHeader.class); + when(blockHeader.getTransactionsRoot()).thenReturn(Hash.EMPTY_TRIE_HASH); + when(blockHeader.getOmmersHash()).thenReturn(Hash.EMPTY_LIST_HASH); + when(blockHeader.getWithdrawalsRoot()).thenReturn(Optional.empty()); + when(blockHeader.getRequestsRoot()).thenReturn(Optional.empty()); + return blockHeader; + } + + private static BlockHeader getNonEmptyBlockHeaderMock( + final String transactionsRootHexString, final String ommersHash) { + final BlockHeader blockHeader = mock(BlockHeader.class); + when(blockHeader.getTransactionsRoot()) + .thenReturn(Hash.fromHexStringLenient(transactionsRootHexString)); + when(blockHeader.getOmmersHash()).thenReturn(Hash.fromHexStringLenient(ommersHash)); + when(blockHeader.getWithdrawalsRoot()).thenReturn(Optional.empty()); + when(blockHeader.getRequestsRoot()).thenReturn(Optional.empty()); + return blockHeader; + } + + private static List getExpectedBlocks( + final BlockHeader header1, final BlockHeader header2) { + final Block block1 = + new Block( + header1, + new BlockBody( + Collections.emptyList(), + Collections.emptyList(), + Optional.empty(), + Optional.empty())); + final Block block2 = + new Block( + header2, + new BlockBody( + Collections.emptyList(), + Collections.emptyList(), + Optional.of(Collections.emptyList()), + Optional.empty())); + + return asList(block1, block2); + } +} From 845b56445aa4d483b06dc4b7f570f1b4733bf0f6 Mon Sep 17 00:00:00 2001 From: "stefan.pingel@consensys.net" Date: Thu, 24 Oct 2024 17:03:40 +1000 Subject: [PATCH 101/106] fix loop Signed-off-by: stefan.pingel@consensys.net --- .../besu/ethereum/eth/sync/tasks/CompleteBlocksWithPeerTask.java | 1 + 1 file changed, 1 insertion(+) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/tasks/CompleteBlocksWithPeerTask.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/tasks/CompleteBlocksWithPeerTask.java index 665cae8ab5a..dfc3405b8da 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/tasks/CompleteBlocksWithPeerTask.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/tasks/CompleteBlocksWithPeerTask.java @@ -115,6 +115,7 @@ public List getBlocks() { block -> { remainingBlocks--; result[nextIndex] = block; + headersToGet.removeFirst(); nextIndex = findNextIndex(nextIndex + 1); }); } From 1b9922f8b8c75e5054fd68efbf6bb8c9c7a945e9 Mon Sep 17 00:00:00 2001 From: Matilda Clerke Date: Fri, 25 Oct 2024 09:17:33 +1100 Subject: [PATCH 102/106] 7311: Wrap peer task system usage in ethScheduler call to match other usages Signed-off-by: Matilda Clerke --- .../CheckpointDownloadBlockStep.java | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointDownloadBlockStep.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointDownloadBlockStep.java index 70e9958beac..bf32358d83b 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointDownloadBlockStep.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointDownloadBlockStep.java @@ -84,31 +84,31 @@ private CompletableFuture> downloadReceipts( final PeerTaskResult peerTaskResult) { final Block block = peerTaskResult.getResult(); if (synchronizerConfiguration.isPeerTaskSystemEnabled()) { - CompletableFuture> futureReceipts = new CompletableFuture<>(); - GetReceiptsFromPeerTask task = - new GetReceiptsFromPeerTask( - List.of(block.getHeader()), new BodyValidator(), currentProtocolSpecSupplier); - PeerTaskExecutorResult>> executorResult = - peerTaskExecutor.execute(task); + return ethContext.getScheduler().scheduleServiceTask(() -> { + GetReceiptsFromPeerTask task = + new GetReceiptsFromPeerTask( + List.of(block.getHeader()), new BodyValidator(), currentProtocolSpecSupplier); + PeerTaskExecutorResult>> executorResult = + peerTaskExecutor.execute(task); - if (executorResult.responseCode() == PeerTaskExecutorResponseCode.SUCCESS) { - List transactionReceipts = - executorResult - .result() - .map((map) -> map.get(block.getHeader())) - .orElseThrow( - () -> - new IllegalStateException("PeerTask response code was success, but empty")); - if (block.getBody().getTransactions().size() != transactionReceipts.size()) { - throw new IllegalStateException( - "PeerTask response code was success, but incorrect number of receipts returned"); + if (executorResult.responseCode() == PeerTaskExecutorResponseCode.SUCCESS) { + List transactionReceipts = + executorResult + .result() + .map((map) -> map.get(block.getHeader())) + .orElseThrow( + () -> + new IllegalStateException("PeerTask response code was success, but empty")); + if (block.getBody().getTransactions().size() != transactionReceipts.size()) { + throw new IllegalStateException( + "PeerTask response code was success, but incorrect number of receipts returned"); + } + BlockWithReceipts blockWithReceipts = new BlockWithReceipts(block, transactionReceipts); + return CompletableFuture.completedFuture(Optional.of(blockWithReceipts)); + } else { + return CompletableFuture.completedFuture(Optional.empty()); } - BlockWithReceipts blockWithReceipts = new BlockWithReceipts(block, transactionReceipts); - futureReceipts.complete(Optional.of(blockWithReceipts)); - } else { - futureReceipts.complete(Optional.empty()); - } - return futureReceipts; + }); } else { final org.hyperledger.besu.ethereum.eth.manager.task.GetReceiptsFromPeerTask From 023445ff6b4b601e2770ba4ffa8814a2914b8a09 Mon Sep 17 00:00:00 2001 From: "stefan.pingel@consensys.net" Date: Fri, 25 Oct 2024 12:51:51 +1000 Subject: [PATCH 103/106] small fixes Signed-off-by: stefan.pingel@consensys.net --- .../peertask/task/GetBodiesFromPeerTask.java | 20 ++++----- .../tasks/CompleteBlocksWithPeerTask.java | 8 +--- .../task/GetBodiesFromPeerTaskTest.java | 42 ++++++++++------- .../tasks/CompleteBlocksWithPeerTaskTest.java | 45 +++++++++---------- 4 files changed, 58 insertions(+), 57 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetBodiesFromPeerTask.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetBodiesFromPeerTask.java index 3688cab107d..a73a4520445 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetBodiesFromPeerTask.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetBodiesFromPeerTask.java @@ -25,14 +25,12 @@ import org.hyperledger.besu.ethereum.eth.messages.GetBlockBodiesMessage; import org.hyperledger.besu.ethereum.mainnet.BodyValidation; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; -import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.SubProtocol; import java.util.ArrayList; import java.util.List; import java.util.function.Predicate; -import java.util.function.Supplier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,19 +40,19 @@ public class GetBodiesFromPeerTask implements PeerTask> { private static final Logger LOG = LoggerFactory.getLogger(GetBodiesFromPeerTask.class); private final List blockHeaders; - private final Supplier currentProtocolSpecSupplier; private final ProtocolSchedule protocolSchedule; private final long requiredBlockchainHeight; private final List blocks = new ArrayList<>(); + private final boolean isPoS; public GetBodiesFromPeerTask( - final List blockHeaders, - final Supplier currentProtocolSpecSupplier, - final ProtocolSchedule protocolSchedule) { + final List blockHeaders, final ProtocolSchedule protocolSchedule) { + if (blockHeaders == null || blockHeaders.isEmpty()) { + throw new IllegalArgumentException("Block headers must not be empty"); + } this.blockHeaders = blockHeaders; - this.currentProtocolSpecSupplier = currentProtocolSpecSupplier; this.protocolSchedule = protocolSchedule; this.requiredBlockchainHeight = @@ -62,6 +60,7 @@ public GetBodiesFromPeerTask( .mapToLong(BlockHeader::getNumber) .max() .orElse(BlockHeader.GENESIS_BLOCK_NUMBER); + this.isPoS = protocolSchedule.getByBlockHeader(blockHeaders.getLast()).isPoS(); } @Override @@ -105,8 +104,7 @@ public List parseResponse(final MessageData messageData) private boolean blockBodyMatchesBlockHeader( final BlockBody blockBody, final BlockHeader blockHeader) { // this method validates that the block body matches the block header by calculating the roots - // of the block body - // and comparing them to the roots in the block header + // of the block body and comparing them to the roots in the block header if (!BodyValidation.transactionsRoot(blockBody.getTransactions()) .equals(blockHeader.getTransactionsRoot())) { return false; @@ -142,9 +140,7 @@ private boolean blockBodyMatchesBlockHeader( @Override public Predicate getPeerRequirementFilter() { return (ethPeer) -> - ethPeer.getProtocolName().equals(getSubProtocol().getName()) - && (currentProtocolSpecSupplier.get().isPoS() - || ethPeer.chainState().getEstimatedHeight() >= requiredBlockchainHeight); + isPoS || ethPeer.chainState().getEstimatedHeight() >= requiredBlockchainHeight; } @Override diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/tasks/CompleteBlocksWithPeerTask.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/tasks/CompleteBlocksWithPeerTask.java index dfc3405b8da..6c86ee7af78 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/tasks/CompleteBlocksWithPeerTask.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/tasks/CompleteBlocksWithPeerTask.java @@ -47,7 +47,7 @@ public class CompleteBlocksWithPeerTask { private final Block[] result; private final int resultSize; private int nextIndex = 0; - private int remainingBlocks = 0; + private int remainingBlocks; public CompleteBlocksWithPeerTask( final ProtocolSchedule protocolSchedule, @@ -97,11 +97,7 @@ public List getBlocks() { .setMessage("Requesting {} bodies from peer") .addArgument(headersToGet.size()) .log(); - final GetBodiesFromPeerTask task = - new GetBodiesFromPeerTask( - headersToGet, - () -> protocolSchedule.getByBlockHeader(headersToGet.getLast()), - protocolSchedule); + final GetBodiesFromPeerTask task = new GetBodiesFromPeerTask(headersToGet, protocolSchedule); final PeerTaskExecutorResult> executionResult = peerTaskExecutor.execute(task); if (executionResult.responseCode() == PeerTaskExecutorResponseCode.SUCCESS && executionResult.result().isPresent()) { diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetBodiesFromPeerTaskTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetBodiesFromPeerTaskTest.java index 463ea97908e..21b9ac59772 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetBodiesFromPeerTaskTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetBodiesFromPeerTaskTest.java @@ -33,6 +33,7 @@ import org.hyperledger.besu.ethereum.eth.messages.EthPV62; import org.hyperledger.besu.ethereum.eth.messages.GetBlockBodiesMessage; import org.hyperledger.besu.ethereum.mainnet.BodyValidation; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput; @@ -45,6 +46,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.tuweni.bytes.Bytes; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.mockito.Mockito; @@ -60,10 +62,21 @@ public class GetBodiesFromPeerTaskTest { public static final List TRANSACTION_LIST = List.of(TX); public static final BlockBody BLOCK_BODY = new BlockBody(TRANSACTION_LIST, Collections.emptyList(), Optional.empty(), Optional.empty()); + private static ProtocolSchedule protocolSchedule; + + @BeforeAll + public static void setup() { + protocolSchedule = mock(ProtocolSchedule.class); + final ProtocolSpec protocolSpec = mock(ProtocolSpec.class); + when(protocolSpec.isPoS()).thenReturn(true); + when(protocolSchedule.getByBlockHeader(Mockito.any())).thenReturn(protocolSpec); + } @Test public void testGetSubProtocol() { - GetBodiesFromPeerTask task = new GetBodiesFromPeerTask(Collections.emptyList(), null, null); + + GetBodiesFromPeerTask task = + new GetBodiesFromPeerTask(List.of(mockBlockHeader(0)), protocolSchedule); Assertions.assertEquals(EthProtocol.get(), task.getSubProtocol()); } @@ -71,7 +84,7 @@ public void testGetSubProtocol() { public void testGetRequestMessage() { GetBodiesFromPeerTask task = new GetBodiesFromPeerTask( - List.of(mockBlockHeader(1), mockBlockHeader(2), mockBlockHeader(3)), null, null); + List.of(mockBlockHeader(1), mockBlockHeader(2), mockBlockHeader(3)), protocolSchedule); MessageData messageData = task.getRequestMessage(); GetBlockBodiesMessage getBlockBodiesMessage = GetBlockBodiesMessage.readFrom(messageData); @@ -93,13 +106,15 @@ public void testGetRequestMessage() { @Test public void testParseResponseWithNullResponseMessage() { - GetBodiesFromPeerTask task = new GetBodiesFromPeerTask(Collections.emptyList(), null, null); - Assertions.assertThrows(InvalidPeerTaskResponseException.class, () -> task.parseResponse(null)); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> new GetBodiesFromPeerTask(Collections.emptyList(), protocolSchedule)); } @Test public void testParseResponseForInvalidResponse() { - GetBodiesFromPeerTask task = new GetBodiesFromPeerTask(List.of(mockBlockHeader(1)), null, null); + GetBodiesFromPeerTask task = + new GetBodiesFromPeerTask(List.of(mockBlockHeader(1)), protocolSchedule); // body does not match header BlockBodiesMessage bodiesMessage = BlockBodiesMessage.create(List.of(BLOCK_BODY)); @@ -113,7 +128,7 @@ public void testParseResponse() throws InvalidPeerTaskResponseException { getNonEmptyBlockHeaderMock(BodyValidation.transactionsRoot(TRANSACTION_LIST).toString()); GetBodiesFromPeerTask task = - new GetBodiesFromPeerTask(List.of(nonEmptyBlockHeaderMock), null, null); + new GetBodiesFromPeerTask(List.of(nonEmptyBlockHeaderMock), protocolSchedule); final BlockBodiesMessage blockBodiesMessage = BlockBodiesMessage.create(List.of(BLOCK_BODY)); @@ -129,32 +144,27 @@ public void testGetPeerRequirementFilter() { BlockHeader blockHeader2 = mockBlockHeader(2); BlockHeader blockHeader3 = mockBlockHeader(3); - ProtocolSpec protocolSpec = Mockito.mock(ProtocolSpec.class); - Mockito.when(protocolSpec.isPoS()).thenReturn(false); - GetBodiesFromPeerTask task = new GetBodiesFromPeerTask( - List.of(blockHeader1, blockHeader2, blockHeader3), () -> protocolSpec, null); + List.of(blockHeader1, blockHeader2, blockHeader3), protocolSchedule); - EthPeer failForIncorrectProtocol = mockPeer("incorrectProtocol", 5); - EthPeer failForShortChainHeight = mockPeer("incorrectProtocol", 1); EthPeer successfulCandidate = mockPeer(EthProtocol.NAME, 5); - Assertions.assertFalse(task.getPeerRequirementFilter().test(failForIncorrectProtocol)); - Assertions.assertFalse(task.getPeerRequirementFilter().test(failForShortChainHeight)); Assertions.assertTrue(task.getPeerRequirementFilter().test(successfulCandidate)); } @Test public void testIsSuccessForPartialSuccess() { - GetBodiesFromPeerTask task = new GetBodiesFromPeerTask(Collections.emptyList(), null, null); + GetBodiesFromPeerTask task = + new GetBodiesFromPeerTask(List.of(mockBlockHeader(1)), protocolSchedule); Assertions.assertFalse(task.isSuccess(Collections.emptyList())); } @Test public void testIsSuccessForFullSuccess() { - GetBodiesFromPeerTask task = new GetBodiesFromPeerTask(Collections.emptyList(), null, null); + GetBodiesFromPeerTask task = + new GetBodiesFromPeerTask(List.of(mockBlockHeader(1)), protocolSchedule); final List blockHeaders = List.of(mock(Block.class)); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/tasks/CompleteBlocksWithPeerTaskTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/tasks/CompleteBlocksWithPeerTaskTest.java index ed6a451df28..d94dd7615b3 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/tasks/CompleteBlocksWithPeerTaskTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/tasks/CompleteBlocksWithPeerTaskTest.java @@ -63,10 +63,10 @@ public void shouldReturnEmptyBlock() { CompleteBlocksWithPeerTask completeBlocksWithPeerTask = new CompleteBlocksWithPeerTask(protocolSchedule, List.of(blockHeader), peerTaskExecutor); - assertThat(completeBlocksWithPeerTask.getBlocks()).isNotEmpty(); - assertThat(completeBlocksWithPeerTask.getBlocks().size()).isEqualTo(1); - assertThat(BlockHeader.hasEmptyBlock(completeBlocksWithPeerTask.getBlocks().get(0).getHeader())) - .isTrue(); + final List blocks = completeBlocksWithPeerTask.getBlocks(); + assertThat(blocks).isNotEmpty(); + assertThat(blocks.size()).isEqualTo(1); + assertThat(BlockHeader.hasEmptyBlock(blocks.get(0).getHeader())).isTrue(); verify(peerTaskExecutor, Mockito.never()).execute(any()); } @@ -120,9 +120,10 @@ public void shouldReturnNonEmptyBlock() { new CompleteBlocksWithPeerTask( protocolSchedule, List.of(nonEmptyBlockHeaderMock), peerTaskExecutor); - assertThat(completeBlocksWithPeerTask.getBlocks()).isNotEmpty(); - assertThat(completeBlocksWithPeerTask.getBlocks().size()).isEqualTo(1); - assertThat(completeBlocksWithPeerTask.getBlocks().get(0)).isEqualTo(block); + final List blocks = completeBlocksWithPeerTask.getBlocks(); + assertThat(blocks).isNotEmpty(); + assertThat(blocks.size()).isEqualTo(1); + assertThat(blocks.get(0)).isEqualTo(block); } @Test @@ -150,14 +151,13 @@ public void shouldReturnBlocksInRightOrderWhenEmptyAndNonEmptyBlocksRequested() emptyBlockHeaderMock), peerTaskExecutor); - assertThat(completeBlocksWithPeerTask.getBlocks()).isNotEmpty(); - assertThat(completeBlocksWithPeerTask.getBlocks().size()).isEqualTo(4); - assertThat(completeBlocksWithPeerTask.getBlocks().get(0)).isEqualTo(block1); - assertThat(BlockHeader.hasEmptyBlock(completeBlocksWithPeerTask.getBlocks().get(1).getHeader())) - .isTrue(); - assertThat(completeBlocksWithPeerTask.getBlocks().get(2)).isEqualTo(block3); - assertThat(BlockHeader.hasEmptyBlock(completeBlocksWithPeerTask.getBlocks().get(3).getHeader())) - .isTrue(); + final List blocks = completeBlocksWithPeerTask.getBlocks(); + assertThat(blocks).isNotEmpty(); + assertThat(blocks.size()).isEqualTo(4); + assertThat(blocks.get(0)).isEqualTo(block1); + assertThat(BlockHeader.hasEmptyBlock(blocks.get(1).getHeader())).isTrue(); + assertThat(blocks.get(2)).isEqualTo(block3); + assertThat(BlockHeader.hasEmptyBlock(blocks.get(3).getHeader())).isTrue(); } @Test @@ -188,14 +188,13 @@ public void shouldRequestMoreBodiesUntilFinished() { emptyBlockHeaderMock), peerTaskExecutor); - assertThat(completeBlocksWithPeerTask.getBlocks()).isNotEmpty(); - assertThat(completeBlocksWithPeerTask.getBlocks().size()).isEqualTo(4); - assertThat(completeBlocksWithPeerTask.getBlocks().get(0)).isEqualTo(block1); - assertThat(BlockHeader.hasEmptyBlock(completeBlocksWithPeerTask.getBlocks().get(1).getHeader())) - .isTrue(); - assertThat(completeBlocksWithPeerTask.getBlocks().get(2)).isEqualTo(block3); - assertThat(BlockHeader.hasEmptyBlock(completeBlocksWithPeerTask.getBlocks().get(3).getHeader())) - .isTrue(); + final List blocks = completeBlocksWithPeerTask.getBlocks(); + assertThat(blocks).isNotEmpty(); + assertThat(blocks.size()).isEqualTo(4); + assertThat(blocks.get(0)).isEqualTo(block1); + assertThat(BlockHeader.hasEmptyBlock(blocks.get(1).getHeader())).isTrue(); + assertThat(blocks.get(2)).isEqualTo(block3); + assertThat(BlockHeader.hasEmptyBlock(blocks.get(3).getHeader())).isTrue(); } private static ProtocolSchedule getProtocolScheduleMock() { From 512ee0b26446ca5360ab1bb64d7c7fecbf919555 Mon Sep 17 00:00:00 2001 From: "stefan.pingel@consensys.net" Date: Fri, 25 Oct 2024 17:14:26 +1000 Subject: [PATCH 104/106] update API change Signed-off-by: stefan.pingel@consensys.net --- .../eth/manager/peertask/task/GetBodiesFromPeerTask.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetBodiesFromPeerTask.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetBodiesFromPeerTask.java index a73a4520445..0b38b17d9b2 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetBodiesFromPeerTask.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetBodiesFromPeerTask.java @@ -75,7 +75,7 @@ public MessageData getRequestMessage() { } @Override - public List parseResponse(final MessageData messageData) + public List processResponse(final MessageData messageData) throws InvalidPeerTaskResponseException { // Blocks returned by this method are in the same order as the headers, but might not be // complete From 39542bb86d75ffa7820c2ca0619e787cdd9436f5 Mon Sep 17 00:00:00 2001 From: "stefan.pingel@consensys.net" Date: Sat, 26 Oct 2024 00:23:15 +1000 Subject: [PATCH 105/106] spotless Signed-off-by: stefan.pingel@consensys.net --- .../CheckpointDownloadBlockStep.java | 49 +++++++++++-------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointDownloadBlockStep.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointDownloadBlockStep.java index bf32358d83b..666ad3e32c0 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointDownloadBlockStep.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointDownloadBlockStep.java @@ -84,31 +84,38 @@ private CompletableFuture> downloadReceipts( final PeerTaskResult peerTaskResult) { final Block block = peerTaskResult.getResult(); if (synchronizerConfiguration.isPeerTaskSystemEnabled()) { - return ethContext.getScheduler().scheduleServiceTask(() -> { - GetReceiptsFromPeerTask task = - new GetReceiptsFromPeerTask( - List.of(block.getHeader()), new BodyValidator(), currentProtocolSpecSupplier); - PeerTaskExecutorResult>> executorResult = - peerTaskExecutor.execute(task); + return ethContext + .getScheduler() + .scheduleServiceTask( + () -> { + GetReceiptsFromPeerTask task = + new GetReceiptsFromPeerTask( + List.of(block.getHeader()), + new BodyValidator(), + currentProtocolSpecSupplier); + PeerTaskExecutorResult>> executorResult = + peerTaskExecutor.execute(task); - if (executorResult.responseCode() == PeerTaskExecutorResponseCode.SUCCESS) { - List transactionReceipts = - executorResult + if (executorResult.responseCode() == PeerTaskExecutorResponseCode.SUCCESS) { + List transactionReceipts = + executorResult .result() .map((map) -> map.get(block.getHeader())) .orElseThrow( - () -> - new IllegalStateException("PeerTask response code was success, but empty")); - if (block.getBody().getTransactions().size() != transactionReceipts.size()) { - throw new IllegalStateException( - "PeerTask response code was success, but incorrect number of receipts returned"); - } - BlockWithReceipts blockWithReceipts = new BlockWithReceipts(block, transactionReceipts); - return CompletableFuture.completedFuture(Optional.of(blockWithReceipts)); - } else { - return CompletableFuture.completedFuture(Optional.empty()); - } - }); + () -> + new IllegalStateException( + "PeerTask response code was success, but empty")); + if (block.getBody().getTransactions().size() != transactionReceipts.size()) { + throw new IllegalStateException( + "PeerTask response code was success, but incorrect number of receipts returned"); + } + BlockWithReceipts blockWithReceipts = + new BlockWithReceipts(block, transactionReceipts); + return CompletableFuture.completedFuture(Optional.of(blockWithReceipts)); + } else { + return CompletableFuture.completedFuture(Optional.empty()); + } + }); } else { final org.hyperledger.besu.ethereum.eth.manager.task.GetReceiptsFromPeerTask From 24401e54fc4110afca97ba9937cdb185ab407189 Mon Sep 17 00:00:00 2001 From: "stefan.pingel@consensys.net" Date: Tue, 19 Nov 2024 16:24:11 +1000 Subject: [PATCH 106/106] spotless Signed-off-by: stefan.pingel@consensys.net --- .../eth/manager/peertask/task/GetReceiptsFromPeerTask.java | 1 - .../eth/sync/tasks/CompleteBlocksWithPeerTaskTest.java | 5 +---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTask.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTask.java index e770b66e72f..7d4b5d585e5 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTask.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/peertask/task/GetReceiptsFromPeerTask.java @@ -36,7 +36,6 @@ import java.util.List; import java.util.Map; import java.util.function.Predicate; -import java.util.function.Supplier; public class GetReceiptsFromPeerTask implements PeerTask>> { diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/tasks/CompleteBlocksWithPeerTaskTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/tasks/CompleteBlocksWithPeerTaskTest.java index a396228a4f6..2853fa47d26 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/tasks/CompleteBlocksWithPeerTaskTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/tasks/CompleteBlocksWithPeerTaskTest.java @@ -229,10 +229,7 @@ private static List getExpectedBlocks( final Block block1 = new Block( header1, - new BlockBody( - Collections.emptyList(), - Collections.emptyList(), - Optional.empty())); + new BlockBody(Collections.emptyList(), Collections.emptyList(), Optional.empty())); final Block block2 = new Block( header2,