diff --git a/CHANGELOG.md b/CHANGELOG.md index 41aea176def..2efa312d263 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,13 +10,16 @@ ### Additions and Improvements - Fine tune already seen txs tracker when a tx is removed from the pool [#7755](https://github.com/hyperledger/besu/pull/7755) +- Support for enabling and configuring TLS/mTLS in WebSocket service. [#7854](https://github.com/hyperledger/besu/pull/7854) - Create and publish Besu BOM (Bill of Materials) [#7615](https://github.com/hyperledger/besu/pull/7615) - Update Java dependencies [#7786](https://github.com/hyperledger/besu/pull/7786) - Add a method to get all the transaction in the pool, to the `TransactionPoolService`, to easily access the transaction pool content from plugins [#7813](https://github.com/hyperledger/besu/pull/7813) - Add a method to check if a metric category is enabled to the plugin API [#7832](https://github.com/hyperledger/besu/pull/7832) +- Add account and state overrides to `eth_call` and `eth_estimateGas` [#7801](https://github.com/hyperledger/besu/pull/7801) ### Bug fixes - Fix registering new metric categories from plugins [#7825](https://github.com/hyperledger/besu/pull/7825) +- Fix CVE-2024-47535 [7878](https://github.com/hyperledger/besu/pull/7878) ## 24.10.0 @@ -102,6 +105,7 @@ This release version has been deprecated release due to CI bug - Remove long-deprecated `perm*whitelist*` methods [#7401](https://github.com/hyperledger/besu/pull/7401) ### Additions and Improvements +- Allow optional loading of `jemalloc` (if installed) by setting the environment variable `BESU_USING_JEMALLOC` to true/false. It that env is not set at all it will behave as if it is set to `true` - Expose set finalized/safe block in plugin api BlockchainService. These method can be used by plugins to set finalized/safe block for a PoA network (such as QBFT, IBFT and Clique).[#7382](https://github.com/hyperledger/besu/pull/7382) - In process RPC service [#7395](https://github.com/hyperledger/besu/pull/7395) - Added support for tracing private transactions using `priv_traceTransaction` API. [#6161](https://github.com/hyperledger/besu/pull/6161) diff --git a/MAINTAINERS.md b/MAINTAINERS.md index afb82b6fc59..6c17df88239 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -22,6 +22,7 @@ | Luis Pinto | lu-pinto | lu-pinto | | Lucas Saldanha | lucassaldanha | lucassaldanha | | Sally MacFarlane | macfarla | macfarla | +| Matilda Clerke | Matilda-Clerke | MatildaClerke | | Karim Taam | matkt | matkt | | Matthew Whitehead| matthew1001 | matthew.whitehead | | Meredith Baxter | mbaxter | mbaxter | diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java index fc2a7d372ea..6f9ed42828a 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java @@ -75,6 +75,7 @@ import org.hyperledger.besu.plugin.services.TransactionSimulationService; import org.hyperledger.besu.plugin.services.metrics.MetricCategoryRegistry; import org.hyperledger.besu.plugin.services.storage.rocksdb.RocksDBPlugin; +import org.hyperledger.besu.plugin.services.transactionpool.TransactionPoolService; import org.hyperledger.besu.services.BesuConfigurationImpl; import org.hyperledger.besu.services.BesuEventsImpl; import org.hyperledger.besu.services.BesuPluginContextImpl; @@ -85,6 +86,7 @@ import org.hyperledger.besu.services.RpcEndpointServiceImpl; import org.hyperledger.besu.services.SecurityModuleServiceImpl; import org.hyperledger.besu.services.StorageServiceImpl; +import org.hyperledger.besu.services.TransactionPoolServiceImpl; import org.hyperledger.besu.services.TransactionPoolValidatorServiceImpl; import org.hyperledger.besu.services.TransactionSelectionServiceImpl; import org.hyperledger.besu.services.TransactionSimulationServiceImpl; @@ -215,6 +217,9 @@ public void startNode(final BesuNode node) { besuController.getTransactionPool(), besuController.getSyncState(), besuController.getProtocolContext().getBadBlockManager())); + besuPluginContext.addService( + TransactionPoolService.class, + new TransactionPoolServiceImpl(besuController.getTransactionPool())); component.rpcEndpointService().init(runner.getInProcessRpcMethods()); @@ -327,7 +332,9 @@ RpcEndpointServiceImpl provideRpcEndpointService() { @Singleton BlockchainServiceImpl provideBlockchainService(final BesuController besuController) { BlockchainServiceImpl retval = new BlockchainServiceImpl(); - retval.init(besuController.getProtocolContext(), besuController.getProtocolSchedule()); + retval.init( + besuController.getProtocolContext().getBlockchain(), + besuController.getProtocolSchedule()); return retval; } diff --git a/acceptance-tests/test-plugins/src/main/java/org/hyperledger/besu/tests/acceptance/plugins/TestMetricsPlugin.java b/acceptance-tests/test-plugins/src/main/java/org/hyperledger/besu/tests/acceptance/plugins/TestMetricsPlugin.java index 7e1ac2ccdec..af179c57cfe 100644 --- a/acceptance-tests/test-plugins/src/main/java/org/hyperledger/besu/tests/acceptance/plugins/TestMetricsPlugin.java +++ b/acceptance-tests/test-plugins/src/main/java/org/hyperledger/besu/tests/acceptance/plugins/TestMetricsPlugin.java @@ -51,7 +51,7 @@ public void start() { .createGauge( TestMetricCategory.TEST_METRIC_CATEGORY, "test_metric", - "Returns 1 on succes", + "Returns 1 on success", () -> 1.0); } diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index 8f4a1472fb3..9126bc8a2a7 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -1233,7 +1233,7 @@ private Runner buildRunner() { private void startPlugins(final Runner runner) { blockchainServiceImpl.init( - besuController.getProtocolContext(), besuController.getProtocolSchedule()); + besuController.getProtocolContext().getBlockchain(), besuController.getProtocolSchedule()); transactionSimulationServiceImpl.init( besuController.getProtocolContext().getBlockchain(), new TransactionSimulator( diff --git a/besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java b/besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java index b3dfa2a054c..a7b43d0d9e5 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java @@ -433,14 +433,18 @@ public String build() { private void detectJemalloc(final List lines) { Optional.ofNullable(Objects.isNull(environment) ? null : environment.get("BESU_USING_JEMALLOC")) .ifPresentOrElse( - t -> { + jemallocEnabled -> { try { - final String version = PlatformDetector.getJemalloc(); - lines.add("jemalloc: " + version); + if (Boolean.parseBoolean(jemallocEnabled)) { + final String version = PlatformDetector.getJemalloc(); + lines.add("jemalloc: " + version); + } else { + logger.warn( + "besu_using_jemalloc is present but is not set to true, jemalloc library not loaded"); + } } catch (final Throwable throwable) { logger.warn( - "BESU_USING_JEMALLOC is present but we failed to load jemalloc library to get the version", - throwable); + "besu_using_jemalloc is present but we failed to load jemalloc library to get the version"); } }, () -> { diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/RpcWebsocketOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/RpcWebsocketOptions.java index d596d4183c7..2dde517ccc7 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/options/RpcWebsocketOptions.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/RpcWebsocketOptions.java @@ -120,6 +120,71 @@ public class RpcWebsocketOptions { arity = "1") private final File rpcWsAuthenticationPublicKeyFile = null; + @CommandLine.Option( + names = {"--rpc-ws-ssl-enabled"}, + description = "Enable SSL/TLS for the WebSocket RPC service") + private final Boolean isRpcWsSslEnabled = false; + + @CommandLine.Option( + names = {"--rpc-ws-ssl-keystore-file"}, + paramLabel = DefaultCommandValues.MANDATORY_FILE_FORMAT_HELP, + description = "Path to the keystore file for the WebSocket RPC service") + private String rpcWsKeyStoreFile = null; + + @CommandLine.Option( + names = {"--rpc-ws-ssl-keystore-password"}, + paramLabel = "", + description = "Password for the WebSocket RPC keystore file") + private String rpcWsKeyStorePassword = null; + + @CommandLine.Option( + names = {"--rpc-ws-ssl-key-file"}, + paramLabel = DefaultCommandValues.MANDATORY_FILE_FORMAT_HELP, + description = "Path to the PEM key file for the WebSocket RPC service") + private String rpcWsKeyFile = null; + + @CommandLine.Option( + names = {"--rpc-ws-ssl-cert-file"}, + paramLabel = DefaultCommandValues.MANDATORY_FILE_FORMAT_HELP, + description = "Path to the PEM cert file for the WebSocket RPC service") + private String rpcWsCertFile = null; + + @CommandLine.Option( + names = {"--rpc-ws-ssl-keystore-type"}, + paramLabel = "", + description = "Type of the WebSocket RPC keystore (JKS, PKCS12, PEM)") + private String rpcWsKeyStoreType = null; + + // For client authentication (mTLS) + @CommandLine.Option( + names = {"--rpc-ws-ssl-client-auth-enabled"}, + description = "Enable client authentication for the WebSocket RPC service") + private final Boolean isRpcWsClientAuthEnabled = false; + + @CommandLine.Option( + names = {"--rpc-ws-ssl-truststore-file"}, + paramLabel = DefaultCommandValues.MANDATORY_FILE_FORMAT_HELP, + description = "Path to the truststore file for the WebSocket RPC service") + private String rpcWsTrustStoreFile = null; + + @CommandLine.Option( + names = {"--rpc-ws-ssl-truststore-password"}, + paramLabel = "", + description = "Password for the WebSocket RPC truststore file") + private String rpcWsTrustStorePassword = null; + + @CommandLine.Option( + names = {"--rpc-ws-ssl-trustcert-file"}, + paramLabel = DefaultCommandValues.MANDATORY_FILE_FORMAT_HELP, + description = "Path to the PEM trustcert file for the WebSocket RPC service") + private String rpcWsTrustCertFile = null; + + @CommandLine.Option( + names = {"--rpc-ws-ssl-truststore-type"}, + paramLabel = "", + description = "Type of the truststore (JKS, PKCS12, PEM)") + private String rpcWsTrustStoreType = null; + /** Default Constructor. */ public RpcWebsocketOptions() {} @@ -184,7 +249,61 @@ private void checkOptionDependencies(final Logger logger, final CommandLine comm "--rpc-ws-authentication-enabled", "--rpc-ws-authentication-credentials-file", "--rpc-ws-authentication-public-key-file", - "--rpc-ws-authentication-jwt-algorithm")); + "--rpc-ws-authentication-jwt-algorithm", + "--rpc-ws-ssl-enabled")); + + CommandLineUtils.checkOptionDependencies( + logger, + commandLine, + "--rpc-ws-ssl-enabled", + !isRpcWsSslEnabled, + List.of( + "--rpc-ws-ssl-keystore-file", + "--rpc-ws-ssl-keystore-type", + "--rpc-ws-ssl-client-auth-enabled")); + + CommandLineUtils.checkOptionDependencies( + logger, + commandLine, + "--rpc-ws-ssl-client-auth-enabled", + !isRpcWsClientAuthEnabled, + List.of( + "--rpc-ws-ssl-truststore-file", + "--rpc-ws-ssl-truststore-type", + "--rpc-ws-ssl-trustcert-file")); + + if (isRpcWsSslEnabled) { + if ("PEM".equalsIgnoreCase(rpcWsKeyStoreType)) { + CommandLineUtils.checkOptionDependencies( + logger, + commandLine, + "--rpc-ws-ssl-key-file", + rpcWsKeyFile == null, + List.of("--rpc-ws-ssl-cert-file")); + CommandLineUtils.checkOptionDependencies( + logger, + commandLine, + "--rpc-ws-ssl-cert-file", + rpcWsCertFile == null, + List.of("--rpc-ws-ssl-key-file")); + } else { + CommandLineUtils.checkOptionDependencies( + logger, + commandLine, + "--rpc-ws-ssl-keystore-file", + rpcWsKeyStoreFile == null, + List.of("--rpc-ws-ssl-keystore-password")); + } + } + + if (isRpcWsClientAuthEnabled && !"PEM".equalsIgnoreCase(rpcWsTrustStoreType)) { + CommandLineUtils.checkOptionDependencies( + logger, + commandLine, + "--rpc-ws-ssl-truststore-file", + rpcWsTrustStoreFile == null, + List.of("--rpc-ws-ssl-truststore-password")); + } if (isRpcWsAuthenticationEnabled) { CommandLineUtils.checkOptionDependencies( @@ -222,6 +341,18 @@ public WebSocketConfiguration webSocketConfiguration( webSocketConfiguration.setAuthenticationPublicKeyFile(rpcWsAuthenticationPublicKeyFile); webSocketConfiguration.setAuthenticationAlgorithm(rpcWebsocketsAuthenticationAlgorithm); webSocketConfiguration.setTimeoutSec(wsTimoutSec); + webSocketConfiguration.setSslEnabled(isRpcWsSslEnabled); + webSocketConfiguration.setKeyStorePath(rpcWsKeyStoreFile); + webSocketConfiguration.setKeyStorePassword(rpcWsKeyStorePassword); + webSocketConfiguration.setKeyStoreType(rpcWsKeyStoreType); + webSocketConfiguration.setClientAuthEnabled(isRpcWsClientAuthEnabled); + webSocketConfiguration.setTrustStorePath(rpcWsTrustStoreFile); + webSocketConfiguration.setTrustStorePassword(rpcWsTrustStorePassword); + webSocketConfiguration.setTrustStoreType(rpcWsTrustStoreType); + webSocketConfiguration.setKeyPath(rpcWsKeyFile); + webSocketConfiguration.setCertPath(rpcWsCertFile); + webSocketConfiguration.setTrustCertPath(rpcWsTrustCertFile); + return webSocketConfiguration; } diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java index cd274a40d91..bc68faf0959 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java @@ -26,7 +26,6 @@ import org.hyperledger.besu.cryptoservices.NodeKey; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.ConsensusContext; -import org.hyperledger.besu.ethereum.ConsensusContextFactory; import org.hyperledger.besu.ethereum.GasLimitCalculator; import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.api.jsonrpc.methods.JsonRpcMethods; @@ -45,6 +44,7 @@ import org.hyperledger.besu.ethereum.core.Difficulty; import org.hyperledger.besu.ethereum.core.MiningConfiguration; import org.hyperledger.besu.ethereum.core.PrivacyParameters; +import org.hyperledger.besu.ethereum.core.Synchronizer; import org.hyperledger.besu.ethereum.eth.EthProtocol; import org.hyperledger.besu.ethereum.eth.EthProtocolConfiguration; import org.hyperledger.besu.ethereum.eth.SnapProtocol; @@ -93,6 +93,7 @@ import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.DiffBasedSubStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; +import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive.WorldStateHealer; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStatePreimageStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; @@ -113,6 +114,7 @@ import java.util.Map; import java.util.Optional; import java.util.OptionalLong; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; import org.slf4j.Logger; @@ -589,17 +591,24 @@ public BesuController build() { .map(BesuComponent::getCachedMerkleTrieLoader) .orElseGet(() -> new BonsaiCachedMerkleTrieLoader(metricsSystem)); + final var worldStateHealerSupplier = new AtomicReference(); + final WorldStateArchive worldStateArchive = createWorldStateArchive( - worldStateStorageCoordinator, blockchain, bonsaiCachedMerkleTrieLoader); + worldStateStorageCoordinator, + blockchain, + bonsaiCachedMerkleTrieLoader, + worldStateHealerSupplier::get); if (maybeStoredGenesisBlockHash.isEmpty()) { genesisState.writeStateTo(worldStateArchive.getMutable()); } + final var consensusContext = + createConsensusContext(blockchain, worldStateArchive, protocolSchedule); + final ProtocolContext protocolContext = - createProtocolContext( - blockchain, worldStateArchive, protocolSchedule, this::createConsensusContext); + createProtocolContext(blockchain, worldStateArchive, consensusContext); validateContext(protocolContext); protocolSchedule.setPublicWorldStateArchiveForPrivacyBlockProcessor( @@ -616,7 +625,7 @@ public BesuController build() { ethereumWireProtocolConfiguration.isLegacyEth64ForkIdEnabled()); final EthPeers ethPeers = new EthPeers( - getSupportedProtocol(), + EthProtocol.NAME, currentProtocolSpecSupplier, clock, metricsSystem, @@ -720,6 +729,8 @@ public BesuController build() { ethProtocolManager, pivotBlockSelector); + worldStateHealerSupplier.set(synchronizer::healWorldState); + ethPeers.setTrailingPeerRequirementsSupplier(synchronizer::calculateTrailingPeerRequirements); if (syncConfig.getSyncMode() == SyncMode.SNAP @@ -730,11 +741,9 @@ public BesuController build() { ethPeers.snapServerPeersNeeded(false); } - protocolContext.setSynchronizer(synchronizer); - final Optional maybeSnapProtocolManager = createSnapProtocolManager( - protocolContext, worldStateStorageCoordinator, ethPeers, snapMessages); + protocolContext, worldStateStorageCoordinator, ethPeers, snapMessages, synchronizer); final MiningCoordinator miningCoordinator = createMiningCoordinator( @@ -979,7 +988,7 @@ protected SubProtocolConfiguration createSubProtocolConfiguration( } /** - * Create mining coordinator mining coordinator. + * Create mining coordinator. * * @param protocolSchedule the protocol schedule * @param protocolContext the protocol context @@ -1022,18 +1031,9 @@ protected void validateContext(final ProtocolContext context) {} * @return the consensus context */ protected abstract ConsensusContext createConsensusContext( - Blockchain blockchain, - WorldStateArchive worldStateArchive, - ProtocolSchedule protocolSchedule); - - /** - * Gets supported protocol. - * - * @return the supported protocol - */ - protected String getSupportedProtocol() { - return EthProtocol.NAME; - } + final Blockchain blockchain, + final WorldStateArchive worldStateArchive, + final ProtocolSchedule protocolSchedule); /** * Create eth protocol manager eth protocol manager. @@ -1084,37 +1084,37 @@ protected EthProtocolManager createEthProtocolManager( * * @param blockchain the blockchain * @param worldStateArchive the world state archive - * @param protocolSchedule the protocol schedule - * @param consensusContextFactory the consensus context factory + * @param consensusContext the consensus context * @return the protocol context */ protected ProtocolContext createProtocolContext( final MutableBlockchain blockchain, final WorldStateArchive worldStateArchive, - final ProtocolSchedule protocolSchedule, - final ConsensusContextFactory consensusContextFactory) { - return ProtocolContext.init( - blockchain, worldStateArchive, protocolSchedule, consensusContextFactory, badBlockManager); + final ConsensusContext consensusContext) { + return new ProtocolContext(blockchain, worldStateArchive, consensusContext, badBlockManager); } private Optional createSnapProtocolManager( final ProtocolContext protocolContext, final WorldStateStorageCoordinator worldStateStorageCoordinator, final EthPeers ethPeers, - final EthMessages snapMessages) { + final EthMessages snapMessages, + final Synchronizer synchronizer) { return Optional.of( new SnapProtocolManager( worldStateStorageCoordinator, syncConfig.getSnapSyncConfiguration(), ethPeers, snapMessages, - protocolContext)); + protocolContext, + synchronizer)); } WorldStateArchive createWorldStateArchive( final WorldStateStorageCoordinator worldStateStorageCoordinator, final Blockchain blockchain, - final BonsaiCachedMerkleTrieLoader bonsaiCachedMerkleTrieLoader) { + final BonsaiCachedMerkleTrieLoader bonsaiCachedMerkleTrieLoader, + final Supplier worldStateHealerSupplier) { return switch (dataStorageConfiguration.getDataStorageFormat()) { case BONSAI -> { final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage = @@ -1129,7 +1129,8 @@ yield new BonsaiWorldStateProvider( .getMaxLayersToLoad()), bonsaiCachedMerkleTrieLoader, besuComponent.map(BesuComponent::getBesuPluginContext).orElse(null), - evmConfiguration); + evmConfiguration, + worldStateHealerSupplier); } case FOREST -> { final WorldStatePreimageStorage preimageStorage = diff --git a/besu/src/main/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilder.java index 508b8cd82db..7948c656e31 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilder.java @@ -20,13 +20,12 @@ import org.hyperledger.besu.consensus.common.CombinedProtocolScheduleFactory; import org.hyperledger.besu.consensus.common.ForkSpec; import org.hyperledger.besu.consensus.common.ForksSchedule; -import org.hyperledger.besu.consensus.common.MigratingContext; +import org.hyperledger.besu.consensus.common.MigratingConsensusContext; import org.hyperledger.besu.consensus.common.MigratingMiningCoordinator; import org.hyperledger.besu.consensus.common.MigratingProtocolContext; import org.hyperledger.besu.cryptoservices.NodeKey; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.ConsensusContext; -import org.hyperledger.besu.ethereum.ConsensusContextFactory; import org.hyperledger.besu.ethereum.GasLimitCalculator; import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.api.jsonrpc.methods.JsonRpcMethods; @@ -171,10 +170,12 @@ protected ProtocolSchedule createProtocolSchedule() { protected ProtocolContext createProtocolContext( final MutableBlockchain blockchain, final WorldStateArchive worldStateArchive, - final ProtocolSchedule protocolSchedule, - final ConsensusContextFactory consensusContextFactory) { - return MigratingProtocolContext.init( - blockchain, worldStateArchive, protocolSchedule, consensusContextFactory, badBlockManager); + final ConsensusContext consensusContext) { + return new MigratingProtocolContext( + blockchain, + worldStateArchive, + consensusContext.as(MigratingConsensusContext.class), + badBlockManager); } @Override @@ -191,10 +192,10 @@ protected ConsensusContext createConsensusContext( e.getValue() .createConsensusContext( blockchain, worldStateArchive, protocolSchedule))) - .collect(Collectors.toList()); + .toList(); final ForksSchedule consensusContextsSchedule = new ForksSchedule<>(consensusContextSpecs); - return new MigratingContext(consensusContextsSchedule); + return new MigratingConsensusContext(consensusContextsSchedule); } @Override @@ -230,11 +231,6 @@ protected void validateContext(final ProtocolContext context) { besuControllerBuilderSchedule.get(GENESIS_BLOCK_NUMBER).validateContext(context); } - @Override - protected String getSupportedProtocol() { - return besuControllerBuilderSchedule.get(0L).getSupportedProtocol(); - } - @Override protected EthProtocolManager createEthProtocolManager( final ProtocolContext protocolContext, diff --git a/besu/src/main/java/org/hyperledger/besu/controller/QbftBesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/QbftBesuControllerBuilder.java index 67ed8ec135a..df9933638c5 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/QbftBesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/QbftBesuControllerBuilder.java @@ -113,10 +113,7 @@ public QbftBesuControllerBuilder() {} @Override protected Supplier bftExtraDataCodec() { - return Suppliers.memoize( - () -> { - return new QbftExtraDataCodec(); - }); + return Suppliers.memoize(QbftExtraDataCodec::new); } @Override diff --git a/besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java index a5826697d45..b72d7400f35 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java @@ -24,7 +24,6 @@ import org.hyperledger.besu.cryptoservices.NodeKey; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.ConsensusContext; -import org.hyperledger.besu.ethereum.ConsensusContextFactory; import org.hyperledger.besu.ethereum.GasLimitCalculator; import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.blockcreation.MiningCoordinator; @@ -197,11 +196,9 @@ protected ProtocolSchedule createProtocolSchedule() { protected ProtocolContext createProtocolContext( final MutableBlockchain blockchain, final WorldStateArchive worldStateArchive, - final ProtocolSchedule protocolSchedule, - final ConsensusContextFactory consensusContextFactory) { + final ConsensusContext consensusContext) { final ProtocolContext protocolContext = - super.createProtocolContext( - blockchain, worldStateArchive, protocolSchedule, consensusContextFactory); + super.createProtocolContext(blockchain, worldStateArchive, consensusContext); transitionProtocolSchedule.setProtocolContext(protocolContext); return protocolContext; } diff --git a/besu/src/main/java/org/hyperledger/besu/services/BlockchainServiceImpl.java b/besu/src/main/java/org/hyperledger/besu/services/BlockchainServiceImpl.java index e981a3e78d6..76b4d7e1509 100644 --- a/besu/src/main/java/org/hyperledger/besu/services/BlockchainServiceImpl.java +++ b/besu/src/main/java/org/hyperledger/besu/services/BlockchainServiceImpl.java @@ -16,7 +16,6 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.Wei; -import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; import org.hyperledger.besu.ethereum.core.Block; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; @@ -40,7 +39,6 @@ @Unstable public class BlockchainServiceImpl implements BlockchainService { - private ProtocolContext protocolContext; private ProtocolSchedule protocolSchedule; private MutableBlockchain blockchain; @@ -50,13 +48,12 @@ public BlockchainServiceImpl() {} /** * Initialize the Blockchain service. * - * @param protocolContext the protocol context + * @param blockchain the blockchain * @param protocolSchedule the protocol schedule */ - public void init(final ProtocolContext protocolContext, final ProtocolSchedule protocolSchedule) { - this.protocolContext = protocolContext; + public void init(final MutableBlockchain blockchain, final ProtocolSchedule protocolSchedule) { this.protocolSchedule = protocolSchedule; - this.blockchain = protocolContext.getBlockchain(); + this.blockchain = blockchain; } /** @@ -67,25 +64,24 @@ public void init(final ProtocolContext protocolContext, final ProtocolSchedule p */ @Override public Optional getBlockByNumber(final long number) { - return protocolContext - .getBlockchain() + return blockchain .getBlockByNumber(number) .map(block -> blockContext(block::getHeader, block::getBody)); } @Override public Hash getChainHeadHash() { - return protocolContext.getBlockchain().getChainHeadHash(); + return blockchain.getChainHeadHash(); } @Override public BlockHeader getChainHeadHeader() { - return protocolContext.getBlockchain().getChainHeadHeader(); + return blockchain.getChainHeadHeader(); } @Override public Optional getNextBlockBaseFee() { - final var chainHeadHeader = protocolContext.getBlockchain().getChainHeadHeader(); + final var chainHeadHeader = blockchain.getChainHeadHeader(); final var protocolSpec = protocolSchedule.getForNextBlockHeader(chainHeadHeader, System.currentTimeMillis()); return Optional.of(protocolSpec.getFeeMarket()) diff --git a/besu/src/main/scripts/unixStartScript.txt b/besu/src/main/scripts/unixStartScript.txt index 3302b9c2fe1..9e5c5a300b0 100644 --- a/besu/src/main/scripts/unixStartScript.txt +++ b/besu/src/main/scripts/unixStartScript.txt @@ -182,19 +182,19 @@ APP_ARGS=`save "\$@"` # Collect all arguments for the java command, following the shell quoting and substitution rules eval set -- \$DEFAULT_JVM_OPTS \$JAVA_OPTS \$${optsEnvironmentVar} <% if ( appNameSystemProperty ) { %>"\"-D${appNameSystemProperty}=\$APP_BASE_NAME\"" <% } %>-classpath "\"\$CLASSPATH\"" <% if ( mainClassName.startsWith('--module ') ) { %>--module-path "\"\$MODULE_PATH\"" <% } %>${mainClassName} "\$APP_ARGS" -unset BESU_USING_JEMALLOC if [ "\$darwin" = "false" -a "\$msys" = "false" ]; then - # check if jemalloc is available - TEST_JEMALLOC=\$(LD_PRELOAD=libjemalloc.so sh -c true 2>&1) - - # if jemalloc is available the output is empty, otherwise the output has an error line - if [ -z "\$TEST_JEMALLOC" ]; then - export LD_PRELOAD=libjemalloc.so - export BESU_USING_JEMALLOC=true - else - # jemalloc not available, as fallback limit malloc to 2 arenas - export MALLOC_ARENA_MAX=2 - fi + if [ "\$BESU_USING_JEMALLOC" != "FALSE" -a "\$BESU_USING_JEMALLOC" != "false" ]; then + # check if jemalloc is available + TEST_JEMALLOC=\$(LD_PRELOAD=libjemalloc.so sh -c true 2>&1) + + # if jemalloc is available the output is empty, otherwise the output has an error line + if [ -z "\$TEST_JEMALLOC" ]; then + export LD_PRELOAD=libjemalloc.so + else + # jemalloc not available, as fallback limit malloc to 2 arenas + export MALLOC_ARENA_MAX=2 + fi + fi fi exec "\$JAVACMD" "\$@" diff --git a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java index ea06f406ed9..eb789478b9a 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java @@ -37,6 +37,7 @@ import static org.mockito.ArgumentMatchers.contains; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.isNotNull; +import static org.mockito.Mockito.argThat; import static org.mockito.Mockito.atLeast; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; @@ -2412,13 +2413,16 @@ public void nativeLibrariesAreEnabledByDefault() { @Test public void logsWarningWhenFailToLoadJemalloc() { assumeTrue(PlatformDetector.getOSType().equals("linux")); - setEnvironmentVariable("BESU_USING_JEMALLOC", "true"); + setEnvironmentVariable("BESU_USING_JEMALLOC", "false"); parseCommand(); verify(mockLogger) .warn( - eq( - "BESU_USING_JEMALLOC is present but we failed to load jemalloc library to get the version"), - any(Throwable.class)); + argThat( + arg -> + arg.equals( + "besu_using_jemalloc is present but is not set to true, jemalloc library not loaded") + || arg.equals( + "besu_using_jemalloc is present but we failed to load jemalloc library to get the version"))); } @Test diff --git a/besu/src/test/java/org/hyperledger/besu/cli/options/MetricsOptionsTest.java b/besu/src/test/java/org/hyperledger/besu/cli/options/MetricsOptionsTest.java index 39e77cbad7e..f68a87643bd 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/options/MetricsOptionsTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/options/MetricsOptionsTest.java @@ -14,12 +14,18 @@ */ package org.hyperledger.besu.cli.options; +import static org.assertj.core.api.Assertions.assertThat; + import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.metrics.MetricCategoryRegistryImpl; import org.hyperledger.besu.metrics.StandardMetricCategory; import org.hyperledger.besu.metrics.prometheus.MetricsConfiguration; +import java.util.EnumSet; +import java.util.stream.Collectors; + import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.junit.jupiter.MockitoExtension; @@ -64,4 +70,22 @@ protected MetricsOptions getOptionsFromBesuCommand(final TestBesuCommand besuCom protected String[] getNonOptionFields() { return new String[] {"metricCategoryRegistry"}; } + + @Test + public void enableRocksDbCategories() { + final var rocksDbMetricsCategories = + EnumSet.of( + BesuMetricCategory.KVSTORE_ROCKSDB, + BesuMetricCategory.KVSTORE_ROCKSDB_STATS, + BesuMetricCategory.KVSTORE_PRIVATE_ROCKSDB, + BesuMetricCategory.KVSTORE_PRIVATE_ROCKSDB_STATS); + + internalTestSuccess( + metricsConfBuilder -> { + assertThat(metricsConfBuilder.build().getMetricCategories()) + .containsExactlyInAnyOrderElementsOf(rocksDbMetricsCategories); + }, + "--metrics-categories", + rocksDbMetricsCategories.stream().map(Enum::name).collect(Collectors.joining(","))); + } } diff --git a/besu/src/test/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilderTest.java b/besu/src/test/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilderTest.java index 47a7aece3a7..30e89416db7 100644 --- a/besu/src/test/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilderTest.java +++ b/besu/src/test/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilderTest.java @@ -24,7 +24,7 @@ import org.hyperledger.besu.config.StubGenesisConfigOptions; import org.hyperledger.besu.consensus.common.ForkSpec; import org.hyperledger.besu.consensus.common.ForksSchedule; -import org.hyperledger.besu.consensus.common.MigratingContext; +import org.hyperledger.besu.consensus.common.MigratingConsensusContext; import org.hyperledger.besu.consensus.common.MigratingMiningCoordinator; import org.hyperledger.besu.consensus.common.bft.blockcreation.BftMiningCoordinator; import org.hyperledger.besu.ethereum.ConsensusContext; @@ -170,8 +170,8 @@ public void createsMigratingMiningCoordinator() { @Test public void createsMigratingContext() { - final ConsensusContext context1 = Mockito.mock(ConsensusContext.class); - final ConsensusContext context2 = Mockito.mock(ConsensusContext.class); + final ConsensusContext context1 = mock(ConsensusContext.class); + final ConsensusContext context2 = mock(ConsensusContext.class); final Map besuControllerBuilderSchedule = new TreeMap<>(); besuControllerBuilderSchedule.put(0L, besuControllerBuilder1); @@ -184,15 +184,14 @@ public void createsMigratingContext() { new ConsensusScheduleBesuControllerBuilder(besuControllerBuilderSchedule); final ConsensusContext consensusContext = controllerBuilder.createConsensusContext( - Mockito.mock(Blockchain.class), - Mockito.mock(WorldStateArchive.class), - Mockito.mock(ProtocolSchedule.class)); + mock(Blockchain.class), mock(WorldStateArchive.class), mock(ProtocolSchedule.class)); - assertThat(consensusContext).isInstanceOf(MigratingContext.class); - final MigratingContext migratingContext = (MigratingContext) consensusContext; + assertThat(consensusContext).isInstanceOf(MigratingConsensusContext.class); + final MigratingConsensusContext migratingConsensusContext = + (MigratingConsensusContext) consensusContext; final ForksSchedule contextSchedule = - migratingContext.getConsensusContextSchedule(); + migratingConsensusContext.getConsensusContextSchedule(); final NavigableSet> expectedConsensusContextSpecs = new TreeSet<>(ForkSpec.COMPARATOR); diff --git a/besu/src/test/resources/everything_config.toml b/besu/src/test/resources/everything_config.toml index e89442f7f78..960b2b5772c 100644 --- a/besu/src/test/resources/everything_config.toml +++ b/besu/src/test/resources/everything_config.toml @@ -120,6 +120,19 @@ rpc-ws-max-frame-size=65535 rpc-ws-authentication-enabled=false rpc-ws-authentication-credentials-file="none" rpc-ws-authentication-jwt-public-key-file="none" +rpc-ws-ssl-enabled=false +rpc-ws-ssl-keystore-file="none.pfx" +rpc-ws-ssl-keystore-password="none.passwd" +rpc-ws-ssl-keystore-type="none" +rpc-ws-ssl-client-auth-enabled=false +rpc-ws-ssl-truststore-file="none.pfx" +rpc-ws-ssl-truststore-password="none.passwd" +rpc-ws-ssl-truststore-type="none" +rpc-ws-ssl-key-file="none.pfx" +rpc-ws-ssl-cert-file="none.pfx" +rpc-ws-ssl-trustcert-file="none.pfx" + + # API api-gas-price-blocks=100 diff --git a/consensus/clique/src/test/java/org/hyperledger/besu/consensus/clique/blockcreation/CliqueBlockCreatorTest.java b/consensus/clique/src/test/java/org/hyperledger/besu/consensus/clique/blockcreation/CliqueBlockCreatorTest.java index fdc068ddef7..12467cda4ee 100644 --- a/consensus/clique/src/test/java/org/hyperledger/besu/consensus/clique/blockcreation/CliqueBlockCreatorTest.java +++ b/consensus/clique/src/test/java/org/hyperledger/besu/consensus/clique/blockcreation/CliqueBlockCreatorTest.java @@ -52,6 +52,7 @@ import org.hyperledger.besu.ethereum.core.ImmutableMiningConfiguration; import org.hyperledger.besu.ethereum.core.ImmutableMiningConfiguration.MutableInitValues; import org.hyperledger.besu.ethereum.core.MiningConfiguration; +import org.hyperledger.besu.ethereum.core.PrivacyParameters; import org.hyperledger.besu.ethereum.core.Util; import org.hyperledger.besu.ethereum.eth.manager.EthContext; import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; @@ -99,11 +100,20 @@ public class CliqueBlockCreatorTest { @BeforeEach void setup() { + final Address otherAddress = Util.publicKeyToAddress(otherKeyPair.getPublicKey()); + validatorList.add(otherAddress); + + validatorProvider = mock(ValidatorProvider.class); + voteProvider = mock(VoteProvider.class); + when(validatorProvider.getVoteProviderAtHead()).thenReturn(Optional.of(voteProvider)); + when(validatorProvider.getValidatorsAfterBlock(any())).thenReturn(validatorList); + protocolSchedule = CliqueProtocolSchedule.create( GenesisConfigFile.DEFAULT.getConfigOptions(), new ForksSchedule<>(List.of()), proposerNodeKey, + PrivacyParameters.DEFAULT, false, EvmConfiguration.DEFAULT, MiningConfiguration.MINING_DISABLED, @@ -111,13 +121,6 @@ void setup() { false, new NoOpMetricsSystem()); - final Address otherAddress = Util.publicKeyToAddress(otherKeyPair.getPublicKey()); - validatorList.add(otherAddress); - - validatorProvider = mock(ValidatorProvider.class); - voteProvider = mock(VoteProvider.class); - when(validatorProvider.getVoteProviderAtHead()).thenReturn(Optional.of(voteProvider)); - when(validatorProvider.getValidatorsAfterBlock(any())).thenReturn(validatorList); final CliqueContext cliqueContext = new CliqueContext(validatorProvider, null, blockInterface); final Block genesis = diff --git a/consensus/clique/src/test/java/org/hyperledger/besu/consensus/clique/blockcreation/CliqueMinerExecutorTest.java b/consensus/clique/src/test/java/org/hyperledger/besu/consensus/clique/blockcreation/CliqueMinerExecutorTest.java index 3d6dbd77db9..7000d145b6b 100644 --- a/consensus/clique/src/test/java/org/hyperledger/besu/consensus/clique/blockcreation/CliqueMinerExecutorTest.java +++ b/consensus/clique/src/test/java/org/hyperledger/besu/consensus/clique/blockcreation/CliqueMinerExecutorTest.java @@ -42,6 +42,7 @@ import org.hyperledger.besu.ethereum.core.ImmutableMiningConfiguration; import org.hyperledger.besu.ethereum.core.ImmutableMiningConfiguration.MutableInitValues; import org.hyperledger.besu.ethereum.core.MiningConfiguration; +import org.hyperledger.besu.ethereum.core.PrivacyParameters; import org.hyperledger.besu.ethereum.core.Util; import org.hyperledger.besu.ethereum.eth.manager.EthContext; import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; @@ -103,6 +104,7 @@ public void setup() { GENESIS_CONFIG_OPTIONS, new ForksSchedule<>(List.of()), proposerNodeKey, + PrivacyParameters.DEFAULT, false, EvmConfiguration.DEFAULT, MiningConfiguration.MINING_DISABLED, diff --git a/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/MigratingContext.java b/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/MigratingConsensusContext.java similarity index 88% rename from consensus/common/src/main/java/org/hyperledger/besu/consensus/common/MigratingContext.java rename to consensus/common/src/main/java/org/hyperledger/besu/consensus/common/MigratingConsensusContext.java index f71e3ac7f36..91ace1a04b3 100644 --- a/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/MigratingContext.java +++ b/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/MigratingConsensusContext.java @@ -17,7 +17,7 @@ import org.hyperledger.besu.ethereum.ConsensusContext; /** The Migrating context. */ -public class MigratingContext implements ConsensusContext { +public class MigratingConsensusContext implements ConsensusContext { private final ForksSchedule consensusContextSchedule; @@ -26,7 +26,7 @@ public class MigratingContext implements ConsensusContext { * * @param consensusContextSchedule the consensus context schedule */ - public MigratingContext(final ForksSchedule consensusContextSchedule) { + public MigratingConsensusContext(final ForksSchedule consensusContextSchedule) { this.consensusContextSchedule = consensusContextSchedule; } diff --git a/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/MigratingProtocolContext.java b/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/MigratingProtocolContext.java index c0557485f3e..27cce3157b2 100644 --- a/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/MigratingProtocolContext.java +++ b/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/MigratingProtocolContext.java @@ -15,11 +15,9 @@ package org.hyperledger.besu.consensus.common; import org.hyperledger.besu.ethereum.ConsensusContext; -import org.hyperledger.besu.ethereum.ConsensusContextFactory; import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.chain.BadBlockManager; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; -import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; /** The Migrating protocol context. */ @@ -32,42 +30,16 @@ public class MigratingProtocolContext extends ProtocolContext { * * @param blockchain the blockchain * @param worldStateArchive the world state archive - * @param consensusContextSchedule the consensus context schedule + * @param migratingConsensusContext the consensus context * @param badBlockManager the cache to use to keep invalid blocks */ public MigratingProtocolContext( final MutableBlockchain blockchain, final WorldStateArchive worldStateArchive, - final ForksSchedule consensusContextSchedule, + final MigratingConsensusContext migratingConsensusContext, final BadBlockManager badBlockManager) { - super(blockchain, worldStateArchive, null, badBlockManager); - this.consensusContextSchedule = consensusContextSchedule; - } - - /** - * Init protocol context. - * - * @param blockchain the blockchain - * @param worldStateArchive the world state archive - * @param protocolSchedule the protocol schedule - * @param consensusContextFactory the consensus context factory - * @param badBlockManager the cache to use to keep invalid blocks - * @return the protocol context - */ - public static ProtocolContext init( - final MutableBlockchain blockchain, - final WorldStateArchive worldStateArchive, - final ProtocolSchedule protocolSchedule, - final ConsensusContextFactory consensusContextFactory, - final BadBlockManager badBlockManager) { - final ConsensusContext consensusContext = - consensusContextFactory.create(blockchain, worldStateArchive, protocolSchedule); - final MigratingContext migratingContext = consensusContext.as(MigratingContext.class); - return new MigratingProtocolContext( - blockchain, - worldStateArchive, - migratingContext.getConsensusContextSchedule(), - badBlockManager); + super(blockchain, worldStateArchive, migratingConsensusContext, badBlockManager); + this.consensusContextSchedule = migratingConsensusContext.getConsensusContextSchedule(); } @Override diff --git a/consensus/common/src/test/java/org/hyperledger/besu/consensus/common/MigratingProtocolContextTest.java b/consensus/common/src/test/java/org/hyperledger/besu/consensus/common/MigratingProtocolContextTest.java index d62de57eac9..39ab9f8cb28 100644 --- a/consensus/common/src/test/java/org/hyperledger/besu/consensus/common/MigratingProtocolContextTest.java +++ b/consensus/common/src/test/java/org/hyperledger/besu/consensus/common/MigratingProtocolContextTest.java @@ -43,9 +43,13 @@ public void returnsContextForSpecificChainHeight() { final ForksSchedule contextSchedule = new ForksSchedule<>(List.of(new ForkSpec<>(0L, context1), new ForkSpec<>(10L, context2))); + final MigratingProtocolContext migratingProtocolContext = new MigratingProtocolContext( - blockchain, worldStateArchive, contextSchedule, new BadBlockManager()); + blockchain, + worldStateArchive, + new MigratingConsensusContext(contextSchedule), + new BadBlockManager()); assertThat(migratingProtocolContext.getConsensusContext(ConsensusContext.class)) .isSameAs(context1); diff --git a/consensus/merge/src/test/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinatorTest.java b/consensus/merge/src/test/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinatorTest.java index 23a5ae16b10..90c4b6866e4 100644 --- a/consensus/merge/src/test/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinatorTest.java +++ b/consensus/merge/src/test/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinatorTest.java @@ -59,7 +59,6 @@ import org.hyperledger.besu.ethereum.core.ImmutableMiningConfiguration.MutableInitValues; import org.hyperledger.besu.ethereum.core.ImmutableMiningConfiguration.Unstable; import org.hyperledger.besu.ethereum.core.MiningConfiguration; -import org.hyperledger.besu.ethereum.core.Synchronizer; import org.hyperledger.besu.ethereum.core.TransactionTestFixture; import org.hyperledger.besu.ethereum.eth.manager.EthContext; import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; @@ -190,7 +189,6 @@ public void setUp() { protocolContext = new ProtocolContext(blockchain, worldStateArchive, mergeContext, badBlockManager); - protocolContext.setSynchronizer(mock(Synchronizer.class)); var mutable = worldStateArchive.getMutable(); genesisState.writeStateTo(mutable); mutable.persist(null); diff --git a/docs/Private-Txns-Migration.md b/docs/Private-Txns-Migration.md index 0ae06dae3ce..042e83c8dd1 100644 --- a/docs/Private-Txns-Migration.md +++ b/docs/Private-Txns-Migration.md @@ -12,7 +12,7 @@ state storage when upgrading to v1.4. It is not possible to upgrade to v1.4 with ## Private transactions created using v1.3.4 or earlier A critical issue for privacy users with private transactions created using Hyperledger Besu v1.3.4 -or earlier has been identified. If you have a network with private transaction created using v1.3.4 +or earlier has been identified. If you have a network with private transactions created using v1.3.4 or earlier, please read the following and take the appropriate steps: https://wiki.hyperledger.org/display/BESU/Critical+Issue+for+Privacy+Users @@ -41,4 +41,4 @@ your Besu database from backup and restart the migration process. ## Migration support If you have a long running network with a large volume of private transactions and/or would like to discuss -the migration process with us before upgrading, contact us at support@pegasys.tech \ No newline at end of file +the migration process with us before upgrading, contact us at support@pegasys.tech diff --git a/docs/trace_rpc_apis.md b/docs/trace_rpc_apis.md index e8db0bcbc5c..7f04d497a7b 100644 --- a/docs/trace_rpc_apis.md +++ b/docs/trace_rpc_apis.md @@ -16,7 +16,7 @@ implementations of Besu might track gas refunds separately. ### Returned Memory from Calls -In the `vmTrace` `ope.ex.mem` fields Besu only reports actual data returned +In the `vmTrace` `op.ex.mem` fields Besu only reports actual data returned from a `RETURN` opcode. Other implementations return the contents of the reserved output space for the call operations. Note two major differences: diff --git a/ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcTestMethodsFactory.java b/ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcTestMethodsFactory.java index 902d2b04870..2a65b1e7cb0 100644 --- a/ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcTestMethodsFactory.java +++ b/ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcTestMethodsFactory.java @@ -19,6 +19,7 @@ import static org.mockito.Mockito.mock; import org.hyperledger.besu.config.StubGenesisConfigOptions; +import org.hyperledger.besu.ethereum.ConsensusContext; import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.api.ImmutableApiConfiguration; import org.hyperledger.besu.ethereum.api.graphql.GraphQLConfiguration; @@ -82,7 +83,9 @@ public JsonRpcTestMethodsFactory(final BlockchainImporter importer) { this.blockchain = createInMemoryBlockchain(importer.getGenesisBlock()); this.stateArchive = createInMemoryWorldStateArchive(); this.importer.getGenesisState().writeStateTo(stateArchive.getMutable()); - this.context = new ProtocolContext(blockchain, stateArchive, null, new BadBlockManager()); + this.context = + new ProtocolContext( + blockchain, stateArchive, mock(ConsensusContext.class), new BadBlockManager()); this.protocolSchedule = importer.getProtocolSchedule(); this.synchronizer = mock(Synchronizer.class); diff --git a/ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/EthGetBlockByNumberLatestDesyncIntegrationTest.java b/ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/EthGetBlockByNumberLatestDesyncIntegrationTest.java index 088eff85007..7e26a4d8126 100644 --- a/ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/EthGetBlockByNumberLatestDesyncIntegrationTest.java +++ b/ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/EthGetBlockByNumberLatestDesyncIntegrationTest.java @@ -20,6 +20,7 @@ import static org.mockito.Mockito.when; import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.ConsensusContext; import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.api.jsonrpc.BlockchainImporter; import org.hyperledger.besu.ethereum.api.jsonrpc.JsonRpcTestMethodsFactory; @@ -67,7 +68,8 @@ public static void setUpOnce() throws Exception { InMemoryKeyValueStorageProvider.createInMemoryBlockchain(importer.getGenesisBlock()); WorldStateArchive state = InMemoryKeyValueStorageProvider.createInMemoryWorldStateArchive(); importer.getGenesisState().writeStateTo(state.getMutable()); - ProtocolContext context = new ProtocolContext(chain, state, null, new BadBlockManager()); + ProtocolContext context = + new ProtocolContext(chain, state, mock(ConsensusContext.class), new BadBlockManager()); for (final Block block : importer.getBlocks()) { final ProtocolSchedule protocolSchedule = importer.getProtocolSchedule(); diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthCall.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthCall.java index 0e0318f9c2b..5f736e8d5a9 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthCall.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthCall.java @@ -23,6 +23,7 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.RpcMethod; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.exception.InvalidJsonRpcParameters; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.exception.InvalidJsonRpcRequestException; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.BlockParameterOrBlockHash; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.JsonCallParameter; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.JsonRpcParameter.JsonRpcParameterException; @@ -40,8 +41,13 @@ import org.hyperledger.besu.ethereum.transaction.TransactionInvalidReason; import org.hyperledger.besu.ethereum.transaction.TransactionSimulator; import org.hyperledger.besu.ethereum.transaction.TransactionSimulatorResult; +import org.hyperledger.besu.ethereum.util.AccountOverrideMap; import org.hyperledger.besu.evm.tracing.OperationTracer; +import java.util.Optional; + +import com.google.common.annotations.VisibleForTesting; + public class EthCall extends AbstractBlockParameterOrBlockHashMethod { private final TransactionSimulator transactionSimulator; @@ -81,10 +87,13 @@ protected Object resultByBlockHash(final JsonRpcRequestContext request, final Ha protected Object resultByBlockHeader( final JsonRpcRequestContext request, final BlockHeader header) { JsonCallParameter callParams = JsonCallParameterUtil.validateAndGetCallParams(request); + Optional maybeStateOverrides = getAddressAccountOverrideMap(request); + // TODO implement for block overrides return transactionSimulator .process( callParams, + maybeStateOverrides, buildTransactionValidationParams(header, callParams), OperationTracer.NO_TRACING, (mutableWorldState, transactionSimulatorResult) -> @@ -108,6 +117,17 @@ protected Object resultByBlockHeader( .orElse(errorResponse(request, INTERNAL_ERROR)); } + @VisibleForTesting + protected Optional getAddressAccountOverrideMap( + final JsonRpcRequestContext request) { + try { + return request.getOptionalParameter(2, AccountOverrideMap.class); + } catch (JsonRpcParameterException e) { + throw new InvalidJsonRpcRequestException( + "Invalid account overrides parameter (index 2)", RpcErrorType.INVALID_CALL_PARAMS, e); + } + } + @Override public JsonRpcResponse response(final JsonRpcRequestContext requestContext) { return (JsonRpcResponse) handleParamTypes(requestContext); diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceBlock.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceBlock.java index adacf01e764..16798160d8e 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceBlock.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceBlock.java @@ -25,7 +25,6 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.Tracer; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.TransactionTrace; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.RpcErrorType; -import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.tracing.flat.FlatTraceGenerator; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.tracing.flat.RewardTraceGenerator; import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; import org.hyperledger.besu.ethereum.api.util.ArrayNodeWrapper; @@ -41,12 +40,11 @@ import org.hyperledger.besu.evm.worldstate.WorldUpdater; import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; -import org.hyperledger.besu.metrics.prometheus.PrometheusMetricsSystem; +import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.plugin.services.metrics.Counter; import org.hyperledger.besu.plugin.services.metrics.LabelledMetric; import org.hyperledger.besu.services.pipeline.Pipeline; -import java.util.List; import java.util.Optional; import java.util.concurrent.ExecutionException; @@ -58,10 +56,21 @@ public class TraceBlock extends AbstractBlockParameterMethod { private static final Logger LOG = LoggerFactory.getLogger(TraceBlock.class); private static final ObjectMapper MAPPER = new ObjectMapper(); protected final ProtocolSchedule protocolSchedule; + private final LabelledMetric outputCounter; - public TraceBlock(final ProtocolSchedule protocolSchedule, final BlockchainQueries queries) { + public TraceBlock( + final ProtocolSchedule protocolSchedule, + final BlockchainQueries queries, + final MetricsSystem metricsSystem) { super(queries); this.protocolSchedule = protocolSchedule; + this.outputCounter = + metricsSystem.createLabelledCounter( + BesuMetricCategory.BLOCKCHAIN, + "transactions_traceblock_pipeline_processed_total", + "Number of transactions processed for each block", + "step", + "action"); } @Override @@ -115,14 +124,6 @@ protected ArrayNodeWrapper traceBlock( final ChainUpdater chainUpdater = new ChainUpdater(traceableState); TransactionSource transactionSource = new TransactionSource(block); - final LabelledMetric outputCounter = - new PrometheusMetricsSystem(BesuMetricCategory.DEFAULT_METRIC_CATEGORIES, false) - .createLabelledCounter( - BesuMetricCategory.BLOCKCHAIN, - "transactions_traceblock_pipeline_processed_total", - "Number of transactions processed for each block", - "step", - "action"); DebugOperationTracer debugOperationTracer = new DebugOperationTracer(new TraceOptions(false, false, true), false); ExecuteTransactionStep executeTransactionStep = @@ -173,18 +174,6 @@ protected ArrayNodeWrapper traceBlock( .orElse(emptyResult()); } - protected void generateTracesFromTransactionTraceAndBlock( - final Optional filterParameter, - final List transactionTraces, - final Block block, - final ArrayNodeWrapper resultArrayNode) { - transactionTraces.forEach( - transactionTrace -> - FlatTraceGenerator.generateFromTransactionTraceAndBlock( - protocolSchedule, transactionTrace, block) - .forEachOrdered(resultArrayNode::addPOJO)); - } - protected void generateRewardsFromBlock( final Optional maybeFilterParameter, final Block block, diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceCallMany.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceCallMany.java index 10d4018bce2..1601241db62 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceCallMany.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceCallMany.java @@ -160,7 +160,12 @@ private JsonNode getSingleCallResult( new DebugOperationTracer(buildTraceOptions(traceTypes), false); final Optional maybeSimulatorResult = transactionSimulator.processWithWorldUpdater( - callParameter, buildTransactionValidationParams(), tracer, header, worldUpdater); + callParameter, + Optional.empty(), + buildTransactionValidationParams(), + tracer, + header, + worldUpdater); LOG.trace("Executing {} call for transaction {}", traceTypeParameter, callParameter); if (maybeSimulatorResult.isEmpty()) { diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceFilter.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceFilter.java index 74bfec87c29..6be65149cb9 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceFilter.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceFilter.java @@ -23,7 +23,6 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.BlockParameter; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.FilterParameter; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.JsonRpcParameter.JsonRpcParameterException; -import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.BlockTracer; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.Tracer; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.TransactionTrace; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcErrorResponse; @@ -45,7 +44,7 @@ import org.hyperledger.besu.ethereum.vm.DebugOperationTracer; import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; -import org.hyperledger.besu.metrics.prometheus.PrometheusMetricsSystem; +import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.plugin.services.metrics.Counter; import org.hyperledger.besu.plugin.services.metrics.LabelledMetric; import org.hyperledger.besu.services.pipeline.Pipeline; @@ -58,7 +57,6 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.function.Function; -import java.util.function.Supplier; import java.util.stream.Stream; import javax.annotation.Nonnull; @@ -68,17 +66,24 @@ import org.slf4j.LoggerFactory; public class TraceFilter extends TraceBlock { - private static final Logger LOG = LoggerFactory.getLogger(TraceFilter.class); private final Long maxRange; + private final LabelledMetric outputCounter; public TraceFilter( - final Supplier blockTracerSupplier, final ProtocolSchedule protocolSchedule, final BlockchainQueries blockchainQueries, - final Long maxRange) { - super(protocolSchedule, blockchainQueries); + final Long maxRange, + final MetricsSystem metricsSystem) { + super(protocolSchedule, blockchainQueries, metricsSystem); this.maxRange = maxRange; + this.outputCounter = + metricsSystem.createLabelledCounter( + BesuMetricCategory.BLOCKCHAIN, + "transactions_tracefilter_pipeline_processed_total", + "Number of transactions processed for trace_filter", + "step", + "action"); } @Override @@ -157,15 +162,6 @@ private JsonRpcResponse traceFilterWithPipeline( final MainnetTransactionProcessor transactionProcessor = protocolSpec.getTransactionProcessor(); final ChainUpdater chainUpdater = new ChainUpdater(traceableState); - final LabelledMetric outputCounter = - new PrometheusMetricsSystem( - BesuMetricCategory.DEFAULT_METRIC_CATEGORIES, false) - .createLabelledCounter( - BesuMetricCategory.BLOCKCHAIN, - "transactions_tracefilter_pipeline_processed_total", - "Number of transactions processed for trace_filter", - "step", - "action"); DebugOperationTracer debugOperationTracer = new DebugOperationTracer(new TraceOptions(false, false, true), false); diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceReplayBlockTransactions.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceReplayBlockTransactions.java index 202d2ddbbbc..51eeebb9592 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceReplayBlockTransactions.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceReplayBlockTransactions.java @@ -39,7 +39,7 @@ import org.hyperledger.besu.ethereum.vm.DebugOperationTracer; import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; -import org.hyperledger.besu.metrics.prometheus.PrometheusMetricsSystem; +import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.plugin.services.metrics.Counter; import org.hyperledger.besu.plugin.services.metrics.LabelledMetric; import org.hyperledger.besu.services.pipeline.Pipeline; @@ -57,13 +57,23 @@ public class TraceReplayBlockTransactions extends AbstractBlockParameterMethod { private static final Logger LOG = LoggerFactory.getLogger(TraceReplayBlockTransactions.class); - private final ProtocolSchedule protocolSchedule; private static final ObjectMapper MAPPER = new ObjectMapper(); + private final ProtocolSchedule protocolSchedule; + private final LabelledMetric outputCounter; public TraceReplayBlockTransactions( - final ProtocolSchedule protocolSchedule, final BlockchainQueries queries) { + final ProtocolSchedule protocolSchedule, + final BlockchainQueries queries, + final MetricsSystem metricsSystem) { super(queries); this.protocolSchedule = protocolSchedule; + this.outputCounter = + metricsSystem.createLabelledCounter( + BesuMetricCategory.BLOCKCHAIN, + "transactions_tracereplayblock_pipeline_processed_total", + "Number of transactions processed for each block", + "step", + "action"); } @Override @@ -131,14 +141,7 @@ private ArrayNode traceBlock(final Block block, final TraceTypeParameter traceTy final ChainUpdater chainUpdater = new ChainUpdater(traceableState); final TransactionSource transactionSource = new TransactionSource(block); - final LabelledMetric outputCounter = - new PrometheusMetricsSystem(BesuMetricCategory.DEFAULT_METRIC_CATEGORIES, false) - .createLabelledCounter( - BesuMetricCategory.BLOCKCHAIN, - "transactions_tracereplayblock_pipeline_processed_total", - "Number of transactions processed for each block", - "step", - "action"); + final DebugOperationTracer debugOperationTracer = new DebugOperationTracer(new TraceOptions(false, false, true), false); final ExecuteTransactionStep executeTransactionStep = diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/JsonRpcMethodsFactory.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/JsonRpcMethodsFactory.java index 7be34f02290..2e68576fca7 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/JsonRpcMethodsFactory.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/JsonRpcMethodsFactory.java @@ -151,7 +151,11 @@ public Map methods( blockchainQueries, protocolSchedule, transactionPool, privacyParameters), new Web3JsonRpcMethods(clientNodeName), new TraceJsonRpcMethods( - blockchainQueries, protocolSchedule, protocolContext, apiConfiguration), + blockchainQueries, + protocolSchedule, + protocolContext, + apiConfiguration, + metricsSystem), new TxPoolJsonRpcMethods(transactionPool), new PluginsJsonRpcMethods(namedPlugins)); diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/TraceJsonRpcMethods.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/TraceJsonRpcMethods.java index 6a36bebfcd5..4638e90a625 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/TraceJsonRpcMethods.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/TraceJsonRpcMethods.java @@ -31,6 +31,7 @@ import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.transaction.TransactionSimulator; +import org.hyperledger.besu.plugin.services.MetricsSystem; import java.util.Map; @@ -38,19 +39,21 @@ public class TraceJsonRpcMethods extends ApiGroupJsonRpcMethods { private final BlockchainQueries blockchainQueries; private final ProtocolSchedule protocolSchedule; - private final ApiConfiguration apiConfiguration; private final ProtocolContext protocolContext; + private final MetricsSystem metricsSystem; TraceJsonRpcMethods( final BlockchainQueries blockchainQueries, final ProtocolSchedule protocolSchedule, final ProtocolContext protocolContext, - final ApiConfiguration apiConfiguration) { + final ApiConfiguration apiConfiguration, + final MetricsSystem metricsSystem) { this.blockchainQueries = blockchainQueries; this.protocolSchedule = protocolSchedule; this.protocolContext = protocolContext; this.apiConfiguration = apiConfiguration; + this.metricsSystem = metricsSystem; } @Override @@ -63,16 +66,16 @@ protected Map create() { final BlockReplay blockReplay = new BlockReplay(protocolSchedule, protocolContext, blockchainQueries.getBlockchain()); return mapOf( - new TraceReplayBlockTransactions(protocolSchedule, blockchainQueries), + new TraceReplayBlockTransactions(protocolSchedule, blockchainQueries, metricsSystem), new TraceFilter( - () -> new BlockTracer(blockReplay), protocolSchedule, blockchainQueries, - apiConfiguration.getMaxTraceFilterRange()), + apiConfiguration.getMaxTraceFilterRange(), + metricsSystem), new TraceGet(() -> new BlockTracer(blockReplay), blockchainQueries, protocolSchedule), new TraceTransaction( () -> new BlockTracer(blockReplay), protocolSchedule, blockchainQueries), - new TraceBlock(protocolSchedule, blockchainQueries), + new TraceBlock(protocolSchedule, blockchainQueries, metricsSystem), new TraceCall( blockchainQueries, protocolSchedule, diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/websocket/WebSocketConfiguration.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/websocket/WebSocketConfiguration.java index e27f7f21cec..42905b96a70 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/websocket/WebSocketConfiguration.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/websocket/WebSocketConfiguration.java @@ -25,6 +25,7 @@ import java.util.Collections; import java.util.List; import java.util.Objects; +import java.util.Optional; import com.google.common.base.MoreObjects; @@ -49,6 +50,21 @@ public class WebSocketConfiguration { private int maxActiveConnections; private int maxFrameSize; + private boolean isSslEnabled = false; + private Optional keyStorePath = Optional.empty(); + private Optional keyStorePassword = Optional.empty(); + private Optional keyStoreType = Optional.of("JKS"); // Default to JKS + + private boolean clientAuthEnabled = false; + private Optional trustStorePath = Optional.empty(); + private Optional trustStorePassword = Optional.empty(); + private Optional trustStoreType = Optional.of("JKS"); // Default to JKS + + // For PEM format + private Optional keyPath = Optional.empty(); + private Optional certPath = Optional.empty(); + private Optional trustCertPath = Optional.empty(); + public static WebSocketConfiguration createDefault() { final WebSocketConfiguration config = new WebSocketConfiguration(); config.setEnabled(false); @@ -159,6 +175,102 @@ public void setTimeoutSec(final long timeoutSec) { this.timeoutSec = timeoutSec; } + public boolean isSslEnabled() { + return isSslEnabled; + } + + public void setSslEnabled(final boolean isSslEnabled) { + this.isSslEnabled = isSslEnabled; + } + + public Optional getKeyStorePath() { + return keyStorePath; + } + + public void setKeyStorePath(final String keyStorePath) { + this.keyStorePath = Optional.ofNullable(keyStorePath); + } + + public Optional getKeyStorePassword() { + return keyStorePassword; + } + + public void setKeyStorePassword(final String keyStorePassword) { + this.keyStorePassword = Optional.ofNullable(keyStorePassword); + } + + // Keystore Type + public Optional getKeyStoreType() { + return keyStoreType; + } + + public void setKeyStoreType(final String keyStoreType) { + this.keyStoreType = Optional.ofNullable(keyStoreType); + } + + // Key Path (for PEM) + public Optional getKeyPath() { + return keyPath; + } + + public void setKeyPath(final String keyPath) { + this.keyPath = Optional.ofNullable(keyPath); + } + + // Cert Path (for PEM) + public Optional getCertPath() { + return certPath; + } + + public void setCertPath(final String certPath) { + this.certPath = Optional.ofNullable(certPath); + } + + // Client Authentication Enabled + public boolean isClientAuthEnabled() { + return clientAuthEnabled; + } + + public void setClientAuthEnabled(final boolean clientAuthEnabled) { + this.clientAuthEnabled = clientAuthEnabled; + } + + // Truststore Path + public Optional getTrustStorePath() { + return trustStorePath; + } + + public void setTrustStorePath(final String trustStorePath) { + this.trustStorePath = Optional.ofNullable(trustStorePath); + } + + // Truststore Password + public Optional getTrustStorePassword() { + return trustStorePassword; + } + + public void setTrustStorePassword(final String trustStorePassword) { + this.trustStorePassword = Optional.ofNullable(trustStorePassword); + } + + // Truststore Type + public Optional getTrustStoreType() { + return trustStoreType; + } + + public void setTrustStoreType(final String trustStoreType) { + this.trustStoreType = Optional.ofNullable(trustStoreType); + } + + // Trust Cert Path (for PEM) + public Optional getTrustCertPath() { + return trustCertPath; + } + + public void setTrustCertPath(final String trustCertPath) { + this.trustCertPath = Optional.ofNullable(trustCertPath); + } + @Override public boolean equals(final Object o) { if (this == o) { diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/websocket/WebSocketService.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/websocket/WebSocketService.java index 59742836ba8..31c86b18d21 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/websocket/WebSocketService.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/websocket/WebSocketService.java @@ -25,6 +25,7 @@ import org.hyperledger.besu.plugin.services.MetricsSystem; import java.net.InetSocketAddress; +import java.util.Locale; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicInteger; @@ -34,6 +35,7 @@ import io.vertx.core.Handler; import io.vertx.core.Vertx; import io.vertx.core.buffer.Buffer; +import io.vertx.core.http.ClientAuth; import io.vertx.core.http.HttpConnection; import io.vertx.core.http.HttpServer; import io.vertx.core.http.HttpServerOptions; @@ -41,6 +43,9 @@ import io.vertx.core.http.HttpServerResponse; import io.vertx.core.http.ServerWebSocket; import io.vertx.core.net.HostAndPort; +import io.vertx.core.net.JksOptions; +import io.vertx.core.net.PemKeyCertOptions; +import io.vertx.core.net.PemTrustOptions; import io.vertx.core.net.SocketAddress; import io.vertx.ext.web.Router; import io.vertx.ext.web.RoutingContext; @@ -103,18 +108,65 @@ public CompletableFuture start() { "Starting Websocket service on {}:{}", configuration.getHost(), configuration.getPort()); final CompletableFuture resultFuture = new CompletableFuture<>(); + HttpServerOptions serverOptions = + new HttpServerOptions() + .setHost(configuration.getHost()) + .setPort(configuration.getPort()) + .setHandle100ContinueAutomatically(true) + .setCompressionSupported(true) + .addWebSocketSubProtocol("undefined") + .setMaxWebSocketFrameSize(configuration.getMaxFrameSize()) + .setMaxWebSocketMessageSize(configuration.getMaxFrameSize() * 4) + .setRegisterWebSocketWriteHandlers(true); + + // Check if SSL/TLS is enabled in the configuration + if (configuration.isSslEnabled()) { + serverOptions.setSsl(true); + + String keystorePath = configuration.getKeyStorePath().orElse(null); + String keystorePassword = configuration.getKeyStorePassword().orElse(null); + String keyPath = configuration.getKeyPath().orElse(null); + String certPath = configuration.getCertPath().orElse(null); + + String keystoreType = configuration.getKeyStoreType().orElse("JKS"); + + switch (keystoreType.toUpperCase(Locale.getDefault())) { + case "PEM": + serverOptions.setKeyCertOptions( + new PemKeyCertOptions().setKeyPath(keyPath).setCertPath(certPath)); + break; + case "JKS": + default: + serverOptions.setKeyCertOptions( + new JksOptions().setPath(keystorePath).setPassword(keystorePassword)); + break; + } + } + + // Set up truststore for client authentication (mTLS) + if (configuration.isClientAuthEnabled()) { + serverOptions.setClientAuth(ClientAuth.REQUIRED); + + String truststorePath = configuration.getTrustStorePath().orElse(null); + String truststorePassword = configuration.getTrustStorePassword().orElse(""); + String truststoreType = configuration.getTrustStoreType().orElse("JKS"); + String trustCertPath = configuration.getTrustCertPath().orElse(null); + + switch (truststoreType.toUpperCase(Locale.getDefault())) { + case "PEM": + serverOptions.setTrustOptions(new PemTrustOptions().addCertPath(trustCertPath)); + break; + case "JKS": + default: + serverOptions.setTrustOptions( + new JksOptions().setPath(truststorePath).setPassword(truststorePassword)); + break; + } + } + httpServer = vertx - .createHttpServer( - new HttpServerOptions() - .setHost(configuration.getHost()) - .setPort(configuration.getPort()) - .setHandle100ContinueAutomatically(true) - .setCompressionSupported(true) - .addWebSocketSubProtocol("undefined") - .setMaxWebSocketFrameSize(configuration.getMaxFrameSize()) - .setMaxWebSocketMessageSize(configuration.getMaxFrameSize() * 4) - .setRegisterWebSocketWriteHandlers(true)) + .createHttpServer(serverOptions) .webSocketHandler(websocketHandler()) .connectionHandler(connectionHandler()) .requestHandler(httpHandler()) diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/graphql/AbstractEthGraphQLHttpServiceTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/graphql/AbstractEthGraphQLHttpServiceTest.java index 2b55b42ba2d..08cfc3d7bd4 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/graphql/AbstractEthGraphQLHttpServiceTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/graphql/AbstractEthGraphQLHttpServiceTest.java @@ -19,6 +19,7 @@ import org.hyperledger.besu.datatypes.TransactionType; import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.ConsensusContext; import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.api.ImmutableApiConfiguration; import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; @@ -110,7 +111,10 @@ public void setupTest() throws Exception { final MutableBlockchain blockchain = blockchainSetupUtil.getBlockchain(); ProtocolContext context = new ProtocolContext( - blockchain, blockchainSetupUtil.getWorldArchive(), null, new BadBlockManager()); + blockchain, + blockchainSetupUtil.getWorldArchive(), + mock(ConsensusContext.class), + new BadBlockManager()); final BlockchainQueries blockchainQueries = new BlockchainQueries( blockchainSetupUtil.getProtocolSchedule(), diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthCallTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthCallTest.java index 8b8ff67cd81..1ba9839b66d 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthCallTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthCallTest.java @@ -51,6 +51,8 @@ import org.hyperledger.besu.ethereum.transaction.PreCloseStateHandler; import org.hyperledger.besu.ethereum.transaction.TransactionSimulator; import org.hyperledger.besu.ethereum.transaction.TransactionSimulatorResult; +import org.hyperledger.besu.ethereum.util.AccountOverride; +import org.hyperledger.besu.ethereum.util.AccountOverrideMap; import java.util.Optional; @@ -92,6 +94,33 @@ public void shouldReturnCorrectMethodName() { assertThat(method.getName()).isEqualTo("eth_call"); } + @Test + public void noAccountOverrides() { + final JsonRpcRequestContext request = ethCallRequest(callParameter(), "latest"); + Optional overrideMap = method.getAddressAccountOverrideMap(request); + assertThat(overrideMap.isPresent()).isFalse(); + } + + @Test + public void someAccountOverrides() { + AccountOverrideMap expectedOverrides = new AccountOverrideMap(); + AccountOverride override = new AccountOverride.Builder().withNonce(88L).build(); + final Address address = Address.fromHexString("0xd9c9cd5f6779558b6e0ed4e6acf6b1947e7fa1f3"); + expectedOverrides.put(address, override); + + final JsonRpcRequestContext request = + ethCallRequestWithStateOverrides(callParameter(), "latest", expectedOverrides); + + Optional maybeOverrideMap = method.getAddressAccountOverrideMap(request); + assertThat(maybeOverrideMap.isPresent()).isTrue(); + AccountOverrideMap overrideMap = maybeOverrideMap.get(); + assertThat(overrideMap.keySet()).hasSize(1); + assertThat(overrideMap.values()).hasSize(1); + + assertThat(overrideMap).containsKey(address); + assertThat(overrideMap).containsValue(override); + } + @Test public void shouldReturnInternalErrorWhenProcessorReturnsEmpty() { final JsonRpcRequestContext request = ethCallRequest(callParameter(), "latest"); @@ -99,7 +128,7 @@ public void shouldReturnInternalErrorWhenProcessorReturnsEmpty() { when(blockchainQueries.getBlockchain()).thenReturn(blockchain); when(blockchain.getChainHead()).thenReturn(chainHead); - when(transactionSimulator.process(any(), any(), any(), any(), any())) + when(transactionSimulator.process(any(), any(), any(), any(), any(), any())) .thenReturn(Optional.empty()); final BlockHeader blockHeader = mock(BlockHeader.class); @@ -109,7 +138,7 @@ public void shouldReturnInternalErrorWhenProcessorReturnsEmpty() { final JsonRpcResponse response = method.response(request); assertThat(response).usingRecursiveComparison().isEqualTo(expectedResponse); - verify(transactionSimulator).process(any(), any(), any(), any(), any()); + verify(transactionSimulator).process(any(), any(), any(), any(), any(), any()); } @Test @@ -130,12 +159,13 @@ public void shouldAcceptRequestWhenMissingOptionalFields() { when(result.isSuccessful()).thenReturn(true); when(result.getValidationResult()).thenReturn(ValidationResult.valid()); when(result.getOutput()).thenReturn(Bytes.of()); - verify(transactionSimulator).process(any(), any(), any(), mapperCaptor.capture(), any()); + verify(transactionSimulator) + .process( + eq(callParameter), eq(Optional.empty()), any(), any(), mapperCaptor.capture(), any()); assertThat(mapperCaptor.getValue().apply(mock(MutableWorldState.class), Optional.of(result))) .isEqualTo(Optional.of(expectedResponse)); assertThat(response).usingRecursiveComparison().isEqualTo(expectedResponse); - verify(transactionSimulator).process(eq(callParameter), any(), any(), any(), any()); } @Test @@ -158,7 +188,8 @@ public void shouldReturnExecutionResultWhenExecutionIsSuccessful() { when(result.getValidationResult()).thenReturn(ValidationResult.valid()); when(result.getOutput()).thenReturn(Bytes.of(1)); verify(transactionSimulator) - .process(eq(callParameter()), any(), any(), mapperCaptor.capture(), any()); + .process( + eq(callParameter()), eq(Optional.empty()), any(), any(), mapperCaptor.capture(), any()); assertThat(mapperCaptor.getValue().apply(mock(MutableWorldState.class), Optional.of(result))) .isEqualTo(Optional.of(expectedResponse)); @@ -196,7 +227,8 @@ public void shouldReturnBasicExecutionRevertErrorWithoutReason() { when(result.isSuccessful()).thenReturn(false); when(result.getValidationResult()).thenReturn(ValidationResult.valid()); when(result.result()).thenReturn(processingResult); - verify(transactionSimulator).process(any(), any(), any(), mapperCaptor.capture(), any()); + verify(transactionSimulator) + .process(any(), eq(Optional.empty()), any(), any(), mapperCaptor.capture(), any()); assertThat(mapperCaptor.getValue().apply(mock(MutableWorldState.class), Optional.of(result))) .isEqualTo(Optional.of(expectedResponse)); @@ -235,7 +267,8 @@ public void shouldReturnExecutionRevertErrorWithABIParseError() { when(result.isSuccessful()).thenReturn(false); when(result.getValidationResult()).thenReturn(ValidationResult.valid()); when(result.result()).thenReturn(processingResult); - verify(transactionSimulator).process(any(), any(), any(), mapperCaptor.capture(), any()); + verify(transactionSimulator) + .process(any(), eq(Optional.empty()), any(), any(), mapperCaptor.capture(), any()); assertThat(mapperCaptor.getValue().apply(mock(MutableWorldState.class), Optional.of(result))) .isEqualTo(Optional.of(expectedResponse)); @@ -277,7 +310,8 @@ public void shouldReturnExecutionRevertErrorWithParsedABI() { when(result.getValidationResult()).thenReturn(ValidationResult.valid()); when(result.result()).thenReturn(processingResult); - verify(transactionSimulator).process(any(), any(), any(), mapperCaptor.capture(), any()); + verify(transactionSimulator) + .process(any(), eq(Optional.empty()), any(), any(), mapperCaptor.capture(), any()); assertThat(mapperCaptor.getValue().apply(mock(MutableWorldState.class), Optional.of(result))) .isEqualTo(Optional.of(expectedResponse)); @@ -291,7 +325,7 @@ public void shouldUseCorrectBlockNumberWhenLatest() { final JsonRpcRequestContext request = ethCallRequest(callParameter(), "latest"); when(blockchainQueries.getBlockchain()).thenReturn(blockchain); when(blockchain.getChainHead()).thenReturn(chainHead); - when(transactionSimulator.process(any(), any(), any(), any(), any())) + when(transactionSimulator.process(any(), eq(Optional.empty()), any(), any(), any(), any())) .thenReturn(Optional.empty()); final BlockHeader blockHeader = mock(BlockHeader.class); @@ -301,7 +335,7 @@ public void shouldUseCorrectBlockNumberWhenLatest() { method.response(request); verify(blockchainQueries, atLeastOnce()).getBlockchain(); - verify(transactionSimulator).process(any(), any(), any(), any(), any()); + verify(transactionSimulator).process(any(), eq(Optional.empty()), any(), any(), any(), any()); } @Test @@ -315,7 +349,7 @@ public void shouldUseCorrectBlockNumberWhenEarliest() { method.response(request); verify(blockchainQueries).getBlockHeaderByHash(eq(Hash.ZERO)); - verify(transactionSimulator).process(any(), any(), any(), any(), any()); + verify(transactionSimulator).process(any(), eq(Optional.empty()), any(), any(), any(), any()); } @Test @@ -323,13 +357,13 @@ public void shouldUseCorrectBlockNumberWhenSafe() { final JsonRpcRequestContext request = ethCallRequest(callParameter(), "safe"); when(blockchainQueries.getBlockHeaderByHash(Hash.ZERO)).thenReturn(Optional.of(blockHeader)); when(blockchainQueries.safeBlockHeader()).thenReturn(Optional.of(blockHeader)); - when(transactionSimulator.process(any(), any(), any(), any(), any())) + when(transactionSimulator.process(any(), eq(Optional.empty()), any(), any(), any(), any())) .thenReturn(Optional.empty()); method.response(request); verify(blockchainQueries).getBlockHeaderByHash(Hash.ZERO); verify(blockchainQueries).safeBlockHeader(); - verify(transactionSimulator).process(any(), any(), any(), any(), any()); + verify(transactionSimulator).process(any(), eq(Optional.empty()), any(), any(), any(), any()); } @Test @@ -337,13 +371,13 @@ public void shouldUseCorrectBlockNumberWhenFinalized() { final JsonRpcRequestContext request = ethCallRequest(callParameter(), "finalized"); when(blockchainQueries.getBlockHeaderByHash(Hash.ZERO)).thenReturn(Optional.of(blockHeader)); when(blockchainQueries.finalizedBlockHeader()).thenReturn(Optional.of(blockHeader)); - when(transactionSimulator.process(any(), any(), any(), any(), any())) + when(transactionSimulator.process(any(), eq(Optional.empty()), any(), any(), any(), any())) .thenReturn(Optional.empty()); method.response(request); verify(blockchainQueries).getBlockHeaderByHash(Hash.ZERO); verify(blockchainQueries).finalizedBlockHeader(); - verify(transactionSimulator).process(any(), any(), any(), any(), any()); + verify(transactionSimulator).process(any(), eq(Optional.empty()), any(), any(), any(), any()); } @Test @@ -353,13 +387,13 @@ public void shouldUseCorrectBlockNumberWhenSpecified() { when(blockchainQueries.getBlockHashByNumber(anyLong())).thenReturn(Optional.of(Hash.ZERO)); when(blockchainQueries.getBlockHeaderByHash(Hash.ZERO)) .thenReturn(Optional.of(mock(BlockHeader.class))); - when(transactionSimulator.process(any(), any(), any(), any(), any())) + when(transactionSimulator.process(any(), eq(Optional.empty()), any(), any(), any(), any())) .thenReturn(Optional.empty()); method.response(request); verify(blockchainQueries).getBlockHeaderByHash(eq(Hash.ZERO)); - verify(transactionSimulator).process(any(), any(), any(), any(), any()); + verify(transactionSimulator).process(any(), eq(Optional.empty()), any(), any(), any(), any()); } @Test @@ -431,7 +465,7 @@ private void internalAutoSelectIsAllowedExceedingBalance( .build(); verify(transactionSimulator) - .process(any(), eq(transactionValidationParams), any(), any(), any()); + .process(any(), eq(Optional.empty()), eq(transactionValidationParams), any(), any(), any()); } private JsonCallParameter callParameter() { @@ -458,8 +492,17 @@ private JsonRpcRequestContext ethCallRequest( new JsonRpcRequest("2.0", "eth_call", new Object[] {callParameter, blockNumberInHex})); } + private JsonRpcRequestContext ethCallRequestWithStateOverrides( + final CallParameter callParameter, + final String blockNumberInHex, + final AccountOverrideMap overrides) { + return new JsonRpcRequestContext( + new JsonRpcRequest( + "2.0", "eth_call", new Object[] {callParameter, blockNumberInHex, overrides})); + } + private void mockTransactionProcessorSuccessResult(final JsonRpcResponse jsonRpcResponse) { - when(transactionSimulator.process(any(), any(), any(), any(), any())) + when(transactionSimulator.process(any(), eq(Optional.empty()), any(), any(), any(), any())) .thenReturn(Optional.of(jsonRpcResponse)); } } diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceFilterTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceFilterTest.java index c5c03f6b114..8f64d2cb0bd 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceFilterTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceFilterTest.java @@ -26,6 +26,7 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.RpcErrorType; import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; +import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import java.util.function.Supplier; @@ -69,7 +70,8 @@ public void shouldFailIfParamsExceedMaxRange( new JsonRpcRequest("2.0", "trace_filter", new Object[] {filterParameter})); method = - new TraceFilter(blockTracerSupplier, protocolSchedule, blockchainQueries, maxFilterRange); + new TraceFilter( + protocolSchedule, blockchainQueries, maxFilterRange, new NoOpMetricsSystem()); final JsonRpcResponse response = method.response(request); assertThat(response).isInstanceOf(JsonRpcErrorResponse.class); diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/websocket/WebSocketServiceTLSTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/websocket/WebSocketServiceTLSTest.java new file mode 100644 index 00000000000..9a227dc4326 --- /dev/null +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/websocket/WebSocketServiceTLSTest.java @@ -0,0 +1,444 @@ +/* + * Copyright contributors to Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.api.jsonrpc.websocket; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; + +import org.hyperledger.besu.ethereum.api.handlers.TimeoutOptions; +import org.hyperledger.besu.ethereum.api.jsonrpc.execution.BaseJsonRpcProcessor; +import org.hyperledger.besu.ethereum.api.jsonrpc.execution.JsonRpcExecutor; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.JsonRpcMethod; +import org.hyperledger.besu.ethereum.api.jsonrpc.websocket.methods.WebSocketMethodsFactory; +import org.hyperledger.besu.ethereum.api.jsonrpc.websocket.subscription.SubscriptionManager; +import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; +import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; + +import java.io.File; +import java.io.FileOutputStream; +import java.security.KeyStore; +import java.util.Base64; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import io.netty.handler.ssl.util.SelfSignedCertificate; +import io.vertx.core.Vertx; +import io.vertx.core.http.WebSocketClient; +import io.vertx.core.http.WebSocketClientOptions; +import io.vertx.core.net.JksOptions; +import io.vertx.core.net.PemTrustOptions; +import io.vertx.junit5.VertxExtension; +import io.vertx.junit5.VertxTestContext; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +@ExtendWith(VertxExtension.class) +public class WebSocketServiceTLSTest { + + private Vertx vertx; + private WebSocketConfiguration config; + private WebSocketMessageHandler webSocketMessageHandlerSpy; + + @BeforeEach + public void setUp() { + vertx = Vertx.vertx(); + config = WebSocketConfiguration.createDefault(); + Map websocketMethods; + config.setPort(0); // Use ephemeral port + config.setHost("localhost"); + websocketMethods = + new WebSocketMethodsFactory( + new SubscriptionManager(new NoOpMetricsSystem()), new HashMap<>()) + .methods(); + webSocketMessageHandlerSpy = + spy( + new WebSocketMessageHandler( + vertx, + new JsonRpcExecutor(new BaseJsonRpcProcessor(), websocketMethods), + mock(EthScheduler.class), + TimeoutOptions.defaultOptions().getTimeoutSeconds())); + } + + @Test + public void shouldAcceptSecureWebSocketConnection(final VertxTestContext testContext) + throws Throwable { + // Generate a self-signed certificate + SelfSignedCertificate ssc = new SelfSignedCertificate(); + + // Create a temporary keystore file + File keystoreFile = File.createTempFile("keystore", ".jks"); + keystoreFile.deleteOnExit(); + + // Create a PKCS12 keystore and load the self-signed certificate + KeyStore keyStore = KeyStore.getInstance("JKS"); + keyStore.load(null, null); + keyStore.setKeyEntry( + "alias", + ssc.key(), + "password".toCharArray(), + new java.security.cert.Certificate[] {ssc.cert()}); + + // Save the keystore to the temporary file + try (FileOutputStream fos = new FileOutputStream(keystoreFile)) { + keyStore.store(fos, "password".toCharArray()); + } + + // Configure WebSocket with SSL enabled + config.setSslEnabled(true); + config.setKeyStorePath(keystoreFile.getAbsolutePath()); + config.setKeyStorePassword("password"); + config.setKeyStoreType("JKS"); + + // Create and start WebSocketService + WebSocketService webSocketService = + new WebSocketService(vertx, config, webSocketMessageHandlerSpy, new NoOpMetricsSystem()); + webSocketService.start().join(); + + // Get the actual port + int port = webSocketService.socketAddress().getPort(); + + // Create a temporary truststore file + File truststoreFile = File.createTempFile("truststore", ".jks"); + truststoreFile.deleteOnExit(); + + // Create a PKCS12 truststore and load the server's certificate + KeyStore trustStore = KeyStore.getInstance("JKS"); + trustStore.load(null, null); + trustStore.setCertificateEntry("alias", ssc.cert()); + + // Save the truststore to the temporary file + try (FileOutputStream fos = new FileOutputStream(truststoreFile)) { + trustStore.store(fos, "password".toCharArray()); + } + + // Configure the HTTP client with the truststore + WebSocketClientOptions clientOptions = + new WebSocketClientOptions() + .setSsl(true) + .setTrustOptions( + new JksOptions().setPath(truststoreFile.getAbsolutePath()).setPassword("password")) + .setVerifyHost(true); + + WebSocketClient webSocketClient = vertx.createWebSocketClient(clientOptions); + webSocketClient + .connect(port, "localhost", "/") + .onSuccess( + ws -> { + assertThat(ws.isSsl()).isTrue(); + ws.close(); + testContext.completeNow(); + }) + .onFailure(testContext::failNow); + + assertThat(testContext.awaitCompletion(5, TimeUnit.SECONDS)).isTrue(); + if (testContext.failed()) { + throw testContext.causeOfFailure(); + } + + // Stop the WebSocketService after the test + webSocketService.stop().join(); + } + + @Test + public void shouldAcceptSecureWebSocketConnectionPEM(final VertxTestContext testContext) + throws Throwable { + // Generate a self-signed certificate + SelfSignedCertificate ssc = new SelfSignedCertificate(); + + // Create temporary PEM files for the certificate and key + File certFile = File.createTempFile("cert", ".pem"); + certFile.deleteOnExit(); + File keyFile = File.createTempFile("key", ".pem"); + keyFile.deleteOnExit(); + + // Write the certificate and key to the PEM files + try (FileOutputStream certOut = new FileOutputStream(certFile); + FileOutputStream keyOut = new FileOutputStream(keyFile)) { + certOut.write("-----BEGIN CERTIFICATE-----\n".getBytes(UTF_8)); + certOut.write( + Base64.getMimeEncoder(64, "\n".getBytes(UTF_8)).encode(ssc.cert().getEncoded())); + certOut.write("\n-----END CERTIFICATE-----\n".getBytes(UTF_8)); + + keyOut.write("-----BEGIN PRIVATE KEY-----\n".getBytes(UTF_8)); + keyOut.write(Base64.getMimeEncoder(64, "\n".getBytes(UTF_8)).encode(ssc.key().getEncoded())); + keyOut.write("\n-----END PRIVATE KEY-----\n".getBytes(UTF_8)); + } + + // Configure WebSocket with SSL enabled using PEM files + config.setSslEnabled(true); + config.setKeyPath(keyFile.getAbsolutePath()); + config.setCertPath(certFile.getAbsolutePath()); + config.setKeyStoreType("PEM"); + + // Create and start WebSocketService + WebSocketService webSocketService = + new WebSocketService(vertx, config, webSocketMessageHandlerSpy, new NoOpMetricsSystem()); + webSocketService.start().join(); + + // Get the actual port + int port = webSocketService.socketAddress().getPort(); + + // Create a temporary PEM file for the trust store + File trustCertFile = File.createTempFile("trust-cert", ".pem"); + trustCertFile.deleteOnExit(); + + // Write the server's certificate to the PEM file + try (FileOutputStream trustCertOut = new FileOutputStream(trustCertFile)) { + trustCertOut.write("-----BEGIN CERTIFICATE-----\n".getBytes(UTF_8)); + trustCertOut.write( + Base64.getMimeEncoder(64, "\n".getBytes(UTF_8)).encode(ssc.cert().getEncoded())); + trustCertOut.write("\n-----END CERTIFICATE-----\n".getBytes(UTF_8)); + } + + // Configure the HTTP client with the trust store using PEM files + WebSocketClientOptions clientOptions = + new WebSocketClientOptions() + .setSsl(true) + .setTrustOptions(new PemTrustOptions().addCertPath(trustCertFile.getAbsolutePath())) + .setVerifyHost(true); + + WebSocketClient webSocketClient = vertx.createWebSocketClient(clientOptions); + webSocketClient + .connect(port, "localhost", "/") + .onSuccess( + ws -> { + assertThat(ws.isSsl()).isTrue(); + ws.close(); + testContext.completeNow(); + }) + .onFailure(testContext::failNow); + + assertThat(testContext.awaitCompletion(5, TimeUnit.SECONDS)).isTrue(); + if (testContext.failed()) { + throw testContext.causeOfFailure(); + } + + // Stop the WebSocketService after the test + webSocketService.stop().join(); + } + + @Test + public void shouldFailConnectionWithWrongCertificateInTrustStore( + final VertxTestContext testContext) throws Throwable { + // Generate a self-signed certificate for the server + SelfSignedCertificate serverCert = new SelfSignedCertificate(); + + // Create a temporary keystore file for the server + File keystoreFile = File.createTempFile("keystore", ".p12"); + keystoreFile.deleteOnExit(); + + // Create a PKCS12 keystore and load the server's self-signed certificate + KeyStore keyStore = KeyStore.getInstance("PKCS12"); + keyStore.load(null, null); + keyStore.setKeyEntry( + "alias", + serverCert.key(), + "password".toCharArray(), + new java.security.cert.Certificate[] {serverCert.cert()}); + + // Save the keystore to the temporary file + try (FileOutputStream fos = new FileOutputStream(keystoreFile)) { + keyStore.store(fos, "password".toCharArray()); + } + + // Configure WebSocket with SSL enabled + config.setSslEnabled(true); + config.setKeyStorePath(keystoreFile.getAbsolutePath()); + config.setKeyStorePassword("password"); + config.setKeyStoreType("PKCS12"); + + // Create and start WebSocketService + WebSocketService webSocketService = + new WebSocketService(vertx, config, webSocketMessageHandlerSpy, new NoOpMetricsSystem()); + webSocketService.start().join(); + + // Get the actual port + int port = webSocketService.socketAddress().getPort(); + + // Generate a different self-signed certificate for the trust store + SelfSignedCertificate wrongCert = new SelfSignedCertificate(); + + // Create a temporary truststore file + File truststoreFile = File.createTempFile("truststore", ".p12"); + truststoreFile.deleteOnExit(); + + // Create a PKCS12 truststore and load the wrong certificate + KeyStore trustStore = KeyStore.getInstance("PKCS12"); + trustStore.load(null, null); + trustStore.setCertificateEntry("alias", wrongCert.cert()); + + // Save the truststore to the temporary file + try (FileOutputStream fos = new FileOutputStream(truststoreFile)) { + trustStore.store(fos, "password".toCharArray()); + } + + // Configure the HTTP client with the truststore containing the wrong certificate + WebSocketClientOptions clientOptions = + new WebSocketClientOptions() + .setSsl(true) + .setTrustOptions( + new JksOptions().setPath(truststoreFile.getAbsolutePath()).setPassword("password")) + .setVerifyHost(true); + + WebSocketClient webSocketClient = vertx.createWebSocketClient(clientOptions); + webSocketClient + .connect(port, "localhost", "/") + .onSuccess( + ws -> { + testContext.failNow(new AssertionError("Connection should have been rejected")); + }) + .onFailure( + throwable -> { + assertThat(throwable).isInstanceOf(Exception.class); + testContext.completeNow(); + }); + + assertThat(testContext.awaitCompletion(5, TimeUnit.SECONDS)).isTrue(); + if (testContext.failed()) { + throw testContext.causeOfFailure(); + } + + // Stop the WebSocketService after the test + webSocketService.stop().join(); + } + + @Test + public void shouldAuthenticateClient(final VertxTestContext testContext) throws Throwable { + // Generate a self-signed certificate for the server + SelfSignedCertificate serverCert = new SelfSignedCertificate(); + + // Generate a self-signed certificate for the client + SelfSignedCertificate clientCert = new SelfSignedCertificate(); + + // Create a temporary keystore file for the server + File serverKeystoreFile = File.createTempFile("keystore", ".p12"); + serverKeystoreFile.deleteOnExit(); + + // Create a temporary truststore file for the server + File serverTruststoreFile = File.createTempFile("truststore", ".p12"); + serverTruststoreFile.deleteOnExit(); + + // Create a temporary keystore file for the client + File clientKeystoreFile = File.createTempFile("client-keystore", ".p12"); + clientKeystoreFile.deleteOnExit(); + + // Create a temporary truststore file for the client + File clientTruststoreFile = File.createTempFile("truststore", ".p12"); + clientTruststoreFile.deleteOnExit(); + + // Create a PKCS12 keystore and load the server's self-signed certificate + KeyStore serverKeyStore = KeyStore.getInstance("PKCS12"); + serverKeyStore.load(null, null); + serverKeyStore.setKeyEntry( + "alias", + serverCert.key(), + "password".toCharArray(), + new java.security.cert.Certificate[] {serverCert.cert()}); + + // Save the keystore to the temporary file + try (FileOutputStream fos = new FileOutputStream(serverKeystoreFile)) { + serverKeyStore.store(fos, "password".toCharArray()); + } + + // Create a PKCS12 truststore and load the client's self-signed certificate + KeyStore serverTrustStore = KeyStore.getInstance("PKCS12"); + serverTrustStore.load(null, null); + serverTrustStore.setCertificateEntry("alias", clientCert.cert()); + + // Save the truststore to the temporary file + try (FileOutputStream fos = new FileOutputStream(serverTruststoreFile)) { + serverTrustStore.store(fos, "password".toCharArray()); + } + + // Create a PKCS12 keystore and load the client's self-signed certificate + KeyStore clientKeyStore = KeyStore.getInstance("PKCS12"); + clientKeyStore.load(null, null); + clientKeyStore.setKeyEntry( + "alias", + clientCert.key(), + "password".toCharArray(), + new java.security.cert.Certificate[] {clientCert.cert()}); + + // Save the client keystore to the temporary file + try (FileOutputStream fos = new FileOutputStream(clientKeystoreFile)) { + clientKeyStore.store(fos, "password".toCharArray()); + } + + // Create a PKCS12 truststore and load the server's self-signed certificate + KeyStore clientTrustStore = KeyStore.getInstance("PKCS12"); + clientTrustStore.load(null, null); + clientTrustStore.setCertificateEntry("alias", serverCert.cert()); + + // Save the truststore to the temporary file + try (FileOutputStream fos = new FileOutputStream(clientTruststoreFile)) { + clientTrustStore.store(fos, "password".toCharArray()); + } + + // Configure WebSocket with SSL and client authentication enabled + config.setSslEnabled(true); + config.setKeyStorePath(serverKeystoreFile.getAbsolutePath()); + config.setKeyStorePassword("password"); + config.setKeyStoreType("PKCS12"); + config.setClientAuthEnabled(true); + config.setTrustStorePath(serverTruststoreFile.getAbsolutePath()); + config.setTrustStorePassword("password"); + config.setTrustStoreType("PKCS12"); + + // Create and start WebSocketService + WebSocketService webSocketService = + new WebSocketService(vertx, config, webSocketMessageHandlerSpy, new NoOpMetricsSystem()); + webSocketService.start().join(); + + // Get the actual port + int port = webSocketService.socketAddress().getPort(); + + // Configure the HTTP client with the client certificate + WebSocketClientOptions clientOptions = + new WebSocketClientOptions() + .setSsl(true) + .setKeyStoreOptions( + new JksOptions() + .setPath(clientKeystoreFile.getAbsolutePath()) + .setPassword("password")) + .setTrustOptions( + new JksOptions() + .setPath(clientTruststoreFile.getAbsolutePath()) + .setPassword("password")) + .setVerifyHost(true); + + WebSocketClient webSocketClient = vertx.createWebSocketClient(clientOptions); + webSocketClient + .connect(port, "localhost", "/") + .onSuccess( + ws -> { + assertThat(ws.isSsl()).isTrue(); + ws.close(); + testContext.completeNow(); + }) + .onFailure(testContext::failNow); + + assertThat(testContext.awaitCompletion(5, TimeUnit.SECONDS)).isTrue(); + if (testContext.failed()) { + throw testContext.causeOfFailure(); + } + + // Stop the WebSocketService after the test + webSocketService.stop().join(); + } +} diff --git a/ethereum/api/src/test/resources/org/hyperledger/besu/ethereum/api/jsonrpc/eth/eth_call_overrides_empty.json b/ethereum/api/src/test/resources/org/hyperledger/besu/ethereum/api/jsonrpc/eth/eth_call_overrides_empty.json new file mode 100644 index 00000000000..6280773f3d0 --- /dev/null +++ b/ethereum/api/src/test/resources/org/hyperledger/besu/ethereum/api/jsonrpc/eth/eth_call_overrides_empty.json @@ -0,0 +1,24 @@ +{ + "request": { + "id": 3, + "jsonrpc": "2.0", + "method": "eth_call", + "params": [ + { + "to": "0x6295ee1b4f6dd65047762f924ecd367c17eabf8f", + "from": "a94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "data": "0x12a7b914" + }, + "latest", + { + "a94f5374fce5edbc8e2a8697c15331677e6ebf0b": {} + } + ] + }, + "response": { + "jsonrpc": "2.0", + "id": 3, + "result": "0x0000000000000000000000000000000000000000000000000000000000000001" + }, + "statusCode": 200 +} \ No newline at end of file diff --git a/ethereum/api/src/test/resources/org/hyperledger/besu/ethereum/api/jsonrpc/eth/eth_call_stateOverride.json b/ethereum/api/src/test/resources/org/hyperledger/besu/ethereum/api/jsonrpc/eth/eth_call_stateOverride.json new file mode 100644 index 00000000000..762202f96b2 --- /dev/null +++ b/ethereum/api/src/test/resources/org/hyperledger/besu/ethereum/api/jsonrpc/eth/eth_call_stateOverride.json @@ -0,0 +1,32 @@ +{ + "request": { + "id": 3, + "jsonrpc": "2.0", + "method": "eth_call", + "params": [ + { + "to": "0x6295ee1b4f6dd65047762f924ecd367c17eabf8f", + "from": "a94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "data": "0x12a7b914" + }, + "latest", + { + "a94f5374fce5edbc8e2a8697c15331677e6ebf0b": { + "balance": "0xde0b6b3a7640000", + "nonce": 88 + }, + "0xb9741079a300Cb3B8f324CdDB847c0d1d273a05E": { + "stateDiff": { + "0x1cf7945003fc5b59d2f6736f0704557aa805c4f2844084ccd1173b8d56946962": "0x000000000000000000000000000000000000000000000000000000110ed03bf7" + } + } + } + ] + }, + "response": { + "jsonrpc": "2.0", + "id": 3, + "result": "0x0000000000000000000000000000000000000000000000000000000000000001" + }, + "statusCode": 200 +} \ No newline at end of file diff --git a/ethereum/api/src/test/resources/org/hyperledger/besu/ethereum/api/jsonrpc/eth/eth_call_stateOverride_insufficientBalance.json b/ethereum/api/src/test/resources/org/hyperledger/besu/ethereum/api/jsonrpc/eth/eth_call_stateOverride_insufficientBalance.json new file mode 100644 index 00000000000..c3832c5689a --- /dev/null +++ b/ethereum/api/src/test/resources/org/hyperledger/besu/ethereum/api/jsonrpc/eth/eth_call_stateOverride_insufficientBalance.json @@ -0,0 +1,34 @@ +{ + "request": { + "id": 3, + "jsonrpc": "2.0", + "method": "eth_call", + "params": [ + { + "to": "0x6295ee1b4f6dd65047762f924ecd367c17eabf8f", + "from": "a94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "value": "0x000002" + }, + "latest", + { + "a94f5374fce5edbc8e2a8697c15331677e6ebf0b": { + "balance": "0x000001" + }, + "0xb9741079a300Cb3B8f324CdDB847c0d1d273a05E": { + "stateDiff": { + "0x1cf7945003fc5b59d2f6736f0704557aa805c4f2844084ccd1173b8d56946962": "0x000000000000000000000000000000000000000000000000000000110ed03bf7" + } + } + } + ] + }, + "response": { + "jsonrpc": "2.0", + "id": 3, + "error" : { + "code" : -32004, + "message" : "Upfront cost exceeds account balance" + } + }, + "statusCode": 200 +} \ No newline at end of file diff --git a/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/BlockMinerTest.java b/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/BlockMinerTest.java index e5b46925aad..5b893d11f24 100644 --- a/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/BlockMinerTest.java +++ b/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/BlockMinerTest.java @@ -22,6 +22,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import org.hyperledger.besu.ethereum.ConsensusContext; import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.blockcreation.BlockCreator.BlockCreationResult; import org.hyperledger.besu.ethereum.blockcreation.txselection.TransactionSelectionResults; @@ -58,7 +59,7 @@ public void blockCreatedIsAddedToBlockChain() throws InterruptedException { headerBuilder.buildHeader(), new BlockBody(Lists.newArrayList(), Lists.newArrayList())); final ProtocolContext protocolContext = - new ProtocolContext(null, null, null, new BadBlockManager()); + new ProtocolContext(null, null, mock(ConsensusContext.class), new BadBlockManager()); final PoWBlockCreator blockCreator = mock(PoWBlockCreator.class); final Function blockCreatorSupplier = @@ -102,7 +103,7 @@ public void failureToImportDoesNotTriggerObservers() throws InterruptedException headerBuilder.buildHeader(), new BlockBody(Lists.newArrayList(), Lists.newArrayList())); final ProtocolContext protocolContext = - new ProtocolContext(null, null, null, new BadBlockManager()); + new ProtocolContext(null, null, mock(ConsensusContext.class), new BadBlockManager()); final PoWBlockCreator blockCreator = mock(PoWBlockCreator.class); final Function blockCreatorSupplier = @@ -150,7 +151,7 @@ public void blockValidationFailureBeforeImportDoesNotImportBlock() throws Interr headerBuilder.buildHeader(), new BlockBody(Lists.newArrayList(), Lists.newArrayList())); final ProtocolContext protocolContext = - new ProtocolContext(null, null, null, new BadBlockManager()); + new ProtocolContext(null, null, mock(ConsensusContext.class), new BadBlockManager()); final PoWBlockCreator blockCreator = mock(PoWBlockCreator.class); final Function blockCreatorSupplier = diff --git a/ethereum/core/build.gradle b/ethereum/core/build.gradle index c5ab4462669..c0f04a76441 100644 --- a/ethereum/core/build.gradle +++ b/ethereum/core/build.gradle @@ -65,8 +65,6 @@ dependencies { implementation 'org.immutables:value-annotations' implementation 'tech.pegasys:jc-kzg-4844' - implementation 'io.prometheus:simpleclient_guava' - implementation 'org.xerial.snappy:snappy-java' annotationProcessor 'org.immutables:value' diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/ConsensusContextFactory.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/ConsensusContextFactory.java deleted file mode 100644 index a9381fa199f..00000000000 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/ConsensusContextFactory.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.besu.ethereum; - -import org.hyperledger.besu.ethereum.chain.Blockchain; -import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; -import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; - -/** The ConsensusContextFactory interface defines a method for creating a consensus context. */ -@FunctionalInterface -public interface ConsensusContextFactory { - - /** - * Creates a consensus context with the given blockchain, world state archive, and protocol - * schedule. - * - * @param blockchain the blockchain - * @param worldStateArchive the world state archive - * @param protocolSchedule the protocol schedule - * @return the created consensus context - */ - ConsensusContext create( - Blockchain blockchain, - WorldStateArchive worldStateArchive, - ProtocolSchedule protocolSchedule); -} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/MainnetBlockValidator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/MainnetBlockValidator.java index b218982c092..39f5cb0f66f 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/MainnetBlockValidator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/MainnetBlockValidator.java @@ -181,7 +181,7 @@ public BlockProcessingResult validateAndProcessBlock( Optional.of(new BlockProcessingOutputs(worldState, receipts, maybeRequests))); } } catch (MerkleTrieException ex) { - context.getSynchronizer().healWorldState(ex.getMaybeAddress(), ex.getLocation()); + context.getWorldStateArchive().heal(ex.getMaybeAddress(), ex.getLocation()); return new BlockProcessingResult(Optional.empty(), ex); } catch (StorageException ex) { var retval = new BlockProcessingResult(Optional.empty(), ex); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/ProtocolContext.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/ProtocolContext.java index 4ca40bc0a9d..33897c3e1ec 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/ProtocolContext.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/ProtocolContext.java @@ -16,8 +16,6 @@ import org.hyperledger.besu.ethereum.chain.BadBlockManager; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; -import org.hyperledger.besu.ethereum.core.Synchronizer; -import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import java.util.Optional; @@ -30,9 +28,8 @@ public class ProtocolContext { private final MutableBlockchain blockchain; private final WorldStateArchive worldStateArchive; - private final BadBlockManager badBlockManager; private final ConsensusContext consensusContext; - private Synchronizer synchronizer; + private final BadBlockManager badBlockManager; /** * Constructs a new ProtocolContext with the given blockchain, world state archive, consensus @@ -40,7 +37,7 @@ public class ProtocolContext { * * @param blockchain the blockchain of the protocol context * @param worldStateArchive the world state archive of the protocol context - * @param consensusContext the consensus context of the protocol context + * @param consensusContext the consensus context * @param badBlockManager the bad block manager of the protocol context */ public ProtocolContext( @@ -54,48 +51,6 @@ public ProtocolContext( this.badBlockManager = badBlockManager; } - /** - * Initializes a new ProtocolContext with the given blockchain, world state archive, protocol - * schedule, consensus context factory, and bad block manager. - * - * @param blockchain the blockchain of the protocol context - * @param worldStateArchive the world state archive of the protocol context - * @param protocolSchedule the protocol schedule of the protocol context - * @param consensusContextFactory the consensus context factory of the protocol context - * @param badBlockManager the bad block manager of the protocol context - * @return the initialized ProtocolContext - */ - public static ProtocolContext init( - final MutableBlockchain blockchain, - final WorldStateArchive worldStateArchive, - final ProtocolSchedule protocolSchedule, - final ConsensusContextFactory consensusContextFactory, - final BadBlockManager badBlockManager) { - return new ProtocolContext( - blockchain, - worldStateArchive, - consensusContextFactory.create(blockchain, worldStateArchive, protocolSchedule), - badBlockManager); - } - - /** - * Gets the synchronizer of the protocol context. - * - * @return the synchronizer of the protocol context - */ - public Synchronizer getSynchronizer() { - return synchronizer; - } - - /** - * Sets the synchronizer of the protocol context. - * - * @param synchronizer the synchronizer to set - */ - public void setSynchronizer(final Synchronizer synchronizer) { - this.synchronizer = synchronizer; - } - /** * Gets the blockchain of the protocol context. * diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/DefaultBlockchain.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/DefaultBlockchain.java index bce94a19153..f6e06f82d2e 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/DefaultBlockchain.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/DefaultBlockchain.java @@ -19,6 +19,7 @@ import static java.util.Collections.emptyList; import static java.util.stream.Collectors.joining; import static java.util.stream.Collectors.toList; +import static org.hyperledger.besu.metrics.BesuMetricCategory.BLOCKCHAIN; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.chain.BlockchainStorage.Updater; @@ -32,7 +33,6 @@ import org.hyperledger.besu.ethereum.core.TransactionReceipt; import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; -import org.hyperledger.besu.metrics.prometheus.PrometheusMetricsSystem; import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.plugin.services.metrics.Counter; import org.hyperledger.besu.util.InvalidConfigurationException; @@ -56,7 +56,6 @@ import com.google.common.cache.CacheBuilder; import com.google.common.collect.Lists; import com.google.common.collect.Streams; -import io.prometheus.client.guava.cache.CacheMetricsCollector; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -134,13 +133,12 @@ private DefaultBlockchain( totalDifficultyCache = Optional.of( CacheBuilder.newBuilder().recordStats().maximumSize(numberOfBlocksToCache).build()); - CacheMetricsCollector cacheMetrics = new CacheMetricsCollector(); - cacheMetrics.addCache("blockHeaders", blockHeadersCache.get()); - cacheMetrics.addCache("blockBodies", blockBodiesCache.get()); - cacheMetrics.addCache("transactionReceipts", transactionReceiptsCache.get()); - cacheMetrics.addCache("totalDifficulty", totalDifficultyCache.get()); - if (metricsSystem instanceof PrometheusMetricsSystem prometheusMetricsSystem) - prometheusMetricsSystem.addCollector(BesuMetricCategory.BLOCKCHAIN, () -> cacheMetrics); + metricsSystem.createGuavaCacheCollector(BLOCKCHAIN, "blockHeaders", blockHeadersCache.get()); + metricsSystem.createGuavaCacheCollector(BLOCKCHAIN, "blockBodies", blockBodiesCache.get()); + metricsSystem.createGuavaCacheCollector( + BLOCKCHAIN, "transactionReceipts", transactionReceiptsCache.get()); + metricsSystem.createGuavaCacheCollector( + BLOCKCHAIN, "totalDifficulty", totalDifficultyCache.get()); } else { blockHeadersCache = Optional.empty(); blockBodiesCache = Optional.empty(); @@ -155,11 +153,11 @@ private DefaultBlockchain( private void createCounters(final MetricsSystem metricsSystem) { gasUsedCounter = metricsSystem.createCounter( - BesuMetricCategory.BLOCKCHAIN, "chain_head_gas_used_counter", "Counter for Gas used"); + BLOCKCHAIN, "chain_head_gas_used_counter", "Counter for Gas used"); numberOfTransactionsCounter = metricsSystem.createCounter( - BesuMetricCategory.BLOCKCHAIN, + BLOCKCHAIN, "chain_head_transaction_count_counter", "Counter for the number of transactions"); } @@ -184,37 +182,37 @@ private void createGauges(final MetricsSystem metricsSystem) { this::getSafeBlockNumber); metricsSystem.createGauge( - BesuMetricCategory.BLOCKCHAIN, + BLOCKCHAIN, "difficulty_total", "Total difficulty of the chainhead", () -> this.getChainHead().getTotalDifficulty().toBigInteger().doubleValue()); metricsSystem.createLongGauge( - BesuMetricCategory.BLOCKCHAIN, + BLOCKCHAIN, "chain_head_timestamp", "Timestamp from the current chain head", () -> getChainHeadHeader().getTimestamp()); metricsSystem.createLongGauge( - BesuMetricCategory.BLOCKCHAIN, + BLOCKCHAIN, "chain_head_gas_used", "Gas used by the current chain head block", () -> getChainHeadHeader().getGasUsed()); metricsSystem.createLongGauge( - BesuMetricCategory.BLOCKCHAIN, + BLOCKCHAIN, "chain_head_gas_limit", "Block gas limit of the current chain head block", () -> getChainHeadHeader().getGasLimit()); metricsSystem.createIntegerGauge( - BesuMetricCategory.BLOCKCHAIN, + BLOCKCHAIN, "chain_head_transaction_count", "Number of transactions in the current chain head block", () -> chainHeadTransactionCount); metricsSystem.createIntegerGauge( - BesuMetricCategory.BLOCKCHAIN, + BLOCKCHAIN, "chain_head_ommer_count", "Number of ommers in the current chain head block", () -> chainHeadOmmerCount); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/transaction/TransactionSimulator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/transaction/TransactionSimulator.java index 321dc965ecd..b0a7fa43257 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/transaction/TransactionSimulator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/transaction/TransactionSimulator.java @@ -34,10 +34,13 @@ import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; import org.hyperledger.besu.ethereum.mainnet.TransactionValidationParams; import org.hyperledger.besu.ethereum.processing.TransactionProcessingResult; +import org.hyperledger.besu.ethereum.util.AccountOverride; +import org.hyperledger.besu.ethereum.util.AccountOverrideMap; import org.hyperledger.besu.ethereum.vm.CachingBlockHashLookup; import org.hyperledger.besu.ethereum.vm.DebugOperationTracer; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.evm.account.Account; +import org.hyperledger.besu.evm.account.MutableAccount; import org.hyperledger.besu.evm.tracing.OperationTracer; import org.hyperledger.besu.evm.worldstate.WorldUpdater; @@ -46,8 +49,10 @@ import java.util.function.Supplier; import javax.annotation.Nonnull; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Suppliers; import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.units.bigints.UInt256; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -152,6 +157,35 @@ public Optional process( final OperationTracer operationTracer, final PreCloseStateHandler preWorldStateCloseGuard, final BlockHeader header) { + return process( + callParams, + Optional.empty(), + transactionValidationParams, + operationTracer, + preWorldStateCloseGuard, + header); + } + + /** + * Processes a transaction simulation with the provided parameters and executes pre-worldstate + * close actions. + * + * @param callParams The call parameters for the transaction. + * @param maybeStateOverrides The map of state overrides to apply to the state for this + * transaction. + * @param transactionValidationParams The validation parameters for the transaction. + * @param operationTracer The tracer for capturing operations during processing. + * @param preWorldStateCloseGuard The pre-worldstate close guard for executing pre-close actions. + * @param header The block header. + * @return An Optional containing the result of the processing. + */ + public Optional process( + final CallParameter callParams, + final Optional maybeStateOverrides, + final TransactionValidationParams transactionValidationParams, + final OperationTracer operationTracer, + final PreCloseStateHandler preWorldStateCloseGuard, + final BlockHeader header) { if (header == null) { return Optional.empty(); } @@ -169,7 +203,12 @@ public Optional process( return preWorldStateCloseGuard.apply( ws, processWithWorldUpdater( - callParams, transactionValidationParams, operationTracer, header, updater)); + callParams, + maybeStateOverrides, + transactionValidationParams, + operationTracer, + header, + updater)); } catch (final Exception e) { return Optional.empty(); @@ -208,6 +247,7 @@ private MutableWorldState getWorldState(final BlockHeader header) { @Nonnull public Optional processWithWorldUpdater( final CallParameter callParams, + final Optional maybeStateOverrides, final TransactionValidationParams transactionValidationParams, final OperationTracer operationTracer, final BlockHeader header, @@ -226,6 +266,12 @@ public Optional processWithWorldUpdater( .blockHeaderFunctions(protocolSpec.getBlockHeaderFunctions()) .buildBlockHeader(); } + if (maybeStateOverrides.isPresent()) { + for (Address accountToOverride : maybeStateOverrides.get().keySet()) { + final AccountOverride overrides = maybeStateOverrides.get().get(accountToOverride); + applyOverrides(updater.getOrCreate(accountToOverride), overrides); + } + } final Account sender = updater.get(senderAddress); final long nonce = sender != null ? sender.getNonce() : 0L; @@ -284,6 +330,24 @@ public Optional processWithWorldUpdater( return Optional.of(new TransactionSimulatorResult(transaction, result)); } + @VisibleForTesting + protected void applyOverrides(final MutableAccount account, final AccountOverride override) { + LOG.debug("applying overrides to state for account {}", account.getAddress()); + override.getNonce().ifPresent(account::setNonce); + if (override.getBalance().isPresent()) { + account.setBalance(override.getBalance().get()); + } + override.getCode().ifPresent(n -> account.setCode(Bytes.fromHexString(n))); + override + .getStateDiff() + .ifPresent( + d -> + d.forEach( + (key, value) -> + account.setStorageValue( + UInt256.fromHexString(key), UInt256.fromHexString(value)))); + } + private long calculateSimulationGasCap( final long userProvidedGasLimit, final long blockGasLimit) { final long simulationGasCap; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiWorldStateProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiWorldStateProvider.java index e4b7ea991f3..96a56da6e20 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiWorldStateProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiWorldStateProvider.java @@ -34,6 +34,7 @@ import java.util.Optional; import java.util.Set; import java.util.function.Function; +import java.util.function.Supplier; import com.google.common.annotations.VisibleForTesting; import org.apache.tuweni.bytes.Bytes; @@ -44,6 +45,7 @@ public class BonsaiWorldStateProvider extends DiffBasedWorldStateProvider { private static final Logger LOG = LoggerFactory.getLogger(BonsaiWorldStateProvider.class); private final BonsaiCachedMerkleTrieLoader bonsaiCachedMerkleTrieLoader; + private final Supplier worldStateHealerSupplier; public BonsaiWorldStateProvider( final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage, @@ -51,9 +53,11 @@ public BonsaiWorldStateProvider( final Optional maxLayersToLoad, final BonsaiCachedMerkleTrieLoader bonsaiCachedMerkleTrieLoader, final BesuContext pluginContext, - final EvmConfiguration evmConfiguration) { + final EvmConfiguration evmConfiguration, + final Supplier worldStateHealerSupplier) { super(worldStateKeyValueStorage, blockchain, maxLayersToLoad, pluginContext); this.bonsaiCachedMerkleTrieLoader = bonsaiCachedMerkleTrieLoader; + this.worldStateHealerSupplier = worldStateHealerSupplier; provideCachedWorldStorageManager( new BonsaiCachedWorldStorageManager( this, worldStateKeyValueStorage, this::cloneBonsaiWorldStateConfig)); @@ -69,9 +73,11 @@ public BonsaiWorldStateProvider( final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage, final Blockchain blockchain, final BonsaiCachedMerkleTrieLoader bonsaiCachedMerkleTrieLoader, - final EvmConfiguration evmConfiguration) { + final EvmConfiguration evmConfiguration, + final Supplier worldStateHealerSupplier) { super(worldStateKeyValueStorage, blockchain, trieLogManager); this.bonsaiCachedMerkleTrieLoader = bonsaiCachedMerkleTrieLoader; + this.worldStateHealerSupplier = worldStateHealerSupplier; provideCachedWorldStorageManager(bonsaiCachedWorldStorageManager); loadPersistedState( new BonsaiWorldState( @@ -151,4 +157,9 @@ public void prepareStateHealing(final Address address, final Bytes location) { private DiffBasedWorldStateConfig cloneBonsaiWorldStateConfig() { return new DiffBasedWorldStateConfig(defaultWorldStateConfig); } + + @Override + public void heal(final Optional
maybeAccountToRepair, final Bytes location) { + worldStateHealerSupplier.get().heal(maybeAccountToRepair, location); + } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedMerkleTrieLoader.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedMerkleTrieLoader.java index 9b9960b5c67..402402bc069 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedMerkleTrieLoader.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedMerkleTrieLoader.java @@ -14,6 +14,8 @@ */ package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache; +import static org.hyperledger.besu.metrics.BesuMetricCategory.BLOCKCHAIN; + import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; @@ -22,9 +24,7 @@ import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.diffbased.common.StorageSubscriber; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; -import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.metrics.ObservableMetricsSystem; -import org.hyperledger.besu.metrics.prometheus.PrometheusMetricsSystem; import java.util.Optional; import java.util.concurrent.CompletableFuture; @@ -33,7 +33,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; -import io.prometheus.client.guava.cache.CacheMetricsCollector; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; @@ -47,12 +46,8 @@ public class BonsaiCachedMerkleTrieLoader implements StorageSubscriber { CacheBuilder.newBuilder().recordStats().maximumSize(STORAGE_CACHE_SIZE).build(); public BonsaiCachedMerkleTrieLoader(final ObservableMetricsSystem metricsSystem) { - - CacheMetricsCollector cacheMetrics = new CacheMetricsCollector(); - cacheMetrics.addCache("accountsNodes", accountNodes); - cacheMetrics.addCache("storageNodes", storageNodes); - if (metricsSystem instanceof PrometheusMetricsSystem prometheusMetricsSystem) - prometheusMetricsSystem.addCollector(BesuMetricCategory.BLOCKCHAIN, () -> cacheMetrics); + metricsSystem.createGuavaCacheCollector(BLOCKCHAIN, "accountsNodes", accountNodes); + metricsSystem.createGuavaCacheCollector(BLOCKCHAIN, "storageNodes", storageNodes); } public void preLoadAccount( diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/forest/ForestWorldStateArchive.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/forest/ForestWorldStateArchive.java index 1cdd079e15a..800d2c98343 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/forest/ForestWorldStateArchive.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/forest/ForestWorldStateArchive.java @@ -112,6 +112,11 @@ public Optional getAccountProof( blockHeader.getStateRoot(), accountAddress, accountStorageKeys)); } + @Override + public void heal(final Optional
maybeAccountToRepair, final Bytes location) { + // no heal needed for Forest + } + @Override public void close() { // no op diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/util/AccountOverride.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/util/AccountOverride.java new file mode 100644 index 00000000000..3bae4af1d84 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/util/AccountOverride.java @@ -0,0 +1,147 @@ +/* + * Copyright contributors to Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.util; + +import org.hyperledger.besu.datatypes.Wei; + +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +import com.fasterxml.jackson.annotation.JsonAnySetter; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +// similar to AccountDiff +// BUT +// there are more fields that need to be added +// stateDiff +// movePrecompileToAddress +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonDeserialize(builder = AccountOverride.Builder.class) +public class AccountOverride { + private static final Logger LOG = LoggerFactory.getLogger(AccountOverride.class); + + private final Optional balance; + private final Optional nonce; + private final Optional code; + private final Optional> stateDiff; + + private AccountOverride( + final Optional balance, + final Optional nonce, + final Optional code, + final Optional> stateDiff) { + this.balance = balance; + this.nonce = nonce; + this.code = code; + this.stateDiff = stateDiff; + } + + public Optional getBalance() { + return balance; + } + + public Optional getNonce() { + return nonce; + } + + public Optional getCode() { + return code; + } + + public Optional> getStateDiff() { + return stateDiff; + } + + public static class Builder { + private Optional balance = Optional.empty(); + private Optional nonce = Optional.empty(); + private Optional code = Optional.empty(); + private Optional> stateDiff = Optional.empty(); + + /** Default constructor. */ + public Builder() {} + + public Builder withBalance(final Wei balance) { + this.balance = Optional.ofNullable(balance); + return this; + } + + public Builder withNonce(final Long nonce) { + this.nonce = Optional.ofNullable(nonce); + return this; + } + + public Builder withCode(final String code) { + this.code = Optional.ofNullable(code); + return this; + } + + public Builder withStateDiff(final Map stateDiff) { + this.stateDiff = Optional.ofNullable(stateDiff); + return this; + } + + public AccountOverride build() { + return new AccountOverride(balance, nonce, code, stateDiff); + } + } + + @JsonAnySetter + public void withUnknownProperties(final String key, final Object value) { + LOG.debug( + "unknown property - {} with value - {} and type - {} caught during serialization", + key, + value, + value != null ? value.getClass() : "NULL"); + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final AccountOverride accountOverride = (AccountOverride) o; + return balance.equals(accountOverride.balance) + && nonce.equals(accountOverride.nonce) + && code.equals(accountOverride.code) + && stateDiff.equals(accountOverride.stateDiff); + } + + @Override + public int hashCode() { + return Objects.hash(balance, nonce, code, stateDiff); + } + + @Override + public String toString() { + return "AccountOverride{" + + "balance=" + + balance + + ", nonce=" + + nonce + + ", code=" + + code + + ", stateDiff=" + + stateDiff + + '}'; + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/util/AccountOverrideMap.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/util/AccountOverrideMap.java new file mode 100644 index 00000000000..30fc808c9bb --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/util/AccountOverrideMap.java @@ -0,0 +1,27 @@ +/* + * Copyright contributors to Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.util; + +import org.hyperledger.besu.datatypes.Address; + +import java.util.HashMap; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class AccountOverrideMap extends HashMap { + + public AccountOverrideMap() {} +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateArchive.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateArchive.java index 23859ac3961..9856a200b21 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateArchive.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateArchive.java @@ -64,4 +64,24 @@ Optional getAccountProof( final Address accountAddress, final List accountStorageKeys, final Function, ? extends Optional> mapper); + + /** + * Heal the world state to fix inconsistency + * + * @param maybeAccountToRepair the optional account to repair + * @param location the location of the inconsistency + */ + void heal(Optional
maybeAccountToRepair, Bytes location); + + /** A world state healer */ + @FunctionalInterface + interface WorldStateHealer { + /** + * Heal the world state to fix inconsistency + * + * @param maybeAccountToRepair the optional account to repair + * @param location the location of the inconsistency + */ + void heal(Optional
maybeAccountToRepair, Bytes location); + } } diff --git a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/BlockchainSetupUtil.java b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/BlockchainSetupUtil.java index 57f11585857..3391a0e162c 100644 --- a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/BlockchainSetupUtil.java +++ b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/BlockchainSetupUtil.java @@ -21,7 +21,6 @@ import static org.mockito.Mockito.mock; import org.hyperledger.besu.config.GenesisConfigFile; -import org.hyperledger.besu.ethereum.ConsensusContext; import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.chain.BadBlockManager; import org.hyperledger.besu.ethereum.chain.Blockchain; @@ -161,15 +160,7 @@ private static ProtocolSchedule mainnetProtocolScheduleProvider( private static ProtocolContext mainnetProtocolContextProvider( final MutableBlockchain blockchain, final WorldStateArchive worldStateArchive) { return new ProtocolContext( - blockchain, - worldStateArchive, - new ConsensusContext() { - @Override - public C as(final Class klass) { - return null; - } - }, - new BadBlockManager()); + blockchain, worldStateArchive, new ConsensusContextFixture(), new BadBlockManager()); } private static BlockchainSetupUtil create( @@ -194,7 +185,6 @@ private static BlockchainSetupUtil create( genesisState.writeStateTo(worldArchive.getMutable()); final ProtocolContext protocolContext = protocolContextProvider.get(blockchain, worldArchive); - protocolContext.setSynchronizer(new DummySynchronizer()); final Path blocksPath = Path.of(chainResources.getBlocksURL().toURI()); final List blocks = new ArrayList<>(); diff --git a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/ConsensusContextFixture.java b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/ConsensusContextFixture.java new file mode 100644 index 00000000000..198b3ab8138 --- /dev/null +++ b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/ConsensusContextFixture.java @@ -0,0 +1,24 @@ +/* + * Copyright contributors to Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.core; + +import org.hyperledger.besu.ethereum.ConsensusContext; + +public class ConsensusContextFixture implements ConsensusContext { + @Override + public C as(final Class klass) { + return klass.cast(this); + } +} diff --git a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/ExecutionContextTestFixture.java b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/ExecutionContextTestFixture.java index b2b979e283c..b4eaec04d3e 100644 --- a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/ExecutionContextTestFixture.java +++ b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/ExecutionContextTestFixture.java @@ -76,7 +76,8 @@ private ExecutionContextTestFixture( else this.stateArchive = createInMemoryWorldStateArchive(); this.protocolSchedule = protocolSchedule; this.protocolContext = - new ProtocolContext(blockchain, stateArchive, null, new BadBlockManager()); + new ProtocolContext( + blockchain, stateArchive, new ConsensusContextFixture(), new BadBlockManager()); genesisState.writeStateTo(stateArchive.getMutable()); } diff --git a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java index 9bac9254940..aa32ee2b05f 100644 --- a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java +++ b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java @@ -14,6 +14,8 @@ */ package org.hyperledger.besu.ethereum.core; +import static org.hyperledger.besu.ethereum.core.WorldStateHealerHelper.throwingWorldStateHealerSupplier; + import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.chain.DefaultBlockchain; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; @@ -105,7 +107,8 @@ public static BonsaiWorldStateProvider createBonsaiInMemoryWorldStateArchive( Optional.empty(), bonsaiCachedMerkleTrieLoader, null, - evmConfiguration); + evmConfiguration, + throwingWorldStateHealerSupplier()); } public static MutableWorldState createInMemoryWorldState() { diff --git a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/WorldStateHealerHelper.java b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/WorldStateHealerHelper.java new file mode 100644 index 00000000000..a94ce247932 --- /dev/null +++ b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/WorldStateHealerHelper.java @@ -0,0 +1,39 @@ +/* + * Copyright contributors to Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.core; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive.WorldStateHealer; + +import java.util.Optional; +import java.util.function.Supplier; + +import org.apache.tuweni.bytes.Bytes; + +public class WorldStateHealerHelper { + + public static WorldStateHealer throwingHealer( + final Optional
maybeAccountToRepair, final Bytes location) { + throw new RuntimeException( + "World state needs to be healed: " + + maybeAccountToRepair.map(address -> "account to repair: " + address).orElse("") + + " location: " + + location.toHexString()); + } + + public static Supplier throwingWorldStateHealerSupplier() { + return () -> WorldStateHealerHelper::throwingHealer; + } +} diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/MainnetBlockValidatorTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/MainnetBlockValidatorTest.java index 013adae51ea..255ea778395 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/MainnetBlockValidatorTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/MainnetBlockValidatorTest.java @@ -31,7 +31,6 @@ import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.BlockchainSetupUtil; import org.hyperledger.besu.ethereum.core.MutableWorldState; -import org.hyperledger.besu.ethereum.core.Synchronizer; import org.hyperledger.besu.ethereum.mainnet.BlockBodyValidator; import org.hyperledger.besu.ethereum.mainnet.BlockHeaderValidator; import org.hyperledger.besu.ethereum.mainnet.BlockProcessor; @@ -91,7 +90,6 @@ public void setup() { when(protocolContext.getBlockchain()).thenReturn(blockchain); when(protocolContext.getWorldStateArchive()).thenReturn(worldStateArchive); - when(protocolContext.getSynchronizer()).thenReturn(mock(Synchronizer.class)); when(worldStateArchive.getMutable(any(BlockHeader.class), anyBoolean())) .thenReturn(Optional.of(worldState)); when(worldStateArchive.getMutable(any(Hash.class), any(Hash.class))) diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/transaction/TransactionSimulatorTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/transaction/TransactionSimulatorTest.java index c6c676c4151..0dfa7e924fa 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/transaction/TransactionSimulatorTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/transaction/TransactionSimulatorTest.java @@ -20,6 +20,7 @@ import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; import org.hyperledger.besu.crypto.SECPSignature; @@ -47,17 +48,21 @@ import org.hyperledger.besu.ethereum.mainnet.feemarket.FeeMarket; import org.hyperledger.besu.ethereum.processing.TransactionProcessingResult; import org.hyperledger.besu.ethereum.processing.TransactionProcessingResult.Status; +import org.hyperledger.besu.ethereum.util.AccountOverride; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.evm.account.Account; +import org.hyperledger.besu.evm.account.MutableAccount; import org.hyperledger.besu.evm.tracing.OperationTracer; import org.hyperledger.besu.evm.worldstate.WorldUpdater; import java.math.BigInteger; +import java.util.Map; import java.util.Optional; import com.google.common.base.Supplier; import com.google.common.base.Suppliers; import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.units.bigints.UInt256; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -100,6 +105,42 @@ public void setUp() { new TransactionSimulator(blockchain, worldStateArchive, protocolSchedule, GAS_CAP); } + @Test + public void testOverrides_whenNoOverrides_noUpdates() { + MutableAccount mutableAccount = mock(MutableAccount.class); + when(mutableAccount.getAddress()).thenReturn(DEFAULT_FROM); // called from logging + AccountOverride.Builder builder = new AccountOverride.Builder(); + AccountOverride override = builder.build(); + transactionSimulator.applyOverrides(mutableAccount, override); + verify(mutableAccount).getAddress(); + verifyNoMoreInteractions(mutableAccount); + } + + @Test + public void testOverrides_whenBalanceOverrides_balanceIsUpdated() { + MutableAccount mutableAccount = mock(MutableAccount.class); + when(mutableAccount.getAddress()).thenReturn(DEFAULT_FROM); + AccountOverride.Builder builder = new AccountOverride.Builder().withBalance(Wei.of(99)); + AccountOverride override = builder.build(); + transactionSimulator.applyOverrides(mutableAccount, override); + verify(mutableAccount).setBalance(eq(Wei.of(99))); + } + + @Test + public void testOverrides_whenStateDiffOverrides_stateIsUpdated() { + MutableAccount mutableAccount = mock(MutableAccount.class); + when(mutableAccount.getAddress()).thenReturn(DEFAULT_FROM); + final String storageKey = "0x01a2"; + final String storageValue = "0x00ff"; + AccountOverride.Builder builder = + new AccountOverride.Builder().withStateDiff(Map.of(storageKey, storageValue)); + AccountOverride override = builder.build(); + transactionSimulator.applyOverrides(mutableAccount, override); + verify(mutableAccount) + .setStorageValue( + eq(UInt256.fromHexString(storageKey)), eq(UInt256.fromHexString(storageValue))); + } + @Test public void shouldReturnEmptyWhenBlockDoesNotExist() { when(blockchain.getBlockHeader(eq(1L))).thenReturn(Optional.empty()); diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/AbstractIsolationTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/AbstractIsolationTests.java index 999d33a5c5e..275adc9ff43 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/AbstractIsolationTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/AbstractIsolationTests.java @@ -15,6 +15,7 @@ package org.hyperledger.besu.ethereum.trie.diffbased.bonsai; import static org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider.createInMemoryBlockchain; +import static org.hyperledger.besu.ethereum.core.WorldStateHealerHelper.throwingWorldStateHealerSupplier; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.RETURNS_DEEP_STUBS; import static org.mockito.Mockito.mock; @@ -29,6 +30,7 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.BlockProcessingResult; +import org.hyperledger.besu.ethereum.ConsensusContext; import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.blockcreation.AbstractBlockCreator; import org.hyperledger.besu.ethereum.chain.BadBlockManager; @@ -167,10 +169,13 @@ public void createStorage() { Optional.of(16L), new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()), null, - EvmConfiguration.DEFAULT); + EvmConfiguration.DEFAULT, + throwingWorldStateHealerSupplier()); var ws = archive.getMutable(); genesisState.writeStateTo(ws); - protocolContext = new ProtocolContext(blockchain, archive, null, new BadBlockManager()); + protocolContext = + new ProtocolContext( + blockchain, archive, mock(ConsensusContext.class), new BadBlockManager()); ethContext = mock(EthContext.class, RETURNS_DEEP_STUBS); when(ethContext.getEthPeers().subscribeConnect(any())).thenReturn(1L); transactionPool = diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiWorldStateProviderTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiWorldStateProviderTest.java index 4ac99377ef0..87cc16c23c3 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiWorldStateProviderTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiWorldStateProviderTest.java @@ -15,6 +15,7 @@ package org.hyperledger.besu.ethereum.trie.diffbased.bonsai; import static org.assertj.core.api.Assertions.assertThat; +import static org.hyperledger.besu.ethereum.core.WorldStateHealerHelper.throwingWorldStateHealerSupplier; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.BLOCKCHAIN; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE; import static org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage.WORLD_BLOCK_HASH_KEY; @@ -111,7 +112,8 @@ void testGetMutableReturnPersistedStateWhenNeeded() { DataStorageConfiguration.DEFAULT_BONSAI_CONFIG), blockchain, new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()), - EvmConfiguration.DEFAULT); + EvmConfiguration.DEFAULT, + throwingWorldStateHealerSupplier()); assertThat(bonsaiWorldStateArchive.getMutable(chainHead, true)) .containsInstanceOf(BonsaiWorldState.class); @@ -129,7 +131,8 @@ void testGetMutableReturnEmptyWhenLoadMoreThanLimitLayersBack() { Optional.of(512L), new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()), null, - EvmConfiguration.DEFAULT); + EvmConfiguration.DEFAULT, + throwingWorldStateHealerSupplier()); final BlockHeader blockHeader = blockBuilder.number(0).buildHeader(); final BlockHeader chainHead = blockBuilder.number(512).buildHeader(); when(blockchain.getChainHeadHeader()).thenReturn(chainHead); @@ -150,7 +153,8 @@ void testGetMutableWhenLoadLessThanLimitLayersBack() { DataStorageConfiguration.DEFAULT_BONSAI_CONFIG), blockchain, new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()), - EvmConfiguration.DEFAULT); + EvmConfiguration.DEFAULT, + throwingWorldStateHealerSupplier()); final BlockHeader blockHeader = blockBuilder.number(0).buildHeader(); final BlockHeader chainHead = blockBuilder.number(511).buildHeader(); final BonsaiWorldState mockWorldState = mock(BonsaiWorldState.class); @@ -185,7 +189,8 @@ void testGetMutableWithStorageInconsistencyRollbackTheState() { worldStateKeyValueStorage, blockchain, new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()), - EvmConfiguration.DEFAULT)); + EvmConfiguration.DEFAULT, + throwingWorldStateHealerSupplier())); final BlockHeader blockHeader = blockBuilder.number(0).buildHeader(); when(blockchain.getBlockHeader(blockHeader.getHash())).thenReturn(Optional.of(blockHeader)); @@ -214,7 +219,8 @@ void testGetMutableWithStorageConsistencyNotRollbackTheState() { worldStateKeyValueStorage, blockchain, new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()), - EvmConfiguration.DEFAULT)); + EvmConfiguration.DEFAULT, + throwingWorldStateHealerSupplier())); final BlockHeader blockHeader = blockBuilder.number(0).buildHeader(); @@ -254,7 +260,8 @@ void testGetMutableWithStorageConsistencyToRollbackAndRollForwardTheState() { worldStateKeyValueStorage, blockchain, new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()), - EvmConfiguration.DEFAULT)); + EvmConfiguration.DEFAULT, + throwingWorldStateHealerSupplier())); // initial persisted state hash key when(blockchain.getBlockHeader(Hash.ZERO)).thenReturn(Optional.of(blockHeaderChainA)); @@ -297,7 +304,8 @@ void testGetMutableWithRollbackNotOverrideTrieLogLayer() { DataStorageConfiguration.DEFAULT_BONSAI_CONFIG), blockchain, new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()), - EvmConfiguration.DEFAULT)); + EvmConfiguration.DEFAULT, + throwingWorldStateHealerSupplier())); // initial persisted state hash key when(blockchain.getBlockHeader(Hash.ZERO)).thenReturn(Optional.of(blockHeaderChainA)); diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/util/AccountOverrideParameterTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/util/AccountOverrideParameterTest.java new file mode 100644 index 00000000000..8e7b5c3f0eb --- /dev/null +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/util/AccountOverrideParameterTest.java @@ -0,0 +1,188 @@ +/* + * Copyright contributors to Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.util; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequest; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext; + +import java.util.Optional; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Test; + +public class AccountOverrideParameterTest { + + private static final String ADDRESS_HEX1 = "0xd9c9cd5f6779558b6e0ed4e6acf6b1947e7fa1f3"; + private static final String ADDRESS_HEX2 = "0xd5E23607D5d73ff2293152f464C3caB005f87696"; + private static final String STORAGE_KEY = + "0x1cf7945003fc5b59d2f6736f0704557aa805c4f2844084ccd1173b8d56946962"; + private static final String STORAGE_VALUE = + "0x000000000000000000000000000000000000000000000000000000110ed03bf7"; + private static final String CODE_STRING = + "0xdbf4257000000000000000000000000000000000000000000000000000000000"; + + @Test + public void jsonDeserializesCorrectly() throws Exception { + final String json = + "{\"jsonrpc\":\"2.0\",\"method\":\"eth_call\",\"params\":[{" + + "\"from\":\"0x0\", \"to\": \"0x0\"}, " + + "\"latest\"," + + "{\"" + + ADDRESS_HEX1 + + "\":" + + "{" + + "\"balance\": \"0x01\"," + + "\"nonce\": 88" + + "}}],\"id\":1}"; + + final JsonRpcRequestContext request = new JsonRpcRequestContext(readJsonAsJsonRpcRequest(json)); + final AccountOverrideMap accountOverrideParam = + request.getRequiredParameter(2, AccountOverrideMap.class); + + final AccountOverride accountOverride = + accountOverrideParam.get(Address.fromHexString(ADDRESS_HEX1)); + + assertThat(accountOverride.getNonce()).isEqualTo(Optional.of(88L)); + assertThat(accountOverride.getBalance()).isEqualTo(Optional.of(Wei.of(1))); + assertFalse(accountOverride.getStateDiff().isPresent()); + } + + @Test + public void jsonWithCodeDeserializesCorrectly() throws Exception { + final String json = + "{\"jsonrpc\":\"2.0\",\"method\":\"eth_call\",\"params\":[{" + + "\"from\":\"0x0\", \"to\": \"0x0\"}, " + + "\"latest\"," + + "{\"" + + ADDRESS_HEX1 + + "\":" + + "{" + + "\"balance\": \"0x01\"," + + "\"code\": \"" + + CODE_STRING + + "\"" + + "}}],\"id\":1}"; + + final JsonRpcRequestContext request = new JsonRpcRequestContext(readJsonAsJsonRpcRequest(json)); + final AccountOverrideMap accountOverrideParam = + request.getRequiredParameter(2, AccountOverrideMap.class); + + final AccountOverride accountOverride = + accountOverrideParam.get(Address.fromHexString(ADDRESS_HEX1)); + + assertFalse(accountOverride.getNonce().isPresent()); + assertThat(accountOverride.getBalance()).isEqualTo(Optional.of(Wei.of(1))); + assertThat(accountOverride.getCode()).isEqualTo(Optional.of(CODE_STRING)); + assertFalse(accountOverride.getStateDiff().isPresent()); + } + + @Test + public void jsonWithStorageOverridesDeserializesCorrectly() throws Exception { + final String json = + "{\"jsonrpc\":\"2.0\",\"method\":\"eth_call\",\"params\":[{" + + "\"from\":\"0x0\", \"to\": \"0x0\"}, " + + "\"latest\"," + + "{\"" + + ADDRESS_HEX1 + + "\":" + + "{" + + "\"balance\": \"0x01\"," + + "\"nonce\": 88," + + "\"stateDiff\": {" + + "\"" + + STORAGE_KEY + + "\": \"" + + STORAGE_VALUE + + "\"" + + "}}}],\"id\":1}"; + + final JsonRpcRequestContext request = new JsonRpcRequestContext(readJsonAsJsonRpcRequest(json)); + + final AccountOverrideMap accountOverrideParam = + request.getRequiredParameter(2, AccountOverrideMap.class); + assertThat(accountOverrideParam.size()).isEqualTo(1); + + final AccountOverride accountOverride = + accountOverrideParam.get(Address.fromHexString(ADDRESS_HEX1)); + assertThat(accountOverride.getNonce()).isEqualTo(Optional.of(88L)); + + assertTrue(accountOverride.getStateDiff().isPresent()); + assertThat(accountOverride.getStateDiff().get().get(STORAGE_KEY)).isEqualTo(STORAGE_VALUE); + } + + @Test + public void jsonWithMultipleAccountOverridesDeserializesCorrectly() throws Exception { + final String json = + "{\"jsonrpc\":\"2.0\",\"method\":\"eth_call\",\"params\":[{" + + "\"from\":\"0x0\", \"to\": \"0x0\"}, " + + "\"latest\"," + + "{\"" + + ADDRESS_HEX1 + + "\":" + + "{" + + "\"balance\": \"0x01\"," + + "\"nonce\": 88," + + "\"stateDiff\": {" + + "\"" + + STORAGE_KEY + + "\": \"" + + STORAGE_VALUE + + "\"" + + "}}," + + "\"" + + ADDRESS_HEX2 + + "\":" + + "{" + + "\"balance\": \"0xFF\"," + + "\"nonce\": 99," + + "\"stateDiff\": {" + + "\"" + + STORAGE_KEY + + "\": \"" + + STORAGE_VALUE + + "\"" + + "}}}],\"id\":1}"; + + final JsonRpcRequestContext request = new JsonRpcRequestContext(readJsonAsJsonRpcRequest(json)); + + final AccountOverrideMap accountOverrideParam = + request.getRequiredParameter(2, AccountOverrideMap.class); + assertThat(accountOverrideParam.size()).isEqualTo(2); + + final AccountOverride accountOverride1 = + accountOverrideParam.get(Address.fromHexString(ADDRESS_HEX1)); + assertThat(accountOverride1.getNonce()).isEqualTo(Optional.of(88L)); + assertThat(accountOverride1.getBalance()).isEqualTo(Optional.of(Wei.fromHexString("0x01"))); + assertTrue(accountOverride1.getStateDiff().isPresent()); + assertThat(accountOverride1.getStateDiff().get().get(STORAGE_KEY)).isEqualTo(STORAGE_VALUE); + + final AccountOverride accountOverride2 = + accountOverrideParam.get(Address.fromHexString(ADDRESS_HEX2)); + assertThat(accountOverride2.getNonce()).isEqualTo(Optional.of(99L)); + assertThat(accountOverride2.getBalance()).isEqualTo(Optional.of(Wei.fromHexString("0xFF"))); + assertTrue(accountOverride2.getStateDiff().isPresent()); + assertThat(accountOverride2.getStateDiff().get().get(STORAGE_KEY)).isEqualTo(STORAGE_VALUE); + } + + private JsonRpcRequest readJsonAsJsonRpcRequest(final String json) throws java.io.IOException { + return new ObjectMapper().readValue(json, JsonRpcRequest.class); + } +} diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/snap/SnapProtocolManager.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/snap/SnapProtocolManager.java index ce639a7a9a0..2e49eba3538 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/snap/SnapProtocolManager.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/snap/SnapProtocolManager.java @@ -15,6 +15,7 @@ package org.hyperledger.besu.ethereum.eth.manager.snap; import org.hyperledger.besu.ethereum.ProtocolContext; +import org.hyperledger.besu.ethereum.core.Synchronizer; import org.hyperledger.besu.ethereum.eth.SnapProtocol; import org.hyperledger.besu.ethereum.eth.manager.EthMessage; import org.hyperledger.besu.ethereum.eth.manager.EthMessages; @@ -53,11 +54,13 @@ public SnapProtocolManager( final SnapSyncConfiguration snapConfig, final EthPeers ethPeers, final EthMessages snapMessages, - final ProtocolContext protocolContext) { + final ProtocolContext protocolContext, + final Synchronizer synchronizer) { this.ethPeers = ethPeers; this.snapMessages = snapMessages; this.supportedCapabilities = calculateCapabilities(); - new SnapServer(snapConfig, snapMessages, worldStateStorageCoordinator, protocolContext); + new SnapServer( + snapConfig, snapMessages, worldStateStorageCoordinator, protocolContext, synchronizer); } private List calculateCapabilities() { diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/snap/SnapServer.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/snap/SnapServer.java index 56c3ae0a4a8..35cc63cde83 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/snap/SnapServer.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/snap/SnapServer.java @@ -16,6 +16,7 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.ProtocolContext; +import org.hyperledger.besu.ethereum.core.Synchronizer; import org.hyperledger.besu.ethereum.eth.manager.EthMessages; import org.hyperledger.besu.ethereum.eth.messages.snap.AccountRangeMessage; import org.hyperledger.besu.ethereum.eth.messages.snap.ByteCodesMessage; @@ -98,7 +99,8 @@ class SnapServer implements BesuEvents.InitialSyncCompletionListener { final SnapSyncConfiguration snapConfig, final EthMessages snapMessages, final WorldStateStorageCoordinator worldStateStorageCoordinator, - final ProtocolContext protocolContext) { + final ProtocolContext protocolContext, + final Synchronizer synchronizer) { this.snapServerEnabled = Optional.ofNullable(snapConfig) .map(SnapSyncConfiguration::isSnapServerEnabled) @@ -110,7 +112,7 @@ class SnapServer implements BesuEvents.InitialSyncCompletionListener { // subscribe to initial sync completed events to start/stop snap server, // not saving the listenerId since we never need to unsubscribe. - protocolContext.getSynchronizer().subscribeInitialSync(this); + synchronizer.subscribeInitialSync(this); } /** diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionBroadcaster.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionBroadcaster.java index c60c064c4c2..35f438d4191 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionBroadcaster.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionBroadcaster.java @@ -37,7 +37,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class TransactionBroadcaster implements TransactionBatchAddedListener { +public class TransactionBroadcaster + implements TransactionBatchAddedListener, PendingTransactionDroppedListener { private static final Logger LOG = LoggerFactory.getLogger(TransactionBroadcaster.class); private static final EnumSet ANNOUNCE_HASH_ONLY_TX_TYPES = EnumSet.of(BLOB); @@ -219,4 +220,9 @@ private void movePeersBetweenLists( destinationList.add(sourceList.remove(i)); } } + + @Override + public void onTransactionDropped(final Transaction transaction, final RemovalReason reason) { + transactionTracker.onTransactionDropped(transaction, reason); + } } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPool.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPool.java index 19bcfe0b36e..3acf49894a3 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPool.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPool.java @@ -139,6 +139,7 @@ public TransactionPool( subscribePendingTransactions(this::mapBlobsOnTransactionAdded); subscribeDroppedTransactions( (transaction, reason) -> unmapBlobsOnTransactionDropped(transaction)); + subscribeDroppedTransactions(transactionBroadcaster); } private void initLogForReplay() { diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/snap/SnapServerGetAccountRangeTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/snap/SnapServerGetAccountRangeTest.java index 6d8180c8c0e..eb97419967b 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/snap/SnapServerGetAccountRangeTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/snap/SnapServerGetAccountRangeTest.java @@ -15,10 +15,12 @@ package org.hyperledger.besu.ethereum.eth.manager.snap; import static org.assertj.core.api.AssertionsForClassTypes.assertThat; +import static org.mockito.Mockito.mock; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.core.BlockchainSetupUtil; +import org.hyperledger.besu.ethereum.core.Synchronizer; import org.hyperledger.besu.ethereum.eth.manager.EthMessages; import org.hyperledger.besu.ethereum.eth.messages.snap.AccountRangeMessage; import org.hyperledger.besu.ethereum.eth.messages.snap.GetAccountRangeMessage; @@ -70,7 +72,8 @@ public void setupTest() { snapSyncConfiguration, new EthMessages(), worldStateStorageCoordinator, - protocolContext) + protocolContext, + mock(Synchronizer.class)) .start(); initAccounts(); } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncTargetManagerTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncTargetManagerTest.java index 84c6ae9ff77..bb2a343d6d2 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncTargetManagerTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncTargetManagerTest.java @@ -18,6 +18,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import org.hyperledger.besu.ethereum.ConsensusContext; import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.chain.BadBlockManager; import org.hyperledger.besu.ethereum.chain.Blockchain; @@ -78,7 +79,8 @@ public void setup(final DataStorageFormat storageFormat) { final ProtocolSchedule protocolSchedule = ProtocolScheduleFixture.MAINNET; final ProtocolContext protocolContext = - new ProtocolContext(localBlockchain, localWorldState, null, new BadBlockManager()); + new ProtocolContext( + localBlockchain, localWorldState, mock(ConsensusContext.class), new BadBlockManager()); ethProtocolManager = EthProtocolManagerTestUtil.create( protocolSchedule, diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/tasks/DetermineCommonAncestorTaskParameterizedTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/tasks/DetermineCommonAncestorTaskParameterizedTest.java index 53f7e636558..59957081ae1 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/tasks/DetermineCommonAncestorTaskParameterizedTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/tasks/DetermineCommonAncestorTaskParameterizedTest.java @@ -19,6 +19,7 @@ import static org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider.createInMemoryWorldStateArchive; import static org.mockito.Mockito.mock; +import org.hyperledger.besu.ethereum.ConsensusContext; import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.chain.BadBlockManager; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; @@ -165,7 +166,11 @@ public void searchesAgainstNetwork( final EthContext ethContext = ethProtocolManager.ethContext(); final ProtocolContext protocolContext = - new ProtocolContext(localBlockchain, worldStateArchive, null, new BadBlockManager()); + new ProtocolContext( + localBlockchain, + worldStateArchive, + mock(ConsensusContext.class), + new BadBlockManager()); final EthTask task = DetermineCommonAncestorTask.create( diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/tasks/DetermineCommonAncestorTaskTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/tasks/DetermineCommonAncestorTaskTest.java index 6ccc3459ac4..cbba77eb34e 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/tasks/DetermineCommonAncestorTaskTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/tasks/DetermineCommonAncestorTaskTest.java @@ -27,6 +27,7 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import org.hyperledger.besu.ethereum.ConsensusContext; import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.chain.BadBlockManager; import org.hyperledger.besu.ethereum.chain.Blockchain; @@ -100,7 +101,11 @@ public void setup() { EthProtocolConfiguration.defaultConfig()); ethContext = ethProtocolManager.ethContext(); protocolContext = - new ProtocolContext(localBlockchain, worldStateArchive, null, new BadBlockManager()); + new ProtocolContext( + localBlockchain, + worldStateArchive, + mock(ConsensusContext.class), + new BadBlockManager()); peerTaskExecutor = Mockito.mock(PeerTaskExecutor.class); } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TestNode.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TestNode.java index 0081dc7d948..d042996ce7a 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TestNode.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TestNode.java @@ -27,6 +27,7 @@ import org.hyperledger.besu.crypto.KeyPair; import org.hyperledger.besu.cryptoservices.NodeKey; import org.hyperledger.besu.cryptoservices.NodeKeyUtils; +import org.hyperledger.besu.ethereum.ConsensusContext; import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.chain.BadBlockManager; import org.hyperledger.besu.ethereum.chain.GenesisState; @@ -135,7 +136,8 @@ public TestNode( final WorldStateArchive worldStateArchive = createInMemoryWorldStateArchive(); genesisState.writeStateTo(worldStateArchive.getMutable()); final ProtocolContext protocolContext = - new ProtocolContext(blockchain, worldStateArchive, null, new BadBlockManager()); + new ProtocolContext( + blockchain, worldStateArchive, mock(ConsensusContext.class), new BadBlockManager()); final SyncState syncState = mock(SyncState.class); final SynchronizerConfiguration syncConfig = mock(SynchronizerConfiguration.class); diff --git a/ethereum/p2p/build.gradle b/ethereum/p2p/build.gradle index 971504a25af..7cfcf2f8d58 100644 --- a/ethereum/p2p/build.gradle +++ b/ethereum/p2p/build.gradle @@ -43,7 +43,6 @@ dependencies { implementation 'com.google.guava:guava' implementation 'dnsjava:dnsjava' implementation 'io.netty:netty-transport-native-unix-common' - implementation 'io.prometheus:simpleclient' implementation 'io.vertx:vertx-core' implementation 'io.tmio:tuweni-bytes' diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BlockchainReferenceTestCaseSpec.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BlockchainReferenceTestCaseSpec.java index 7ab0091090b..771ae4672a0 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BlockchainReferenceTestCaseSpec.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BlockchainReferenceTestCaseSpec.java @@ -27,6 +27,7 @@ import org.hyperledger.besu.ethereum.core.BlockBody; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.BlockHeaderFunctions; +import org.hyperledger.besu.ethereum.core.ConsensusContextFixture; import org.hyperledger.besu.ethereum.core.Difficulty; import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; import org.hyperledger.besu.ethereum.core.MutableWorldState; @@ -108,7 +109,11 @@ public BlockchainReferenceTestCaseSpec( this.blockchain = buildBlockchain(genesisBlockHeader); this.sealEngine = sealEngine; this.protocolContext = - new ProtocolContext(this.blockchain, this.worldStateArchive, null, new BadBlockManager()); + new ProtocolContext( + this.blockchain, + this.worldStateArchive, + new ConsensusContextFixture(), + new BadBlockManager()); } public String getNetwork() { diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index d4407855f09..5c143f63a6b 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -1932,6 +1932,14 @@ + + + + + + + + @@ -1942,6 +1950,11 @@ + + + + + @@ -1955,6 +1968,14 @@ + + + + + + + + @@ -1963,6 +1984,14 @@ + + + + + + + + @@ -1971,6 +2000,14 @@ + + + + + + + + @@ -1979,6 +2016,14 @@ + + + + + + + + @@ -1992,6 +2037,14 @@ + + + + + + + + @@ -2005,6 +2058,14 @@ + + + + + + + + @@ -2013,6 +2074,14 @@ + + + + + + + + @@ -2021,6 +2090,14 @@ + + + + + + + + @@ -2029,6 +2106,14 @@ + + + + + + + + @@ -2037,6 +2122,14 @@ + + + + + + + + @@ -2045,6 +2138,14 @@ + + + + + + + + @@ -2053,6 +2154,14 @@ + + + + + + + + @@ -2061,6 +2170,14 @@ + + + + + + + + @@ -2074,6 +2191,14 @@ + + + + + + + + @@ -2087,6 +2212,14 @@ + + + + + + + + @@ -2100,6 +2233,14 @@ + + + + + + + + @@ -2108,6 +2249,14 @@ + + + + + + + + @@ -2118,6 +2267,11 @@ + + + + + @@ -2131,6 +2285,14 @@ + + + + + + + + @@ -2144,6 +2306,14 @@ + + + + + + + + @@ -2152,6 +2322,14 @@ + + + + + + + + @@ -2163,6 +2341,17 @@ + + + + + + + + + + + @@ -2176,6 +2365,14 @@ + + + + + + + + @@ -2184,11 +2381,24 @@ + + + + + + + + + + + + + @@ -2202,6 +2412,14 @@ + + + + + + + + @@ -2210,6 +2428,14 @@ + + + + + + + + @@ -2218,6 +2444,14 @@ + + + + + + + + @@ -2240,6 +2474,23 @@ + + + + + + + + + + + + + + + + + @@ -2259,6 +2510,20 @@ + + + + + + + + + + + + + + @@ -2272,6 +2537,14 @@ + + + + + + + + @@ -2280,6 +2553,14 @@ + + + + + + + + @@ -2288,6 +2569,14 @@ + + + + + + + + @@ -2296,6 +2585,14 @@ + + + + + + + + diff --git a/metrics/core/build.gradle b/metrics/core/build.gradle index 1438a198ae5..160093c7490 100644 --- a/metrics/core/build.gradle +++ b/metrics/core/build.gradle @@ -57,6 +57,7 @@ dependencies { implementation 'io.prometheus:simpleclient' implementation 'io.prometheus:simpleclient_common' + implementation 'io.prometheus:simpleclient_guava' implementation 'io.prometheus:simpleclient_hotspot' implementation 'io.prometheus:simpleclient_pushgateway' implementation 'io.vertx:vertx-core' diff --git a/metrics/core/src/main/java/org/hyperledger/besu/metrics/MetricCategoryRegistryImpl.java b/metrics/core/src/main/java/org/hyperledger/besu/metrics/MetricCategoryRegistryImpl.java index 6596268acb7..43feef12ac5 100644 --- a/metrics/core/src/main/java/org/hyperledger/besu/metrics/MetricCategoryRegistryImpl.java +++ b/metrics/core/src/main/java/org/hyperledger/besu/metrics/MetricCategoryRegistryImpl.java @@ -40,7 +40,8 @@ public MetricCategoryRegistryImpl() {} * @param categoryEnum the category enum */ public & MetricCategory> void addCategories(final Class categoryEnum) { - EnumSet.allOf(categoryEnum).forEach(this::addMetricCategory); + EnumSet.allOf(categoryEnum) + .forEach(category -> metricCategories.put(category.name(), category)); } /** diff --git a/metrics/core/src/main/java/org/hyperledger/besu/metrics/ObservableMetricsSystem.java b/metrics/core/src/main/java/org/hyperledger/besu/metrics/ObservableMetricsSystem.java index 8090d28fc1d..98e7a8b634f 100644 --- a/metrics/core/src/main/java/org/hyperledger/besu/metrics/ObservableMetricsSystem.java +++ b/metrics/core/src/main/java/org/hyperledger/besu/metrics/ObservableMetricsSystem.java @@ -17,42 +17,25 @@ import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.plugin.services.metrics.MetricCategory; -import java.util.Set; import java.util.stream.Stream; -/** The interface Observable metrics system. */ +/** The observable metrics system is used to inspect metrics for debug reasons */ public interface ObservableMetricsSystem extends MetricsSystem { - /** - * Stream observations. + * Stream observations by category * * @param category the category - * @return the stream + * @return the observations stream */ Stream streamObservations(MetricCategory category); /** - * Stream observations. + * Stream observations * - * @return the stream + * @return the observations stream */ Stream streamObservations(); - /** - * Provides an immutable view into the metric categories enabled for metric collection. - * - * @return the set of enabled metric categories. - */ - Set getEnabledCategories(); - - /** - * Checks if a particular category of metrics is enabled. - * - * @param category the category to check - * @return true if the category is enabled, false otherwise - */ - default boolean isCategoryEnabled(final MetricCategory category) { - return getEnabledCategories().stream() - .anyMatch(metricCategory -> metricCategory.getName().equals(category.getName())); - } + /** Unregister all the collectors and perform other cleanup tasks */ + void shutdown(); } diff --git a/metrics/core/src/main/java/org/hyperledger/besu/metrics/noop/NoOpMetricsSystem.java b/metrics/core/src/main/java/org/hyperledger/besu/metrics/noop/NoOpMetricsSystem.java index 5f876fa4d80..d3b6c4aadc5 100644 --- a/metrics/core/src/main/java/org/hyperledger/besu/metrics/noop/NoOpMetricsSystem.java +++ b/metrics/core/src/main/java/org/hyperledger/besu/metrics/noop/NoOpMetricsSystem.java @@ -17,6 +17,7 @@ import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.metrics.Observation; import org.hyperledger.besu.plugin.services.metrics.Counter; +import org.hyperledger.besu.plugin.services.metrics.ExternalSummary; import org.hyperledger.besu.plugin.services.metrics.LabelledGauge; import org.hyperledger.besu.plugin.services.metrics.LabelledMetric; import org.hyperledger.besu.plugin.services.metrics.MetricCategory; @@ -27,9 +28,11 @@ import java.util.List; import java.util.Set; import java.util.function.DoubleSupplier; +import java.util.function.Supplier; import java.util.stream.Stream; import com.google.common.base.Preconditions; +import com.google.common.cache.Cache; /** The NoOp metrics system. */ public class NoOpMetricsSystem implements ObservableMetricsSystem { @@ -113,6 +116,13 @@ public LabelledMetric createSimpleLabelledTimer( return getOperationTimerLabelledMetric(labelNames.length); } + @Override + public void trackExternalSummary( + final MetricCategory category, + final String name, + final String help, + final Supplier summarySupplier) {} + @Override public LabelledMetric createLabelledTimer( final MetricCategory category, @@ -144,6 +154,10 @@ public void createGauge( final String help, final DoubleSupplier valueSupplier) {} + @Override + public void createGuavaCacheCollector( + final MetricCategory category, final String name, final Cache cache) {} + @Override public LabelledGauge createLabelledGauge( final MetricCategory category, @@ -187,6 +201,9 @@ public Set getEnabledCategories() { return Collections.emptySet(); } + @Override + public void shutdown() {} + /** * The Label counting NoOp metric. * diff --git a/metrics/core/src/main/java/org/hyperledger/besu/metrics/opentelemetry/OpenTelemetrySystem.java b/metrics/core/src/main/java/org/hyperledger/besu/metrics/opentelemetry/OpenTelemetrySystem.java index a399b283734..a17c773813c 100644 --- a/metrics/core/src/main/java/org/hyperledger/besu/metrics/opentelemetry/OpenTelemetrySystem.java +++ b/metrics/core/src/main/java/org/hyperledger/besu/metrics/opentelemetry/OpenTelemetrySystem.java @@ -20,6 +20,7 @@ import org.hyperledger.besu.metrics.StandardMetricCategory; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.metrics.Counter; +import org.hyperledger.besu.plugin.services.metrics.ExternalSummary; import org.hyperledger.besu.plugin.services.metrics.LabelledGauge; import org.hyperledger.besu.plugin.services.metrics.LabelledMetric; import org.hyperledger.besu.plugin.services.metrics.MetricCategory; @@ -40,9 +41,11 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.function.DoubleSupplier; +import java.util.function.Supplier; import java.util.stream.Stream; import javax.inject.Singleton; +import com.google.common.cache.Cache; import com.google.common.collect.ImmutableSet; import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.api.common.Attributes; @@ -242,6 +245,13 @@ public LabelledMetric createSimpleLabelledTimer( return createLabelledTimer(category, name, help, labelNames); } + @Override + public void trackExternalSummary( + final MetricCategory category, + final String name, + final String help, + final Supplier summarySupplier) {} + @Override public LabelledMetric createLabelledTimer( final MetricCategory category, @@ -277,6 +287,10 @@ public void createGauge( } } + @Override + public void createGuavaCacheCollector( + final MetricCategory category, final String name, final Cache cache) {} + @Override public LabelledGauge createLabelledGauge( final MetricCategory category, @@ -376,6 +390,7 @@ private void collectGC() { } /** Shuts down the OpenTelemetry exporters, blocking until they have completed orderly. */ + @Override public void shutdown() { final CompletableResultCode result = CompletableResultCode.ofAll( diff --git a/metrics/core/src/main/java/org/hyperledger/besu/metrics/prometheus/PrometheusMetricsSystem.java b/metrics/core/src/main/java/org/hyperledger/besu/metrics/prometheus/PrometheusMetricsSystem.java index 653f448311f..b001eb0b3be 100644 --- a/metrics/core/src/main/java/org/hyperledger/besu/metrics/prometheus/PrometheusMetricsSystem.java +++ b/metrics/core/src/main/java/org/hyperledger/besu/metrics/prometheus/PrometheusMetricsSystem.java @@ -18,6 +18,7 @@ import org.hyperledger.besu.metrics.Observation; import org.hyperledger.besu.metrics.StandardMetricCategory; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; +import org.hyperledger.besu.plugin.services.metrics.ExternalSummary; import org.hyperledger.besu.plugin.services.metrics.LabelledGauge; import org.hyperledger.besu.plugin.services.metrics.LabelledMetric; import org.hyperledger.besu.plugin.services.metrics.MetricCategory; @@ -34,6 +35,7 @@ import java.util.function.Supplier; import java.util.stream.Stream; +import com.google.common.cache.Cache; import com.google.common.collect.ImmutableSet; import io.prometheus.client.Collector; import io.prometheus.client.Collector.MetricFamilySamples; @@ -42,6 +44,7 @@ import io.prometheus.client.Counter; import io.prometheus.client.Histogram; import io.prometheus.client.Summary; +import io.prometheus.client.guava.cache.CacheMetricsCollector; import io.prometheus.client.hotspot.BufferPoolsExports; import io.prometheus.client.hotspot.ClassLoadingExports; import io.prometheus.client.hotspot.GarbageCollectorExports; @@ -52,6 +55,7 @@ /** The Prometheus metrics system. */ public class PrometheusMetricsSystem implements ObservableMetricsSystem { + private static final List EXTERNAL_SUMMARY_LABELS = List.of("quantile"); private final Map> collectors = new ConcurrentHashMap<>(); private final CollectorRegistry registry = new CollectorRegistry(true); @@ -60,6 +64,9 @@ public class PrometheusMetricsSystem implements ObservableMetricsSystem { private final Map> cachedTimers = new ConcurrentHashMap<>(); private final Set totalSuffixedCounters = new ConcurrentHashSet<>(); + private final Map guavaCacheCollectors = + new ConcurrentHashMap<>(); + private final Set guavaCacheNames = new ConcurrentHashSet<>(); private final Set enabledCategories; private final boolean timersEnabled; @@ -78,12 +85,16 @@ public PrometheusMetricsSystem( /** Init. */ public void init() { - addCollector(StandardMetricCategory.PROCESS, StandardExports::new); - addCollector(StandardMetricCategory.JVM, MemoryPoolsExports::new); - addCollector(StandardMetricCategory.JVM, BufferPoolsExports::new); - addCollector(StandardMetricCategory.JVM, GarbageCollectorExports::new); - addCollector(StandardMetricCategory.JVM, ThreadExports::new); - addCollector(StandardMetricCategory.JVM, ClassLoadingExports::new); + if (isCategoryEnabled(StandardMetricCategory.PROCESS)) { + registerCollector(StandardMetricCategory.PROCESS, new StandardExports()); + } + if (isCategoryEnabled(StandardMetricCategory.JVM)) { + registerCollector(StandardMetricCategory.JVM, new MemoryPoolsExports()); + registerCollector(StandardMetricCategory.JVM, new BufferPoolsExports()); + registerCollector(StandardMetricCategory.JVM, new GarbageCollectorExports()); + registerCollector(StandardMetricCategory.JVM, new ThreadExports()); + registerCollector(StandardMetricCategory.JVM, new ClassLoadingExports()); + } } @Override @@ -103,7 +114,7 @@ public LabelledMetric crea (k) -> { if (isCategoryEnabled(category)) { final Counter counter = Counter.build(metricName, help).labelNames(labelNames).create(); - addCollectorUnchecked(category, counter); + registerCollector(category, counter); return new PrometheusCounter(counter); } else { return NoOpMetricsSystem.getCounterLabelledMetric(labelNames.length); @@ -132,7 +143,7 @@ public LabelledMetric createLabelledTimer( .quantile(1.0, 0) .labelNames(labelNames) .create(); - addCollectorUnchecked(category, summary); + registerCollector(category, summary); return new PrometheusTimer(summary); } else { return NoOpMetricsSystem.getOperationTimerLabelledMetric(labelNames.length); @@ -153,7 +164,7 @@ public LabelledMetric createSimpleLabelledTimer( if (timersEnabled && isCategoryEnabled(category)) { final Histogram histogram = Histogram.build(metricName, help).labelNames(labelNames).buckets(1D).create(); - addCollectorUnchecked(category, histogram); + registerCollector(category, histogram); return new PrometheusSimpleTimer(histogram); } else { return NoOpMetricsSystem.getOperationTimerLabelledMetric(labelNames.length); @@ -170,7 +181,61 @@ public void createGauge( final String metricName = convertToPrometheusName(category, name); if (isCategoryEnabled(category)) { final Collector collector = new CurrentValueCollector(metricName, help, valueSupplier); - addCollectorUnchecked(category, collector); + registerCollector(category, collector); + } + } + + @Override + public void trackExternalSummary( + final MetricCategory category, + final String name, + final String help, + final Supplier summarySupplier) { + if (isCategoryEnabled(category)) { + final var externalSummaryCollector = + new Collector() { + @Override + public List collect() { + final var externalSummary = summarySupplier.get(); + + final var quantileValues = + externalSummary.quantiles().stream() + .map( + quantile -> + new Sample( + name, + EXTERNAL_SUMMARY_LABELS, + List.of(Double.toString(quantile.quantile())), + quantile.value())) + .toList(); + + return List.of( + new MetricFamilySamples( + name, Type.SUMMARY, "RocksDB histogram for " + name, quantileValues)); + } + }; + + registerCollector(category, externalSummaryCollector); + } + } + + @Override + public void createGuavaCacheCollector( + final MetricCategory category, final String name, final Cache cache) { + if (isCategoryEnabled(category)) { + if (guavaCacheNames.contains(name)) { + throw new IllegalStateException("Cache already registered: " + name); + } + guavaCacheNames.add(name); + final var guavaCacheCollector = + guavaCacheCollectors.computeIfAbsent( + category, + unused -> { + final var cmc = new CacheMetricsCollector(); + registerCollector(category, cmc); + return cmc; + }); + guavaCacheCollector.addCache(name, cache); } } @@ -183,46 +248,33 @@ public LabelledGauge createLabelledGauge( final String metricName = convertToPrometheusName(category, name); if (isCategoryEnabled(category)) { final PrometheusGauge gauge = new PrometheusGauge(metricName, help, List.of(labelNames)); - addCollectorUnchecked(category, gauge); + registerCollector(category, gauge); return gauge; } return NoOpMetricsSystem.getLabelledGauge(labelNames.length); } - /** - * Add collector. - * - * @param category the category - * @param metricSupplier the metric supplier - */ - public void addCollector( - final MetricCategory category, final Supplier metricSupplier) { - if (isCategoryEnabled(category)) { - addCollectorUnchecked(category, metricSupplier.get()); - } - } - - private void addCollectorUnchecked(final MetricCategory category, final Collector metric) { - final Collection metrics = + private void registerCollector(final MetricCategory category, final Collector collector) { + final Collection categoryCollectors = this.collectors.computeIfAbsent( category, key -> Collections.newSetFromMap(new ConcurrentHashMap<>())); final List newSamples = - metric.collect().stream().map(metricFamilySamples -> metricFamilySamples.name).toList(); + collector.collect().stream().map(metricFamilySamples -> metricFamilySamples.name).toList(); - metrics.stream() + categoryCollectors.stream() .filter( - collector -> - collector.collect().stream() + c -> + c.collect().stream() .anyMatch(metricFamilySamples -> newSamples.contains(metricFamilySamples.name))) .findFirst() .ifPresent( - collector -> { - metrics.remove(collector); - registry.unregister(collector); + c -> { + categoryCollectors.remove(c); + registry.unregister(c); }); - metrics.add(metric.register(registry)); + categoryCollectors.add(collector.register(registry)); } @Override @@ -237,6 +289,16 @@ public Stream streamObservations() { return collectors.keySet().stream().flatMap(this::streamObservations); } + @Override + public void shutdown() { + registry.clear(); + collectors.clear(); + cachedCounters.clear(); + cachedTimers.clear(); + guavaCacheCollectors.clear(); + guavaCacheNames.clear(); + } + private Stream convertSamplesToObservations( final MetricCategory category, final MetricFamilySamples familySamples) { return familySamples.samples.stream() diff --git a/metrics/core/src/test-support/java/org/hyperledger/besu/metrics/StubMetricsSystem.java b/metrics/core/src/test-support/java/org/hyperledger/besu/metrics/StubMetricsSystem.java index 2e0ea006db0..13f16ef0a0c 100644 --- a/metrics/core/src/test-support/java/org/hyperledger/besu/metrics/StubMetricsSystem.java +++ b/metrics/core/src/test-support/java/org/hyperledger/besu/metrics/StubMetricsSystem.java @@ -18,6 +18,7 @@ import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.metrics.Counter; +import org.hyperledger.besu.plugin.services.metrics.ExternalSummary; import org.hyperledger.besu.plugin.services.metrics.LabelledGauge; import org.hyperledger.besu.plugin.services.metrics.LabelledMetric; import org.hyperledger.besu.plugin.services.metrics.MetricCategory; @@ -29,8 +30,11 @@ import java.util.Map; import java.util.Set; import java.util.function.DoubleSupplier; +import java.util.function.Supplier; import java.util.stream.Stream; +import com.google.common.cache.Cache; + public class StubMetricsSystem implements ObservableMetricsSystem { private final Map counters = new HashMap<>(); @@ -84,6 +88,13 @@ public LabelledMetric createSimpleLabelledTimer( return labelValues -> NoOpMetricsSystem.NO_OP_OPERATION_TIMER; } + @Override + public void trackExternalSummary( + final MetricCategory category, + final String name, + final String help, + final Supplier summarySupplier) {} + @Override public void createGauge( final MetricCategory category, @@ -93,6 +104,10 @@ public void createGauge( gauges.put(name, valueSupplier); } + @Override + public void createGuavaCacheCollector( + final MetricCategory category, final String name, final Cache cache) {} + public double getGaugeValue(final String name) { final DoubleSupplier gauge = gauges.get(name); if (gauge == null) { @@ -116,6 +131,12 @@ public Set getEnabledCategories() { return Collections.emptySet(); } + @Override + public void shutdown() { + counters.clear(); + gauges.clear(); + } + public static class StubLabelledCounter implements LabelledMetric { private final Map, StubCounter> metrics = new HashMap<>(); diff --git a/metrics/rocksdb/build.gradle b/metrics/rocksdb/build.gradle index d46488daf8f..de312ab708a 100644 --- a/metrics/rocksdb/build.gradle +++ b/metrics/rocksdb/build.gradle @@ -40,7 +40,5 @@ dependencies { implementation project(':metrics:core') implementation project(':plugin-api') - implementation 'com.google.guava:guava' - implementation 'io.prometheus:simpleclient' implementation 'org.rocksdb:rocksdbjni' } diff --git a/metrics/rocksdb/src/main/java/org/hyperledger/besu/metrics/rocksdb/RocksDBStats.java b/metrics/rocksdb/src/main/java/org/hyperledger/besu/metrics/rocksdb/RocksDBStats.java index edb837985dd..c3c5fef2c11 100644 --- a/metrics/rocksdb/src/main/java/org/hyperledger/besu/metrics/rocksdb/RocksDBStats.java +++ b/metrics/rocksdb/src/main/java/org/hyperledger/besu/metrics/rocksdb/RocksDBStats.java @@ -16,15 +16,14 @@ import static org.hyperledger.besu.metrics.BesuMetricCategory.KVSTORE_ROCKSDB_STATS; -import org.hyperledger.besu.metrics.prometheus.PrometheusMetricsSystem; +import org.hyperledger.besu.plugin.services.MetricsSystem; +import org.hyperledger.besu.plugin.services.metrics.ExternalSummary; +import org.hyperledger.besu.plugin.services.metrics.ExternalSummary.Quantile; import org.hyperledger.besu.plugin.services.metrics.MetricCategory; -import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.Locale; -import io.prometheus.client.Collector; import org.rocksdb.HistogramData; import org.rocksdb.HistogramType; import org.rocksdb.Statistics; @@ -32,22 +31,9 @@ /** The Rocks db stats. */ public class RocksDBStats { - - /** The Labels. */ - static final List LABELS = Collections.singletonList("quantile"); - - /** The Label 50. */ - static final List LABEL_50 = Collections.singletonList("0.5"); - - /** The Label 95. */ - static final List LABEL_95 = Collections.singletonList("0.95"); - - /** The Label 99. */ - static final List LABEL_99 = Collections.singletonList("0.99"); - - /** The constant TICKERS. */ + /** The constant TICKER_TYPES. */ // Tickers - RocksDB equivalent of counters - static final TickerType[] TICKERS = { + static final TickerType[] TICKER_TYPES = { TickerType.BLOCK_CACHE_ADD, TickerType.BLOCK_CACHE_HIT, TickerType.BLOCK_CACHE_ADD_FAILURES, @@ -133,9 +119,9 @@ public class RocksDBStats { TickerType.NUMBER_MULTIGET_KEYS_FOUND, }; - /** The constant HISTOGRAMS. */ + /** The constant HISTOGRAM_TYPES. */ // Histograms - treated as prometheus summaries - static final HistogramType[] HISTOGRAMS = { + static final HistogramType[] HISTOGRAM_TYPES = { HistogramType.DB_GET, HistogramType.DB_WRITE, HistogramType.COMPACTION_TIME, @@ -175,47 +161,40 @@ private RocksDBStats() {} * @param category the category */ public static void registerRocksDBMetrics( - final Statistics stats, - final PrometheusMetricsSystem metricsSystem, - final MetricCategory category) { - if (!metricsSystem.isCategoryEnabled(category)) { - return; - } - for (final TickerType ticker : TICKERS) { - final String promCounterName = ticker.name().toLowerCase(Locale.ROOT); + final Statistics stats, final MetricsSystem metricsSystem, final MetricCategory category) { + + for (final var tickerType : TICKER_TYPES) { + final String promCounterName = tickerType.name().toLowerCase(Locale.ROOT); metricsSystem.createLongGauge( category, promCounterName, - "RocksDB reported statistics for " + ticker.name(), - () -> stats.getTickerCount(ticker)); + "RocksDB reported statistics for " + tickerType.name(), + () -> stats.getTickerCount(tickerType)); } - for (final HistogramType histogram : HISTOGRAMS) { - metricsSystem.addCollector(category, () -> histogramToCollector(stats, histogram)); + for (final var histogramType : HISTOGRAM_TYPES) { + + metricsSystem.trackExternalSummary( + KVSTORE_ROCKSDB_STATS, + KVSTORE_ROCKSDB_STATS.getName() + "_" + histogramType.name().toLowerCase(Locale.ROOT), + "RocksDB histogram for " + histogramType.name(), + () -> provideExternalSummary(stats, histogramType)); } } - private static Collector histogramToCollector( - final Statistics stats, final HistogramType histogram) { - return new Collector() { - final String metricName = - KVSTORE_ROCKSDB_STATS.getName() + "_" + histogram.name().toLowerCase(Locale.ROOT); + private static ExternalSummary provideExternalSummary( + final Statistics stats, final HistogramType histogramType) { + + final HistogramData data = stats.getHistogramData(histogramType); - @Override - public List collect() { - final HistogramData data = stats.getHistogramData(histogram); - return Collections.singletonList( - new MetricFamilySamples( - metricName, - Type.SUMMARY, - "RocksDB histogram for " + metricName, - Arrays.asList( - new MetricFamilySamples.Sample(metricName, LABELS, LABEL_50, data.getMedian()), - new MetricFamilySamples.Sample( - metricName, LABELS, LABEL_95, data.getPercentile95()), - new MetricFamilySamples.Sample( - metricName, LABELS, LABEL_99, data.getPercentile99())))); - } - }; + return new ExternalSummary( + data.getCount(), + data.getSum(), + List.of( + new Quantile(0.0, data.getMin()), + new Quantile(0.5, data.getMedian()), + new Quantile(0.95, data.getPercentile95()), + new Quantile(0.99, data.getPercentile99()), + new Quantile(1.0, data.getMax()))); } } diff --git a/platform/build.gradle b/platform/build.gradle index d614603f17f..dad2d8e1a09 100644 --- a/platform/build.gradle +++ b/platform/build.gradle @@ -28,7 +28,7 @@ javaPlatform { dependencies { api platform('com.fasterxml.jackson:jackson-bom:2.18.0') api platform('io.grpc:grpc-bom:1.68.0') - api platform('io.netty:netty-bom:4.1.114.Final') + api platform('io.netty:netty-bom:4.1.115.Final') api platform('io.opentelemetry:opentelemetry-bom:1.43.0') api platform('io.prometheus:simpleclient_bom:0.16.0') api platform('io.vertx:vertx-stack-depchain:4.5.10') diff --git a/plugin-api/build.gradle b/plugin-api/build.gradle index c7ec6728674..2d3d5191df4 100644 --- a/plugin-api/build.gradle +++ b/plugin-api/build.gradle @@ -71,7 +71,7 @@ Calculated : ${currentHash} tasks.register('checkAPIChanges', FileStateChecker) { description = "Checks that the API for the Plugin-API project does not change without deliberate thought" files = sourceSets.main.allJava.files - knownHash = '8rPIE3fYl48RPRQXxYhMk559e/r+wHSKU9bGSJmruKQ=' + knownHash = 'aYWbsgPoKTGDgq9d4QUBvQEaZYbKNJGMiBufzyKnusA=' } check.dependsOn('checkAPIChanges') diff --git a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/MetricsSystem.java b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/MetricsSystem.java index 80e02a6dba7..744bca4eb3e 100644 --- a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/MetricsSystem.java +++ b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/MetricsSystem.java @@ -15,14 +15,19 @@ package org.hyperledger.besu.plugin.services; import org.hyperledger.besu.plugin.services.metrics.Counter; +import org.hyperledger.besu.plugin.services.metrics.ExternalSummary; import org.hyperledger.besu.plugin.services.metrics.LabelledGauge; import org.hyperledger.besu.plugin.services.metrics.LabelledMetric; import org.hyperledger.besu.plugin.services.metrics.MetricCategory; import org.hyperledger.besu.plugin.services.metrics.OperationTimer; +import java.util.Set; import java.util.function.DoubleSupplier; import java.util.function.IntSupplier; import java.util.function.LongSupplier; +import java.util.function.Supplier; + +import com.google.common.cache.Cache; /** An interface for creating various Metrics components. */ public interface MetricsSystem extends BesuService { @@ -159,4 +164,45 @@ default void createLongGauge( final LongSupplier valueSupplier) { createGauge(category, name, help, () -> (double) valueSupplier.getAsLong()); } + + /** + * Track a summary that is computed externally to this metric system. Useful when existing + * libraries calculate the summary data on their own, and we want to export that summary via the + * configured metric system. A notable example are RocksDB statistics. + * + * @param category The {@link MetricCategory} this external summary is assigned to. + * @param name A name for the metric. + * @param help A human readable description of the metric. + * @param summarySupplier A supplier to retrieve the summary data when needed. + */ + void trackExternalSummary( + MetricCategory category, String name, String help, Supplier summarySupplier); + + /** + * Collect metrics from Guava cache. + * + * @param category The {@link MetricCategory} this Guava cache is assigned to. + * @param name the name to identify this Guava cache, must be unique. + * @param cache the Guava cache + */ + void createGuavaCacheCollector(MetricCategory category, String name, Cache cache); + + /** + * Provides an immutable view into the metric categories enabled for metric collection. + * + * @return the set of enabled metric categories. + */ + Set getEnabledCategories(); + + /** + * Checks if a particular category of metrics is enabled. + * + * @param category the category to check + * @return true if the category is enabled, false otherwise + */ + default boolean isCategoryEnabled(final MetricCategory category) { + return getEnabledCategories().stream() + .map(MetricCategory::getName) + .anyMatch(category.getName()::equals); + } } diff --git a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/metrics/ExternalSummary.java b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/metrics/ExternalSummary.java new file mode 100644 index 00000000000..5e7e38b76af --- /dev/null +++ b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/metrics/ExternalSummary.java @@ -0,0 +1,36 @@ +/* + * Copyright contributors to Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.plugin.services.metrics; + +import java.util.List; + +/** + * Record of summary data the is kept outside the metric system. Useful when existing libraries + * calculate the summary data on their own, and we want to export that summary via the configured + * metric system. A notable example are RocksDB statistics. + * + * @param count the number of observations + * @param sum the sum of all the observations + * @param quantiles a list of quantiles with values + */ +public record ExternalSummary(long count, double sum, List quantiles) { + /** + * Represent a single quantile and its value + * + * @param quantile the quantile + * @param value the value + */ + public record Quantile(double quantile, double value) {} +} diff --git a/plugins/rocksdb/build.gradle b/plugins/rocksdb/build.gradle index d487cf9824c..a2315398e13 100644 --- a/plugins/rocksdb/build.gradle +++ b/plugins/rocksdb/build.gradle @@ -46,7 +46,6 @@ dependencies { implementation 'com.google.guava:guava' implementation 'info.picocli:picocli' implementation 'io.opentelemetry:opentelemetry-api' - implementation 'io.prometheus:simpleclient' implementation 'io.tmio:tuweni-bytes' implementation 'org.rocksdb:rocksdbjni' implementation project(path: ':ethereum:core') diff --git a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBMetricsFactory.java b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBMetricsFactory.java index bf781d300a7..f5c666184b6 100644 --- a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBMetricsFactory.java +++ b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBMetricsFactory.java @@ -15,7 +15,6 @@ package org.hyperledger.besu.plugin.services.storage.rocksdb; import org.hyperledger.besu.metrics.BesuMetricCategory; -import org.hyperledger.besu.metrics.prometheus.PrometheusMetricsSystem; import org.hyperledger.besu.metrics.rocksdb.RocksDBStats; import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.plugin.services.metrics.Counter; @@ -107,10 +106,7 @@ public RocksDBMetrics create( "database") .labels(rocksDbConfiguration.getLabel()); - if (metricsSystem instanceof PrometheusMetricsSystem) { - RocksDBStats.registerRocksDBMetrics( - stats, (PrometheusMetricsSystem) metricsSystem, statsDbMetricCategory); - } + RocksDBStats.registerRocksDBMetrics(stats, metricsSystem, statsDbMetricCategory); metricsSystem.createLongGauge( rocksDbMetricCategory, diff --git a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorageTest.java b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorageTest.java index e28828c396e..4800e82abcd 100644 --- a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorageTest.java +++ b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorageTest.java @@ -264,7 +264,7 @@ public void dbWillBeBackwardIncompatibleAfterExperimentalSegmentsAreAdded( List.of()); store.close(); - // Create new db without ignoring experimental colum family will add column to db + // Create new db without ignoring experimental column family will add column to db store = createSegmentedStore( testPath, diff --git a/testfuzz/src/main/java/org/hyperledger/besu/testfuzz/javafuzz/FuzzTarget.java b/testfuzz/src/main/java/org/hyperledger/besu/testfuzz/javafuzz/FuzzTarget.java index 26f3522e116..bee6bca7769 100644 --- a/testfuzz/src/main/java/org/hyperledger/besu/testfuzz/javafuzz/FuzzTarget.java +++ b/testfuzz/src/main/java/org/hyperledger/besu/testfuzz/javafuzz/FuzzTarget.java @@ -25,7 +25,7 @@ public interface FuzzTarget { /** * The target to fuzz * - * @param data data proviced by the fuzzer + * @param data data provided by the fuzzer */ void fuzz(byte[] data); }