From db8bd9d2af40a64c835240abb929a0e5b4046d68 Mon Sep 17 00:00:00 2001 From: Danno Ferrin Date: Wed, 20 Sep 2023 22:35:31 -0600 Subject: [PATCH] Update reference tests to 12.4 (#5899) * Update reference tests to 12.4 Update reference tests to 12.4 * Some in-memory storage changed to ConcurrentMap * exclude cancun from all EIP tests, EIP-4788 still in flux * Add new fields to ReferenceTestEnv, and re-order the reflected constructor for clarity and ease of development. Signed-off-by: Danno Ferrin * spotless Signed-off-by: Danno Ferrin --------- Signed-off-by: Danno Ferrin Co-authored-by: Sally MacFarlane --- ethereum/referencetests/build.gradle | 2 +- .../referencetests/ReferenceTestEnv.java | 45 ++++++++++--------- .../src/reference-test/external-resources | 2 +- .../vm/BlockchainReferenceTestTools.java | 3 ++ .../kvstore/InMemoryKeyValueStorage.java | 8 ++-- .../kvstore/LayeredKeyValueStorage.java | 3 +- .../SegmentedInMemoryKeyValueStorage.java | 16 ++++--- 7 files changed, 47 insertions(+), 32 deletions(-) diff --git a/ethereum/referencetests/build.gradle b/ethereum/referencetests/build.gradle index cf1a6e1f8a8..dc3e0165017 100644 --- a/ethereum/referencetests/build.gradle +++ b/ethereum/referencetests/build.gradle @@ -198,7 +198,7 @@ tasks.register('validateReferenceTestSubmodule') { description = "Checks that the reference tests submodule is not accidentally changed" doLast { def result = new ByteArrayOutputStream() - def expectedHash = '06e276776bc87817c38f6efb492bf6f4527fa904' + def expectedHash = '661356317ac6df52208d54187e692472a25a01f8' def submodulePath = java.nio.file.Path.of("${rootProject.projectDir}", "ethereum/referencetests/src/reference-test/external-resources").toAbsolutePath() try { exec { diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestEnv.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestEnv.java index 5f7af60719a..bac137b2a58 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestEnv.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestEnv.java @@ -97,33 +97,35 @@ Withdrawal asWithdrawal() { */ @JsonCreator public ReferenceTestEnv( + @JsonProperty("beaconRoot") final String beaconRoot, + @JsonProperty("blockHashes") final Map blockHashes, + @JsonProperty("ommers") final List _ommers, + @JsonProperty("previousHash") final String previousHash, + @JsonProperty("withdrawals") final List withdrawals, + @JsonProperty("currentBaseFee") final String baseFee, + @JsonProperty("currentBeaconRoot") final String currentBeaconRoot, + @JsonProperty("currentBlobGasUsed") final String currentBlobGasUsed, @JsonProperty("currentCoinbase") final String coinbase, + @JsonProperty("currentDataGasUsed") final String currentDataGasUsed, @JsonProperty("currentDifficulty") final String difficulty, + @JsonProperty("currentExcessBlobGas") final String currentExcessBlobGas, + @JsonProperty("currentExcessDataGas") final String currentExcessDataGas, @JsonProperty("currentGasLimit") final String gasLimit, @JsonProperty("currentNumber") final String number, - @JsonProperty("currentBaseFee") final String baseFee, - @JsonProperty("currentTimestamp") final String timestamp, @JsonProperty("currentRandom") final String random, @JsonProperty("currentStateRoot") final String stateRoot, - @JsonProperty("previousHash") final String previousHash, - @JsonProperty("parentDifficulty") final String parentDifficulty, + @JsonProperty("currentTimestamp") final String timestamp, + @JsonProperty("currentWithdrawalsRoot") final String currentWithdrawalsRoot, @JsonProperty("parentBaseFee") final String parentBaseFee, - @JsonProperty("parentGasUsed") final String parentGasUsed, - @JsonProperty("parentGasLimit") final String parentGasLimit, - @JsonProperty("parentTimestamp") final String parentTimestamp, - @JsonProperty("ommers") final List _ommers, - @JsonProperty("parentUncleHash") final String _parentUncleHash, - @JsonProperty("withdrawals") final List withdrawals, - @JsonProperty("blockHashes") final Map blockHashes, - @JsonProperty("currentExcessBlobGas") final String currentExcessBlobGas, - @JsonProperty("currentBlobGasUsed") final String currentBlobGasUsed, - @JsonProperty("currentExcessDataGas") final String currentExcessDataGas, - @JsonProperty("currentDataGasUsed") final String currentDataGasUsed, - @JsonProperty("parentExcessBlobGas") final String parentExcessBlobGas, @JsonProperty("parentBlobGasUsed") final String parentBlobGasUsed, - @JsonProperty("parentExcessDataGas") final String parentExcessDataGas, @JsonProperty("parentDataGasUsed") final String parentDataGasUsed, - @JsonProperty("beaconRoot") final String beaconRoot) { + @JsonProperty("parentDifficulty") final String parentDifficulty, + @JsonProperty("parentExcessBlobGas") final String parentExcessBlobGas, + @JsonProperty("parentExcessDataGas") final String parentExcessDataGas, + @JsonProperty("parentGasLimit") final String parentGasLimit, + @JsonProperty("parentGasUsed") final String parentGasUsed, + @JsonProperty("parentTimestamp") final String parentTimestamp, + @JsonProperty("parentUncleHash") final String _parentUncleHash) { super( generateTestBlockHash(previousHash, number), Hash.EMPTY_LIST_HASH, // ommersHash @@ -141,7 +143,7 @@ public ReferenceTestEnv( Optional.ofNullable(baseFee).map(Wei::fromHexString).orElse(null), Optional.ofNullable(random).map(Difficulty::fromHexString).orElse(Difficulty.ZERO), 0L, - null, // withdrawalsRoot + currentWithdrawalsRoot == null ? null : Hash.fromHexString(currentWithdrawalsRoot), currentBlobGasUsed == null ? currentDataGasUsed == null ? null : Long.decode(currentDataGasUsed) : Long.decode(currentBlobGasUsed), @@ -172,7 +174,10 @@ public ReferenceTestEnv( Map.entry( Long.decode(entry.getKey()), Hash.fromHexString(entry.getValue()))) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - this.beaconRoot = beaconRoot == null ? null : Hash.fromHexString(beaconRoot); + this.beaconRoot = + beaconRoot == null + ? (currentBeaconRoot == null ? null : Hash.fromHexString(currentBeaconRoot)) + : Hash.fromHexString(beaconRoot); } @Override diff --git a/ethereum/referencetests/src/reference-test/external-resources b/ethereum/referencetests/src/reference-test/external-resources index 06e276776bc..661356317ac 160000 --- a/ethereum/referencetests/src/reference-test/external-resources +++ b/ethereum/referencetests/src/reference-test/external-resources @@ -1 +1 @@ -Subproject commit 06e276776bc87817c38f6efb492bf6f4527fa904 +Subproject commit 661356317ac6df52208d54187e692472a25a01f8 diff --git a/ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/vm/BlockchainReferenceTestTools.java b/ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/vm/BlockchainReferenceTestTools.java index 3a11761e708..266f8bb6a38 100644 --- a/ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/vm/BlockchainReferenceTestTools.java +++ b/ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/vm/BlockchainReferenceTestTools.java @@ -86,6 +86,9 @@ public class BlockchainReferenceTestTools { params.ignore("blobhashListBounds5"); params.ignore("blockWithAllTransactionTypes"); + // EIP-4788 is still in flux and the current fill is not against the final address + params.ignore("[Cancun]"); + // EOF tests are written against an older version of the spec params.ignore("/stEOF/"); } diff --git a/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/InMemoryKeyValueStorage.java b/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/InMemoryKeyValueStorage.java index 28e4436fe6e..3b1139cd179 100644 --- a/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/InMemoryKeyValueStorage.java +++ b/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/InMemoryKeyValueStorage.java @@ -18,9 +18,10 @@ import java.io.PrintStream; import java.nio.charset.StandardCharsets; -import java.util.HashMap; import java.util.Map; import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.locks.ReadWriteLock; import org.apache.tuweni.bytes.Bytes; @@ -51,9 +52,10 @@ public boolean containsStaticData() { } }; - private static Map>> asSegmentMap( + private static ConcurrentMap>> asSegmentMap( final Map> initialMap) { - final Map>> segmentMap = new HashMap<>(); + final ConcurrentMap>> segmentMap = + new ConcurrentHashMap<>(); segmentMap.put(SEGMENT_IDENTIFIER, initialMap); return segmentMap; } diff --git a/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/LayeredKeyValueStorage.java b/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/LayeredKeyValueStorage.java index ffc57530682..ddaf567433f 100644 --- a/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/LayeredKeyValueStorage.java +++ b/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/LayeredKeyValueStorage.java @@ -25,6 +25,7 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.locks.Lock; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -59,7 +60,7 @@ public LayeredKeyValueStorage(final SegmentedKeyValueStorage parent) { * @param parent the parent key value storage for this layered storage. */ public LayeredKeyValueStorage( - final Map>> map, + final ConcurrentMap>> map, final SegmentedKeyValueStorage parent) { super(map); this.parent = parent; diff --git a/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/SegmentedInMemoryKeyValueStorage.java b/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/SegmentedInMemoryKeyValueStorage.java index 66d70bac4dc..fd48384c366 100644 --- a/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/SegmentedInMemoryKeyValueStorage.java +++ b/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/SegmentedInMemoryKeyValueStorage.java @@ -30,6 +30,8 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -46,14 +48,14 @@ public class SegmentedInMemoryKeyValueStorage implements SnappedKeyValueStorage, SnappableKeyValueStorage, SegmentedKeyValueStorage { /** protected access for the backing hash map. */ - final Map>> hashValueStore; + final ConcurrentMap>> hashValueStore; /** protected access to the rw lock. */ protected final ReadWriteLock rwLock = new ReentrantReadWriteLock(); /** Instantiates a new In memory key value storage. */ public SegmentedInMemoryKeyValueStorage() { - this(new HashMap<>()); + this(new ConcurrentHashMap<>()); } /** @@ -62,7 +64,7 @@ public SegmentedInMemoryKeyValueStorage() { * @param hashValueStore the hash value store */ protected SegmentedInMemoryKeyValueStorage( - final Map>> hashValueStore) { + final ConcurrentMap>> hashValueStore) { this.hashValueStore = hashValueStore; } @@ -76,8 +78,8 @@ public SegmentedInMemoryKeyValueStorage(final List segments) segments.stream() .collect( Collectors - .>>toMap( - s -> s, s -> new HashMap<>()))); + .>> + toConcurrentMap(s -> s, s -> new ConcurrentHashMap<>()))); } @Override @@ -214,7 +216,9 @@ public SegmentedInMemoryKeyValueStorage takeSnapshot() { // need to clone the submaps also: return new SegmentedInMemoryKeyValueStorage( hashValueStore.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, e -> new HashMap<>(e.getValue())))); + .collect( + Collectors.toConcurrentMap( + Map.Entry::getKey, e -> new ConcurrentHashMap<>(e.getValue())))); } @Override