Merge branch 'master' into feat/total-difficulty-strategy

src/Nethermind/Nethermind.Blockchain.Test/FullPruning/FullPruningDiskTest.cs

@@ -132,7 +132,7 @@ protected override async Task RunFullPruning(CancellationToken cancellationToken
         }
     }
 
-    [Test, MaxTime(Timeout.MaxTestTime), Retry(5)]
+    [Test, MaxTime(Timeout.LongTestTime)]
     public async Task prune_on_disk_multiple_times()
     {
         using PruningTestBlockchain chain = await PruningTestBlockchain.Create(new PruningConfig { FullPruningMinimumDelayHours = 0 });
@@ -142,7 +142,7 @@ public async Task prune_on_disk_multiple_times()
         }
     }
 
-    [Test, MaxTime(Timeout.MaxTestTime), Retry(5)]
+    [Test, MaxTime(Timeout.LongTestTime)]
     public async Task prune_on_disk_only_once()
     {
         using PruningTestBlockchain chain = await PruningTestBlockchain.Create(new PruningConfig { FullPruningMinimumDelayHours = 10 });

src/Nethermind/Nethermind.Blockchain.Test/Timeout.cs

@@ -4,6 +4,7 @@
 namespace Nethermind.Blockchain.Test;
 internal class Timeout
 {
+    public const int LongTestTime = 60_000;
     public const int MaxTestTime = 10_000;
     public const int MaxWaitTime = 1_000;
 }

src/Nethermind/Nethermind.Core.Test/TestMemColumnDb.cs

@@ -31,4 +31,5 @@ public IColumnsWriteBatch<TKey> StartWriteBatch()
         return new InMemoryColumnWriteBatch<TKey>(this);
     }
     public void Dispose() { }
+    public void Flush(bool onlyWal = false) { }
 }

src/Nethermind/Nethermind.Core.Test/TestMemDb.cs

@@ -109,7 +109,7 @@ public override IWriteBatch StartWriteBatch()
         return new InMemoryWriteBatch(this);
     }
 
-    public override void Flush()
+    public override void Flush(bool onlyWal)
     {
         FlushCount++;
     }

src/Nethermind/Nethermind.Db.Rocks/ColumnDb.cs

@@ -128,9 +128,9 @@ public bool KeyExists(ReadOnlySpan<byte> key)
         return _mainDb.KeyExistsWithColumn(key, _columnFamily);
     }
 
-    public void Flush()
+    public void Flush(bool onlyWal)
     {
-        _mainDb.Flush();
+        _mainDb.Flush(onlyWal);
     }
 
     public void Compact()

src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs

@@ -1321,23 +1321,28 @@ private void FlushOnTooManyWrites()
         }
     }
 
-    public void Flush()
+    public void Flush(bool onlyWal = false)
     {
         ObjectDisposedException.ThrowIf(_isDisposing, this);
 
-        InnerFlush();
+        InnerFlush(onlyWal);
     }
 
     public virtual void Compact()
     {
         _db.CompactRange(Keccak.Zero.BytesToArray(), Keccak.MaxValue.BytesToArray());
     }
 
-    private void InnerFlush()
+    private void InnerFlush(bool onlyWal)
     {
         try
         {
-            _rocksDbNative.rocksdb_flush(_db.Handle, FlushOptions.DefaultFlushOptions.Handle);
+            _rocksDbNative.rocksdb_flush_wal(_db.Handle, true);
+
+            if (!onlyWal)
+            {
+                _rocksDbNative.rocksdb_flush(_db.Handle, FlushOptions.DefaultFlushOptions.Handle);
+            }
         }
         catch (RocksDbSharpException e)
         {
@@ -1439,7 +1444,7 @@ public void Dispose()
             dbMetricsUpdater.Dispose();
         }
 
-        InnerFlush();
+        InnerFlush(false);
         ReleaseUnmanagedResources();
 
         _dbsByPath.Remove(_fullPath!, out _);

src/Nethermind/Nethermind.Db.Rpc/RpcColumnsDb.cs

@@ -46,5 +46,6 @@ public IColumnsWriteBatch<T> StartWriteBatch()
             return new InMemoryColumnWriteBatch<T>(this);
         }
         public void Dispose() { }
+        public void Flush(bool onlyWal = false) { }
     }
 }

src/Nethermind/Nethermind.Db.Rpc/RpcDb.cs

@@ -73,6 +73,8 @@ public bool KeyExists(ReadOnlySpan<byte> key)
 
         public IDb Innermost => this; // record db is just a helper DB here
         public void Flush() { }
+        public void Flush(bool onlyWal = false) { }
+
         public void Clear() { }
 
         public IEnumerable<KeyValuePair<byte[], byte[]>> GetAll(bool ordered = false) => _recordDb.GetAll();

src/Nethermind/Nethermind.Db/CompressingDb.cs

@@ -137,7 +137,7 @@ public IEnumerable<byte[]> GetAllValues(bool ordered = false) =>
 
             public bool KeyExists(ReadOnlySpan<byte> key) => _wrapped.KeyExists(key);
 
-            public void Flush() => _wrapped.Flush();
+            public void Flush(bool onlyWal) => _wrapped.Flush(onlyWal);
 
             public void Clear() => _wrapped.Clear();
 

src/Nethermind/Nethermind.Db/FullPruning/FullPruningDb.cs

@@ -144,11 +144,11 @@ public void Remove(ReadOnlySpan<byte> key)
         public IDb Innermost => this;
 
         // we need to flush both DB's
-        public void Flush()
+        public void Flush(bool onlyWal)
         {
-            _currentDb.Flush();
+            _currentDb.Flush(onlyWal);
             IDb? cloningDb = _pruningContext?.CloningDb;
-            cloningDb?.Flush();
+            cloningDb?.Flush(onlyWal);
         }
 
         // we need to clear both DB's

src/Nethermind/Nethermind.Db/IDb.cs

@@ -23,7 +23,7 @@ public interface IDbMeta
     {
         DbMetric GatherMetric(bool includeSharedCache = false) => new DbMetric();
 
-        void Flush() { }
+        void Flush(bool onlyWal = false);
         void Clear() { }
         void Compact() { }
 

src/Nethermind/Nethermind.Db/MemColumnsDb.cs

@@ -35,5 +35,6 @@ public IColumnsWriteBatch<TKey> StartWriteBatch()
             return new InMemoryColumnWriteBatch<TKey>(this);
         }
         public void Dispose() { }
+        public void Flush(bool onlyWal = false) { }
     }
 }

src/Nethermind/Nethermind.Db/MemDb.cs

@@ -77,9 +77,7 @@ public bool KeyExists(ReadOnlySpan<byte> key)
 
         public IDb Innermost => this;
 
-        public virtual void Flush()
-        {
-        }
+        public virtual void Flush(bool onlyWal = false) { }
 
         public void Clear()
         {

src/Nethermind/Nethermind.Db/NullDb.cs

@@ -43,7 +43,8 @@ public bool KeyExists(ReadOnlySpan<byte> key)
             return false;
         }
 
-        public void Flush() { }
+        public void Flush(bool onlyWal = false) { }
+
         public void Clear() { }
 
         public IEnumerable<KeyValuePair<byte[], byte[]>> GetAll(bool ordered = false) => Enumerable.Empty<KeyValuePair<byte[], byte[]>>();

src/Nethermind/Nethermind.Db/ReadOnlyColumnsDb.cs

@@ -44,5 +44,7 @@ public void Dispose()
                 readOnlyColumn.Value.Dispose();
             }
         }
+
+        public void Flush(bool onlyWal = false) { }
     }
 }

src/Nethermind/Nethermind.Db/ReadOnlyDb.cs

@@ -67,11 +67,7 @@ public void Remove(ReadOnlySpan<byte> key) { }
 
         public bool KeyExists(ReadOnlySpan<byte> key) => _memDb.KeyExists(key) || wrappedDb.KeyExists(key);
 
-        public void Flush()
-        {
-            wrappedDb.Flush();
-            _memDb.Flush();
-        }
+        public void Flush(bool onlyWal) { }
 
         public void Clear() => throw new InvalidOperationException();
 

src/Nethermind/Nethermind.Db/SimpleFilePublicKeyDb.cs

@@ -85,7 +85,8 @@ public bool KeyExists(ReadOnlySpan<byte> key)
             return _cache.ContainsKey(key);
         }
 
-        public void Flush() { }
+        public void Flush(bool onlyWal = false) { }
+
         public void Clear()
         {
             File.Delete(DbPath);

src/Nethermind/Nethermind.ExternalSigner.Plugin/ClefSignerPlugin.cs

@@ -22,6 +22,8 @@ public class ClefSignerPlugin : INethermindPlugin
 
     public string Author => "Nethermind";
 
+    public bool MustInitialize => true;
+
     public ValueTask DisposeAsync() => ValueTask.CompletedTask;
 
     public async Task Init(INethermindApi nethermindApi)

src/Nethermind/Nethermind.Init/InitializeStateDb.cs

@@ -119,9 +119,9 @@ public Task Execute(CancellationToken cancellationToken)
             var minimumWriteBufferMb = 0.2 * pruningConfig.CacheMb;
             if (totalWriteBufferMb < minimumWriteBufferMb)
             {
-                int minimumWriteBufferNumber = (int)Math.Ceiling((minimumWriteBufferMb * 1.MB()) / dbConfig.StateDbWriteBufferSize);
+                long minimumWriteBufferSize = (int)Math.Ceiling((minimumWriteBufferMb * 1.MB()) / dbConfig.StateDbWriteBufferNumber);
 
-                if (_logger.IsWarn) _logger.Warn($"Detected {totalWriteBufferMb}MB of maximum write buffer size. Write buffer size should be at least 20% of pruning cache MB or memory pruning may slow down. Try setting `--Db.{nameof(dbConfig.WriteBufferNumber)} {minimumWriteBufferNumber}`.");
+                if (_logger.IsWarn) _logger.Warn($"Detected {totalWriteBufferMb}MB of maximum write buffer size. Write buffer size should be at least 20% of pruning cache MB or memory pruning may slow down. Try setting `--Db.{nameof(dbConfig.StateDbWriteBufferSize)} {minimumWriteBufferSize}`.");
             }
 
             pruningStrategy = Prune

src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs

@@ -379,7 +379,7 @@ public async Task<string> NewPayloadV3_should_verify_blob_versioned_hashes_again
                  Substitute.For<IGetPayloadBodiesByRangeV2Handler>(),
                  Substitute.For<IHandler<TransitionConfigurationV1, TransitionConfigurationV1>>(),
                  Substitute.For<IHandler<IEnumerable<string>, IEnumerable<string>>>(),
-                 Substitute.For<IAsyncHandler<byte[][], GetBlobsV1Result>>(),
+                 Substitute.For<IAsyncHandler<byte[][], IEnumerable<BlobAndProofV1?>>>(),
                  chain.SpecProvider,
                  new GCKeeper(NoGCStrategy.Instance, chain.LogManager),
                  Substitute.For<ILogManager>()));
@@ -539,7 +539,7 @@ public async Task GetBlobsV1_should_throw_if_more_than_128_requested_blobs([Valu
             request.Add(Bytes.FromHexString(i.ToString("X64")));
         }
 
-        ResultWrapper<GetBlobsV1Result> result = await rpcModule.engine_getBlobsV1(request.ToArray());
+        ResultWrapper<IEnumerable<BlobAndProofV1?>> result = await rpcModule.engine_getBlobsV1(request.ToArray());
 
         if (requestSize > 128)
         {
@@ -549,7 +549,7 @@ public async Task GetBlobsV1_should_throw_if_more_than_128_requested_blobs([Valu
         else
         {
             result.Result.Should().Be(Result.Success);
-            result.Data.BlobsAndProofs.Should().HaveCount(requestSize);
+            result.Data.Should().HaveCount(requestSize);
         }
     }
 
@@ -559,10 +559,10 @@ public async Task GetBlobsV1_should_handle_empty_request()
         MergeTestBlockchain chain = await CreateBlockchain(releaseSpec: Cancun.Instance);
         IEngineRpcModule rpcModule = CreateEngineModule(chain, null, TimeSpan.FromDays(1));
 
-        ResultWrapper<GetBlobsV1Result> result = await rpcModule.engine_getBlobsV1([]);
+        ResultWrapper<IEnumerable<BlobAndProofV1?>> result = await rpcModule.engine_getBlobsV1([]);
 
         result.Result.Should().Be(Result.Success);
-        result.Data.Should().BeEquivalentTo(new GetBlobsV1Result(ArraySegment<BlobAndProofV1?>.Empty));
+        result.Data.Should().BeEquivalentTo(ArraySegment<BlobAndProofV1?>.Empty);
     }
 
     [Test]
@@ -580,11 +580,11 @@ public async Task GetBlobsV1_should_return_requested_blobs([Values(1, 2, 3, 4, 5
 
         chain.TxPool.SubmitTx(blobTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted);
 
-        ResultWrapper<GetBlobsV1Result> result = await rpcModule.engine_getBlobsV1(blobTx.BlobVersionedHashes!);
+        ResultWrapper<IEnumerable<BlobAndProofV1?>> result = await rpcModule.engine_getBlobsV1(blobTx.BlobVersionedHashes!);
 
         ShardBlobNetworkWrapper wrapper = (ShardBlobNetworkWrapper)blobTx.NetworkWrapper!;
-        result.Data.BlobsAndProofs.Select(b => b!.Blob).Should().BeEquivalentTo(wrapper.Blobs);
-        result.Data.BlobsAndProofs.Select(b => b!.Proof).Should().BeEquivalentTo(wrapper.Proofs);
+        result.Data.Select(b => b!.Blob).Should().BeEquivalentTo(wrapper.Blobs);
+        result.Data.Select(b => b!.Proof).Should().BeEquivalentTo(wrapper.Proofs);
     }
 
     [Test]
@@ -602,10 +602,10 @@ public async Task GetBlobsV1_should_return_nulls_when_blobs_not_found([Values(1,
             .SignedAndResolved(chain.EthereumEcdsa, TestItem.PrivateKeyA).TestObject;
 
         // requesting hashes that are not present in TxPool
-        ResultWrapper<GetBlobsV1Result> result = await rpcModule.engine_getBlobsV1(blobTx.BlobVersionedHashes!);
+        ResultWrapper<IEnumerable<BlobAndProofV1?>> result = await rpcModule.engine_getBlobsV1(blobTx.BlobVersionedHashes!);
 
-        result.Data.BlobsAndProofs.Should().HaveCount(numberOfRequestedBlobs);
-        result.Data.BlobsAndProofs.Should().AllBeEquivalentTo<BlobAndProofV1?>(null);
+        result.Data.Should().HaveCount(numberOfRequestedBlobs);
+        result.Data.Should().AllBeEquivalentTo<BlobAndProofV1?>(null);
     }
 
     [Test]
@@ -638,12 +638,11 @@ public async Task GetBlobsV1_should_return_mix_of_blobs_and_nulls([Values(1, 2,
                 : null);
             blobVersionedHashesRequest.Add(addActualHash ? blobTx.BlobVersionedHashes![actualIndex++]! : Bytes.FromHexString(i.ToString("X64")));
         }
-        GetBlobsV1Result expected = new(blobsAndProofs.ToArray());
 
-        ResultWrapper<GetBlobsV1Result> result = await rpcModule.engine_getBlobsV1(blobVersionedHashesRequest.ToArray());
+        ResultWrapper<IEnumerable<BlobAndProofV1?>> result = await rpcModule.engine_getBlobsV1(blobVersionedHashesRequest.ToArray());
 
-        result.Data.Should().BeEquivalentTo(expected);
-        BlobAndProofV1?[] resultBlobsAndProofs = result.Data.BlobsAndProofs.ToArray();
+        result.Data.Should().BeEquivalentTo(blobsAndProofs);
+        BlobAndProofV1?[] resultBlobsAndProofs = result.Data.ToArray();
         resultBlobsAndProofs.Length.Should().Be(requestSize);
         for (int i = 0; i < requestSize; i++)
         {

src/Nethermind/Nethermind.Merge.Plugin/EngineRpcModule.Cancun.cs

@@ -1,6 +1,7 @@
 // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited
 // SPDX-License-Identifier: LGPL-3.0-only
 
+using System.Collections.Generic;
 using System.Threading.Tasks;
 using Nethermind.Consensus;
 using Nethermind.Consensus.Producers;
@@ -14,7 +15,7 @@ namespace Nethermind.Merge.Plugin;
 public partial class EngineRpcModule : IEngineRpcModule
 {
     private readonly IAsyncHandler<byte[], GetPayloadV3Result?> _getPayloadHandlerV3;
-    private readonly IAsyncHandler<byte[][], GetBlobsV1Result> _getBlobsHandler;
+    private readonly IAsyncHandler<byte[][], IEnumerable<BlobAndProofV1?>> _getBlobsHandler;
 
     public Task<ResultWrapper<ForkchoiceUpdatedV1Result>> engine_forkchoiceUpdatedV3(ForkchoiceStateV1 forkchoiceState, PayloadAttributes? payloadAttributes = null)
         => ForkchoiceUpdated(forkchoiceState, payloadAttributes, EngineApiVersions.Cancun);
@@ -25,6 +26,6 @@ public Task<ResultWrapper<PayloadStatusV1>> engine_newPayloadV3(ExecutionPayload
     public async Task<ResultWrapper<GetPayloadV3Result?>> engine_getPayloadV3(byte[] payloadId) =>
         await _getPayloadHandlerV3.HandleAsync(payloadId);
 
-    public async Task<ResultWrapper<GetBlobsV1Result>> engine_getBlobsV1(byte[][] blobVersionedHashes) =>
+    public async Task<ResultWrapper<IEnumerable<BlobAndProofV1?>>> engine_getBlobsV1(byte[][] blobVersionedHashes) =>
         await _getBlobsHandler.HandleAsync(blobVersionedHashes);
 }

src/Nethermind/Nethermind.Merge.Plugin/EngineRpcModule.cs

@@ -33,7 +33,7 @@ public EngineRpcModule(
         IGetPayloadBodiesByRangeV2Handler executionGetPayloadBodiesByRangeV2Handler,
         IHandler<TransitionConfigurationV1, TransitionConfigurationV1> transitionConfigurationHandler,
         IHandler<IEnumerable<string>, IEnumerable<string>> capabilitiesHandler,
-        IAsyncHandler<byte[][], GetBlobsV1Result> getBlobsHandler,
+        IAsyncHandler<byte[][], IEnumerable<BlobAndProofV1?>> getBlobsHandler,
         ISpecProvider specProvider,
         GCKeeper gcKeeper,
         ILogManager logManager)

src/Nethermind/Nethermind.Merge.Plugin/Handlers/GetBlobsHandler.cs

@@ -9,28 +9,36 @@
 
 namespace Nethermind.Merge.Plugin.Handlers;
 
-public class GetBlobsHandler(ITxPool txPool) : IAsyncHandler<byte[][], GetBlobsV1Result>
+public class GetBlobsHandler(ITxPool txPool) : IAsyncHandler<byte[][], IEnumerable<BlobAndProofV1?>>
 {
     private const int MaxRequest = 128;
 
-    public Task<ResultWrapper<GetBlobsV1Result>> HandleAsync(byte[][] request)
+    public Task<ResultWrapper<IEnumerable<BlobAndProofV1?>>> HandleAsync(byte[][] request)
     {
         if (request.Length > MaxRequest)
         {
             var error = $"The number of requested blobs must not exceed {MaxRequest}";
-            return ResultWrapper<GetBlobsV1Result>.Fail(error, MergeErrorCodes.TooLargeRequest);
+            return ResultWrapper<IEnumerable<BlobAndProofV1?>>.Fail(error, MergeErrorCodes.TooLargeRequest);
         }
 
-        return ResultWrapper<GetBlobsV1Result>.Success(new GetBlobsV1Result(GetBlobsAndProofs(request)));
+        return ResultWrapper<IEnumerable<BlobAndProofV1?>>.Success(GetBlobsAndProofs(request));
     }
 
     private IEnumerable<BlobAndProofV1?> GetBlobsAndProofs(byte[][] request)
     {
+        Metrics.NumberOfRequestedBlobs += request.Length;
+
         foreach (byte[] requestedBlobVersionedHash in request)
         {
-            yield return txPool.TryGetBlobAndProof(requestedBlobVersionedHash, out byte[]? blob, out byte[]? proof)
-                ? new BlobAndProofV1(blob, proof)
-                : null;
+            if (txPool.TryGetBlobAndProof(requestedBlobVersionedHash, out byte[]? blob, out byte[]? proof))
+            {
+                Metrics.NumberOfSentBlobs++;
+                yield return new BlobAndProofV1(blob, proof);
+            }
+            else
+            {
+                yield return null;
+            }
         }
     }
 }