From 7059b45599cee73f4e6cfde96c4175e025b12d00 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Thu, 28 Mar 2024 23:01:53 +0800 Subject: [PATCH] Perf/dont redownload downloaded code (#6873) --- .../RecreateStateFromAccountRangesTests.cs | 52 +++++++++++++++++++ .../ParallelSync/SyncFeed.cs | 2 +- .../SnapSync/ISnapProvider.cs | 1 + .../SnapSync/SnapProvider.cs | 22 +++++++- .../SnapSync/SnapSyncFeed.cs | 6 +++ 5 files changed, 81 insertions(+), 2 deletions(-) diff --git a/src/Nethermind/Nethermind.State.Test/SnapSync/RecreateStateFromAccountRangesTests.cs b/src/Nethermind/Nethermind.State.Test/SnapSync/RecreateStateFromAccountRangesTests.cs index 8bf5129c1d9..769114bd898 100644 --- a/src/Nethermind/Nethermind.State.Test/SnapSync/RecreateStateFromAccountRangesTests.cs +++ b/src/Nethermind/Nethermind.State.Test/SnapSync/RecreateStateFromAccountRangesTests.cs @@ -6,6 +6,8 @@ using System; using System.Collections.Generic; using System.Linq; +using FluentAssertions; +using Nethermind.Blockchain; using Nethermind.Core; using Nethermind.Core.Crypto; using Nethermind.Core.Test.Builders; @@ -13,6 +15,7 @@ using Nethermind.Logging; using Nethermind.State; using Nethermind.State.Proofs; +using Nethermind.State.Snap; using Nethermind.Synchronization.SnapSync; using Nethermind.Trie; using Nethermind.Trie.Pruning; @@ -264,6 +267,55 @@ public void MissingAccountFromRange() Assert.IsFalse(db.KeyExists(rootHash)); } + [Test] + public void Will_not_redownload_persisted_code() + { + MemDb db = new(); + MemDb codeDb = new(); + DbProvider dbProvider = new(); + dbProvider.RegisterDb(DbNames.State, db); + dbProvider.RegisterDb(DbNames.Code, codeDb); + + BlockTree tree = Build.A.BlockTree().OfChainLength(5).TestObject; + using ProgressTracker progressTracker = new(tree, dbProvider.GetDb(DbNames.State), LimboLogs.Instance, + accountRangePartitionCount: 1); + SnapProvider snapProvider = CreateSnapProvider(progressTracker, dbProvider); + + PathWithAccount[] accountsWithPath = + [ + new PathWithAccount(new Hash256("0000000000000000000000000000000000000000000000000000000001112345"), + new Account(0, 0, Keccak.EmptyTreeHash, TestItem.Keccaks[0])), + new PathWithAccount(new Hash256("0000000000000000000000000000000000000000000000000000000001113456"), + new Account(0, 0, Keccak.EmptyTreeHash, TestItem.Keccaks[1])), + new PathWithAccount(new Hash256("0000000000000000000000000000000000000000000000000000000001114567"), + new Account(0, 0, Keccak.EmptyTreeHash, TestItem.Keccaks[2])), + new PathWithAccount(new Hash256("0000000000000000000000000000000000000000000000000000000001123456"), + new Account(0, 0, Keccak.EmptyTreeHash, TestItem.Keccaks[3])), + new PathWithAccount(new Hash256("0000000000000000000000000000000000000000000000000000000001123457"), + new Account(0, 0, Keccak.EmptyTreeHash, TestItem.Keccaks[4])) + ]; + + codeDb[TestItem.Keccaks[1].Bytes] = [1]; + codeDb[TestItem.Keccaks[2].Bytes] = [1]; + + StateTree stateTree = new StateTree(); + foreach (PathWithAccount pathWithAccount in accountsWithPath) + { + stateTree.Set(pathWithAccount.Path, pathWithAccount.Account); + } + + stateTree.UpdateRootHash(); + + snapProvider.AddAccountRange(1, + stateTree.RootHash, + accountsWithPath[0].Path, + accountsWithPath); + + progressTracker.IsFinished(out SnapSyncBatch nextRequest).Should().BeFalse(); + progressTracker.IsFinished(out nextRequest).Should().BeFalse(); + nextRequest.CodesRequest.Count.Should().Be(3); + } + private SnapProvider CreateSnapProvider(ProgressTracker progressTracker, IDbProvider dbProvider) { try diff --git a/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncFeed.cs b/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncFeed.cs index 1d887c70c36..d3d258f80ec 100644 --- a/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncFeed.cs +++ b/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncFeed.cs @@ -36,7 +36,7 @@ private void ChangeState(SyncFeedState newState) public void Activate() => ChangeState(SyncFeedState.Active); - public void Finish() + public virtual void Finish() { ChangeState(SyncFeedState.Finished); GC.Collect(2, GCCollectionMode.Aggressive, true, true); diff --git a/src/Nethermind/Nethermind.Synchronization/SnapSync/ISnapProvider.cs b/src/Nethermind/Nethermind.Synchronization/SnapSync/ISnapProvider.cs index c55ff822c79..ce1ae75d614 100644 --- a/src/Nethermind/Nethermind.Synchronization/SnapSync/ISnapProvider.cs +++ b/src/Nethermind/Nethermind.Synchronization/SnapSync/ISnapProvider.cs @@ -26,5 +26,6 @@ public interface ISnapProvider bool IsSnapGetRangesFinished(); void UpdatePivot(); + void Dispose(); } } diff --git a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs index 5d520236bde..aa74716db65 100644 --- a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs +++ b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs @@ -8,6 +8,7 @@ using System.Threading; using Microsoft.Extensions.ObjectPool; using Nethermind.Core; +using Nethermind.Core.Caching; using Nethermind.Core.Collections; using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; @@ -29,6 +30,9 @@ public class SnapProvider : ISnapProvider private readonly ProgressTracker _progressTracker; + // This is actually close to 97% effective. + private readonly LruKeyCache _codeExistKeyCache = new(1024 * 16, ""); + public SnapProvider(ProgressTracker progressTracker, IDb codeDb, INodeStorage nodeStorage, ILogManager logManager) { _codeDb = codeDb ?? throw new ArgumentNullException(nameof(codeDb)); @@ -88,7 +92,18 @@ public AddRangeResult AddAccountRange(long blockNumber, in ValueHash256 expected _progressTracker.EnqueueAccountStorage(item); } - _progressTracker.EnqueueCodeHashes(CollectionsMarshal.AsSpan(codeHashes)); + + using ArrayPoolList filteredCodeHashes = codeHashes.AsParallel().Where((code) => + { + if (_codeExistKeyCache.Get(code)) return false; + + bool exist = _codeDb.KeyExists(code.Bytes); + if (exist) _codeExistKeyCache.Set(code); + return !exist; + }).ToPooledList(codeHashes.Count); + + _progressTracker.EnqueueCodeHashes(filteredCodeHashes.AsSpan()); + _progressTracker.UpdateAccountRangePartitionProgress(effectiveHashLimit, accounts[^1].Path, moreChildrenToRight); } else if (result == AddRangeResult.MissingRootHashInProofs) @@ -326,6 +341,11 @@ public void UpdatePivot() _progressTracker.UpdatePivot(); } + public void Dispose() + { + _codeExistKeyCache.Clear(); + } + private class TrieStorePoolPolicy : IPooledObjectPolicy { private readonly INodeStorage _stateDb; diff --git a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapSyncFeed.cs b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapSyncFeed.cs index abcdf0b9f09..aa56e36552f 100644 --- a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapSyncFeed.cs +++ b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapSyncFeed.cs @@ -215,6 +215,12 @@ public override void SyncModeSelectorOnChanged(SyncMode current) } } + public override void Finish() + { + _snapProvider.Dispose(); + base.Finish(); + } + public override bool IsFinished => _snapProvider.IsSnapGetRangesFinished(); } }