Merge pull request #108 from Nexus-Mods/better-benchmarks-and-fix-for…

…-empty-tx

Fix for empty transactions + Some read benchmarks

benchmarks/NexusMods.MnemonicDB.Benchmarks/Benchmarks/ColdStartBenchmarks.cs

@@ -0,0 +1,98 @@
+using System.Linq;
+using System.Threading.Tasks;
+using BenchmarkDotNet.Attributes;
+using NexusMods.MnemonicDB.Abstractions;
+using NexusMods.MnemonicDB.Abstractions.ElementComparers;
+using NexusMods.MnemonicDB.Abstractions.IndexSegments;
+using NexusMods.MnemonicDB.TestModel;
+using NexusMods.Paths;
+
+namespace NexusMods.MnemonicDB.Benchmarks.Benchmarks;
+
+[MemoryDiagnoser]
+public class ColdStartBenchmarks : ABenchmark
+{
+    [GlobalSetup]
+    public async ValueTask GlobalSetup()
+    {
+        await InitializeAsync();
+        await InsertData();
+    }
+
+    private async Task InsertData()
+    {
+        foreach (var loadout in Enumerable.Range(0, 10))
+        {
+            using var tx = Connection.BeginTransaction();
+            var loadoutEntity = new Loadout.New(tx)
+            {
+                Name = $"Loadout {loadout}"
+            };
+            foreach (var mod in Enumerable.Range(0, 1000))
+            {
+                var modEntity = new Mod.New(tx)
+                {
+                    Name = $"Mod {mod}",
+                    Source = new System.Uri($"http://mod{mod}.com"),
+                    LoadoutId = loadoutEntity,
+                    OptionalHash = Hashing.xxHash3.Hash.FromLong(0)
+                };
+                foreach (var file in Enumerable.Range(0, 100))
+                {
+                    _ = new File.New(tx)
+                    {
+                        Path = $"File {file}",
+                        ModId = modEntity,
+                        Size = Size.FromLong(file),
+                        Hash = Hashing.xxHash3.Hash.FromLong(file)
+                    };
+                }
+            }
+            await tx.Commit();
+        }
+    }
+
+    [IterationSetup]
+    public void IterationSetup()
+    {
+        Connection.Db.ClearIndexCache();
+    }
+
+    [Benchmark]
+    public Size TotalSizeEAVT()
+    {
+        var loadout = Loadout.FindByName(Connection.Db, "Loadout 5").First();
+        var totalSize = Size.FromLong(0);
+
+        foreach (var mod in loadout.Mods)
+        {
+            foreach (var file in mod.Files)
+            {
+                totalSize += file.Size;
+            }
+        }
+
+        return totalSize;
+    }
+
+
+    [Benchmark]
+    public Size TotalSizeAEVT()
+    {
+        var loadoutId = Connection.Db.Datoms(Loadout.Name, "Loadout 5").First();
+        var modIds = Connection.Db.Datoms(Mod.LoadoutId, loadoutId.E).Select(e => e.E).ToHashSet();
+        var fileIds = Connection.Db.Datoms(File.ModId).ToLookup(d => ValueTag.Reference.Read<EntityId>(d.ValueSpan), d => d.E);
+        var fileSizes = Connection.Db.Datoms(File.Size)
+            .ToDictionary(d => d.E, d => Size.From(ValueTag.UInt64.Read<ulong>(d.ValueSpan)));
+
+        var totalSize = Size.FromLong(0);
+        foreach (var modId in modIds)
+        {
+            foreach (var fileId in fileIds[modId])
+            {
+                totalSize += fileSizes[fileId];
+            }
+        }
+        return totalSize;
+    }
+}

benchmarks/NexusMods.MnemonicDB.Benchmarks/Program.cs

@@ -29,7 +29,7 @@
 
 #else
 
-BenchmarkRunner.Run<ReadThenWriteBenchmarks>(config: DefaultConfig.Instance.WithOption(ConfigOptions.DisableOptimizationsValidator, true));
+BenchmarkRunner.Run<ColdStartBenchmarks>();
 #endif
 
 

src/NexusMods.MnemonicDB.Abstractions/IDb.cs

@@ -85,4 +85,9 @@ public interface IDb : IEquatable<IDb>
     /// </summary>
     TReturn AnalyzerData<TAnalyzer, TReturn>() 
         where TAnalyzer : IAnalyzer<TReturn>;
+
+    /// <summary>
+    /// Clears the internal cache of the database.
+    /// </summary>
+    void ClearIndexCache();
 }

src/NexusMods.MnemonicDB/Caching/IndexSegmentCache.cs

@@ -283,5 +283,12 @@ private void UpdateEntry(CacheKey key, IndexSegment segment)
                 return;
         }
     }
-
+
+    /// <summary>
+    /// Clears the cache.
+    /// </summary>
+    public void Clear()
+    {
+        _root = new CacheRoot(ImmutableDictionary<CacheKey, CacheValue>.Empty);
+    }
 }

src/NexusMods.MnemonicDB/Db.cs

@@ -114,6 +114,11 @@ TReturn IDb.AnalyzerData<TAnalyzer, TReturn>()
         throw new KeyNotFoundException($"Analyzer {typeof(TAnalyzer).Name} not found");
     }
 
+    public void ClearIndexCache()
+    {
+        _cache.Clear();
+    }
+
     public IndexSegment Datoms<TValue>(IWritableAttribute<TValue> attribute, TValue value)
     {
         return Datoms(SliceDescriptor.Create(attribute, value, AttributeCache));

src/NexusMods.MnemonicDB/Storage/DatomStore.cs

@@ -103,7 +103,8 @@ public DatomStore(
 
         Backend.Init(settings.Path);
 
-        if (bootstrap) Bootstrap();
+        if (bootstrap) 
+            Bootstrap();
     }
 
     /// <inheritdoc />
@@ -308,7 +309,7 @@ private StoreResult Log(IInternalTxFunctionImpl pendingTransaction)
 
         return new StoreResult
         {
-            AssignedTxId = _thisTx,
+            AssignedTxId = _asOfTx,
             Remaps = _remaps.ToFrozenDictionary(),
             Snapshot = CurrentSnapshot
         };

tests/NexusMods.MnemonicDB.Tests/ComplexModelTests.cs

@@ -173,7 +173,15 @@ public async Task CanRestartStorage(int modCount, int filesPerMod, int extraFile
         await extraTx.Commit();
 
         Logger.LogInformation("Restarting storage");
+        Connection.Db.RecentlyAdded.Should().NotBeEmpty("the last transaction added data");
+
+        var lastTxId = Connection.TxId;
         await RestartDatomStore();
+
+        Connection.TxId.Should().Be(lastTxId, "the transaction id should be the same after a restart");
+        Connection.Db.BasisTxId.Should().Be(lastTxId, "the basis transaction id should be the same after a restart");
+
+        Connection.Db.RecentlyAdded.Should().NotBeEmpty("the restarted database should populate the recently added");
         Logger.LogInformation("Storage restarted");