Update mnemonicdb (reopening #2064) (#2074)

* Update to the new MnemonicDB API, still needs a fix in the MnemonicDB repo before this works

* Fix tuple remapping

* Try and fix the package reference errors (someday I should read up on how this works)

* Not sure why this existed

* Fix sorting error on RedModInstallerTests by sorting the data we hand to Verify

* cleanup the project file a bit

* Update to the newest version of MnemonicDB so that we can fix the deadlock issue introduced in the previous version

---------

Co-authored-by: halgari <[email protected]>

Directory.Packages.props

@@ -12,8 +12,8 @@
     <PackageVersion Include="Nerdbank.FullDuplexStream" Version="1.1.12" />
     <PackageVersion Include="Nerdbank.Streams" Version="2.11.74" />
     <PackageVersion Include="NexusMods.Paths" Version="0.10.0" />
-    <PackageVersion Include="NexusMods.MnemonicDB.Abstractions" Version="0.9.81" />
-    <PackageVersion Include="NexusMods.MnemonicDB" Version="0.9.81" />
+    <PackageVersion Include="NexusMods.MnemonicDB.Abstractions" Version="0.9.86" />
+    <PackageVersion Include="NexusMods.MnemonicDB" Version="0.9.86" />
     <PackageVersion Include="NexusMods.Hashing.xxHash64" Version="2.0.1" />
     <PackageVersion Include="NexusMods.Paths.Extensions.Nx" Version="0.10.0" />
     <PackageVersion Include="NexusMods.Paths.TestingHelpers" Version="0.9.5" />
@@ -124,7 +124,7 @@
     <PackageVersion Include="Humanizer" Version="2.14.1" />
     <PackageVersion Include="ini-parser-netstandard" Version="2.5.2" />
     <PackageVersion Include="Mutagen.Bethesda.Skyrim" Version="0.44.0" />
-    <PackageVersion Include="NexusMods.MnemonicDB.SourceGenerator" Version="0.9.81" />
+    <PackageVersion Include="NexusMods.MnemonicDB.SourceGenerator" Version="0.9.86" />
     <PackageVersion Include="NLog.Extensions.Logging" Version="5.3.11" />
     <PackageVersion Include="OneOf" Version="3.0.271" />
     <PackageVersion Include="ReactiveUI.Fody" Version="19.5.41" />

src/Abstractions/NexusMods.Abstractions.FileStore.Nx/Models/NxFileEntryAttribute.cs

@@ -11,7 +11,7 @@ namespace NexusMods.Abstractions.FileStore.Nx.Models;
 public class NxFileEntryAttribute(string ns, string name) : BlobAttribute<FileEntry>(ns, name)
 {
     /// <inheritdoc />
-    protected override unsafe FileEntry FromLowLevel(ReadOnlySpan<byte> value, ValueTags tags, RegistryId registryId)
+    protected override unsafe FileEntry FromLowLevel(ReadOnlySpan<byte> value, ValueTags tags, AttributeResolver resolver)
     {
         fixed (byte* ptr = value)
         {

src/Abstractions/NexusMods.Abstractions.Loadouts.Synchronizers/ALoadoutSynchronizer.cs

@@ -1164,9 +1164,9 @@ public virtual bool IsIgnoredPath(GamePath path)
         Memory<byte> buffer = System.GC.AllocateUninitializedArray<byte>(32);
 
         // Cache some attribute ids
-        var registry = baseDb.Registry;
-        var nameId = Loadout.Name.GetDbId(registry.Id);
-        var shortNameId = Loadout.ShortName.GetDbId(registry.Id);
+        var cache = baseDb.AttributeCache;
+        var nameId = cache.GetAttributeId(Loadout.Name.Id);
+        var shortNameId = cache.GetAttributeId(Loadout.ShortName.Id);
 
         // Generate a new name and short name
         var newShortName = LoadoutNameProvider.GetNewShortName(Loadout.All(baseDb)
@@ -1219,11 +1219,10 @@ public virtual bool IsIgnoredPath(GamePath path)
 
                 // Create the new datom and reference the copied value
                 var prefix = new KeyPrefix(newId, datom.A, TxId.Tmp, false, datom.Prefix.ValueTag);
-                var newDatom = new Datom(prefix, buffer[..datom.ValueSpan.Length], registry);
+                var newDatom = new Datom(prefix, buffer[..datom.ValueSpan.Length]);
 
                 // Remap any entity ids in the value
-                var attr = registry.GetAttribute(datom.A);
-                attr.Remap(remapFn, buffer[..datom.ValueSpan.Length].Span);
+                ValueHelpers.Remap(remapFn, datom.Prefix, buffer[..datom.ValueSpan.Length].Span);
 
                 // Add the new datom
                 tx.Add(newDatom);

src/Abstractions/NexusMods.Abstractions.Media/ImageDataAttribute.cs

@@ -11,7 +11,7 @@ namespace NexusMods.Abstractions.Media;
 public class ImageDataAttribute(string ns, string name) : BlobAttribute<ImageData>(ns, name)
 {
     /// <inheritdoc/>
-    protected override ImageData FromLowLevel(ReadOnlySpan<byte> value, ValueTags tags, RegistryId registryId)
+    protected override ImageData FromLowLevel(ReadOnlySpan<byte> value, ValueTags tags, AttributeResolver resolver)
     {
         Debug.Assert(sizeof(ImageDataCompression) == 1);
         var compression = (ImageDataCompression)value[0];

src/Abstractions/NexusMods.Abstractions.Media/ImageMetadataAttribute.cs

@@ -13,7 +13,7 @@ public class ImageMetadataAttribute(string ns, string name) : BlobAttribute<Imag
     private static readonly int ImageMetadataSize = Marshal.SizeOf<ImageMetadata>();
 
     /// <inheritdoc/>
-    protected override ImageMetadata FromLowLevel(ReadOnlySpan<byte> value, ValueTags tags, RegistryId registryId)
+    protected override ImageMetadata FromLowLevel(ReadOnlySpan<byte> value, ValueTags tags, AttributeResolver resolver)
     {
         return ImageMetadata.Read(value);
     }

src/Abstractions/NexusMods.Abstractions.MnemonicDB.Analyzers/TreeAnalyzer.cs

@@ -30,13 +30,14 @@ public object Analyze(IDb? dbOld, IDb dbNew)
 
             var db = current.IsRetract ? dbOld : dbNew;
             var entity = db.Get(current.E);
+            var resolver = dbNew.Connection.AttributeResolver;
             foreach (var datom in entity)
             {
-                var resolved = dbNew.Registry.GetAttribute(datom.A);
-                if (resolved is not ReferenceAttribute reference) 
+                var resolved = resolver.Resolve(datom);
+                if (resolved.A is not ReferenceAttribute reference) 
                     continue;
-                
-                var parent = reference.ReadValue(datom.ValueSpan, datom.Prefix.ValueTag, entity.RegistryId);
+
+                var parent = reference.ReadValue(datom.ValueSpan, datom.Prefix.ValueTag, resolver);
                 remaining.Push((parent, current.IsRetract));
             }
         }

src/Abstractions/NexusMods.Abstractions.MnemonicDB.Attributes/AbsolutePathAttribute.cs

@@ -18,8 +18,9 @@ protected override string ToLowLevel(AbsolutePath value)
     }
 
     /// <inheritdoc />
-    protected override AbsolutePath FromLowLevel(string value, ValueTags tag, RegistryId registryId)
+    protected override AbsolutePath FromLowLevel(string value, ValueTags tag, AttributeResolver resolver)
+
     {
-        return GetServiceProvider(registryId).GetRequiredService<IFileSystem>().FromUnsanitizedFullPath(value);
+        return resolver.ServiceProvider.GetRequiredService<IFileSystem>().FromUnsanitizedFullPath(value);
     }
 }

src/Abstractions/NexusMods.Abstractions.MnemonicDB.Attributes/BooleanAttribute.cs

@@ -16,7 +16,7 @@ protected override byte ToLowLevel(bool value)
     }
 
     /// <inheritdoc />
-    protected override bool FromLowLevel(byte value, ValueTags tags, RegistryId registry)
+    protected override bool FromLowLevel(byte value, ValueTags tags, AttributeResolver resolver)
     {
         return value == 1;
     }

src/Abstractions/NexusMods.Abstractions.MnemonicDB.Attributes/ByteAttribute.cs

@@ -14,7 +14,7 @@ protected override byte ToLowLevel(byte value)
         return value;
     }
 
-    protected override byte FromLowLevel(byte value, ValueTags tags, RegistryId registry)
+    protected override byte FromLowLevel(byte value, ValueTags tags, AttributeResolver resolver)
     {
         return value;
     }

src/Abstractions/NexusMods.Abstractions.MnemonicDB.Attributes/ClassAttribute.cs

@@ -18,7 +18,7 @@ public class ClassAttribute<TType>(string ns, string name) : ScalarAttribute<UIn
     protected override UInt128 ToLowLevel(UInt128 id) => id;
 
     /// <inheritdoc />
-    protected override UInt128 FromLowLevel(UInt128 value, ValueTags tags, RegistryId registry) => value;
+    protected override UInt128 FromLowLevel(UInt128 value, ValueTags tags, AttributeResolver resolver) => value;
 
     /// <summary>
     /// Gets the instance of the class from the DI container.

src/Abstractions/NexusMods.Abstractions.MnemonicDB.Attributes/DateTimeAttribute.cs

@@ -16,5 +16,6 @@ public class DateTimeAttribute(string ns, string name) : ScalarAttribute<DateTim
     protected override long ToLowLevel(DateTime value) => value.ToBinary();
 
     /// <inheritdoc />
-    protected override DateTime FromLowLevel(long value, ValueTags tags, RegistryId registryId) => DateTime.FromBinary(value);
+    protected override DateTime FromLowLevel(long value, ValueTags tags, AttributeResolver resolver)
+        => DateTime.FromBinary(value);
 }

src/Abstractions/NexusMods.Abstractions.MnemonicDB.Attributes/EnumAttribute.cs

@@ -19,7 +19,7 @@ protected override int ToLowLevel(T value)
     }
 
     /// <inheritdoc />
-    protected override T FromLowLevel(int value, ValueTags tags, RegistryId registryId)
+    protected override T FromLowLevel(int value, ValueTags tags, AttributeResolver resolver)
     {
         // Same as ToLowLevel, the cast to object is removed by the JIT
         return (T)(object)value;
@@ -41,7 +41,7 @@ protected override int ToLowLevel(T value)
     }
 
     /// <inheritdoc />
-    protected override T FromLowLevel(byte value, ValueTags tags, RegistryId registryId)
+    protected override T FromLowLevel(byte value, ValueTags tags, AttributeResolver resolver)
     {
         // Same as ToLowLevel, the cast to object is removed by the JIT
         return (T)(object)value;

src/Abstractions/NexusMods.Abstractions.MnemonicDB.Attributes/GameDomainAttribute.cs

@@ -17,7 +17,7 @@ protected override string ToLowLevel(GameDomain value)
     }
 
     /// <inheritdoc />
-    protected override GameDomain FromLowLevel(string value, ValueTags tag, RegistryId registryId)
+    protected override GameDomain FromLowLevel(string value, ValueTags tag, AttributeResolver resolver)
     {
         return GameDomain.From(value);
     }

src/Abstractions/NexusMods.Abstractions.MnemonicDB.Attributes/GamePathAttribute.cs

@@ -20,7 +20,7 @@ protected override string ToLowLevel(GamePath value)
     }
 
     /// <inheritdoc />
-    protected override GamePath FromLowLevel(string value, ValueTags tags, RegistryId registryId)
+    protected override GamePath FromLowLevel(string value, ValueTags tags, AttributeResolver resolver)
     {
         var parts = value.Split('|');
         Debug.Assert(parts.Length == 2);

src/Abstractions/NexusMods.Abstractions.MnemonicDB.Attributes/GameStoreAttribute.cs

@@ -17,7 +17,7 @@ protected override string ToLowLevel(GameStore value)
     }
 
     /// <inheritdoc />
-    protected override GameStore FromLowLevel(string value, ValueTags tag, RegistryId registryId)
+    protected override GameStore FromLowLevel(string value, ValueTags tag, AttributeResolver resolver)
     {
         return GameStore.From(value);
     }

src/Abstractions/NexusMods.Abstractions.MnemonicDB.Attributes/HashAttribute.cs

@@ -17,7 +17,7 @@ protected override ulong ToLowLevel(Hash value)
     }
 
     /// <inheritdoc />
-    protected override Hash FromLowLevel(ulong value, ValueTags tags, RegistryId registryId)
+    protected override Hash FromLowLevel(ulong value, ValueTags tags, AttributeResolver resolver)
     {
         return Hash.From(value);
     }

src/Abstractions/NexusMods.Abstractions.MnemonicDB.Attributes/RelativePathAttribute.cs

@@ -17,7 +17,7 @@ protected override string ToLowLevel(RelativePath value)
     }
 
     /// <inheritdoc />
-    protected override RelativePath FromLowLevel(string value, ValueTags tags, RegistryId registryId)
+    protected override RelativePath FromLowLevel(string value, ValueTags tags, AttributeResolver resolver)
     {
         return RelativePath.FromUnsanitizedInput(value);
     }

src/Abstractions/NexusMods.Abstractions.MnemonicDB.Attributes/SizeAttribute.cs

@@ -12,5 +12,5 @@ public class SizeAttribute(string ns, string name) : ScalarAttribute<Size, ulong
 {
     protected override ulong ToLowLevel(Size value) => value.Value;
 
-    protected override Size FromLowLevel(ulong value, ValueTags tags, RegistryId registryId) => Size.From(value);
+    protected override Size FromLowLevel(ulong value, ValueTags tags, AttributeResolver resolver) => Size.From(value);
 }

src/Abstractions/NexusMods.Abstractions.MnemonicDB.Attributes/StringsAttribute.cs

@@ -11,5 +11,5 @@ public class StringsAttribute(string ns, string name) : CollectionAttribute<stri
 {
     protected override string ToLowLevel(string value) => value;
 
-    protected override string FromLowLevel(string value, ValueTags tag, RegistryId registryId) => value;
+    protected override string FromLowLevel(string value, ValueTags tags, AttributeResolver resolver) => value;
 }

src/Abstractions/NexusMods.Abstractions.MnemonicDB.Attributes/TxIdAttribute.cs

@@ -16,7 +16,7 @@ protected override ulong ToLowLevel(TxId value)
     }
 
     /// <inheritdoc />
-    protected override TxId FromLowLevel(ulong value, ValueTags tags, RegistryId registryId)
+    protected override TxId FromLowLevel(ulong value, ValueTags tags, AttributeResolver resolver)
     {
         return TxId.From(value);
     }

src/Abstractions/NexusMods.Abstractions.MnemonicDB.Attributes/ULongAttribute.cs

@@ -11,5 +11,5 @@ public class ULongAttribute(string ns, string name) : ScalarAttribute<ulong, ulo
 {
     protected override ulong ToLowLevel(ulong value) => value;
 
-    protected override ulong FromLowLevel(ulong value, ValueTags tags, RegistryId registryId) => value;
+    protected override ulong FromLowLevel(ulong value, ValueTags tags, AttributeResolver resolver) => value;
 }

src/Abstractions/NexusMods.Abstractions.MnemonicDB.Attributes/UriAttribute.cs

@@ -16,7 +16,7 @@ protected override string ToLowLevel(Uri value)
     }
 
     /// <inheritdoc />
-    protected override Uri FromLowLevel(string value, ValueTags tag, RegistryId registryId)
+    protected override Uri FromLowLevel(string value, ValueTags tag, AttributeResolver resolver)
     {
         return new Uri(value);
     }

src/Abstractions/NexusMods.Abstractions.MnemonicDB.Attributes/UrisAttribute.cs

@@ -16,7 +16,7 @@ protected override string ToLowLevel(Uri value)
     }
 
     /// <inheritdoc />
-    protected override Uri FromLowLevel(string value, ValueTags tag, RegistryId registryId)
+    protected override Uri FromLowLevel(string value, ValueTags tag, AttributeResolver resolver)
     {
         return new Uri(value);
     }

src/Abstractions/NexusMods.Abstractions.NexusModsLibrary/Attributes/CollectionsSlugAttribute.cs

@@ -17,7 +17,7 @@ protected override string ToLowLevel(CollectionSlug value)
     }
 
     /// <inheritdoc />
-    protected override CollectionSlug FromLowLevel(string value, ValueTags tag, RegistryId registryId)
+    protected override CollectionSlug FromLowLevel(string value, ValueTags tag, AttributeResolver resolver)
     {
         return CollectionSlug.From(value);
     }

src/Abstractions/NexusMods.Abstractions.NexusModsLibrary/Attributes/FloatAttribute.cs

@@ -11,7 +11,7 @@ protected override float ToLowLevel(float value)
         return value;
     }
 
-    protected override float FromLowLevel(float value, ValueTags tags, RegistryId registryId)
+    protected override float FromLowLevel(float value, ValueTags tags, AttributeResolver resolver)
     {
         return value;
     }

src/Abstractions/NexusMods.Abstractions.NexusModsLibrary/Attributes/MemoryAttribute.cs

@@ -11,7 +11,7 @@ namespace NexusMods.Abstractions.NexusModsLibrary.Attributes;
 public class MemoryAttribute(string ns, string name) : HashedBlobAttribute<Memory<byte>>(ns, name)
 {
     /// <inheritdoc />
-    protected override Memory<byte> FromLowLevel(ReadOnlySpan<byte> value, ValueTags tags, RegistryId registryId)
+    protected override Memory<byte> FromLowLevel(ReadOnlySpan<byte> value, ValueTags tags, AttributeResolver resolver)
     {
         return new Memory<byte>(value.ToArray());
     }

src/Abstractions/NexusMods.Abstractions.NexusModsLibrary/Attributes/RevisionIdAttribute.cs

@@ -17,7 +17,7 @@ protected override ulong ToLowLevel(RevisionId value)
     }
 
     /// <inheritdoc />
-    protected override RevisionId FromLowLevel(ulong value, ValueTags tags, RegistryId registryId)
+    protected override RevisionId FromLowLevel(ulong value, ValueTags tags, AttributeResolver resolver)
     {
         return RevisionId.From(value);
     }

src/Abstractions/NexusMods.Abstractions.NexusModsLibrary/Attributes/RevisionNumberAttribute.cs

@@ -17,7 +17,7 @@ protected override ulong ToLowLevel(RevisionNumber value)
     }
 
     /// <inheritdoc />
-    protected override RevisionNumber FromLowLevel(ulong value, ValueTags tags, RegistryId registryId)
+    protected override RevisionNumber FromLowLevel(ulong value, ValueTags tags, AttributeResolver resolver)
     {
         return RevisionNumber.From(value);
     }

src/Abstractions/NexusMods.Abstractions.NexusWebApi/Types/FileId.cs

@@ -29,5 +29,5 @@ public class FileIdAttribute(string ns, string name) :
     protected override ulong ToLowLevel(FileId value) => value.Value;
 
     /// <inheritdoc />
-    protected override FileId FromLowLevel(ulong value, ValueTags tags, RegistryId registryId) => FileId.From(value);
+    protected override FileId FromLowLevel(ulong value, ValueTags tags, AttributeResolver resolver) => FileId.From(value);
 }

src/Abstractions/NexusMods.Abstractions.NexusWebApi/Types/ModId.cs

@@ -27,5 +27,5 @@ public class ModIdAttribute(string ns, string name)
     protected override ulong ToLowLevel(ModId value) => value.Value;
 
     /// <inheritdoc />
-    protected override ModId FromLowLevel(ulong value, ValueTags tags, RegistryId registryId) => ModId.From(value);
+    protected override ModId FromLowLevel(ulong value, ValueTags tags, AttributeResolver resolver) => ModId.From(value);
 } 

src/NexusMods.App/TelemetryProvider.cs

@@ -34,7 +34,7 @@ public TelemetryProvider(IServiceProvider serviceProvider)
 
         // download size
         _connection.ObserveDatoms(DownloadAnalysis.Size)
-            .Transform(d => (SizeAttribute.ReadDatom)d.Resolved)
+            .Transform(d => (SizeAttribute.ReadDatom)d.Resolved(_connection))
             .RemoveKey()
             .QueryWhenChanged(datoms => datoms.Sum(d => d.V))
             .SubscribeWithErrorLogging(totalDownloadSize => _downloadSize = Size.From((ulong)totalDownloadSize))

src/NexusMods.DataModel/NxFileStore.cs

@@ -16,6 +16,7 @@
 using NexusMods.Archives.Nx.Utilities;
 using NexusMods.DataModel.ChunkedStreams;
 using NexusMods.Hashing.xxHash64;
+using NexusMods.MnemonicDB;
 using NexusMods.MnemonicDB.Abstractions;
 using NexusMods.Paths;
 using NexusMods.Paths.Utilities;
@@ -307,7 +308,7 @@ public HashSet<ulong> GetFileHashes()
         var fileHashes = new HashSet<ulong>();
 
         // Replace this once we redo the IFileStore. Instead that can likely query MneumonicDB directly.
-        fileHashes.AddRange(_conn.Db.Datoms(ArchivedFile.Hash).Resolved().OfType<HashAttribute.ReadDatom>().Select(d => d.V.Value));
+        fileHashes.AddRange(_conn.Db.Datoms(ArchivedFile.Hash).Resolved(_conn).OfType<HashAttribute.ReadDatom>().Select(d => d.V.Value));
 
         return fileHashes;
     }