Merge pull request #100 from Nexus-Mods/value-handlers

Fix tuple 3 handling, and add an API for value remapping from user code

CHANGELOG.md

@@ -1,5 +1,9 @@
 ## Changelog
 
+### 0.9.84 - 20/09/2024
+* Fixed a bug with Tuple3 values that had a reference in the first position.
+* Added a user accessible remap function for values
+
 ### 0.9.83 - 20/09/2024
 * Optimized the interface with RocksDB used all throughout the library. Results in a 30% speedup on search operations
 inside RocksDB.

src/NexusMods.MnemonicDB.Abstractions/PartitionId.cs

@@ -51,7 +51,20 @@ public static PartitionId User(byte id)
     public EntityId MaxValue => EntityId.From(((ulong)Value << 56) | 0x00FFFFFFFFFFFFFF);
 
     /// <inheritdoc />
-    public override string ToString() => $"PartId:{Value:x}";
+    public override string ToString()
+    {
+        if (Value == Attribute)
+            return $"PartId:({Attribute.Value:x})Attribute";
+        if (Value == Transactions)
+            return $"PartId:({Transactions.Value:x})Transactions";
+        if (Value == Entity)
+            return $"PartId:({Entity.Value:x})Entity";
+        if (Value == Temp)
+            return $"PartId:({Temp.Value:x})Temp";
+        if (Value > Temp)
+            return $"PartId:({Value:x})User";
+        return $"PartId:{Value:x}";
+    }
 
     /// <summary>
     /// Encode a partition id and entity id pair

src/NexusMods.MnemonicDB.Abstractions/ValueHelpers.cs

@@ -0,0 +1,66 @@
+using System;
+using System.Runtime.InteropServices;
+using NexusMods.MnemonicDB.Abstractions.ElementComparers;
+using NexusMods.MnemonicDB.Abstractions.Internals;
+using Reloaded.Memory.Extensions;
+
+namespace NexusMods.MnemonicDB.Abstractions;
+
+/// <summary>
+/// Static methods that help with reading, writing and formatting values
+/// </summary>
+public static class ValueHelpers
+{
+    /// <summary>
+    /// Remaps the values of this attribute, if required. 
+    /// </summary>
+    public static void Remap(Func<EntityId, EntityId> remapFn, in KeyPrefix prefix, Span<byte> valueSpan)
+    {
+        switch (prefix.ValueTag)
+        {
+            case ValueTags.Reference:
+                var oldId = MemoryMarshal.Read<EntityId>(valueSpan);
+                var newId = remapFn(oldId);
+                MemoryMarshal.Write(valueSpan, newId);
+                break;
+            case ValueTags.Tuple2:
+            {
+                var tag1 = (ValueTags)valueSpan[0];
+                var tag2 = (ValueTags)valueSpan[1];
+                if (tag1 == ValueTags.Reference)
+                {
+                    var entityId = MemoryMarshal.Read<EntityId>(valueSpan.SliceFast(2));
+                    var newEntityId = remapFn(entityId);
+                    MemoryMarshal.Write(valueSpan.SliceFast(2), newEntityId);
+                }
+                if (tag2 == ValueTags.Reference)
+                {
+                    throw new NotSupportedException("This attribute does not support remapping of the second element.");
+                }
+                break;
+            }
+            case ValueTags.Tuple3:
+            {
+                var tag1 = (ValueTags)valueSpan[0];
+                var tag2 = (ValueTags)valueSpan[1];
+                var tag3 = (ValueTags)valueSpan[2];
+                if (tag1 == ValueTags.Reference)
+                {
+                    var entityId = MemoryMarshal.Read<EntityId>(valueSpan.SliceFast(3));
+                    var newEntityId = remapFn(entityId);
+                    MemoryMarshal.Write(valueSpan.SliceFast(3), newEntityId);
+                }
+                if (tag2 == ValueTags.Reference)
+                {
+                    throw new NotSupportedException("This attribute does not support remapping of the second element.");
+                }
+                if (tag3 == ValueTags.Reference)
+                {
+                    throw new NotSupportedException("This attribute does not support remapping of the third element.");
+                }
+                break;
+            } 
+        }
+    }
+
+}

src/NexusMods.MnemonicDB/Storage/DatomStore.cs

@@ -52,6 +52,11 @@ public class DatomStore : IDatomStore
     private IDb? _currentDb = null;
 
     private static readonly TimeSpan TransactionTimeout = TimeSpan.FromMinutes(120);
+
+    /// <summary>
+    /// Cached function to remap temporary entity ids to real entity ids
+    /// </summary>
+    private readonly Func<EntityId, EntityId> _remapFunc;
 
     /// <summary>
     /// Used to remap temporary entity ids to real entity ids, this is cleared after each transaction
@@ -76,6 +81,7 @@ public class DatomStore : IDatomStore
     /// </summary>
     public DatomStore(ILogger<DatomStore> logger, DatomStoreSettings settings, IStoreBackend backend)
     {
+        _remapFunc = Remap;
         _attributeCache = backend.AttributeCache;
         _pendingTransactions = new BlockingCollection<PendingTransaction>(new ConcurrentQueue<PendingTransaction>());
 
@@ -400,7 +406,7 @@ private void Log(PendingTransaction pendingTransaction, out StoreResult result)
                 var valueSpan = datom.ValueSpan;
                 var span = _writer.GetSpan(valueSpan.Length);
                 valueSpan.CopyTo(span);
-                Remap(in keyPrefix, span);
+                ValueHelpers.Remap(_remapFunc, in keyPrefix, span);
                 _writer.Advance(valueSpan.Length);
             }
 
@@ -450,35 +456,6 @@ private void Log(PendingTransaction pendingTransaction, out StoreResult result)
         };
     }
 
-    private void Remap(in KeyPrefix prefix, Span<byte> valueSpan)
-    {
-        switch (prefix.ValueTag)
-        {
-            case ValueTags.Reference:
-                var oldId = MemoryMarshal.Read<EntityId>(valueSpan);
-                var newId = Remap(oldId);
-                MemoryMarshal.Write(valueSpan, newId);
-                break;
-            case ValueTags.Tuple2:
-            {
-                var tag1 = (ValueTags)valueSpan[0];
-                var tag2 = (ValueTags)valueSpan[1];
-                if (tag1 == ValueTags.Reference)
-                {
-                    var entityId = MemoryMarshal.Read<EntityId>(valueSpan.SliceFast(2));
-                    var newEntityId = Remap(entityId);
-                    MemoryMarshal.Write(valueSpan.SliceFast(2), newEntityId);
-                }
-                if (tag2 == ValueTags.Reference)
-                {
-                    throw new NotSupportedException("This attribute does not support remapping of the second element.");
-                }
-                break;
-            }
-        }
-
-    }
-
     /// <summary>
     /// Updates the data in _prevWriter to be a retraction of the data in that write.
     /// </summary>

tests/NexusMods.MnemonicDB.TestModel/Attributes/Tuple3TestAttribute.cs

@@ -0,0 +1,18 @@
+using NexusMods.MnemonicDB.Abstractions;
+using NexusMods.MnemonicDB.Abstractions.Attributes;
+using NexusMods.MnemonicDB.Abstractions.ElementComparers;
+
+namespace NexusMods.MnemonicDB.TestModel.Attributes;
+
+public class Tuple3TestAttribute(string ns, string name) : TupleAttribute<EntityId, ulong, int, int, string, string>(ValueTags.Reference, ValueTags.Int32, ValueTags.Ascii, ns, name)
+{
+    protected override (EntityId, int, string) FromLowLevel((ulong, int, string) value)
+    {
+        return (EntityId.From(value.Item1), value.Item2, value.Item3);
+    }
+
+    protected override (ulong, int, string) ToLowLevel((EntityId, int, string) value)
+    {
+        return (value.Item1.Value, value.Item2, value.Item3);
+    }
+}

tests/NexusMods.MnemonicDB.TestModel/File.cs

@@ -20,5 +20,5 @@ public partial class File : IModelDefinition
     /// <summary>
     /// Tuple3 test
     /// </summary>
-    public static readonly Int3Attribute TupleTest = new(Namespace, nameof(TupleTest)) { IsIndexed = true, IsOptional = true};
+    public static readonly Tuple3TestAttribute TupleTest = new(Namespace, nameof(TupleTest)) { IsIndexed = true, IsOptional = true};
 }

tests/NexusMods.MnemonicDB.Tests/DbTests.CanUseTuple3Attributes.verified.txt

@@ -1,27 +1,27 @@
-+ | 0200000000000001 | TupleTest                | (1, 1, 1)                                        | 0100000000000003
-+ | 0200000000000002 | TupleTest                | (1, 1, 2)                                        | 0100000000000003
-+ | 0200000000000003 | TupleTest                | (1, 1, 3)                                        | 0100000000000003
-+ | 0200000000000004 | TupleTest                | (1, 2, 1)                                        | 0100000000000003
-+ | 0200000000000005 | TupleTest                | (1, 2, 2)                                        | 0100000000000003
-+ | 0200000000000006 | TupleTest                | (1, 2, 3)                                        | 0100000000000003
-+ | 0200000000000007 | TupleTest                | (1, 3, 1)                                        | 0100000000000003
-+ | 0200000000000008 | TupleTest                | (1, 3, 2)                                        | 0100000000000003
-+ | 0200000000000009 | TupleTest                | (1, 3, 3)                                        | 0100000000000003
-+ | 020000000000000A | TupleTest                | (2, 1, 1)                                        | 0100000000000003
-+ | 020000000000000B | TupleTest                | (2, 1, 2)                                        | 0100000000000003
-+ | 020000000000000C | TupleTest                | (2, 1, 3)                                        | 0100000000000003
-+ | 020000000000000D | TupleTest                | (2, 2, 1)                                        | 0100000000000003
-+ | 020000000000000E | TupleTest                | (2, 2, 2)                                        | 0100000000000003
-+ | 020000000000000F | TupleTest                | (2, 2, 3)                                        | 0100000000000003
-+ | 0200000000000010 | TupleTest                | (2, 3, 1)                                        | 0100000000000003
-+ | 0200000000000011 | TupleTest                | (2, 3, 2)                                        | 0100000000000003
-+ | 0200000000000012 | TupleTest                | (2, 3, 3)                                        | 0100000000000003
-+ | 0200000000000013 | TupleTest                | (3, 1, 1)                                        | 0100000000000003
-+ | 0200000000000014 | TupleTest                | (3, 1, 2)                                        | 0100000000000003
-+ | 0200000000000015 | TupleTest                | (3, 1, 3)                                        | 0100000000000003
-+ | 0200000000000016 | TupleTest                | (3, 2, 1)                                        | 0100000000000003
-+ | 0200000000000017 | TupleTest                | (3, 2, 2)                                        | 0100000000000003
-+ | 0200000000000018 | TupleTest                | (3, 2, 3)                                        | 0100000000000003
-+ | 0200000000000019 | TupleTest                | (3, 3, 1)                                        | 0100000000000003
-+ | 020000000000001A | TupleTest                | (3, 3, 2)                                        | 0100000000000003
-+ | 020000000000001B | TupleTest                | (3, 3, 3)                                        | 0100000000000003
++ | 0200000000000001 | TupleTest                | (EId:200000000000001, 1, 1)                      | 0100000000000003
++ | 0200000000000002 | TupleTest                | (EId:200000000000002, 1, 2)                      | 0100000000000003
++ | 0200000000000003 | TupleTest                | (EId:200000000000003, 1, 3)                      | 0100000000000003
++ | 0200000000000004 | TupleTest                | (EId:200000000000004, 2, 1)                      | 0100000000000003
++ | 0200000000000005 | TupleTest                | (EId:200000000000005, 2, 2)                      | 0100000000000003
++ | 0200000000000006 | TupleTest                | (EId:200000000000006, 2, 3)                      | 0100000000000003
++ | 0200000000000007 | TupleTest                | (EId:200000000000007, 3, 1)                      | 0100000000000003
++ | 0200000000000008 | TupleTest                | (EId:200000000000008, 3, 2)                      | 0100000000000003
++ | 0200000000000009 | TupleTest                | (EId:200000000000009, 3, 3)                      | 0100000000000003
++ | 020000000000000A | TupleTest                | (EId:20000000000000A, 1, 1)                      | 0100000000000003
++ | 020000000000000B | TupleTest                | (EId:20000000000000B, 1, 2)                      | 0100000000000003
++ | 020000000000000C | TupleTest                | (EId:20000000000000C, 1, 3)                      | 0100000000000003
++ | 020000000000000D | TupleTest                | (EId:20000000000000D, 2, 1)                      | 0100000000000003
++ | 020000000000000E | TupleTest                | (EId:20000000000000E, 2, 2)                      | 0100000000000003
++ | 020000000000000F | TupleTest                | (EId:20000000000000F, 2, 3)                      | 0100000000000003
++ | 0200000000000010 | TupleTest                | (EId:200000000000010, 3, 1)                      | 0100000000000003
++ | 0200000000000011 | TupleTest                | (EId:200000000000011, 3, 2)                      | 0100000000000003
++ | 0200000000000012 | TupleTest                | (EId:200000000000012, 3, 3)                      | 0100000000000003
++ | 0200000000000013 | TupleTest                | (EId:200000000000013, 1, 1)                      | 0100000000000003
++ | 0200000000000014 | TupleTest                | (EId:200000000000014, 1, 2)                      | 0100000000000003
++ | 0200000000000015 | TupleTest                | (EId:200000000000015, 1, 3)                      | 0100000000000003
++ | 0200000000000016 | TupleTest                | (EId:200000000000016, 2, 1)                      | 0100000000000003
++ | 0200000000000017 | TupleTest                | (EId:200000000000017, 2, 2)                      | 0100000000000003
++ | 0200000000000018 | TupleTest                | (EId:200000000000018, 2, 3)                      | 0100000000000003
++ | 0200000000000019 | TupleTest                | (EId:200000000000019, 3, 1)                      | 0100000000000003
++ | 020000000000001A | TupleTest                | (EId:20000000000001A, 3, 2)                      | 0100000000000003
++ | 020000000000001B | TupleTest                | (EId:20000000000001B, 3, 3)                      | 0100000000000003

tests/NexusMods.MnemonicDB.Tests/DbTests.cs

@@ -15,6 +15,7 @@
 using NexusMods.MnemonicDB.Abstractions.TxFunctions;
 using NexusMods.MnemonicDB.TestModel;
 using NexusMods.MnemonicDB.TestModel.Analyzers;
+using NexusMods.MnemonicDB.TestModel.Attributes;
 using NexusMods.Paths;
 using File = NexusMods.MnemonicDB.TestModel.File;
 
@@ -1002,12 +1003,17 @@ public async Task CanUseTuple3Attributes()
         foreach (var c in new[]{1, 2, 3})
         {
             var tmpId = tx.TempId();
-            tx.Add(tmpId, File.TupleTest, (a, b, c.ToString()));
+            tx.Add(tmpId, File.TupleTest, (tmpId, b, c.ToString()));
         }
 
         var results = await tx.Commit();
+
+        var resolved = results.Db.Datoms(File.TupleTest).Resolved(Connection).ToArray();
+
+        resolved.Select(v => ((Tuple3TestAttribute.ReadDatom)v).V.Item1)
+            .Should().AllSatisfy(id => id.Partition.Should().NotBe(PartitionId.Temp));
 
-        await VerifyTable(results.Db.Datoms(File.TupleTest).Resolved(Connection));
+        await VerifyTable(resolved);
     }
 
     [Fact]