Merge branch 'Nexus-Mods:main' into dev

NexusMods.App.sln

@@ -253,8 +253,11 @@ EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.Collections", "src\NexusMods.Collections\NexusMods.Collections.csproj", "{A9FD538A-E101-4AEA-A98E-35DCED950AEE}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.Collections.Tests", "tests\NexusMods.Collections.Tests\NexusMods.Collections.Tests.csproj", "{8C817874-7A88-450E-B216-851A1B03684C}"
+EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.Abstractions.Media", "src\Abstractions\NexusMods.Abstractions.Media\NexusMods.Abstractions.Media.csproj", "{5CB6D02C-07D0-4C0D-BF5C-4E2E958A0612}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.Games.Larian", "src\Games\NexusMods.Games.Larian\NexusMods.Games.Larian.csproj", "{2A35EBB5-1CA6-4F5D-8CE8-352146C82C28}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -661,6 +664,10 @@ Global
 		{5CB6D02C-07D0-4C0D-BF5C-4E2E958A0612}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{5CB6D02C-07D0-4C0D-BF5C-4E2E958A0612}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{5CB6D02C-07D0-4C0D-BF5C-4E2E958A0612}.Release|Any CPU.Build.0 = Release|Any CPU
+		{2A35EBB5-1CA6-4F5D-8CE8-352146C82C28}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{2A35EBB5-1CA6-4F5D-8CE8-352146C82C28}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{2A35EBB5-1CA6-4F5D-8CE8-352146C82C28}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{2A35EBB5-1CA6-4F5D-8CE8-352146C82C28}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -779,6 +786,7 @@ Global
 		{A9FD538A-E101-4AEA-A98E-35DCED950AEE} = {E7BAE287-D505-4D6D-A090-665A64309B2D}
 		{8C817874-7A88-450E-B216-851A1B03684C} = {52AF9D62-7D5B-4AD0-BA12-86F2AA67428B}
 		{5CB6D02C-07D0-4C0D-BF5C-4E2E958A0612} = {0CB73565-1207-4A56-A79F-6A8E9BBD795C}
+		{2A35EBB5-1CA6-4F5D-8CE8-352146C82C28} = {70D38D24-79AE-4600-8E83-17F3C11BA81F}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {9F9F8352-34DD-42C0-8564-EE9AF34A3501}

benchmarks/NexusMods.Benchmarks/Benchmarks/Loadouts/Harness/DummyFileStore.cs

@@ -11,7 +11,7 @@ public ValueTask<bool> HaveFile(Hash hash)
         return ValueTask.FromResult(false);
     }
 
-    public Task BackupFiles(IEnumerable<ArchivedFileEntry> backups, CancellationToken token = default)
+    public Task BackupFiles(IEnumerable<ArchivedFileEntry> backups, bool deduplicate = true, CancellationToken token = default)
     {
         return Task.CompletedTask;
     }

docs/developers/decisions/backend/0017-garbage-collector-design.md

@@ -174,6 +174,58 @@ However, if [`NxFileStore`][nx-file-store] is actively extracting a file from an
 the GC should not delete the source `.nx` archive, a lock must be placed to prevent
 that edge case from happening.
 
+#### Edge Case: Duplicate Items
+
+!!! danger "Sometimes we may have a duplicate item in two distinct `.nx` archives."
+
+Sometimes developers may make an error while working on the App and not put the correct
+safety procedures (i.e. taking a write lock) to ensure that no duplicates are created
+within the Nx Archives.
+
+This is dangerous, because there are two sources of truth
+for where a given hash is stored, *the archives* and [*the DataStore*](#updating-the-datastore).  
+
+##### Reproduction
+
+To understand the dangers involved, let's try reproducing the bug.
+
+!!! note "The bug is fixed today."
+
+    Today, the 'bug' is fixed as of commit ( `34de799623cf9688e6c3dcabca7fe029426583ed` ) , however prior to it;
+    the bug could be reproduced in the way listed below.
+
+We will make an assumption that we have an un-patched GC and faulty code that
+creates duplicates.
+
+Create 2 archives:
+
+- dummy-1 (zip)
+- dummy-2 (zip)
+
+Inside `dummy-1`, add 1 file.
+Inside `dummy-2`, add the file from `dummy-1`, and an additional file.
+
+Add the `dummy-1` and `dummy-2` archives from disk in the following order:
+
+- dummy-1
+- dummy-2
+
+Adding `dummy-2` creates a duplicate file due to an error by the programmer,
+the `DataStore` entry for the duplicate hash will be re-routed to `dummy-2` when
+it previously pointed to `dummy-1`.
+
+If we now delete `dummy-1`; the error should have a 50-50 chance of reproducing;
+depending on order archives are processed. There is a risk retracting the item in the
+DataStore will retract the item in the wrong archive/container.
+
+!!! note "There is some RNG involved."
+
+    Reproduction is non-determinstic, due to nature of data.
+    Bigger archives with duplicates are more likely to yield errors.
+
+    So you can replace `dummy-1` with `SMAPI 4` and `dummy-2` with `SMAPI 3` 
+    for a more reliable reproduction.
+
 ## Core Code Design
 
 !!! abstract "The main modular 'core' of the GC lives as `ArchiveGarbageCollector<TParsedHeaderState, TFileEntryWrapper>`"

src/Abstractions/NexusMods.Abstractions.IO/IFileStore.cs

@@ -17,12 +17,32 @@ public interface IFileStore
     public ValueTask<bool> HaveFile(Hash hash);
 
     /// <summary>
-    /// Backup the given files. If the size or hash do not match during the
-    /// backup process a exception may be thrown.
+    /// Backup the given set of files.
+    ///
+    /// If the size or hash do not match during the
+    /// backup process an exception may be thrown.
     /// </summary>
-    /// <param name="backups"></param>
+    /// <param name="backups">The files to back up.</param>
+    /// <param name="deduplicate">Ensures no duplicate files are stored.</param>
     /// <param name="token"></param>
-    Task BackupFiles(IEnumerable<ArchivedFileEntry> backups, CancellationToken token = default);
+    /// <remarks>
+    /// Backing up duplicates should generally be avoided, as it encurs a performance and
+    /// disk space penalty. However accidentally creating a duplicate is not
+    /// considered a failure case; the Garbage Collector is equipped to deal
+    /// with duplicates.
+    ///
+    /// As a default <paramref name="deduplicate"/> is set to 'true' to avoid duplicates,
+    /// however it is slightly more efficient to set <paramref name="deduplicate"/> to 'false'
+    /// and manually check for duplicates with <see cref="HaveFile"/> API when constructing
+    /// the <paramref name="backups"/> collection.
+    ///
+    /// The <see cref="BackupFiles"/> itself is thread safe, but duplicates may be made
+    /// if called from duplicate threads at once. This can prevent with taking a lock
+    /// via <see cref="Lock"/> (and `using` statement). That said, the probability of duplicates
+    /// being made without a lock is so low that it is generally recommended not to lock
+    /// to instead maximize performance. The Garbage Collector will remove any duplicates down the road.
+    /// </remarks>
+    Task BackupFiles(IEnumerable<ArchivedFileEntry> backups, bool deduplicate = true, CancellationToken token = default);
 
     /// <summary>
     /// Extract the given files to the given disk locations, provide as a less-abstract interface incase

src/Abstractions/NexusMods.Abstractions.Loadouts.Synchronizers/ALoadoutSynchronizer.cs

@@ -808,7 +808,8 @@ await Parallel.ForEachAsync(files, async (file, _) =>
             }
         );
 
-        await _fileStore.BackupFiles(archivedFiles);
+        // PERFORMANCE: We deduplicate above with the HaveFile call.
+        await _fileStore.BackupFiles(archivedFiles, deduplicate: false);
     }
 
     private async Task<DiskState> GetOrCreateInitialDiskState(GameInstallation installation)

src/Abstractions/NexusMods.Abstractions.NexusModsLibrary/Attributes/FloatAttribute.cs

@@ -0,0 +1,18 @@
+using NexusMods.MnemonicDB.Abstractions;
+using NexusMods.MnemonicDB.Abstractions.Attributes;
+using NexusMods.MnemonicDB.Abstractions.ElementComparers;
+
+namespace NexusMods.Abstractions.NexusModsLibrary.Attributes;
+
+public class FloatAttribute(string ns, string name) : ScalarAttribute<float, float>(ValueTags.Float32, ns, name)
+{
+    protected override float ToLowLevel(float value)
+    {
+        return value;
+    }
+
+    protected override float FromLowLevel(float value, ValueTags tags, RegistryId registryId)
+    {
+        return value;
+    }
+}

src/Abstractions/NexusMods.Abstractions.NexusModsLibrary/Attributes/MemoryAttribute.cs

@@ -0,0 +1,24 @@
+using System.Buffers;
+using NexusMods.MnemonicDB.Abstractions;
+using NexusMods.MnemonicDB.Abstractions.Attributes;
+using NexusMods.MnemonicDB.Abstractions.ElementComparers;
+
+namespace NexusMods.Abstractions.NexusModsLibrary.Attributes;
+
+/// <summary>
+/// A hashed blob attribute for <see cref="Memory{T}"/>.
+/// </summary>
+public class MemoryAttribute(string ns, string name) : HashedBlobAttribute<Memory<byte>>(ns, name)
+{
+    /// <inheritdoc />
+    protected override Memory<byte> FromLowLevel(ReadOnlySpan<byte> value, ValueTags tags, RegistryId registryId)
+    {
+        return new Memory<byte>(value.ToArray());
+    }
+
+    /// <inheritdoc />
+    protected override void WriteValue<TWriter>(Memory<byte> value, TWriter writer)
+    {
+        writer.Write(value.Span);
+    }
+}

src/Abstractions/NexusMods.Abstractions.NexusModsLibrary/CollectionTag.cs

@@ -0,0 +1,23 @@
+using NexusMods.Abstractions.MnemonicDB.Attributes;
+using NexusMods.MnemonicDB.Abstractions.Attributes;
+using NexusMods.MnemonicDB.Abstractions.Models;
+
+namespace NexusMods.Abstractions.NexusModsLibrary;
+
+/// <summary>
+/// A tag for a collection.
+/// </summary>
+public partial class CollectionTag : IModelDefinition
+{
+    private const string Namespace = "NexusMods.Abstractions.NexusModsLibrary.CollectionTag";
+
+    /// <summary>
+    /// The name of the collection tag.
+    /// </summary>
+    public static readonly StringAttribute Name = new(Namespace, nameof(Name)) { IsIndexed = true };
+
+    /// <summary>
+    /// The Nexus mods id of the collection tag.
+    /// </summary>
+    public static readonly ULongAttribute NexusId = new(Namespace, nameof(NexusId)) { IsIndexed = true };
+}

src/Abstractions/NexusMods.Abstractions.NexusModsLibrary/GraphQLResolver.cs

@@ -0,0 +1,90 @@
+using NexusMods.MnemonicDB.Abstractions;
+using NexusMods.MnemonicDB.Abstractions.Attributes;
+using NexusMods.MnemonicDB.Abstractions.Models;
+using Splat.ModeDetection;
+
+namespace NexusMods.Abstractions.NexusModsLibrary;
+
+/// <summary>
+/// A helper for upserting entities in the database. When created, you must define a "pimary key" attribute and value,
+/// these are used to determine if the entity already exists in the database. If it does, the existing entity is updated,
+/// otherwise a new entity is created.
+///
+/// For each attribute you want to add to the entity, call the Add method with the attribute and value and any duplicate values
+/// will not be added.
+/// </summary>
+// ReSharper disable once InconsistentNaming
+public readonly struct GraphQLResolver(ITransaction Tx, ReadOnlyModel Model)
+{
+    /// <summary>
+    /// Create a new resolver using the given primary key attribute and value.
+    /// </summary>
+    public static GraphQLResolver Create<THighLevel, TLowLevel>(IDb referenceDb, ITransaction tx, ScalarAttribute<THighLevel, TLowLevel> primaryKeyAttribute, THighLevel primaryKeyValue) where THighLevel : notnull
+    {
+        var existing = referenceDb.Datoms(primaryKeyAttribute, primaryKeyValue);
+        var exists = existing.Count > 0;
+        var id = existing.Count == 0 ? tx.TempId() : existing[0].E;
+        if (!exists)
+            tx.Add(id, primaryKeyAttribute, primaryKeyValue);
+        return new GraphQLResolver(tx, new ReadOnlyModel(referenceDb, id));
+    }
+
+    /// <summary>
+    /// The id of the entity, may be temporary if this is a new entity.
+    /// </summary>
+    public EntityId Id => Model.Id;
+
+    /// <summary>
+    /// Add a value to the entity. If the value already exists, it will not be added again.
+    /// </summary>
+    public void Add<THighLevel, TLowLevel>(ScalarAttribute<THighLevel, TLowLevel> attribute, THighLevel value) 
+        where THighLevel : notnull
+    {
+        if (attribute.TryGet(Model, out var foundValue))
+        {
+            // Deduplicate values
+            if (foundValue.Equals(value))
+                return;
+        }
+        // Else add the value
+        Tx.Add(Model.Id, attribute, value);
+    }
+
+    /// <summary>
+    /// Add a value to the entity. If the value already exists, it will not be added again.
+    /// </summary>
+    public void Add<TOther>(ReferencesAttribute<TOther> attribute, EntityId id) 
+        where TOther : IModelDefinition
+    {
+        if (PartitionId.Temp == id.Partition)
+        {
+            Tx.Add(Model.Id, attribute, id);
+            return;
+        }
+
+        if (attribute.Get(Model).Contains(id))
+            return;
+
+        // Else add the value
+        Tx.Add(Model.Id, attribute, id);
+    }
+
+    /// <summary>
+    /// Add a value to the entity. If the value already exists, it will not be added again.
+    /// </summary>
+    public void Add<TOther>(ReferenceAttribute<TOther> attribute, EntityId id) 
+        where TOther : IModelDefinition
+    {
+        if (PartitionId.Temp == id.Partition)
+        {
+            Tx.Add(Model.Id, attribute, id);
+            return;
+        }
+
+        if (attribute.Get(Model).Equals(id))
+            return;
+
+        // Else add the value
+        Tx.Add(Model.Id, attribute, id);
+    }
+}

src/Abstractions/NexusMods.Abstractions.NexusModsLibrary/Models/CollectionMetadata.cs

@@ -0,0 +1,54 @@
+using NexusMods.Abstractions.MnemonicDB.Attributes;
+using NexusMods.Abstractions.NexusModsLibrary.Attributes;
+using NexusMods.MnemonicDB.Abstractions.Attributes;
+using NexusMods.MnemonicDB.Abstractions.Models;
+
+namespace NexusMods.Abstractions.NexusModsLibrary.Models;
+
+/// <summary>
+/// Metadata about a collection on Nexus Mods.
+/// </summary>
+public partial class CollectionMetadata : IModelDefinition
+{
+    private const string Namespace = "NexusMods.Library.NexusModsCollectionMetadata";
+
+    /// <summary>
+    /// The collection slug.
+    /// </summary>
+    public static readonly CollectionsSlugAttribute Slug = new(Namespace, nameof(Slug)) { IsIndexed = true };
+
+    /// <summary>
+    /// The name of the collection.
+    /// </summary>
+    public static readonly StringAttribute Name = new(Namespace, nameof(Name));
+
+    /// <summary>
+    /// The short description of the collection
+    /// </summary>
+    public static readonly StringAttribute Summary = new(Namespace, nameof(Summary));
+
+    /// <summary>
+    /// The Curating user of the collection.
+    /// </summary>
+    public static readonly ReferenceAttribute<User> Author = new(Namespace, nameof(Author));
+
+    /// <summary>
+    /// The revisions of the collection.
+    /// </summary>
+    public static readonly BackReferenceAttribute<CollectionRevisionMetadata> Revisions = new(CollectionRevisionMetadata.Collection);
+
+    /// <summary>
+    /// The tags on the collection.
+    /// </summary>
+    public static readonly ReferencesAttribute<CollectionTag> Tags = new(Namespace, nameof(Tags));
+
+    /// <summary>
+    /// The number of endorsements the collection has.
+    /// </summary>
+    public static readonly ULongAttribute Endorsements = new(Namespace, nameof(Endorsements));
+
+    /// <summary>
+    /// The collections' image.
+    /// </summary>
+    public static readonly MemoryAttribute TileImage = new(Namespace, nameof(TileImage));
+}