Added: Vanilla/Original Game State Reuse (#1259)

* Added: Support for reusing an initial game state between game additions.

* Added: A Stable Hash method for Strings & Separate Table for Initial States

* Improved: Find Speed of GetInitialState

* Added: Proper look into MneumonicDB History

* Updated: Now gets correct attribute value as of transaction

* Fixed: Now new loadouts get fresh game folder

* Changed: Use separate table for Initial States, Removing History from Main States Again

* Added: 'Fast Apply' operation on subsequent game manage.

* Fix: Build after merge with Main

* Removed: Legacy Note, we now support multi locations

* Renamed: `AEntity` to `Entity` (MnemonicDB Breaking Change)

* Added: Simple test for reuse of original game state

src/Abstractions/NexusMods.Abstractions.DiskState/IDiskStateRegistry.cs

@@ -11,23 +11,40 @@ public interface IDiskStateRegistry
     /// <summary>
     /// Saves a disk state to the data store for the given game installation
     /// </summary>
+    /// <param name="installation">The game installation associated with the disk state</param>
+    /// <param name="diskState">The disk state to save</param>
     Task SaveState(GameInstallation installation, DiskStateTree diskState);
 
     /// <summary>
     /// Gets the disk state associated with a specific game installation, returns false if no state is found
     /// </summary>
-    /// <returns></returns>
+    /// <param name="gameInstallation">The game installation to retrieve the disk state for</param>
+    /// <returns>The disk state associated with the game installation, or null if not found</returns>
     DiskStateTree? GetState(GameInstallation gameInstallation);
 
     /// <summary>
     /// Gets the Loadout Revision Id of the last applied state for a given game installation
     /// </summary>
-    /// <param name="gameInstallation"></param>
-    /// <returns></returns>
+    /// <param name="gameInstallation">The game installation to retrieve the last applied loadout for</param>
+    /// <returns>The Loadout Revision Id of the last applied state, or null if not found</returns>
     IId? GetLastAppliedLoadout(GameInstallation gameInstallation);
 
     /// <summary>
     /// Observable of all the last applied revisions for all game installations
     /// </summary>
     IObservable<(GameInstallation gameInstallation, IId loadoutRevision)> LastAppliedRevisionObservable { get; }
+
+    /// <summary>
+    /// Saves the initial disk state for a given game installation.
+    /// </summary>
+    /// <param name="installation">The game installation associated with the initial disk state</param>
+    /// <param name="diskState">The initial disk state to save</param>
+    Task SaveInitialState(GameInstallation installation, DiskStateTree diskState);
+
+    /// <summary>
+    /// Retrieves the initial disk state for a given game installation.
+    /// </summary>
+    /// <param name="installation">The game installation to retrieve the initial disk state for</param>
+    /// <returns>The initial disk state, or null if not found</returns>
+    DiskStateTree? GetInitialState(GameInstallation installation);
 }

src/Abstractions/NexusMods.Abstractions.GameLocators/GameDomain.cs

@@ -1,3 +1,4 @@
+using NexusMods.Extensions.Hashing;
 using TransparentValueObjects;
 
 namespace NexusMods.Abstractions.Games.DTO;
@@ -18,4 +19,10 @@ namespace NexusMods.Abstractions.Games.DTO;
     /// Unknown.
     /// </summary>
     public static GameDomain DefaultValue { get; } = From("Unknown");
+
+    /// <summary>
+    /// Retrieves the 'stable' hash of a domain.
+    /// One that does not change between application runs.
+    /// </summary>
+    public ulong GetStableHash() => Value.AsSpan().GetStableHash();
 }

src/Abstractions/NexusMods.Abstractions.GameLocators/NexusMods.Abstractions.GameLocators.csproj

@@ -6,5 +6,8 @@
         <PackageReference Include="TransparentValueObjects" PrivateAssets="all" OutputItemType="Analyzer" ReferenceOutputAssembly="false" />
         <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" />
     </ItemGroup>
+    <ItemGroup>
+      <ProjectReference Include="..\..\Extensions\NexusMods.Extensions.Hashing\NexusMods.Extensions.Hashing.csproj" />
+    </ItemGroup>
 
 </Project>

src/Abstractions/NexusMods.Abstractions.Loadouts.Synchronizers/ALoadoutSynchronizer.cs

@@ -2,7 +2,6 @@
 using System.Diagnostics;
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
-using System.Runtime.InteropServices;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using NexusMods.Abstractions.DataModel.Entities.Sorting;
@@ -124,14 +123,14 @@ public ValueTask<FileTree> FlattenedLoadoutToFileTree(FlattenedLoadout flattened
 
 
     /// <inheritdoc />
-    public async Task<DiskStateTree> FileTreeToDisk(FileTree fileTree, Loadout loadout, FlattenedLoadout flattenedLoadout, DiskStateTree prevState, GameInstallation installation)
+    public async Task<DiskStateTree> FileTreeToDisk(FileTree fileTree, Loadout loadout, FlattenedLoadout flattenedLoadout, DiskStateTree prevState, GameInstallation installation, bool skipIngest = false)
     {
         // Return the new tree
         return await FileTreeToDiskImpl(fileTree, loadout, flattenedLoadout, prevState, installation, true);
     }
 
     [MethodImpl(MethodImplOptions.AggressiveInlining)]
-    internal async Task<DiskStateTree> FileTreeToDiskImpl(FileTree fileTree, Loadout loadout, FlattenedLoadout flattenedLoadout, DiskStateTree prevState, GameInstallation installation, bool fixFileMode)
+    internal async Task<DiskStateTree> FileTreeToDiskImpl(FileTree fileTree, Loadout loadout, FlattenedLoadout flattenedLoadout, DiskStateTree prevState, GameInstallation installation, bool fixFileMode, bool skipIngest = false)
     {
         List<KeyValuePair<GamePath, HashedEntryWithName>> toDelete = new();
         List<KeyValuePair<AbsolutePath, IGeneratedFile>> toWrite = new();
@@ -151,13 +150,19 @@ internal async Task<DiskStateTree> FileTreeToDiskImpl(FileTree fileTree, Loadout
                 if (!prevState.TryGetValue(gamePath, out var prevEntry))
                 {
                     // File is new, and not in the previous state, so we need to abort and do an ingest
+                    if (skipIngest)
+                        continue;
+
                     HandleNeedIngest(entry);
                     throw new UnreachableException("HandleNeedIngest should have thrown");
                 }
 
                 if (prevEntry.Item.Value.Hash != entry.Hash)
                 {
                     // File has changed, so we need to abort and do an ingest
+                    if (skipIngest)
+                        continue;
+
                     HandleNeedIngest(entry);
                     throw new UnreachableException("HandleNeedIngest should have thrown");
                 }
@@ -209,6 +214,9 @@ internal async Task<DiskStateTree> FileTreeToDiskImpl(FileTree fileTree, Loadout
             if (prevState.TryGetValue(path, out var prevEntry))
             {
                 // File is in new tree, was in prev disk state, but wasn't found on disk
+                if (skipIngest)
+                    continue;
+
                 HandleNeedIngest(prevEntry.Item.Value.ToHashedEntry(absolutePath));
                 throw new UnreachableException("HandleNeedIngest should have thrown");
             }
@@ -560,13 +568,17 @@ public virtual Loadout MergeLoadouts(Loadout loadoutA, Loadout loadoutB)
     /// Applies a loadout to the game folder.
     /// </summary>
     /// <param name="loadout"></param>
+    /// <param name="forceSkipIngest">
+    ///     Skips checking if an ingest is needed.
+    ///     Force overrides current locations to intended tree
+    /// </param>
     /// <returns></returns>
-    public virtual async Task<DiskStateTree> Apply(Loadout loadout)
+    public virtual async Task<DiskStateTree> Apply(Loadout loadout, bool forceSkipIngest = false)
     {
         var flattened = await LoadoutToFlattenedLoadout(loadout);
         var fileTree = await FlattenedLoadoutToFileTree(flattened, loadout);
         var prevState = _diskStateRegistry.GetState(loadout.Installation)!;
-        var diskState = await FileTreeToDisk(fileTree, loadout, flattened, prevState, loadout.Installation);
+        var diskState = await FileTreeToDisk(fileTree, loadout, flattened, prevState, loadout.Installation, forceSkipIngest);
         diskState.LoadoutRevision = loadout.DataStoreId;
         await _diskStateRegistry.SaveState(loadout.Installation, diskState);
         return diskState;
@@ -745,8 +757,8 @@ await Parallel.ForEachAsync(fileTree.GetAllDescendentFiles(), async (file, cance
     /// <inheritdoc />
     public virtual async Task<Loadout> Manage(GameInstallation installation, string? suggestedName = null)
     {
-        var initialState = await GetInitialDiskState(installation);
-
+        var (isCached, initialState) = await GetOrCreateInitialDiskState(installation);
+        
         var loadoutId = LoadoutId.Create();
         var gameFiles = new Mod()
         {
@@ -767,13 +779,15 @@ public virtual async Task<Loadout> Manage(GameInstallation installation, string?
                     });
                 }))
         };
-
-
-        await BackupNewFiles(installation, FileTree.Create(gameFiles.Files.Select(kv =>
-        {
-            var storedFile = (kv.Value as StoredFile)!;
-            return KeyValuePair.Create(storedFile.To, kv.Value);
-        } )));
+
+        var fileTree = FileTree.Create(gameFiles.Files.Select(kv =>
+                {
+                    var storedFile = (kv.Value as StoredFile)!;
+                    return KeyValuePair.Create(storedFile.To, kv.Value);
+                }
+            )
+        );
+        await BackupNewFiles(installation, fileTree);
 
         var loadout = _loadoutRegistry.Alter(loadoutId, "Initial loadout",  loadout => loadout
             with
@@ -784,8 +798,23 @@ await BackupNewFiles(installation, FileTree.Create(gameFiles.Files.Select(kv =>
             });
 
         initialState.LoadoutRevision = loadout.DataStoreId;
+
+        // Reset the game folder to initial state if making a new loadout.
+        // We must do this before saving state, as Apply does a diff against
+        // the last state. Which will be a state from previous loadout.
+        if (isCached)
+        {
+            // This is a 'fast apply' operation, that avoids recomputing the file tree.
+            // And avoids a double save-state.
+            var flattened = await LoadoutToFlattenedLoadout(loadout);
+            var prevState = _diskStateRegistry.GetState(loadout.Installation)!;
+            await FileTreeToDisk(fileTree, loadout, flattened, prevState, loadout.Installation, true);
+
+            // Note: DiskState returned from `FileTreeToDisk` and `initialState`
+            // are the same in terms of content!!
+        }
+
         await _diskStateRegistry.SaveState(loadout.Installation, initialState);
-
         return loadout;
     }
 
@@ -844,9 +873,15 @@ protected virtual async Task<IEnumerable<Mod>> SortMods(Loadout loadout)
     /// </summary>
     /// <param name="installation"></param>
     /// <returns></returns>
-    public virtual ValueTask<DiskStateTree> GetInitialDiskState(GameInstallation installation)
+    public virtual async ValueTask<(bool isCachedState, DiskStateTree tree)> GetOrCreateInitialDiskState(GameInstallation installation)
     {
-        return _hashCache.IndexDiskState(installation);
+        var initialState = _diskStateRegistry.GetInitialState(installation);
+        if (initialState != null)
+            return (true, initialState);
+
+        var indexedState = await _hashCache.IndexDiskState(installation);
+        await _diskStateRegistry.SaveInitialState(installation, indexedState);
+        return (false, indexedState);
     }
     #endregion
 

src/Abstractions/NexusMods.Abstractions.Loadouts.Synchronizers/ILoadoutSynchronizer.cs

@@ -37,8 +37,12 @@ public interface ILoadoutSynchronizer
     /// Applies a loadout to the game folder.
     /// </summary>
     /// <param name="loadout"></param>
+    /// <param name="forceSkipIngest">
+    ///     Skips checking if an ingest is needed.
+    ///     Force overrides current locations to intended tree
+    /// </param>
     /// <returns>The new DiskState after the files were applied</returns>
-    Task<DiskStateTree> Apply(Loadout loadout);
+    Task<DiskStateTree> Apply(Loadout loadout, bool forceSkipIngest = false);
 
     /// <summary>
     /// Ingests changes from the game folder into the loadout.

src/Abstractions/NexusMods.Abstractions.Loadouts.Synchronizers/IStandardizedLoadoutSyncronizer.cs

@@ -35,8 +35,12 @@ public interface IStandardizedLoadoutSynchronizer : ILoadoutSynchronizer
     /// <param name="flattenedLoadout"></param>
     /// <param name="prevState"></param>
     /// <param name="installation"></param>
+    /// <param name="skipIngest">
+    ///     Skips checking if an ingest is needed.
+    ///     Force overrides current locations to intended tree.
+    /// </param>
     Task<DiskStateTree> FileTreeToDisk(FileTree fileTree, Loadout loadout, FlattenedLoadout flattenedLoadout,
-        DiskStateTree prevState, GameInstallation installation);
+        DiskStateTree prevState, GameInstallation installation, bool skipIngest = false);
 
     #endregion
 

src/Abstractions/NexusMods.Abstractions.Serialization/DataModel/EntityCategory.cs

@@ -121,5 +121,11 @@ public enum EntityCategory : byte
     /// <summary>
     /// Persisted workspaces.
     /// </summary>
-    Workspaces = 19
+    Workspaces = 19,
+
+    /// <summary>
+    /// Initial disk states for games.
+    /// Stored separately from <see cref="LoadoutRoots"/> to avoid conflicts.
+    /// </summary>
+    InitialDiskStates = 20,
 }

src/NexusMods.DataModel/Attributes/DiskState.cs

@@ -6,14 +6,12 @@
 using NexusMods.MnemonicDB.Abstractions;
 using NexusMods.MnemonicDB.Abstractions.Attributes;
 using NexusMods.MnemonicDB.Abstractions.Models;
-using NexusMods.Paths;
 
 namespace NexusMods.DataModel.Attributes;
 
 /// <summary>
 /// MnemonicDB attributes for the DiskStateTree registry.
 /// </summary>b
-/// 
 public static class DiskState
 {
     private static readonly string Namespace = "NexusMods.DataModel.DiskStateRegistry";
@@ -39,7 +37,6 @@ public static class DiskState
     /// </summary>
     public static readonly DiskStateAttribute State = new(Namespace, nameof(State)) { NoHistory = true };
 
-
     [PublicAPI]
     [SuppressMessage("ReSharper", "MemberHidesStaticFromOuterClass")]
     internal class Model(ITransaction tx) : Entity(tx, (byte)IdPartitions.DiskState)
@@ -81,3 +78,66 @@ public NexusMods.Abstractions.DiskState.DiskStateTree DiskState
         }
     }
 }
+
+/// <summary>
+///     A sibling of <see cref="DiskState"/>, but only for the initial state.
+/// </summary>
+/// <remarks>
+///     We don't want to keep history in <see cref="DiskState"/> as that is only
+///     supposed to hold the latest state, so in order to keep things clean,
+///     we separated this out to the class.
+///
+///     This will also make cleaning out loadouts in MneumonicDB easier in the future.
+/// </remarks>
+public static class InitialDiskState
+{
+    private static readonly string Namespace = "NexusMods.DataModel.InitialDiskStates";
+
+    /// <summary>
+    /// The associated game id.
+    /// </summary>
+    public static readonly GameDomainAttribute Game = new(Namespace, nameof(Game)) { IsIndexed = true, NoHistory = true };
+
+    /// <summary>
+    /// The game's root folder. Stored as a string, since AbsolutePaths require an IFileSystem, and we don't know or care
+    /// what filesystem is being used when reading/writing the data from the database.
+    /// </summary>
+    public static readonly StringAttribute Root = new(Namespace, nameof(Root)) { IsIndexed = true, NoHistory = true };
+
+    /// <summary>
+    /// The state of the disk.
+    /// </summary>
+    public static readonly DiskStateAttribute State = new(Namespace, nameof(State)) { NoHistory = true };
+
+    [PublicAPI]
+    [SuppressMessage("ReSharper", "MemberHidesStaticFromOuterClass")]
+    internal class Model(ITransaction tx) : Entity(tx)
+    {
+        /// <summary>
+        /// The associated game type
+        /// </summary>
+        public GameDomain Game
+        {
+            get => Attributes.InitialDiskState.Game.Get(this);
+            set => Attributes.InitialDiskState.Game.Add(this, value);
+        }
+
+        /// <summary>
+        /// The associated game root folder
+        /// </summary>
+        public string Root
+        {
+            get => Attributes.InitialDiskState.Root.Get(this);
+            set => Attributes.InitialDiskState.Root.Add(this, value);
+        }
+
+        /// <summary>
+        /// The state of the disk.
+        /// </summary>
+        public NexusMods.Abstractions.DiskState.DiskStateTree DiskState
+        {
+            get => State.Get(this);
+            set => State.Add(this, value);
+        }
+    }
+}