Revert "Fixed: Adding Library Files no longer Adds Duplicates" (#2032)

benchmarks/NexusMods.Benchmarks/Benchmarks/Loadouts/Harness/DummyFileStore.cs

@@ -11,7 +11,7 @@ public ValueTask<bool> HaveFile(Hash hash)
         return ValueTask.FromResult(false);
     }
 
-    public Task BackupFiles(IEnumerable<ArchivedFileEntry> backups, bool deduplicate = true, CancellationToken token = default)
+    public Task BackupFiles(IEnumerable<ArchivedFileEntry> backups, CancellationToken token = default)
     {
         return Task.CompletedTask;
     }

src/Abstractions/NexusMods.Abstractions.IO/IFileStore.cs

@@ -17,23 +17,12 @@ public interface IFileStore
     public ValueTask<bool> HaveFile(Hash hash);
 
     /// <summary>
-    /// Backup the given set of NEW files.
-    ///
-    /// If the size or hash do not match during the
-    /// backup process an exception may be thrown.
+    /// Backup the given files. If the size or hash do not match during the
+    /// backup process a exception may be thrown.
     /// </summary>
-    /// <param name="backups">
-    ///     The files to back up.
-    ///     These should not contain any already stored files unless <paramref name="deduplicate"/>
-    ///     is set to true.
-    /// </param>
-    /// <param name="deduplicate">
-    ///     Ensures no duplicate files are stored.
-    ///     Only set this to false if you are certain there are no duplicates
-    ///     with existing files in the input.
-    /// </param>
+    /// <param name="backups"></param>
     /// <param name="token"></param>
-    Task BackupFiles(IEnumerable<ArchivedFileEntry> backups, bool deduplicate = true, CancellationToken token = default);
+    Task BackupFiles(IEnumerable<ArchivedFileEntry> backups, CancellationToken token = default);
 
     /// <summary>
     /// Extract the given files to the given disk locations, provide as a less-abstract interface incase

src/Abstractions/NexusMods.Abstractions.Loadouts.Synchronizers/ALoadoutSynchronizer.cs

@@ -808,8 +808,7 @@ await Parallel.ForEachAsync(files, async (file, _) =>
             }
         );
 
-        // SAFETY: We deduplicate above with the HaveFile call.
-        await _fileStore.BackupFiles(archivedFiles, deduplicate: false);
+        await _fileStore.BackupFiles(archivedFiles);
     }
 
     private async Task<DiskState> GetOrCreateInitialDiskState(GameInstallation installation)

src/NexusMods.App.UI/Pages/TextEdit/TextEditorPageViewModel.cs

@@ -94,8 +94,7 @@ public TextEditorPageViewModel(
 
             using (var streamFactory = new MemoryStreamFactory(filePath.Path, new MemoryStream(bytes, writable: false)))
             {
-                if (!await fileStore.HaveFile(hash))
-                    await fileStore.BackupFiles([new ArchivedFileEntry(streamFactory, hash, size)], deduplicate: false);
+                await fileStore.BackupFiles([new ArchivedFileEntry(streamFactory, hash, size)]);
             }
 
             // update the file

src/NexusMods.DataModel/NxFileStore.cs

@@ -67,7 +67,7 @@ public ValueTask<bool> HaveFile(Hash hash)
     }
 
     /// <inheritdoc />
-    public async Task BackupFiles(IEnumerable<ArchivedFileEntry> backups, bool deduplicate = true, CancellationToken token = default)
+    public async Task BackupFiles(IEnumerable<ArchivedFileEntry> backups, CancellationToken token = default)
     {
         var hasAnyFiles = backups.Any();
         if (hasAnyFiles == false)
@@ -76,11 +76,9 @@ public async Task BackupFiles(IEnumerable<ArchivedFileEntry> backups, bool dedup
         var builder = new NxPackerBuilder();
         var distinct = backups.DistinctBy(d => d.Hash).ToArray();
         var streams = new List<Stream>();
+        _logger.LogDebug("Backing up {Count} files of {Size} in size", distinct.Length, distinct.Sum(s => s.Size));
         foreach (var backup in distinct)
         {
-            if (await IsDuplicate(deduplicate, backup))
-                continue;
-
             var stream = await backup.StreamFactory.GetStreamAsync();
             streams.Add(stream);
             builder.AddFile(stream, new AddFileParams
@@ -89,7 +87,6 @@ public async Task BackupFiles(IEnumerable<ArchivedFileEntry> backups, bool dedup
             });
         }
 
-        _logger.LogDebug("Backing up {Count} files of {Size} in size", distinct.Length, distinct.Sum(s => s.Size));
         var guid = Guid.NewGuid();
         var id = guid.ToString();
         var outputPath = _archiveLocations.First().Combine(id).AppendExtension(KnownExtensions.Tmp);
@@ -111,24 +108,6 @@ public async Task BackupFiles(IEnumerable<ArchivedFileEntry> backups, bool dedup
         await UpdateIndexes(unpacker, finalPath);
     }
 
-    private async Task<bool> IsDuplicate(bool deduplicate, ArchivedFileEntry backup)
-    {
-        // Extra sanity test for debug builds, else take hot path since
-        // this is supposed to be a speedy-ish API.
-        #if DEBUG
-        var haveFile = await HaveFile(backup.Hash);
-        if (!haveFile) 
-            return false;
-
-        if (!deduplicate)
-            throw new Exception("Writing duplicate but deduplicate is disabled. This is a bug.");
-
-        return true;
-        #else
-        return deduplicate && await HaveFile(backup.Hash);
-        #endif
-    }
-
     private async Task UpdateIndexes(NxUnpacker unpacker, AbsolutePath finalPath)
     {
         using var lck = _lock.ReadLock();

src/NexusMods.Library/AddLibraryFileJob.cs

@@ -17,6 +17,7 @@ internal class AddLibraryFileJob : IJobDefinitionWithStart<AddLibraryFileJob, Li
     public required ITransaction Transaction { get; init; }
     public required AbsolutePath FilePath { get; init; }
     private ConcurrentBag<TemporaryPath> ExtractionDirectories { get; } = [];
+    private ConcurrentBag<ArchivedFileEntry> ToArchive { get; } = [];
 
     public static IJobTask<AddLibraryFileJob, LibraryFile.New> Create(IServiceProvider provider, ITransaction transaction, AbsolutePath filePath, bool doCommit, bool doBackup)
     {
@@ -42,19 +43,20 @@ internal class AddLibraryFileJob : IJobDefinitionWithStart<AddLibraryFileJob, Li
     {
         if (!FilePath.FileExists)
             throw new Exception($"File '{FilePath}' does not exist.");
-
-        var toArchive = new List<ArchivedFileEntry>();
-        var result = await AnalyzeOne(context, FilePath, toArchive);
-        await FileStore.BackupFiles(toArchive, deduplicate: false, context.CancellationToken);
-        return result;
+        
+        var topFile = await AnalyzeOne(context, FilePath);
+
+        await FileStore.BackupFiles(ToArchive, context.CancellationToken);
+        return topFile;
     }
 
-    private async Task<LibraryFile.New> AnalyzeOne(IJobContext<AddLibraryFileJob> context, AbsolutePath filePath, List<ArchivedFileEntry> toArchive)
+    private async Task<LibraryFile.New> AnalyzeOne(IJobContext<AddLibraryFileJob> context, AbsolutePath filePath)
     {
         var isArchive = await CheckIfArchiveAsync(filePath);
         var hash = await filePath.XxHash64Async();
 
         var libraryFile = CreateLibraryFile(Transaction, filePath, hash);
+
         if (isArchive)
         {
             var libraryArchive = new LibraryArchive.New(Transaction, libraryFile.Id)
@@ -72,7 +74,7 @@ internal class AddLibraryFileJob : IJobDefinitionWithStart<AddLibraryFileJob, Li
 
             foreach (var extracted in extractedFiles)
             {
-                var subFile = await AnalyzeOne(context, extracted, toArchive);
+                var subFile = await AnalyzeOne(context, extracted);
                 var path = extracted.RelativeTo(extractionFolder.Path);
                 _ = new LibraryArchiveFileEntry.New(Transaction, subFile.Id)
                 {
@@ -85,13 +87,13 @@ internal class AddLibraryFileJob : IJobDefinitionWithStart<AddLibraryFileJob, Li
         else
         {
             var size = filePath.FileInfo.Size;
-            if (!await FileStore.HaveFile(hash))
-                toArchive.Add(new ArchivedFileEntry(new NativeFileStreamFactory(filePath), hash, size)); 
+            ToArchive.Add(new ArchivedFileEntry(new NativeFileStreamFactory(filePath), hash, size)); 
         }
 
         return libraryFile;
     }
 
+
     private async Task<bool> CheckIfArchiveAsync(AbsolutePath filePath)
     {
         await using var stream = filePath.Open(FileMode.Open, FileAccess.Read, FileShare.None);

tests/NexusMods.Collections.Tests/CollectionInstallTests.CanInstallBasicCollection.verified.txt

@@ -2,7 +2,7 @@
   mods: [
     Game Files,
     Halgari's Helper,
-    My Mods
+    My Collection
   ],
   files: [
     {Game}/archive/pc/mod/_1_Ves_HanakoFixedBodyNaked.archive,