Direct download in collections (#2195)

* WIP

* Add support for direct downloads in collections

* Update the collection test files so that they include the new xxHash3 values

* Delete src/NexusMods.Library/AddDirectDownloadJob.cs

src/Abstractions/NexusMods.Abstractions.Collections/Attributes/Md5Attribute.cs

@@ -0,0 +1,24 @@
+using NexusMods.Abstractions.Collections.Types;
+using NexusMods.MnemonicDB.Abstractions;
+using NexusMods.MnemonicDB.Abstractions.Attributes;
+using NexusMods.MnemonicDB.Abstractions.ElementComparers;
+
+namespace NexusMods.Abstractions.Collections.Attributes;
+
+/// <summary>
+/// An attribute representing an MD5 hash value.
+/// </summary>
+public class Md5Attribute(string ns, string name) : ScalarAttribute<Md5HashValue, UInt128>(ValueTag.UInt128, ns, name)
+{
+    /// <inheritdoc />
+    protected override UInt128 ToLowLevel(Md5HashValue value)
+    {
+        return value.Value;
+    }
+
+    /// <inheritdoc />
+    protected override Md5HashValue FromLowLevel(UInt128 value, AttributeResolver resolver)
+    {
+        return Md5HashValue.From(value);
+    }
+}

src/Abstractions/NexusMods.Abstractions.Collections/DirectDownloadLibraryFile.cs

@@ -0,0 +1,25 @@
+using NexusMods.Abstractions.Collections.Attributes;
+using NexusMods.Abstractions.Library.Models;
+using NexusMods.MnemonicDB.Abstractions.Attributes;
+using NexusMods.MnemonicDB.Abstractions.Models;
+
+namespace NexusMods.Abstractions.Collections;
+
+/// <summary>
+/// A direct downloaded file from a collection
+/// </summary>
+[Include<LibraryFile>]
+public partial class DirectDownloadLibraryFile : IModelDefinition
+{
+    private const string Namespace = "NexusMods.Abstractions.Collections";
+
+    /// <summary>
+    /// The MD5 hash value of the downloaded file.
+    /// </summary>
+    public static readonly Md5Attribute Md5 = new(Namespace, nameof(Md5)) { IsIndexed = true };
+
+    /// <summary>
+    /// A user-friendly name of the file.
+    /// </summary>
+    public static readonly StringAttribute LogicalFileName = new(Namespace, nameof(LogicalFileName));
+}

src/Abstractions/NexusMods.Abstractions.Collections/Json/ModSource.cs

@@ -1,5 +1,5 @@
 using System.Text.Json.Serialization;
-using NexusMods.Abstractions.NexusWebApi.Types;
+using NexusMods.Abstractions.Collections.Types;
 using NexusMods.Abstractions.NexusWebApi.Types.V2;
 using NexusMods.Paths;
 
@@ -14,6 +14,24 @@ public class ModSource
     [JsonPropertyName("modId")]
     public ModId ModId { get; init; }
 
+    /// <summary>
+    /// MD5 hash a direct download
+    /// </summary>
+    [JsonPropertyName("md5")]
+    public Md5HashValue Md5 { get; init; }
+
+    /// <summary>
+    /// If this is a direct download, this is the URL to download the mod from
+    /// </summary>
+    [JsonPropertyName("url")]
+    public Uri? Url { get; init; }
+
+    /// <summary>
+    /// The name of the mod in the installed loadout
+    /// </summary>
+    [JsonPropertyName("logicalFilename")]
+    public string? LogicalFilename { get; init; }
+
     [JsonPropertyName("fileId")]
     public FileId FileId { get; init; }
 

src/Abstractions/NexusMods.Abstractions.Collections/Json/ModSourceType.cs

@@ -6,6 +6,7 @@ namespace NexusMods.Abstractions.Collections.Json;
 /// </summary>
 public enum ModSourceType
 {
+    direct,
     nexus, 
     bundle,
 }

src/Networking/NexusMods.Networking.HttpDownloader/HttpDownloadJob.cs

@@ -44,11 +44,11 @@ public record HttpDownloadJob : IJobDefinitionWithStart<HttpDownloadJob, Absolut
     /// The destination of the download.
     /// </summary>
     public required AbsolutePath Destination { get; init; }
-
+    
     /// <summary>
     /// Only exists for extension by derived classes.
     /// </summary>
-    public ValueTask AddMetadata(ITransaction transaction, LibraryFile.New libraryFile)
+    public virtual ValueTask AddMetadata(ITransaction transaction, LibraryFile.New libraryFile)
     {
         return ValueTask.CompletedTask;
     }

src/NexusMods.Collections/DirectDownloadJob.cs

@@ -0,0 +1,65 @@
+using System.Runtime.InteropServices;
+using System.Security.Cryptography;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using NexusMods.Abstractions.Collections;
+using NexusMods.Abstractions.Collections.Types;
+using NexusMods.Abstractions.Jobs;
+using NexusMods.Abstractions.Library.Models;
+using NexusMods.MnemonicDB.Abstractions;
+using NexusMods.Networking.HttpDownloader;
+using NexusMods.Paths;
+
+namespace NexusMods.Collections;
+
+public record DirectDownloadJob : HttpDownloadJob
+{
+    /// <summary>
+    /// The expected MD5 hash value of the downloaded file.
+    /// </summary>
+    public required Md5HashValue ExpectedMd5 { get; init; }
+
+    /// <summary>
+    /// The user-friendly name of the file.
+    /// </summary>
+    public required string LogicalFileName { get; init; }
+
+    /// <summary>
+    /// Create a new download job for the given URL, the job will fail if the downloaded file does not
+    /// match the expected MD5 hash.
+    /// </summary>
+    public static IJobTask<DirectDownloadJob, AbsolutePath> Create(IServiceProvider provider, Uri uri, 
+        Md5HashValue expectedMd5, string logicalFileName)
+    {
+        var monitor = provider.GetRequiredService<IJobMonitor>();
+        var tempFileManager = provider.GetRequiredService<TemporaryFileManager>();
+        var job = new DirectDownloadJob
+        {
+            Logger = provider.GetRequiredService<ILogger<DirectDownloadJob>>(),
+            ExpectedMd5 = expectedMd5,
+            LogicalFileName = logicalFileName,
+            DownloadPageUri = uri,
+            Destination = tempFileManager.CreateFile(),
+            Uri = uri,
+        };
+
+        return monitor.Begin<DirectDownloadJob, AbsolutePath>(job);
+    }
+
+
+    /// <inheritdoc />
+    public override async ValueTask AddMetadata(ITransaction tx, LibraryFile.New libraryFile)
+    {
+        await using (var fileStream = Destination.Read())
+        {
+            var algo = MD5.Create();
+            var hash = await algo.ComputeHashAsync(fileStream);
+            var md5Actual = Md5HashValue.From(hash);
+            if (md5Actual != ExpectedMd5)
+                throw new InvalidOperationException($"MD5 hash mismatch. Expected: {ExpectedMd5}, Actual: {md5Actual}");
+        }
+
+        tx.Add(libraryFile, DirectDownloadLibraryFile.Md5, ExpectedMd5);
+        tx.Add(libraryFile, DirectDownloadLibraryFile.LogicalFileName, LogicalFileName);
+    }
+}

src/NexusMods.Collections/InstallCollectionJob.cs

@@ -20,7 +20,9 @@
 using NexusMods.Hashing.xxHash3;
 using NexusMods.Abstractions.NexusWebApi.Types.V2;
 using NexusMods.Abstractions.NexusWebApi.Types.V2.Uid;
+using NexusMods.Extensions.BCL;
 using NexusMods.MnemonicDB.Abstractions;
+using NexusMods.MnemonicDB.Abstractions.TxFunctions;
 using NexusMods.Networking.NexusWebApi;
 using NexusMods.Networking.NexusWebApi.V1Interop;
 using NexusMods.Paths;
@@ -121,16 +123,27 @@ public class InstallCollectionJob : IJobDefinitionWithStart<InstallCollectionJob
         var groupResult = await tx.Commit();
         var groupRemapped = groupResult.Remap(group);
 
+        var insalled = new ConcurrentBag<(ModInstructions, LoadoutItemGroup.ReadOnly)>();
+
         // Now install the mods
         await Parallel.ForEachAsync(toInstall, context.CancellationToken, async (file, _) =>
             {
                 // Bit strange, but Install Mod will want to find the collection group, so we'll have to rebase entity it will get the DB from
                 if (file.LibraryFile.HasValue) 
                     file = (file.Mod, file.LibraryFile!.Value.Rebase());
-                await InstallMod(TargetLoadout, file, groupRemapped.AsCollectionGroup().AsLoadoutItemGroup(), archive);
+                var mod = await InstallMod(TargetLoadout, file, groupRemapped.AsCollectionGroup().AsLoadoutItemGroup(), archive);
+                insalled.Add((file, mod));
             }
         );
 
+        using var tx2 = Connection.BeginTransaction();
+        foreach (var (instructions, modGroup) in insalled)
+        {
+            tx2.Add(modGroup.Id, LoadoutItem.Name, instructions.Mod.Name);
+        }
+
+        await tx2.Commit();
+
         return groupRemapped;
     }
 
@@ -405,13 +418,29 @@ private async Task<ModInstructions> EnsureDownloaded(Mod mod)
     {
         return mod.Source.Type switch
         {
+            ModSourceType.direct => await EnsureDirectMod(mod),
             ModSourceType.nexus => await EnsureNexusModDownloaded(mod),
             // Nothing to downoad for a bundle
             ModSourceType.bundle => (mod, null),
             _ => throw new NotSupportedException($"The mod source type '{mod.Source.Type}' is not supported.")
         };
     }
 
+
+    private async Task<ModInstructions> EnsureDirectMod(Mod mod)
+    {
+        var db = Connection.Db;
+        if (DirectDownloadLibraryFile.FindByMd5(db, mod.Source.Md5).TryGetFirst(out var found))
+            return (mod, found.AsLibraryFile());
+
+        await using var tempPath = TemporaryFileManager.CreateFile();
+
+        var job = DirectDownloadJob.Create(SerivceProvider, mod.Source.Url!, mod.Source.Md5, mod.Name);
+        var libraryFile = await LibraryService.AddDownload(job);
+
+        return (mod, libraryFile);
+    }
+
     private async Task<ModInstructions> EnsureNexusModDownloaded(Mod mod)
     {
         var db = Connection.Db;

src/NexusMods.Collections/Services.cs

@@ -10,6 +10,7 @@ public static class Services
     public static IServiceCollection AddNexusModsCollections(this IServiceCollection services)
     {
         return services.AddNexusCollectionLoadoutGroupModel()
+            .AddDirectDownloadLibraryFileModel()
             .AddNexusCollectionBundledLoadoutGroupModel()
             .AddCollectionVerbs();
     }

src/NexusMods.Library/LibraryService.cs

@@ -46,7 +46,7 @@ public LibraryService(IServiceProvider serviceProvider)
     {
         return AddLocalFileJob.Create(_serviceProvider, absolutePath);
     }
-
+    
     public IJobTask<IInstallLoadoutItemJob, LoadoutItemGroup.ReadOnly> InstallItem(LibraryItem.ReadOnly libraryItem, LoadoutId targetLoadout, Optional<LoadoutItemGroupId> parent = default, ILibraryItemInstaller? itemInstaller = null)
     {
         if (!parent.HasValue)