Merge pull request #2117 from erri120/feat/collection-images-pipeline

Add image pipeline

benchmarks/NexusMods.Benchmarks/Benchmarks/Loadouts/Harness/DummyFileStore.cs

@@ -16,6 +16,11 @@ public Task BackupFiles(IEnumerable<ArchivedFileEntry> backups, bool deduplicate
         return Task.CompletedTask;
     }
 
+    public Task BackupFiles(string archiveName, IEnumerable<ArchivedFileEntry> files, CancellationToken cancellationToken = default)
+    {
+        return Task.CompletedTask;
+    }
+
     public Task ExtractFiles(IEnumerable<(Hash Hash, AbsolutePath Dest)> files, CancellationToken token = default)
     {
         return Task.CompletedTask;

src/Abstractions/NexusMods.Abstractions.IO/IFileStore.cs

@@ -44,6 +44,12 @@ public interface IFileStore
     /// </remarks>
     Task BackupFiles(IEnumerable<ArchivedFileEntry> backups, bool deduplicate = true, CancellationToken token = default);
 
+    /// <summary>
+    /// Similar to <see cref="BackupFiles(System.Collections.Generic.IEnumerable{NexusMods.Abstractions.IO.ArchivedFileEntry},bool,System.Threading.CancellationToken)"/>
+    /// except the same archive is used.
+    /// </summary>
+    Task BackupFiles(string archiveName, IEnumerable<ArchivedFileEntry> files, CancellationToken cancellationToken = default);
+
     /// <summary>
     /// Extract the given files to the given disk locations, provide as a less-abstract interface incase
     /// the extractor needs more direct access (such as memory mapping).

src/Abstractions/NexusMods.Abstractions.NexusModsLibrary/Models/CollectionMetadata.cs

@@ -1,5 +1,6 @@
 using NexusMods.Abstractions.MnemonicDB.Attributes;
 using NexusMods.Abstractions.NexusModsLibrary.Attributes;
+using NexusMods.Abstractions.Resources.DB;
 using NexusMods.MnemonicDB.Abstractions.Attributes;
 using NexusMods.MnemonicDB.Abstractions.Models;
 
@@ -46,14 +47,24 @@ public partial class CollectionMetadata : IModelDefinition
     /// The number of endorsements the collection has.
     /// </summary>
     public static readonly ULongAttribute Endorsements = new(Namespace, nameof(Endorsements));
-    
+
     /// <summary>
-    /// The collections' image.
+    /// The tile image uri.
     /// </summary>
-    public static readonly MemoryAttribute TileImage = new(Namespace, nameof(TileImage));
-
+    public static readonly UriAttribute TileImageUri = new(Namespace, nameof(TileImageUri)) { IsOptional = true };
+
+    /// <summary>
+    /// The background image uri.
+    /// </summary>
+    public static readonly UriAttribute BackgroundImageUri = new(Namespace, nameof(BackgroundImageUri)) { IsOptional = true };
+
+    /// <summary>
+    /// The tile image resource.
+    /// </summary>
+    public static readonly ReferenceAttribute<PersistedDbResource> TileImageResource = new(Namespace, nameof(TileImageResource)) { IsOptional = true };
+
     /// <summary>
-    /// The collections' image.
+    /// The background image resource.
     /// </summary>
-    public static readonly MemoryAttribute BackgroundImage = new(Namespace, nameof(BackgroundImage));
+    public static readonly ReferenceAttribute<PersistedDbResource> BackgroundImageResource = new(Namespace, nameof(BackgroundImageResource)) { IsOptional = true };
 }

src/Abstractions/NexusMods.Abstractions.NexusModsLibrary/NexusModsModPageMetadata.cs

@@ -36,7 +36,7 @@ public partial class NexusModsModPageMetadata : IModelDefinition
     /// </summary>
     public static readonly UriAttribute FullSizedPictureUri = new(Namespace, nameof(FullSizedPictureUri)) { IsOptional = true };
 
-    public static readonly ReferenceAttribute<PersistedResource> ThumbnailResource = new(Namespace, nameof(ThumbnailResource)) { IsOptional = true };
+    public static readonly ReferenceAttribute<PersistedDbResource> ThumbnailResource = new(Namespace, nameof(ThumbnailResource)) { IsOptional = true };
 
     /// <summary>
     /// Uri for the thumbnail of the full sized picture.

src/Abstractions/NexusMods.Abstractions.Resources.DB/PersistedResource.cs → src/Abstractions/NexusMods.Abstractions.Resources.DB/PersistedDbResource.cs

@@ -4,19 +4,34 @@
 
 namespace NexusMods.Abstractions.Resources.DB;
 
+/// <summary>
+/// Represents a resources persisted in the database.
+/// </summary>
 [PublicAPI]
-public partial class PersistedResource : IModelDefinition
+public partial class PersistedDbResource : IModelDefinition
 {
     private const string Namespace = "NexusMods.Resources.PersistedResource";
 
+    /// <summary>
+    /// The raw data.
+    /// </summary>
     public static readonly BytesAttribute Data = new(Namespace, nameof(Data));
 
+    /// <summary>
+    /// The expiration date.
+    /// </summary>
     public static readonly DateTimeAttribute ExpiresAt = new(Namespace, nameof(ExpiresAt));
 
+    /// <summary>
+    /// The resource identifier as a hash.
+    /// </summary>
     public static readonly HashAttribute ResourceIdentifierHash = new(Namespace, nameof(ResourceIdentifierHash));
 
     public partial struct ReadOnly
     {
+        /// <summary>
+        /// Whether the resource is expired.
+        /// </summary>
         public bool IsExpired => DateTime.UtcNow > ExpiresAt;
     }
 }

src/Abstractions/NexusMods.Abstractions.Resources.DB/PersistedResourceLoader.cs → src/Abstractions/NexusMods.Abstractions.Resources.DB/PersistedDbResourceLoader.cs

@@ -6,43 +6,45 @@
 
 namespace NexusMods.Abstractions.Resources.DB;
 
+/// <summary>
+/// Loads persisted resources from the database.
+/// </summary>
 [PublicAPI]
-public sealed class PersistedResourceLoader<TResourceIdentifier, TData> : IResourceLoader<TResourceIdentifier, TData>
+public sealed class PersistedDbResourceLoader<TResourceIdentifier> : IResourceLoader<TResourceIdentifier, byte[]>
     where TResourceIdentifier : notnull
-    where TData : notnull
 {
-    public delegate byte[] DataToBytes(TData data);
-    public delegate TData BytesToData(byte[] bytes);
+    /// <summary>
+    /// Converts the identifier to a hash.
+    /// </summary>
     public delegate Hash IdentifierToHash(TResourceIdentifier resourceIdentifier);
 
+    /// <summary>
+    /// Converts the identifier into an entity id.
+    /// </summary>
     public delegate EntityId IdentifierToEntityId(TResourceIdentifier resourceIdentifier);
 
     private readonly IConnection _connection;
-    private readonly IResourceLoader<TResourceIdentifier, TData> _innerLoader;
-    private readonly ReferenceAttribute<PersistedResource> _referenceAttribute;
-    private readonly DataToBytes _dataToBytes;
-    private readonly BytesToData _bytesToData;
+    private readonly IResourceLoader<TResourceIdentifier, byte[]> _innerLoader;
+    private readonly ReferenceAttribute<PersistedDbResource> _referenceAttribute;
     private readonly IdentifierToHash _identifierToHash;
     private readonly IdentifierToEntityId _identifierToEntityId;
     private readonly AttributeId _referenceAttributeId;
     private readonly Optional<PartitionId> _partitionId;
 
-    public PersistedResourceLoader(
+    /// <summary>
+    /// Constructor.
+    /// </summary>
+    public PersistedDbResourceLoader(
         IConnection connection,
-        ReferenceAttribute<PersistedResource> referenceAttribute,
+        ReferenceAttribute<PersistedDbResource> referenceAttribute,
         IdentifierToHash identifierToHash,
-        DataToBytes dataToBytes,
-        BytesToData bytesToData,
         IdentifierToEntityId identifierToEntityId,
         Optional<PartitionId> partitionId,
-        IResourceLoader<TResourceIdentifier, TData> innerLoader)
+        IResourceLoader<TResourceIdentifier, byte[]> innerLoader)
     {
         _connection = connection;
         _innerLoader = innerLoader;
 
-        _dataToBytes = dataToBytes;
-        _bytesToData = bytesToData;
-
         _identifierToHash = identifierToHash;
         _identifierToEntityId = identifierToEntityId;
 
@@ -52,7 +54,7 @@ public PersistedResourceLoader(
     }
 
     /// <inheritdoc/>
-    public ValueTask<Resource<TData>> LoadResourceAsync(TResourceIdentifier resourceIdentifier, CancellationToken cancellationToken)
+    public ValueTask<Resource<byte[]>> LoadResourceAsync(TResourceIdentifier resourceIdentifier, CancellationToken cancellationToken)
     {
         var entityId = _identifierToEntityId(resourceIdentifier);
         var tuple = (entityId, resourceIdentifier);
@@ -62,7 +64,7 @@ public ValueTask<Resource<TData>> LoadResourceAsync(TResourceIdentifier resource
         return SaveResource(tuple, cancellationToken);
     }
 
-    private Resource<TData>? LoadResource(ValueTuple<EntityId, TResourceIdentifier> resourceIdentifier)
+    private Resource<byte[]>? LoadResource(ValueTuple<EntityId, TResourceIdentifier> resourceIdentifier)
     {
         var db = _connection.Db;
         var (entityId, innerResourceIdentifier) = resourceIdentifier;
@@ -76,7 +78,7 @@ public ValueTask<Resource<TData>> LoadResourceAsync(TResourceIdentifier resource
         }
 
         if (!persistedResourceId.HasValue) return null;
-        var persistedResource = PersistedResource.Load(db, persistedResourceId.Value);
+        var persistedResource = PersistedDbResource.Load(db, persistedResourceId.Value);
 
         if (!persistedResource.IsValid()) return null;
         if (persistedResource.IsExpired) return null;
@@ -85,26 +87,24 @@ public ValueTask<Resource<TData>> LoadResourceAsync(TResourceIdentifier resource
         if (!persistedResource.ResourceIdentifierHash.Equals(hash)) return null;
 
         var bytes = persistedResource.Data;
-        var data = _bytesToData(bytes);
 
-        return new Resource<TData>
+        return new Resource<byte[]>
         {
-            Data = data,
+            Data = bytes,
             ExpiresAt = persistedResource.ExpiresAt,
         };
     }
 
-    private async ValueTask<Resource<TData>> SaveResource(ValueTuple<EntityId, TResourceIdentifier> resourceIdentifier, CancellationToken cancellationToken)
+    private async ValueTask<Resource<byte[]>> SaveResource(ValueTuple<EntityId, TResourceIdentifier> resourceIdentifier, CancellationToken cancellationToken)
     {
         var resource = await _innerLoader.LoadResourceAsync(resourceIdentifier.Item2, cancellationToken);
-        var bytes = _dataToBytes(resource.Data);
 
         using var tx = _connection.BeginTransaction();
         var tmpId = _partitionId.HasValue ? tx.TempId(_partitionId.Value) : tx.TempId();
 
-        var persisted = new PersistedResource.New(tx, tmpId)
+        var persisted = new PersistedDbResource.New(tx, tmpId)
         {
-            Data = bytes,
+            Data = resource.Data,
             ExpiresAt = resource.ExpiresAt,
             ResourceIdentifierHash = _identifierToHash(resourceIdentifier.Item2),
         };
@@ -122,47 +122,49 @@ private async ValueTask<Resource<TData>> SaveResource(ValueTuple<EntityId, TReso
 [PublicAPI]
 public static partial class ExtensionsMethods
 {
-    public static IResourceLoader<TResourceIdentifier, byte[]> Persist<TResourceIdentifier>(
+    /// <summary>
+    /// Persist the resource in the database.
+    /// </summary>
+    public static IResourceLoader<TResourceIdentifier, byte[]> PersistInDb<TResourceIdentifier>(
         this IResourceLoader<TResourceIdentifier, byte[]> inner,
         IConnection connection,
-        ReferenceAttribute<PersistedResource> referenceAttribute,
-        PersistedResourceLoader<TResourceIdentifier, byte[]>.IdentifierToHash identifierToHash,
-        PersistedResourceLoader<TResourceIdentifier, byte[]>.IdentifierToEntityId identifierToEntityId,
+        ReferenceAttribute<PersistedDbResource> referenceAttribute,
+        PersistedDbResourceLoader<TResourceIdentifier>.IdentifierToHash identifierToHash,
+        PersistedDbResourceLoader<TResourceIdentifier>.IdentifierToEntityId identifierToEntityId,
         Optional<PartitionId> partitionId)
         where TResourceIdentifier : notnull
     {
-        return inner.Persist(
-            connection: connection,
-            referenceAttribute: referenceAttribute,
-            identifierToHash: identifierToHash,
-            identifierToEntityId: identifierToEntityId,
-            dataToBytes: static bytes => bytes,
-            bytesToData: static bytes => bytes,
-            partitionId: partitionId
+        return inner.Then(
+            state: (connection, referenceAttribute, identifierToHash, identifierToEntityId, partitionId),
+            factory: static (input, inner) => new PersistedDbResourceLoader<TResourceIdentifier>(
+                connection: input.connection,
+                referenceAttribute: input.referenceAttribute,
+                identifierToHash: input.identifierToHash,
+                identifierToEntityId: input.identifierToEntityId,
+                partitionId: input.partitionId,
+                innerLoader: inner
+            )
         );
     }
 
-    public static IResourceLoader<TResourceIdentifier, TData> Persist<TResourceIdentifier, TData>(
-        this IResourceLoader<TResourceIdentifier, TData> inner,
+    /// <summary>
+    /// Persist the resource in the database.
+    /// </summary>
+    public static IResourceLoader<ValueTuple<EntityId, TResourceIdentifier>, byte[]> PersistInDb<TResourceIdentifier>(
+        this IResourceLoader<ValueTuple<EntityId, TResourceIdentifier>, byte[]> inner,
         IConnection connection,
-        ReferenceAttribute<PersistedResource> referenceAttribute,
-        PersistedResourceLoader<TResourceIdentifier, TData>.IdentifierToHash identifierToHash,
-        PersistedResourceLoader<TResourceIdentifier, TData>.IdentifierToEntityId identifierToEntityId,
-        PersistedResourceLoader<TResourceIdentifier, TData>.DataToBytes dataToBytes,
-        PersistedResourceLoader<TResourceIdentifier, TData>.BytesToData bytesToData,
+        ReferenceAttribute<PersistedDbResource> referenceAttribute,
+        Func<TResourceIdentifier, Hash> identifierToHash,
         Optional<PartitionId> partitionId)
         where TResourceIdentifier : notnull
-        where TData : notnull
     {
         return inner.Then(
-            state: (connection, referenceAttribute, identifierToHash, identifierToEntityId, dataToBytes, bytesToData, partitionId),
-            factory: static (input, inner) => new PersistedResourceLoader<TResourceIdentifier, TData>(
+            state: (connection, referenceAttribute, identifierToHash, partitionId),
+            factory: static (input, inner) => new PersistedDbResourceLoader<ValueTuple<EntityId, TResourceIdentifier>>(
                 connection: input.connection,
                 referenceAttribute: input.referenceAttribute,
-                identifierToHash: input.identifierToHash,
-                identifierToEntityId: input.identifierToEntityId,
-                dataToBytes: input.dataToBytes,
-                bytesToData: input.bytesToData,
+                identifierToHash: tuple => input.identifierToHash(tuple.Item2),
+                identifierToEntityId: static tuple => tuple.Item1,
                 partitionId: input.partitionId,
                 innerLoader: inner
             )

src/Abstractions/NexusMods.Abstractions.Resources.IO/PersistedFileResourceLoader.cs

@@ -0,0 +1,92 @@
+using JetBrains.Annotations;
+using NexusMods.Hashing.xxHash64;
+using NexusMods.Paths;
+
+namespace NexusMods.Abstractions.Resources.IO;
+
+/// <summary>
+/// Loads a persisted file from disk.
+/// </summary>
+[PublicAPI]
+public sealed class PersistedFileResourceLoader<TResourceIdentifier> : IResourceLoader<TResourceIdentifier, byte[]>
+    where TResourceIdentifier : notnull
+{
+    /// <summary>
+    /// Converts the resource identifier to a hash.
+    /// </summary>
+    public delegate Hash IdentifierToHash(TResourceIdentifier resourceIdentifier);
+
+    private readonly AbsolutePath _directory;
+    private readonly Extension _extension;
+    private readonly IdentifierToHash _identifierToHash;
+    private readonly IResourceLoader<TResourceIdentifier, byte[]> _innerLoader;
+
+    /// <summary>
+    /// Constructor.
+    /// </summary>
+    public PersistedFileResourceLoader(
+        AbsolutePath directory,
+        Extension extension,
+        IdentifierToHash identifierToHash,
+        IResourceLoader<TResourceIdentifier, byte[]> innerLoader)
+    {
+        _directory = directory;
+        _extension = extension;
+        _identifierToHash = identifierToHash;
+        _innerLoader = innerLoader;
+    }
+
+    /// <inheritdoc/>
+    public async ValueTask<Resource<byte[]>> LoadResourceAsync(TResourceIdentifier resourceIdentifier, CancellationToken cancellationToken)
+    {
+        var hash = _identifierToHash(resourceIdentifier);
+        var path = _directory.Combine($"{hash.ToHex()}{_extension.ToString()}");
+
+        var hasFile = path.FileExists;
+        await using var stream = path.Open(FileMode.Create, FileAccess.ReadWrite, FileShare.None);
+
+        if (hasFile)
+        {
+            var bytes = GC.AllocateUninitializedArray<byte>(length: (int)stream.Length);
+            await stream.ReadExactlyAsync(bytes, cancellationToken);
+
+            return new Resource<byte[]>
+            {
+                Data = bytes,
+            };
+        }
+
+        var resource = await _innerLoader.LoadResourceAsync(resourceIdentifier, cancellationToken);
+        await stream.WriteAsync(resource.Data, cancellationToken);
+
+        return resource;
+    }
+}
+
+/// <summary>
+/// Extension methods.
+/// </summary>
+[PublicAPI]
+public static class ExtensionMethods
+{
+    /// <summary>
+    /// Persist the resource on disk.
+    /// </summary>
+    public static IResourceLoader<TResourceIdentifier, byte[]> PersistOnDisk<TResourceIdentifier>(
+        this IResourceLoader<TResourceIdentifier, byte[]> inner,
+        AbsolutePath directory,
+        Extension extension,
+        PersistedFileResourceLoader<TResourceIdentifier>.IdentifierToHash identifierToHash)
+        where TResourceIdentifier : notnull
+    {
+        return inner.Then(
+            state: (directory, extension, identifierToHash),
+            factory: static (input, inner) => new PersistedFileResourceLoader<TResourceIdentifier>(
+                directory: input.directory,
+                extension: input.extension,
+                identifierToHash: input.identifierToHash,
+                innerLoader: inner
+            )
+        );
+    }
+}