[4/4] Query for Updates from Nexus, and Fix Miscellaneous Bugs in Dat…

…aStore (#2113)

* WIP: Documentation for Update Detection

* Added: Additional Edge Cases to Update Logic Docs

* Added: Extra case of `Archived in the Middle`

* Fixed: Indentation for `file_updates` field.

* Finalized 'Updating Mods' doc with simplified Implementation requested.

* Fixed: Minor Notes from older Research Doc

* Fixed: Added Missing 'Updating Mods' mkdocs sidebar item.

* [Working WIP] Added: Initial Implementation of Generic Page Caching System used by Mod Pages

* Tech Debt Reduction: Add additional V2 GraphQL Types and Correct Sizes of Existing Types

* Added: Missing 'UInt32' types in attribute definitions

* Improved: Accuracy of documentation for FileId struct.

* Added: Method for constructing UidForMod and UidForFile from GraphQL API Results

* Rename: ICanGetUid to ICanGetUidForMod

* Added: Tests for UidForModTests and UidForFileTests

* Removed: Unused Tests.cs file

* Added: Mixin for V1 API Results to ModUpdates Library

* Added: Mod Page Metadata now is ready for handling update info with V2 GameId and Friends

* Added: Mixin for page metadata.

* NexusModsModPageMetadata: Correctly Use uid as 'primary key'

* Update: Use GameId from uid field of NexusModsModPageMetadata

* Added: Fetch Mod Page Metadata from the DB

* Added: Note about field in PageMetadataMixin

* Use Uid in NexusModsFileMetadata, and Add Relevant Constructors for UidForFile and UidForMod

* Added: Code for running actual update check, and relevant constructs.

* V1: Fix field names on ModUpdate structure to align with Nexus V1 API

* Added: Remaining Fixups to make the Update Check 'work'

* Added: A note regarding adding more tests.

* Added: Small note to ModUpdateMixin about choice of field.

* Removed: FilesUpdatedAt field, as it is now currently unused.

* Improve: Clarify last updated date is in UTC

* Updated Note: It's no longer messy, but we still need to upgrade to V2.

* Improved: Now also updates mod pages.

* Merge interfaces into IModFeedItem

* Fix some merge conflicts

---------

Co-authored-by: halgari <[email protected]>

src/Abstractions/NexusMods.Abstractions.Collections/Json/Mod.cs

@@ -1,5 +1,6 @@
 using System.Text.Json.Serialization;
 using NexusMods.Abstractions.Games.DTO;
+using NexusMods.Abstractions.NexusWebApi.Types.V2;
 
 namespace NexusMods.Abstractions.Collections.Json;
 
@@ -17,6 +18,9 @@ public class Mod
     [JsonPropertyName("optional")]
     public bool Optional { get; init; }
 
+    /// <summary>
+    ///     TODO: Deprecate this with <see cref="GameId"/>
+    /// </summary>
     [JsonPropertyName("domainName")]
     public required GameDomain DomainName { get; init; }
 

src/Abstractions/NexusMods.Abstractions.NexusModsLibrary/NexusModsFileMetadata.cs

@@ -2,6 +2,7 @@
 using NexusMods.Abstractions.MnemonicDB.Attributes;
 using NexusMods.Abstractions.NexusWebApi.Types;
 using NexusMods.Abstractions.NexusWebApi.Types.V2;
+using NexusMods.Abstractions.NexusWebApi.Types.V2.Uid;
 using NexusMods.MnemonicDB.Abstractions.Attributes;
 using NexusMods.MnemonicDB.Abstractions.Models;
 
@@ -16,9 +17,9 @@ public partial class NexusModsFileMetadata : IModelDefinition
     private const string Namespace = "NexusMods.Library.NexusModsFileMetadata";
 
     /// <summary>
-    /// The ID of the file.
+    /// Unique identifier for the file on Nexus Mods.
     /// </summary>
-    public static readonly FileIdAttribute FileId = new(Namespace, nameof(FileId)) { IsIndexed = true };
+    public static readonly UidForFileAttribute Uid = new(Namespace, nameof(Uid)) { IsIndexed = true };
 
     /// <summary>
     /// The name of the file.
@@ -31,7 +32,12 @@ public partial class NexusModsFileMetadata : IModelDefinition
     public static readonly StringAttribute Version = new(Namespace, nameof(Version));
 
     /// <summary>
-    /// The size of the file in bytes, this is optional in the NexusMods API for whatever reason.
+    /// The date the file was uploaded at.
+    /// </summary>
+    public static readonly DateTimeAttribute UploadedAt = new(Namespace, nameof(UploadedAt));
+
+    /// <summary>
+    /// The size in bytes of the file.
     /// </summary>
     public static readonly SizeAttribute Size = new(Namespace, nameof(Size)) { IsOptional = true };
 

src/Abstractions/NexusMods.Abstractions.NexusModsLibrary/NexusModsModPageMetadata.cs

@@ -1,8 +1,8 @@
 using JetBrains.Annotations;
 using NexusMods.Abstractions.MnemonicDB.Attributes;
-using NexusMods.Abstractions.NexusWebApi.Types;
 using NexusMods.Abstractions.Resources.DB;
 using NexusMods.Abstractions.NexusWebApi.Types.V2;
+using NexusMods.Abstractions.NexusWebApi.Types.V2.Uid;
 using NexusMods.Abstractions.Telemetry;
 using NexusMods.MnemonicDB.Abstractions.Attributes;
 using NexusMods.MnemonicDB.Abstractions.Models;
@@ -20,7 +20,7 @@ public partial class NexusModsModPageMetadata : IModelDefinition
     /// <summary>
     /// The ID of the mod page.
     /// </summary>
-    public static readonly ModIdAttribute ModId = new(Namespace, nameof(ModId)) { IsIndexed = true };
+    public static readonly UidForModAttribute Uid = new(Namespace, nameof(Uid)) { IsIndexed = true };
 
     /// <summary>
     /// The name of the mod page.
@@ -30,8 +30,17 @@ public partial class NexusModsModPageMetadata : IModelDefinition
     /// <summary>
     /// The game of the mod page.
     /// </summary>
+    /// <remarks>
+    ///     This will be deprecated in the future, since V2 API only needs <see cref="Uid"/>
+    ///     which contains the <see cref="GameId"/> The <see cref="GameDomain"/> is a legacy field of the V1 API.
+    /// </remarks>
     public static readonly GameDomainAttribute GameDomain = new(Namespace, nameof(GameDomain)) { IsIndexed = true };
 
+    /// <summary>
+    /// The last time the mod page was updated (UTC). This is useful for cache invalidation.
+    /// </summary>
+    public static readonly DateTimeAttribute UpdatedAt = new(Namespace, nameof(UpdatedAt));
+
     /// <summary>
     /// Uri for the full sized picture of the mod.
     /// </summary>
@@ -51,6 +60,6 @@ public partial class NexusModsModPageMetadata : IModelDefinition
 
     public partial struct ReadOnly
     {
-        public Uri GetUri() => NexusModsUrlBuilder.CreateGenericUri($"https://nexusmods.com/{GameDomain}/mods/{ModId}");
+        public Uri GetUri() => NexusModsUrlBuilder.CreateGenericUri($"https://nexusmods.com/{GameDomain}/mods/{Uid.ModId}");
     }
 }

src/Abstractions/NexusMods.Abstractions.NexusWebApi/DTOs/ModUpdate.cs

@@ -33,7 +33,7 @@ public class ModUpdate : IJsonArraySerializable<ModUpdate>
     /// <remarks>
     ///    Expressed as a Unix timestamp.
     /// </remarks>
-    [JsonPropertyName("LatestFileUpdated")]
+    [JsonPropertyName("latest_file_update")]
     public long LatestFileUpdated { get; set; }
 
     /// <summary>
@@ -47,7 +47,7 @@ public class ModUpdate : IJsonArraySerializable<ModUpdate>
     /// <remarks>
     ///    Expressed as a Unix timestamp.
     /// </remarks>
-    [JsonPropertyName("LatestModActivity")]
+    [JsonPropertyName("latest_mod_activity")]
     public long LatestModActivity { get; set; }
 
     /// <summary>

src/Abstractions/NexusMods.Abstractions.NexusWebApi/Types/V2/GameId.cs

@@ -1,3 +1,7 @@
+using NexusMods.Abstractions.Games.DTO;
+using NexusMods.MnemonicDB.Abstractions;
+using NexusMods.MnemonicDB.Abstractions.Attributes;
+using NexusMods.MnemonicDB.Abstractions.ElementComparers;
 using TransparentValueObjects;
 namespace NexusMods.Abstractions.NexusWebApi.Types.V2;
 
@@ -9,4 +13,50 @@ namespace NexusMods.Abstractions.NexusWebApi.Types.V2;
 {
     /// <inheritdoc/>
     public static GameId DefaultValue => From(default(uint));
+
+    /// <summary>
+    /// Maps a given <see cref="GameDomain"/> to a <see cref="GameId"/> using known mappings.
+    /// This is a TEMPORARY API, until full migration to V2 is complete.
+    /// After that it should be REMOVED.
+    /// </summary>
+    public static GameId FromGameDomain(GameDomain domain)
+    {
+        return domain.Value switch
+        {
+            "stardewvalley" => (GameId)1704,
+            "cyberpunk2077" => (GameId)3333,
+            "baldursgate3" => (GameId)3474,
+            _ => throw new ArgumentOutOfRangeException(nameof(domain), domain, null),
+        };
+    }
+
+    /// <summary>
+    /// Maps a given <see cref="GameId"/> to a <see cref="GameDomain"/> using known mappings.
+    /// This is a TEMPORARY API, until full migration to V2 is complete.
+    /// After that it should be REMOVED.
+    /// </summary>
+    public GameDomain ToGameDomain()
+    {
+        var value = Value;
+        return value switch
+        {
+            1704 => GameDomain.From("stardewvalley"),
+            3333 => GameDomain.From("cyberpunk2077"),
+            3474 => GameDomain.From("baldursgate3"),
+            _ => throw new ArgumentOutOfRangeException(nameof(value), value, null),
+        };
+    }
+}
+
+/// <summary>
+/// Game ID attribute, for game identifiers from the GraphQL (V2) API.
+/// </summary>
+public class GameIdAttribute(string ns, string name) 
+    : ScalarAttribute<GameId, uint>(ValueTags.UInt32, ns, name)
+{
+    /// <inheritdoc />
+    protected override uint ToLowLevel(GameId value) => value.Value;
+
+    /// <inheritdoc />
+    protected override GameId FromLowLevel(uint value, ValueTags tags, AttributeResolver resolver) => GameId.From(value);
 }

src/Abstractions/NexusMods.Abstractions.NexusWebApi/Types/V2/Uid/UidForFile.cs

@@ -1,5 +1,8 @@
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
+using NexusMods.MnemonicDB.Abstractions;
+using NexusMods.MnemonicDB.Abstractions.Attributes;
+using NexusMods.MnemonicDB.Abstractions.ElementComparers;
 namespace NexusMods.Abstractions.NexusWebApi.Types.V2.Uid;
 
 /// <summary>
@@ -25,6 +28,13 @@ public struct UidForFile
     /// </summary>
     public GameId GameId;
 
+    /// <summary/>
+    public UidForFile(FileId fileId, GameId gameId)
+    {
+        FileId = fileId;
+        GameId = gameId;
+    }
+
     /// <summary>
     /// Decodes a Nexus Mods API result which contains an 'uid' field into a <see cref="UidForFile"/>.
     /// </summary>
@@ -44,3 +54,17 @@ public struct UidForFile
     /// </summary>
     public static UidForFile FromUlong(ulong value) => Unsafe.As<ulong, UidForFile>(ref value);
 }
+
+/// <summary>
+/// Attribute that uniquely identifies a file on Nexus Mods.
+/// See <see cref="UidForFile"/> for more details.
+/// </summary>
+public class UidForFileAttribute(string ns, string name) 
+    : ScalarAttribute<UidForFile, ulong>(ValueTags.UInt64, ns, name)
+{
+    /// <inheritdoc />
+    protected override ulong ToLowLevel(UidForFile value) => value.AsUlong;
+
+    /// <inheritdoc />
+    protected override UidForFile FromLowLevel(ulong value, ValueTags tags, AttributeResolver resolver) => UidForFile.FromUlong(value);
+} 

src/Abstractions/NexusMods.Abstractions.NexusWebApi/Types/V2/Uid/UidForMod.cs

@@ -1,5 +1,8 @@
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
+using NexusMods.MnemonicDB.Abstractions;
+using NexusMods.MnemonicDB.Abstractions.Attributes;
+using NexusMods.MnemonicDB.Abstractions.ElementComparers;
 namespace NexusMods.Abstractions.NexusWebApi.Types.V2.Uid;
 
 /// <summary>
@@ -36,6 +39,11 @@ public struct UidForMod
     /// This throws if <param name="uid"/> is not a valid number.
     /// </remarks>
     public static UidForMod FromV2Api(string uid) => FromUlong(ulong.Parse(uid));
+
+    /// <summary>
+    /// Converts the UID to a string accepted by the V2 API.
+    /// </summary>
+    public string ToV2Api() => AsUlong.ToString();
 
     /// <summary>
     /// Reinterprets the current <see cref="UidForMod"/> as a single <see cref="ulong"/>.
@@ -47,3 +55,17 @@ public struct UidForMod
     /// </summary>
     public static UidForMod FromUlong(ulong value) => Unsafe.As<ulong, UidForMod>(ref value);
 }
+
+/// <summary>
+/// Attribute that uniquely identifies a mod on Nexus Mods.
+/// See <see cref="UidForMod"/> for more details.
+/// </summary>
+public class UidForModAttribute(string ns, string name) 
+    : ScalarAttribute<UidForMod, ulong>(ValueTags.UInt64, ns, name)
+{
+    /// <inheritdoc />
+    protected override ulong ToLowLevel(UidForMod value) => value.AsUlong;
+
+    /// <inheritdoc />
+    protected override UidForMod FromLowLevel(ulong value, ValueTags tags, AttributeResolver resolver) => UidForMod.FromUlong(value);
+} 

src/Networking/NexusMods.Networking.ModUpdates/IModFeedItem.cs

@@ -0,0 +1,22 @@
+using NexusMods.Abstractions.NexusWebApi.Types.V2.Uid;
+namespace NexusMods.Networking.ModUpdates;
+
+/// <summary>
+/// Represents an individual item from a 'mod feed'; with the 'mod feed' being
+/// the result of an API call that returns one or more mods from the Nexus API.
+/// (Either V1 or V2 API)
+/// </summary>
+public interface IModFeedItem
+{
+    /// <summary>
+    /// Returns a unique identifier for the given item, based on the ID format
+    /// used in the NexusMods V2 API.
+    /// </summary>
+    public UidForMod GetModPageId();
+
+    /// <summary>
+    /// Retrieves the time the item was last updated.
+    /// This date is in UTC.
+    /// </summary>
+    public DateTime GetLastUpdatedDateUtc();
+}

src/Networking/NexusMods.Networking.ModUpdates/Mixins/ModFeedItemUpdateMixin.cs

@@ -0,0 +1,40 @@
+using NexusMods.Abstractions.NexusWebApi.DTOs;
+using NexusMods.Abstractions.NexusWebApi.Types.V2;
+using NexusMods.Abstractions.NexusWebApi.Types.V2.Uid;
+namespace NexusMods.Networking.ModUpdates.Mixins;
+
+/// <summary>
+/// Implements the (V1) mod update API mixin.
+/// </summary>
+public readonly struct ModFeedItemUpdateMixin : IModFeedItem
+{
+    private readonly DateTime _lastUpdatedDate;
+    private readonly GameId _gameId;
+    private readonly ModId _modId;
+
+    /// <summary/>
+    private ModFeedItemUpdateMixin(ModUpdate update, GameId gameId)
+    {
+        // Note(sewer): V2 doesn't have 'last file updated' field, so we have to use 'last mod page update' time.
+        // Well, this whole struct is, will be making that ticket to backend, and replace
+        // this when V2 gets relevant API.
+        _lastUpdatedDate = update.LatestModActivityUtc;
+        _gameId = gameId;
+        _modId = update.ModId;
+    }
+
+    /// <summary>
+    /// Transforms the result of a V1 API call for mod updates into the Mixin.
+    /// </summary>
+    public static IEnumerable<ModFeedItemUpdateMixin> FromUpdateResults(IEnumerable<ModUpdate> updates, GameId gameId) => updates.Select(update => new ModFeedItemUpdateMixin(update, gameId));
+
+    /// <inheritdoc />
+    public DateTime GetLastUpdatedDateUtc() => _lastUpdatedDate;
+
+    /// <inheritdoc />
+    public UidForMod GetModPageId() => new()
+    {
+        GameId = _gameId,
+        ModId = _modId, 
+    };
+}

src/Networking/NexusMods.Networking.ModUpdates/Mixins/PageMetadataMixin.cs

@@ -0,0 +1,38 @@
+using NexusMods.Abstractions.NexusModsLibrary;
+using NexusMods.Abstractions.NexusWebApi.Types.V2.Uid;
+using NexusMods.MnemonicDB.Abstractions;
+namespace NexusMods.Networking.ModUpdates.Mixins;
+
+/// <summary>
+/// Implements the MnemonicDB mod page mixin based on V2 API Results.
+/// </summary>
+public struct PageMetadataMixin : IModFeedItem
+{
+    private readonly NexusModsModPageMetadata.ReadOnly _metadata;
+
+    private PageMetadataMixin(NexusModsModPageMetadata.ReadOnly metadata) => _metadata = metadata;
+
+    /// <inheritodc/>
+    public UidForMod GetModPageId() => new()
+    {
+        GameId = _metadata.Uid.GameId,
+        ModId = _metadata.Uid.ModId, 
+    };
+
+    /// <summary/>
+    public EntityId GetModPageEntityId() => _metadata.Id;
+
+    /// <inheritodc/>
+    public DateTime GetLastUpdatedDateUtc() => _metadata.UpdatedAt; // <= TODO: Change this with 'last file updated at' when V2 supports this field.
+
+    /// <summary>
+    /// Returns the database entries containing page metadata(s) as a mixin.
+    /// </summary>
+    public static IEnumerable<PageMetadataMixin> EnumerateDatabaseEntries(IDb db) => NexusModsModPageMetadata.All(db).Select(only => new PageMetadataMixin(only));
+
+    /// <summary/>
+    public static implicit operator NexusModsModPageMetadata.ReadOnly(PageMetadataMixin mixin) => mixin._metadata;
+
+    /// <summary/>
+    public static implicit operator PageMetadataMixin(NexusModsModPageMetadata.ReadOnly metadata) => new(metadata);
+}

src/Networking/NexusMods.Networking.ModUpdates/MultiFeedCacheUpdater.cs

@@ -1,5 +1,4 @@
 using NexusMods.Abstractions.NexusWebApi.Types.V2;
-using NexusMods.Networking.ModUpdates.Traits;
 namespace NexusMods.Networking.ModUpdates;
 
 /// <summary>
@@ -12,7 +11,7 @@ namespace NexusMods.Networking.ModUpdates;
 /// you to use mods and API responses which are sourced from multiple feeds (games),
 /// as opposed to a single feed.
 /// </summary>
-public class MultiFeedCacheUpdater<TUpdateableItem> where TUpdateableItem : ICanGetLastUpdatedTimestamp, ICanGetUidForMod
+public class MultiFeedCacheUpdater<TUpdateableItem> where TUpdateableItem : IModFeedItem
 {
     private readonly Dictionary<GameId, PerFeedCacheUpdater<TUpdateableItem>> _updaters;
 
@@ -41,7 +40,7 @@ public MultiFeedCacheUpdater(TUpdateableItem[] items, TimeSpan expiry)
         var groupedList = new List<(GameId, List<TUpdateableItem>)>();
         foreach (var item in items)
         {
-            var gameId = item.GetUniqueId().GameId;
+            var gameId = item.GetModPageId().GameId;
 
             // Get or Update List for this GameId.
             var found = false;
@@ -73,15 +72,15 @@ public MultiFeedCacheUpdater(TUpdateableItem[] items, TimeSpan expiry)
     /// include items corresponding to multiple feeds (games); the feed source
     /// is automatically detected.
     ///
-    /// Wrap elements in a struct that implements <see cref="ICanGetLastUpdatedTimestamp"/>
-    /// and <see cref="ICanGetUidForMod"/> if necessary. 
+    /// Wrap elements in a struct that implements <see cref="IModFeedItem"/>
+    /// and <see cref="IModFeedItem"/> if necessary. 
     /// </param>
-    public void Update<T>(IEnumerable<T> items) where T : ICanGetLastUpdatedTimestamp, ICanGetUidForMod
+    public void Update<T>(IEnumerable<T> items) where T : IModFeedItem
     {
         foreach (var item in items)
         {
             // Determine feed
-            var feed = item.GetUniqueId().GameId;
+            var feed = item.GetModPageId().GameId;
 
             // The result may contain items from feeds which we are not tracking.
             // For instance, results for other games. This is not an error, we