From d0a202b66253e5f438d8f6239c69df17e40642b1 Mon Sep 17 00:00:00 2001 From: "Sewer." Date: Thu, 3 Oct 2024 16:50:29 +0100 Subject: [PATCH] [2/4] Added: Caching system for updates. (#2092) * WIP: Documentation for Update Detection * Added: Additional Edge Cases to Update Logic Docs * Added: Extra case of `Archived in the Middle` * Fixed: Indentation for `file_updates` field. * Finalized 'Updating Mods' doc with simplified Implementation requested. * Fixed: Minor Notes from older Research Doc * Fixed: Added Missing 'Updating Mods' mkdocs sidebar item. * [Working WIP] Added: Initial Implementation of Generic Page Caching System used by Mod Pages * Removed: Unused Tests.cs file --- NexusMods.App.sln | 14 ++ .../MultiFeedCacheUpdater.cs | 110 +++++++++ .../NexusMods.Networking.ModUpdates.csproj | 16 ++ .../PerFeedCacheUpdater.cs | 155 +++++++++++++ .../PerFeedCacheUpdaterResult.cs | 44 ++++ .../Private/CacheUpdaterAction.cs | 24 ++ .../Structures/GameId.cs | 8 + .../Structures/ModId.cs | 10 + .../Structures/Uid.cs | 33 +++ .../Traits/ICanGetLastUpdatedTimestamp.cs | 12 + .../Traits/ICanGetUid.cs | 17 ++ .../GlobalUsings.cs | 1 + .../Helpers/TestItem.cs | 25 ++ .../MultiFeedCacheUpdaterTests.cs | 214 ++++++++++++++++++ ...xusMods.Networking.ModUpdates.Tests.csproj | 9 + .../PerFeedCacheUpdaterTests.cs | 180 +++++++++++++++ .../Startup.cs | 11 + 17 files changed, 883 insertions(+) create mode 100644 src/Networking/NexusMods.Networking.ModUpdates/MultiFeedCacheUpdater.cs create mode 100644 src/Networking/NexusMods.Networking.ModUpdates/NexusMods.Networking.ModUpdates.csproj create mode 100644 src/Networking/NexusMods.Networking.ModUpdates/PerFeedCacheUpdater.cs create mode 100644 src/Networking/NexusMods.Networking.ModUpdates/PerFeedCacheUpdaterResult.cs create mode 100644 src/Networking/NexusMods.Networking.ModUpdates/Private/CacheUpdaterAction.cs create mode 100644 src/Networking/NexusMods.Networking.ModUpdates/Structures/GameId.cs create mode 100644 src/Networking/NexusMods.Networking.ModUpdates/Structures/ModId.cs create mode 100644 src/Networking/NexusMods.Networking.ModUpdates/Structures/Uid.cs create mode 100644 src/Networking/NexusMods.Networking.ModUpdates/Traits/ICanGetLastUpdatedTimestamp.cs create mode 100644 src/Networking/NexusMods.Networking.ModUpdates/Traits/ICanGetUid.cs create mode 100644 tests/Networking/NexusMods.Networking.ModUpdates.Tests/GlobalUsings.cs create mode 100644 tests/Networking/NexusMods.Networking.ModUpdates.Tests/Helpers/TestItem.cs create mode 100644 tests/Networking/NexusMods.Networking.ModUpdates.Tests/MultiFeedCacheUpdaterTests.cs create mode 100644 tests/Networking/NexusMods.Networking.ModUpdates.Tests/NexusMods.Networking.ModUpdates.Tests.csproj create mode 100644 tests/Networking/NexusMods.Networking.ModUpdates.Tests/PerFeedCacheUpdaterTests.cs create mode 100644 tests/Networking/NexusMods.Networking.ModUpdates.Tests/Startup.cs diff --git a/NexusMods.App.sln b/NexusMods.App.sln index da34ba8182..8104537d53 100644 --- a/NexusMods.App.sln +++ b/NexusMods.App.sln @@ -256,6 +256,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.Collections.Tests EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.Games.Larian", "src\Games\NexusMods.Games.Larian\NexusMods.Games.Larian.csproj", "{2A35EBB5-1CA6-4F5D-8CE8-352146C82C28}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.Networking.ModUpdates", "src\Networking\NexusMods.Networking.ModUpdates\NexusMods.Networking.ModUpdates.csproj", "{8B246C04-F372-47F6-9397-F658915429A8}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.Networking.ModUpdates.Tests", "tests\Networking\NexusMods.Networking.ModUpdates.Tests\NexusMods.Networking.ModUpdates.Tests.csproj", "{CDA2C52B-A9A7-446B-9D2F-D7B75C1905EF}" +EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.Games.Larian.Tests", "tests\Games\NexusMods.Games.Larian.Tests\NexusMods.Games.Larian.Tests.csproj", "{425F7A13-99A2-4231-B0C1-C56EB819C174}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.Media", "src\NexusMods.Media\NexusMods.Media.csproj", "{CEC177AB-4FF0-4F8A-81B8-1E756D892416}" @@ -676,6 +680,14 @@ Global {2A35EBB5-1CA6-4F5D-8CE8-352146C82C28}.Debug|Any CPU.Build.0 = Debug|Any CPU {2A35EBB5-1CA6-4F5D-8CE8-352146C82C28}.Release|Any CPU.ActiveCfg = Release|Any CPU {2A35EBB5-1CA6-4F5D-8CE8-352146C82C28}.Release|Any CPU.Build.0 = Release|Any CPU + {8B246C04-F372-47F6-9397-F658915429A8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8B246C04-F372-47F6-9397-F658915429A8}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8B246C04-F372-47F6-9397-F658915429A8}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8B246C04-F372-47F6-9397-F658915429A8}.Release|Any CPU.Build.0 = Release|Any CPU + {CDA2C52B-A9A7-446B-9D2F-D7B75C1905EF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CDA2C52B-A9A7-446B-9D2F-D7B75C1905EF}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CDA2C52B-A9A7-446B-9D2F-D7B75C1905EF}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CDA2C52B-A9A7-446B-9D2F-D7B75C1905EF}.Release|Any CPU.Build.0 = Release|Any CPU {425F7A13-99A2-4231-B0C1-C56EB819C174}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {425F7A13-99A2-4231-B0C1-C56EB819C174}.Debug|Any CPU.Build.0 = Debug|Any CPU {425F7A13-99A2-4231-B0C1-C56EB819C174}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -822,6 +834,8 @@ Global {A9FD538A-E101-4AEA-A98E-35DCED950AEE} = {E7BAE287-D505-4D6D-A090-665A64309B2D} {8C817874-7A88-450E-B216-851A1B03684C} = {52AF9D62-7D5B-4AD0-BA12-86F2AA67428B} {2A35EBB5-1CA6-4F5D-8CE8-352146C82C28} = {70D38D24-79AE-4600-8E83-17F3C11BA81F} + {8B246C04-F372-47F6-9397-F658915429A8} = {D7E9D8F5-8AC8-4ADA-B219-C549084AD84C} + {CDA2C52B-A9A7-446B-9D2F-D7B75C1905EF} = {897C4198-884F-448A-B0B0-C2A6D971EAE0} {425F7A13-99A2-4231-B0C1-C56EB819C174} = {05B06AC1-7F2B-492F-983E-5BC63CDBF20D} {CEC177AB-4FF0-4F8A-81B8-1E756D892416} = {E7BAE287-D505-4D6D-A090-665A64309B2D} {8744F914-BF51-4276-AFDA-9CBD750B8187} = {0CB73565-1207-4A56-A79F-6A8E9BBD795C} diff --git a/src/Networking/NexusMods.Networking.ModUpdates/MultiFeedCacheUpdater.cs b/src/Networking/NexusMods.Networking.ModUpdates/MultiFeedCacheUpdater.cs new file mode 100644 index 0000000000..29c1bedf4e --- /dev/null +++ b/src/Networking/NexusMods.Networking.ModUpdates/MultiFeedCacheUpdater.cs @@ -0,0 +1,110 @@ +using NexusMods.Networking.ModUpdates.Structures; +using NexusMods.Networking.ModUpdates.Traits; +namespace NexusMods.Networking.ModUpdates; + +/// +/// This is a helper struct which combines multiple +/// instances into a single interface. This allows for a cache update operation across +/// multiple mod feeds (games). +/// +/// For usage instructions, see ; the API and concepts +/// here are similar, except for the difference that this class' public API allows +/// you to use mods and API responses which are sourced from multiple feeds (games), +/// as opposed to a single feed. +/// +public class MultiFeedCacheUpdater where TUpdateableItem : ICanGetLastUpdatedTimestamp, ICanGetUid +{ + private readonly Dictionary> _updaters; + + /// + /// Creates a cache updater from a given list of items for which we want to + /// check for updates. + /// + /// The items to check for updates. + /// + /// Maximum age before an item has to be re-checked for updates. + /// The max for Nexus Mods API is 1 month. + /// + /// + /// In order to ensure accuracy, the age field should include the lifespan of + /// the as well as the cache + /// time on the server's end. That should prevent any technical possible + /// eventual consistency errors due to race conditions. Although + /// + public MultiFeedCacheUpdater(TUpdateableItem[] items, TimeSpan expiry) + { + _updaters = new Dictionary>(); + + // First group the items by their GameId + // We will use a list of groups, the assumption being that the number + // of feeds is low (usually less than 5) + var groupedList = new List<(GameId, List)>(); + foreach (var item in items) + { + var gameId = item.GetUniqueId().GameId; + + // Get or Update List for this GameId. + var found = false; + foreach (var (key, value) in groupedList) + { + if (key != gameId) + continue; + + value.Add(item); + found = true; + break; + } + + if (!found) + groupedList.Add((gameId, [item])); + } + + // Create a PerFeedCacheUpdater for each group + foreach (var (key, value) in groupedList) + _updaters[key] = new PerFeedCacheUpdater(value.ToArray(), expiry); + } + + /// + /// Updates the internal state of the + /// provided the results of the 'most recently updated mods for game' endpoint. + /// + /// + /// The items returned by the 'most recently updated mods' endpoint. This can + /// include items corresponding to multiple feeds (games); the feed source + /// is automatically detected. + /// + /// Wrap elements in a struct that implements + /// and if necessary. + /// + public void Update(IEnumerable items) where T : ICanGetLastUpdatedTimestamp, ICanGetUid + { + foreach (var item in items) + { + // Determine feed + var feed = item.GetUniqueId().GameId; + + // The result may contain items from feeds which we are not tracking. + // For instance, results for other games. This is not an error, we + // just need to filter the items out. + if (!_updaters.TryGetValue(feed, out var updater)) + continue; + + updater.UpdateSingleItem(item); + } + } + + /// + /// Determines the actions needed to taken on the items in the ; + /// returning the items whose actions have to be taken grouped by the action that needs performed. + /// + /// The results of multiple feeds are flattened here; everything is returned as a single result. + /// + public PerFeedCacheUpdaterResult BuildFlattened() + { + var result = new PerFeedCacheUpdaterResult(); + foreach (var updater in _updaters) + result.AddFrom(updater.Value.Build()); + + return result; + } +} diff --git a/src/Networking/NexusMods.Networking.ModUpdates/NexusMods.Networking.ModUpdates.csproj b/src/Networking/NexusMods.Networking.ModUpdates/NexusMods.Networking.ModUpdates.csproj new file mode 100644 index 0000000000..91abef7d7e --- /dev/null +++ b/src/Networking/NexusMods.Networking.ModUpdates/NexusMods.Networking.ModUpdates.csproj @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/src/Networking/NexusMods.Networking.ModUpdates/PerFeedCacheUpdater.cs b/src/Networking/NexusMods.Networking.ModUpdates/PerFeedCacheUpdater.cs new file mode 100644 index 0000000000..3189f1ee7c --- /dev/null +++ b/src/Networking/NexusMods.Networking.ModUpdates/PerFeedCacheUpdater.cs @@ -0,0 +1,155 @@ +using System.Diagnostics; +using NexusMods.Networking.ModUpdates.Private; +using NexusMods.Networking.ModUpdates.Structures; +using NexusMods.Networking.ModUpdates.Traits; +namespace NexusMods.Networking.ModUpdates; + +/// +/// This is a helper struct which helps us make use of the 'most recently updated mods for game' +/// API endpoint to use as a cache. (Where 'game' is a 'feed') +/// For full info on internals, see the NexusMods.App project documentation. +/// +/// This API consists of the following: +/// +/// 1. Input [Constructor]: A set of items with a 'last update time' (see ) +/// and a 'unique id' (see ) that are relevant to the current 'feed' (game). +/// +/// 2. Update [Method]: Submit results from API endpoint returning 'most recently updated mods for game'. +/// This updates the internal state of the . +/// +/// 3. Output [Info]: The outputs items with 2 categories: +/// - Up-to-date mods. These should have their timestamp updated. +/// - Out of date mods. These require re-querying the data from external source. +/// +/// +/// Within the Nexus Mods App: +/// +/// - 'Input' is our set of locally cached mod pages. +/// - 'Update' is our results of `updated.json` for a given game domain. +/// - 'Output' are the pages we need to update. +/// +/// The 'Feed' in the context of the Nexus App is the individual game's 'updated.json' endpoint; +/// i.e. a 'Game Mod Feed' +/// +public class PerFeedCacheUpdater where TUpdateableItem : ICanGetLastUpdatedTimestamp, ICanGetUid +{ + private readonly TUpdateableItem[] _items; + private readonly Dictionary _itemToIndex; + private readonly CacheUpdaterAction[] _actions; + + /// + /// Creates a (per-feed) cache updater from a given list of items for which + /// we want to check for updates. + /// + /// The items to check for updates. + /// + /// Maximum age before an item has to be re-checked for updates. + /// The max for Nexus Mods API is 1 month. + /// + /// + /// In order to ensure accuracy, the age field should include the lifespan of + /// the as well as the cache + /// time on the server's end. That should prevent any technical possible + /// eventual consistency errors due to race conditions. Although + /// + public PerFeedCacheUpdater(TUpdateableItem[] items, TimeSpan expiry) + { + _items = items; + DebugVerifyAllItemsAreFromSameGame(); + + _actions = new CacheUpdaterAction[items.Length]; + _itemToIndex = new Dictionary(items.Length); + for (var x = 0; x < _items.Length; x++) + _itemToIndex[_items[x].GetUniqueId().ModId] = x; + + // Set the action to refresh cache for any mods which exceed max age. + var utcNow = DateTime.UtcNow; + var minCachedDate = utcNow - expiry; + for (var x = 0; x < _items.Length; x++) + { + if (_items[x].GetLastUpdatedDate() < minCachedDate) + _actions[x] = CacheUpdaterAction.NeedsUpdate; + } + } + + /// + /// Updates the internal state of the + /// provided the results of the 'most recently updated mods for game' endpoint. + /// + /// + /// The items returned by the 'most recently updated mods for game' endpoint. + /// Wrap elements in a struct that implements + /// and if necessary. + /// + public void Update(IEnumerable items) where T : ICanGetLastUpdatedTimestamp, ICanGetUid + { + foreach (var item in items) + UpdateSingleItem(item); + } + + internal void UpdateSingleItem(T item) where T : ICanGetLastUpdatedTimestamp, ICanGetUid + { + // Try to get index of the item. + // Not all the items from the update feed are locally stored, thus we need to + // make sure we actually have this item. + if (!_itemToIndex.TryGetValue(item.GetUniqueId().ModId, out var index)) + return; + + var existingItem = _items[index]; + + // If the file timestamp is newer than our cached copy, the item needs updating. + if (item.GetLastUpdatedDate() > existingItem.GetLastUpdatedDate()) + _actions[index] = CacheUpdaterAction.NeedsUpdate; + else + _actions[index] = CacheUpdaterAction.UpdateLastCheckedTimestamp; + } + + /// + /// Determines the actions needed to taken on the items in the ; + /// returning the items whose actions have to be taken grouped by the action that needs performed. + /// + public PerFeedCacheUpdaterResult Build() + { + // We now have files in 3 categories: + // - Up-to-date mods. (Determined in `Update` method) a.k.a. CacheUpdaterAction.UpdateLastCheckedTimestamp + // - Out of date mods. (Determined in `Update` method and constructor) a.k.a. CacheUpdaterAction.NeedsUpdate + // - Undetermined Mods. (Mods with CacheUpdaterAction.Default) + // - This is the case for mods that are not in the `updated.json` payload + // which for some reason are in our cache and are not out of date (not greater than expiry). + // - Alternatively, they are in our cache but not in the `updated.json` payload. + // - This can be the case if the expiry field is changed between calls, or the maxAge of the + // request whose results we feed to `Update` is inconsistent due to programmer error. + // - We return these in a separate category, but the consumer should treat them as 'Out of Date' + var result = new PerFeedCacheUpdaterResult(); + for (var x = 0; x < _actions.Length; x++) + { + switch (_actions[x]) + { + case CacheUpdaterAction.Default: + result.UndeterminedItems.Add(_items[x]); + break; + case CacheUpdaterAction.NeedsUpdate: + result.OutOfDateItems.Add(_items[x]); + break; + case CacheUpdaterAction.UpdateLastCheckedTimestamp: + result.UpToDateItems.Add(_items[x]); + break; + default: + throw new ArgumentOutOfRangeException(); + } + } + + return result; + } + + [Conditional("DEBUG")] + private void DebugVerifyAllItemsAreFromSameGame() + { + if (_items.Length == 0) return; + + var firstGameId = _items[0].GetUniqueId().GameId; + var allSame = _items.All(x => x.GetUniqueId().GameId == firstGameId); + if (!allSame) + throw new ArgumentException("All items must have the same game id", nameof(_items)); + } +} diff --git a/src/Networking/NexusMods.Networking.ModUpdates/PerFeedCacheUpdaterResult.cs b/src/Networking/NexusMods.Networking.ModUpdates/PerFeedCacheUpdaterResult.cs new file mode 100644 index 0000000000..1f14548a1c --- /dev/null +++ b/src/Networking/NexusMods.Networking.ModUpdates/PerFeedCacheUpdaterResult.cs @@ -0,0 +1,44 @@ +using NexusMods.Networking.ModUpdates.Traits; +namespace NexusMods.Networking.ModUpdates; + +/// +/// Stores the result of updating the 'mod page' cache for a given feed or sets +/// of feeds. +/// +/// Wrapper for item supported by the cache updater. +public class PerFeedCacheUpdaterResult where TUpdateableItem : ICanGetLastUpdatedTimestamp, ICanGetUid +{ + /// + /// This is a list of items that is 'out of date'. + /// For these items, we need to fetch updated info from the Nexus Servers and update the timestamp. + /// + public List OutOfDateItems { get; init; } = new(); + + /// + /// These are the items that are 'up-to-date'. + /// Just update the timestamp on these items and you're good. + /// + public List UpToDateItems { get; init; } = new(); + + /// + /// These are the items that are 'undetermined'. + /// + /// These should be treated the same as the items in ; + /// however having items here is indicative of a possible programmer error. + /// (Due to inconsistent expiry parameter between Nexus API call and cache updater). + /// + /// Consider logging these items. + /// + public List UndeterminedItems { get; init; } = new(); + + /// + /// Adds items from another + /// into the current one. + /// + public void AddFrom(PerFeedCacheUpdaterResult other) + { + OutOfDateItems.AddRange(other.OutOfDateItems); + UpToDateItems.AddRange(other.UpToDateItems); + UndeterminedItems.AddRange(other.UndeterminedItems); + } +} diff --git a/src/Networking/NexusMods.Networking.ModUpdates/Private/CacheUpdaterAction.cs b/src/Networking/NexusMods.Networking.ModUpdates/Private/CacheUpdaterAction.cs new file mode 100644 index 0000000000..7026bff0fe --- /dev/null +++ b/src/Networking/NexusMods.Networking.ModUpdates/Private/CacheUpdaterAction.cs @@ -0,0 +1,24 @@ +namespace NexusMods.Networking.ModUpdates.Private; + +/// +/// Defines the actions that need to be taken on all elements submitted to the . +/// +internal enum CacheUpdaterAction : byte +{ + /// + /// This defaults to . + /// Either the entry is missing from the remote, or a programmer error has occurred. + /// + Default = 0, + + /// + /// The item needs to be updated in the local cache. + /// + NeedsUpdate = 1, + + /// + /// The item's 'last checked timestamp' needs to be updated. + /// (The item is already up to date) + /// + UpdateLastCheckedTimestamp = 2, +} diff --git a/src/Networking/NexusMods.Networking.ModUpdates/Structures/GameId.cs b/src/Networking/NexusMods.Networking.ModUpdates/Structures/GameId.cs new file mode 100644 index 0000000000..8dfbbd207e --- /dev/null +++ b/src/Networking/NexusMods.Networking.ModUpdates/Structures/GameId.cs @@ -0,0 +1,8 @@ +using TransparentValueObjects; +namespace NexusMods.Networking.ModUpdates.Structures; + +/// +/// Represents the unique ID of an individual game in the Nexus Backend. +/// +[ValueObject] // Do not modify. Unsafe code relies on this. +public readonly partial struct GameId { } diff --git a/src/Networking/NexusMods.Networking.ModUpdates/Structures/ModId.cs b/src/Networking/NexusMods.Networking.ModUpdates/Structures/ModId.cs new file mode 100644 index 0000000000..4b2a82b295 --- /dev/null +++ b/src/Networking/NexusMods.Networking.ModUpdates/Structures/ModId.cs @@ -0,0 +1,10 @@ +using TransparentValueObjects; +namespace NexusMods.Networking.ModUpdates.Structures; + +/// +/// Represents the unique ID of an individual mod in the Nexus Backend. +/// This is specific to the it belongs to; +/// forming a composite key. +/// +[ValueObject] // Do not modify. Unsafe code relies on this. +public readonly partial struct ModId { } diff --git a/src/Networking/NexusMods.Networking.ModUpdates/Structures/Uid.cs b/src/Networking/NexusMods.Networking.ModUpdates/Structures/Uid.cs new file mode 100644 index 0000000000..b72c404b44 --- /dev/null +++ b/src/Networking/NexusMods.Networking.ModUpdates/Structures/Uid.cs @@ -0,0 +1,33 @@ +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +namespace NexusMods.Networking.ModUpdates.Structures; + +/// +/// This represents a unique ID of an individual mod page as stored on Nexus Mods. +/// +/// +/// Not tested on Big Endian architectures. +/// +[StructLayout(LayoutKind.Sequential, Pack = 1)] +public struct Uid +{ + /// + /// Unique identifier for the mod, within the specific . + /// + public ModId ModId; + + /// + /// Unique identifier for the game. + /// + public GameId GameId; + + /// + /// Reinterprets the current as a single . + /// + public ulong AsUlong => Unsafe.As(ref this); + + /// + /// Reinterprets a given into a . + /// + public static Uid FromUlong(ulong value) => Unsafe.As(ref value); +} diff --git a/src/Networking/NexusMods.Networking.ModUpdates/Traits/ICanGetLastUpdatedTimestamp.cs b/src/Networking/NexusMods.Networking.ModUpdates/Traits/ICanGetLastUpdatedTimestamp.cs new file mode 100644 index 0000000000..37fc2c1737 --- /dev/null +++ b/src/Networking/NexusMods.Networking.ModUpdates/Traits/ICanGetLastUpdatedTimestamp.cs @@ -0,0 +1,12 @@ +namespace NexusMods.Networking.ModUpdates.Traits; + +/// +/// This interface marks an item which has the time it was last updated. +/// +public interface ICanGetLastUpdatedTimestamp +{ + /// + /// Retrieves the time the item was last updated. + /// + public DateTime GetLastUpdatedDate(); +} diff --git a/src/Networking/NexusMods.Networking.ModUpdates/Traits/ICanGetUid.cs b/src/Networking/NexusMods.Networking.ModUpdates/Traits/ICanGetUid.cs new file mode 100644 index 0000000000..b7e341ebe3 --- /dev/null +++ b/src/Networking/NexusMods.Networking.ModUpdates/Traits/ICanGetUid.cs @@ -0,0 +1,17 @@ +using NexusMods.Networking.ModUpdates.Structures; +namespace NexusMods.Networking.ModUpdates.Traits; + +/// +/// A trait representing an item which has a unique ID. +/// In this case, a 'unique ID' refers to a NexusMods Mod Page ID. +/// +/// This ID must be truly unique and belong to a specific item of its type. +/// +public interface ICanGetUid +{ + /// + /// Returns a unique identifier for the given item, based on the ID format + /// used in the NexusMods V2 API. + /// + public Uid GetUniqueId(); +} diff --git a/tests/Networking/NexusMods.Networking.ModUpdates.Tests/GlobalUsings.cs b/tests/Networking/NexusMods.Networking.ModUpdates.Tests/GlobalUsings.cs new file mode 100644 index 0000000000..c802f4480b --- /dev/null +++ b/tests/Networking/NexusMods.Networking.ModUpdates.Tests/GlobalUsings.cs @@ -0,0 +1 @@ +global using Xunit; diff --git a/tests/Networking/NexusMods.Networking.ModUpdates.Tests/Helpers/TestItem.cs b/tests/Networking/NexusMods.Networking.ModUpdates.Tests/Helpers/TestItem.cs new file mode 100644 index 0000000000..1536926193 --- /dev/null +++ b/tests/Networking/NexusMods.Networking.ModUpdates.Tests/Helpers/TestItem.cs @@ -0,0 +1,25 @@ +using NexusMods.Networking.ModUpdates.Structures; +using NexusMods.Networking.ModUpdates.Traits; +namespace NexusMods.Networking.ModUpdates.Tests; + +// Helper class to simulate updateable items +public class TestItem : ICanGetLastUpdatedTimestamp, ICanGetUid +{ + public DateTime LastUpdated { get; set; } + public Uid Uid { get; set; } + + public DateTime GetLastUpdatedDate() => LastUpdated; + public Uid GetUniqueId() => Uid; + + // Helper method to create a test item + public static TestItem Create(uint gameId, uint modId, DateTime lastUpdated) + { + return new TestItem + { + Uid = new Uid { GameId = GameId.From(gameId), ModId = ModId.From(modId) }, + LastUpdated = lastUpdated, + }; + } +} + + diff --git a/tests/Networking/NexusMods.Networking.ModUpdates.Tests/MultiFeedCacheUpdaterTests.cs b/tests/Networking/NexusMods.Networking.ModUpdates.Tests/MultiFeedCacheUpdaterTests.cs new file mode 100644 index 0000000000..3f578e0d6e --- /dev/null +++ b/tests/Networking/NexusMods.Networking.ModUpdates.Tests/MultiFeedCacheUpdaterTests.cs @@ -0,0 +1,214 @@ +using FluentAssertions; +using static NexusMods.Networking.ModUpdates.Tests.TestItem; + +namespace NexusMods.Networking.ModUpdates.Tests; + +public class MultiFeedCacheUpdaterTests +{ + [Fact] + public void Constructor_WithEmptyItems_ShouldNotThrow() + { + // Arrange & Act + Action act = () => new MultiFeedCacheUpdater([], TimeSpan.FromDays(30)); + + // Assert + act.Should().NotThrow(); + } + + [Fact] + public void Constructor_ShouldSetOldItemsToNeedUpdate() + { + // Items which have a 'last checked date' older than expiry + // should be marked as 'Out of Date' across multiple feeds. + + // Arrange + var now = DateTime.UtcNow; + var items = new[] + { + Create(1, 1, now.AddDays(-40)), + Create(1, 2, now.AddDays(-20)), + Create(2, 1, now.AddDays(-35)), + Create(2, 2, now.AddDays(-25)), + }; + + // Act + var updater = new MultiFeedCacheUpdater(items, TimeSpan.FromDays(30)); + var result = updater.BuildFlattened(); + + // Assert + // items [0] and [2] should be out of date + result.OutOfDateItems.Should().HaveCount(2); + result.OutOfDateItems.Should().Contain(items[0]); + result.OutOfDateItems.Should().Contain(items[2]); + } + + [Fact] + public void Update_ShouldMarkMissingItemsAsUndetermined() + { + // Items that are missing from the 'updated' payload + // may be 'inaccessible', such as deleted or taken down + // due to a DMCA. We should notice the mods which fall + // under this category across multiple feeds. + + // Arrange + var now = DateTime.UtcNow; + var items = new[] + { + Create(1, 1, now.AddDays(-10)), + Create(1, 2, now.AddDays(-5)), + Create(2, 1, now.AddDays(-15)), + Create(2, 2, now.AddDays(-8)), + }; + + var updater = new MultiFeedCacheUpdater(items, TimeSpan.FromDays(30)); + + var updateItems = new[] + { + Create(1, 1, now.AddDays(-8)), // Newer, needs update + Create(1, 2, now.AddDays(-7)), // Older, up-to-date + Create(2, 1, now.AddDays(-13)), // Newer, needs update + }; + + // Act + updater.Update(updateItems); + var result = updater.BuildFlattened(); + + // Assert + // item[3] is not in 'update' feed/result, but is within + // the expiry date. This means the mod page may have been archived + // or taken down due to a DMCA. + result.UndeterminedItems.Should().ContainSingle(); + result.UndeterminedItems.Should().Contain(items[3]); + } + + [Fact] + public void Update_ShouldMarkItemsAsUpToDateOrNeedingUpdateAcrossMultipleFeeds() + { + // The MultiFeedCacheUpdater correctly compares the 'lastUpdated' field + // of the update entry with the 'lastUpdated' field of the cached item + // across multiple feeds. + + // Arrange + var now = DateTime.UtcNow; + var items = new[] + { + Create(1, 1, now.AddDays(-10)), + Create(1, 2, now.AddDays(-5)), + Create(2, 1, now.AddDays(-12)), + Create(2, 2, now.AddDays(-7)), + }; + + var updater = new MultiFeedCacheUpdater(items, TimeSpan.FromDays(30)); + var updateItems = new[] + { + Create(1, 1, now.AddDays(-8)), // Newer, needs update + Create(1, 2, now.AddDays(-7)), // Older, up-to-date + Create(2, 1, now.AddDays(-10)), // Newer, needs update + Create(2, 2, now.AddDays(-9)), // Older, up-to-date + }; + + // Act + updater.Update(updateItems); + var result = updater.BuildFlattened(); + + // Assert + // items[0] and items[2] are out of date, items[1] and items[3] are up-to-date + result.OutOfDateItems.Should().HaveCount(2); + result.OutOfDateItems.Should().Contain(items[0]); + result.OutOfDateItems.Should().Contain(items[2]); + + result.UpToDateItems.Should().HaveCount(2); + result.UpToDateItems.Should().Contain(items[1]); + result.UpToDateItems.Should().Contain(items[3]); + } + + [Fact] + public void Update_HavingExtraItemsInUpdatePayloadHasNoSideEffects() + { + // Copy of Update_ShouldMarkItemsAsUpToDateOrNeedingUpdateAcrossMultipleFeeds, + // but with extra item(s) in update payload. These items are ones we don't have cached; + // therefore they should be ignored without side effects. + + // Arrange + var now = DateTime.UtcNow; + var items = new[] + { + Create(1, 1, now.AddDays(-10)), + Create(1, 2, now.AddDays(-5)), + Create(2, 1, now.AddDays(-12)), + Create(2, 2, now.AddDays(-7)), + }; + + var updater = new MultiFeedCacheUpdater(items, TimeSpan.FromDays(30)); + + var updateItems = new[] + { + Create(1, 5, now), // New item, should be ignored + Create(2, 6, now), // New item, should be ignored + Create(1, 1, now.AddDays(-8)), // Newer, needs update + Create(1, 2, now.AddDays(-7)), // Older, up-to-date + Create(2, 1, now.AddDays(-10)), // Newer, needs update + Create(2, 2, now.AddDays(-9)), // Older, up-to-date + Create(1, 7, now), // New item, should be ignored + Create(2, 8, now), // New item, should be ignored + }; + + // Act + updater.Update(updateItems); + var result = updater.BuildFlattened(); + + // Assert + // items[0] and items[2] are out of date, items[1] and items[3] are up-to-date + result.OutOfDateItems.Should().HaveCount(2); + result.OutOfDateItems.Should().Contain(items[0]); + result.OutOfDateItems.Should().Contain(items[2]); + + result.UpToDateItems.Should().HaveCount(2); + result.UpToDateItems.Should().Contain(items[1]); + result.UpToDateItems.Should().Contain(items[3]); + } + + [Fact] + public void Update_ShouldHandleUpdatesFromDifferentFeeds() + { + // Ensure that the MultiFeedCacheUpdater correctly handles updates + // from different feeds separately. + + // Arrange + var now = DateTime.UtcNow; + var items = new[] + { + Create(1, 1, now.AddDays(-10)), + Create(1, 2, now.AddDays(-5)), + Create(2, 1, now.AddDays(-12)), + Create(2, 2, now.AddDays(-7)), + Create(3, 1, now.AddDays(-15)), + }; + + var updater = new MultiFeedCacheUpdater(items, TimeSpan.FromDays(30)); + + var updateItems = new[] + { + Create(1, 1, now.AddDays(-8)), // Feed 1: Newer, needs update + Create(1, 2, now.AddDays(-7)), // Feed 1: Older, up-to-date + Create(2, 1, now.AddDays(-10)), // Feed 2: Newer, needs update + Create(3, 1, now.AddDays(-16)), // Feed 3: Older, up-to-date + }; + + // Act + updater.Update(updateItems); + var result = updater.BuildFlattened(); + + // Assert + result.OutOfDateItems.Should().HaveCount(2); + result.OutOfDateItems.Should().Contain(items[0]); + result.OutOfDateItems.Should().Contain(items[2]); + + result.UpToDateItems.Should().HaveCount(2); + result.UpToDateItems.Should().Contain(items[1]); + result.UpToDateItems.Should().Contain(items[4]); + + result.UndeterminedItems.Should().ContainSingle(); + result.UndeterminedItems.Should().Contain(items[3]); + } +} diff --git a/tests/Networking/NexusMods.Networking.ModUpdates.Tests/NexusMods.Networking.ModUpdates.Tests.csproj b/tests/Networking/NexusMods.Networking.ModUpdates.Tests/NexusMods.Networking.ModUpdates.Tests.csproj new file mode 100644 index 0000000000..ca2fea394b --- /dev/null +++ b/tests/Networking/NexusMods.Networking.ModUpdates.Tests/NexusMods.Networking.ModUpdates.Tests.csproj @@ -0,0 +1,9 @@ + + + NexusMods.Networking.ModUpdates.Tests + + + + + + diff --git a/tests/Networking/NexusMods.Networking.ModUpdates.Tests/PerFeedCacheUpdaterTests.cs b/tests/Networking/NexusMods.Networking.ModUpdates.Tests/PerFeedCacheUpdaterTests.cs new file mode 100644 index 0000000000..e6acd44711 --- /dev/null +++ b/tests/Networking/NexusMods.Networking.ModUpdates.Tests/PerFeedCacheUpdaterTests.cs @@ -0,0 +1,180 @@ +using System.Diagnostics; +using FluentAssertions; +using static NexusMods.Networking.ModUpdates.Tests.TestItem; + +namespace NexusMods.Networking.ModUpdates.Tests; + +public class PerFeedCacheUpdaterTests +{ + [Fact] + public void Constructor_WithEmptyItems_ShouldNotThrow() + { + // Arrange & Act + Action act = () => new PerFeedCacheUpdater([], TimeSpan.FromDays(30)); + + // Assert + act.Should().NotThrow(); + } + + [Fact] + [Conditional("DEBUG")] + public void Constructor_WithItemsFromDifferentGames_ShouldThrowArgumentException_InDebug() + { + // Arrange + var items = new[] + { + Create(1, 1, DateTime.UtcNow), + Create(2, 1, DateTime.UtcNow), + }; + + // Act + // ReSharper disable once HeapView.ObjectAllocation.Evident + // ReSharper disable once ObjectCreationAsStatement + Action act = () => new PerFeedCacheUpdater(items, TimeSpan.FromDays(30)); + + // Assert + act.Should().Throw(); + } + + [Fact] + public void Constructor_ShouldSetOldItemsToNeedUpdate() + { + // Items which have a 'last checked date' older than expiry + // should be marked as 'Out of Date'. + + // Arrange + var now = DateTime.UtcNow; + var items = new[] + { + Create(1, 1, now.AddDays(-40)), + Create(1, 2, now.AddDays(-20)), + Create(1, 3, now.AddDays(-35)), + }; + + // Act + var updater = new PerFeedCacheUpdater(items, TimeSpan.FromDays(30)); + var result = updater.Build(); + + // Assert + // input items [0] and [2] should be out of date + result.OutOfDateItems.Should().HaveCount(2); + result.OutOfDateItems.Should().Contain(items[0]); + result.OutOfDateItems.Should().Contain(items[2]); + } + + [Fact] + public void Update_ShouldMarkMissingItemsAsUndetermined() + { + // Items that are missing from the 'updated' payload + // may be 'inaccessible', such as deleted or taken down + // due to a DMCA. We should notice the mods which fall + // under this category. The external code doing the actual + // mod page querying can determine what to do with these based + // on extra info from the API. + + // Arrange + var now = DateTime.UtcNow; + var items = new[] + { + Create(1, 1, now.AddDays(-10)), + Create(1, 2, now.AddDays(-5)), + Create(1, 3, now.AddDays(-15)), + }; + + var updater = new PerFeedCacheUpdater(items, TimeSpan.FromDays(30)); + + var updateItems = new[] + { + Create(1, 1, now.AddDays(-8)), // Newer, needs update + Create(1, 2, now.AddDays(-7)), // Older, up-to-date + }; + + // Act + updater.Update(updateItems); + var result = updater.Build(); + + // Assert + // item[2] is not in 'update' feed/result, but is within + // the expiry date. This means the mod page may have been archived + // or taken down due to a DMCA. + result.UndeterminedItems.Should().ContainSingle(); + result.UndeterminedItems.Should().Contain(items[2]); + } + + [Fact] + public void Update_ShouldMarkItemsAsUpToDateOrNeedingUpdate() + { + // The PerFeedCacheUpdater correctly compares the 'lastUpdated' field + // of the update entry with the 'lastUpdated' field of the cached item. + // Here we test whether the update function correctly detects if an item + // is older or newer. + + // Arrange + var now = DateTime.UtcNow; + var items = new[] + { + Create(1, 1, now.AddDays(-10)), + Create(1, 2, now.AddDays(-5)), + }; + + var updater = new PerFeedCacheUpdater(items, TimeSpan.FromDays(30)); + var updateItems = new[] + { + Create(1, 1, now.AddDays(-8)), // Newer, needs update + Create(1, 2, now.AddDays(-7)), // Older, up-to-date + }; + + // Act + updater.Update(updateItems); + var result = updater.Build(); + + // Assert + // item[0] is out of date, item[1] is up-to-date + result.OutOfDateItems.Should().ContainSingle(); + result.OutOfDateItems.Should().Contain(items[0]); + + result.UpToDateItems.Should().ContainSingle(); + result.UpToDateItems.Should().Contain(items[1]); + } + + [Fact] + public void Update_HavingExtraItemsInUpdatePayloadHasNoSideEffects() + { + // Copy of Update_ShouldMarkItemsAsUpToDateOrNeedingUpdate, but with extra + // item(s) in update payload. These items are ones we don't have cached; + // therefore they should be ignored without side effects. + + // Arrange + var now = DateTime.UtcNow; + var items = new[] + { + Create(1, 1, now.AddDays(-10)), + Create(1, 2, now.AddDays(-5)), + Create(1, 3, now.AddDays(-15)), + }; + + var updater = new PerFeedCacheUpdater(items, TimeSpan.FromDays(30)); + + var updateItems = new[] + { + Create(1, 5, now), // New item, should be ignored + Create(1, 6, now), // New item, should be ignored + Create(1, 1, now.AddDays(-8)), // Newer, needs update + Create(1, 2, now.AddDays(-7)), // Older, up-to-date + Create(1, 4, now), // New item, should be ignored + Create(1, 7, now), // New item, should be ignored + }; + + // Act + updater.Update(updateItems); + var result = updater.Build(); + + // Assert + // item[0] is out of date, item[1] is up-to-date + result.OutOfDateItems.Should().ContainSingle(); + result.OutOfDateItems.Should().Contain(items[0]); + + result.UpToDateItems.Should().ContainSingle(); + result.UpToDateItems.Should().Contain(items[1]); + } +} diff --git a/tests/Networking/NexusMods.Networking.ModUpdates.Tests/Startup.cs b/tests/Networking/NexusMods.Networking.ModUpdates.Tests/Startup.cs new file mode 100644 index 0000000000..b8a5a63a3a --- /dev/null +++ b/tests/Networking/NexusMods.Networking.ModUpdates.Tests/Startup.cs @@ -0,0 +1,11 @@ +using Microsoft.Extensions.DependencyInjection; +namespace NexusMods.Networking.ModUpdates.Tests; + +public class Startup +{ + public void ConfigureServices(IServiceCollection container) + { + + } +} +