-
Notifications
You must be signed in to change notification settings - Fork 52
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[2/4] Added: Caching system for updates. (#2092)
* WIP: Documentation for Update Detection * Added: Additional Edge Cases to Update Logic Docs * Added: Extra case of `Archived in the Middle` * Fixed: Indentation for `file_updates` field. * Finalized 'Updating Mods' doc with simplified Implementation requested. * Fixed: Minor Notes from older Research Doc * Fixed: Added Missing 'Updating Mods' mkdocs sidebar item. * [Working WIP] Added: Initial Implementation of Generic Page Caching System used by Mod Pages * Removed: Unused Tests.cs file
- Loading branch information
Showing
17 changed files
with
883 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
110 changes: 110 additions & 0 deletions
110
src/Networking/NexusMods.Networking.ModUpdates/MultiFeedCacheUpdater.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,110 @@ | ||
| using NexusMods.Networking.ModUpdates.Structures; | ||
| using NexusMods.Networking.ModUpdates.Traits; | ||
| namespace NexusMods.Networking.ModUpdates; | ||
|
|
||
| /// <summary> | ||
| /// This is a helper struct which combines multiple <see cref="PerFeedCacheUpdater{TUpdateableItem}"/> | ||
| /// instances into a single interface. This allows for a cache update operation across | ||
| /// multiple mod feeds (games). | ||
| /// | ||
| /// For usage instructions, see <see cref="PerFeedCacheUpdater{TUpdateableItem}"/>; the API and concepts | ||
| /// here are similar, except for the difference that this class' public API allows | ||
| /// you to use mods and API responses which are sourced from multiple feeds (games), | ||
| /// as opposed to a single feed. | ||
| /// </summary> | ||
| public class MultiFeedCacheUpdater<TUpdateableItem> where TUpdateableItem : ICanGetLastUpdatedTimestamp, ICanGetUid | ||
| { | ||
| private readonly Dictionary<GameId, PerFeedCacheUpdater<TUpdateableItem>> _updaters; | ||
|
|
||
| /// <summary> | ||
| /// Creates a cache updater from a given list of items for which we want to | ||
| /// check for updates. | ||
| /// </summary> | ||
| /// <param name="items">The items to check for updates.</param> | ||
| /// <param name="expiry"> | ||
| /// Maximum age before an item has to be re-checked for updates. | ||
| /// The max for Nexus Mods API is 1 month. | ||
| /// </param> | ||
| /// <remarks> | ||
| /// In order to ensure accuracy, the age field should include the lifespan of | ||
| /// the <see cref="PerFeedCacheUpdater{TUpdateableItem}"/> as well as the cache | ||
| /// time on the server's end. That should prevent any technical possible | ||
| /// eventual consistency errors due to race conditions. Although | ||
| /// </remarks> | ||
| public MultiFeedCacheUpdater(TUpdateableItem[] items, TimeSpan expiry) | ||
| { | ||
| _updaters = new Dictionary<GameId, PerFeedCacheUpdater<TUpdateableItem>>(); | ||
|
|
||
| // First group the items by their GameId | ||
| // We will use a list of groups, the assumption being that the number | ||
| // of feeds is low (usually less than 5) | ||
| var groupedList = new List<(GameId, List<TUpdateableItem>)>(); | ||
| foreach (var item in items) | ||
| { | ||
| var gameId = item.GetUniqueId().GameId; | ||
|
|
||
| // Get or Update List for this GameId. | ||
| var found = false; | ||
| foreach (var (key, value) in groupedList) | ||
| { | ||
| if (key != gameId) | ||
| continue; | ||
|
|
||
| value.Add(item); | ||
| found = true; | ||
| break; | ||
| } | ||
|
|
||
| if (!found) | ||
| groupedList.Add((gameId, [item])); | ||
| } | ||
|
|
||
| // Create a PerFeedCacheUpdater for each group | ||
| foreach (var (key, value) in groupedList) | ||
| _updaters[key] = new PerFeedCacheUpdater<TUpdateableItem>(value.ToArray(), expiry); | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Updates the internal state of the <see cref="PerFeedCacheUpdater{TUpdateableItem}"/> | ||
| /// provided the results of the 'most recently updated mods for game' endpoint. | ||
| /// </summary> | ||
| /// <param name="items"> | ||
| /// The items returned by the 'most recently updated mods' endpoint. This can | ||
| /// include items corresponding to multiple feeds (games); the feed source | ||
| /// is automatically detected. | ||
| /// | ||
| /// Wrap elements in a struct that implements <see cref="ICanGetLastUpdatedTimestamp"/> | ||
| /// and <see cref="ICanGetUid"/> if necessary. | ||
| /// </param> | ||
| public void Update<T>(IEnumerable<T> items) where T : ICanGetLastUpdatedTimestamp, ICanGetUid | ||
| { | ||
| foreach (var item in items) | ||
| { | ||
| // Determine feed | ||
| var feed = item.GetUniqueId().GameId; | ||
|
|
||
| // The result may contain items from feeds which we are not tracking. | ||
| // For instance, results for other games. This is not an error, we | ||
| // just need to filter the items out. | ||
| if (!_updaters.TryGetValue(feed, out var updater)) | ||
| continue; | ||
|
|
||
| updater.UpdateSingleItem(item); | ||
| } | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Determines the actions needed to taken on the items in the <see cref="MultiFeedCacheUpdater{TUpdateableItem}"/>; | ||
| /// returning the items whose actions have to be taken grouped by the action that needs performed. | ||
| /// | ||
| /// The results of multiple feeds are flattened here; everything is returned as a single result. | ||
| /// </summary> | ||
| public PerFeedCacheUpdaterResult<TUpdateableItem> BuildFlattened() | ||
| { | ||
| var result = new PerFeedCacheUpdaterResult<TUpdateableItem>(); | ||
| foreach (var updater in _updaters) | ||
| result.AddFrom(updater.Value.Build()); | ||
|
|
||
| return result; | ||
| } | ||
| } |
16 changes: 16 additions & 0 deletions
16
src/Networking/NexusMods.Networking.ModUpdates/NexusMods.Networking.ModUpdates.csproj
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| <Project Sdk="Microsoft.NET.Sdk"> | ||
| <!-- NuGet Package Shared Details --> | ||
| <Import Project="$([MSBuild]::GetPathOfFileAbove('NuGet.Build.props', '$(MSBuildThisFileDirectory)../'))" /> | ||
|
|
||
| <ItemGroup> | ||
| <PackageReference Include="TransparentValueObjects"/> | ||
| </ItemGroup> | ||
|
|
||
| <ItemGroup> | ||
| <Folder Include="Private\" /> | ||
| </ItemGroup> | ||
|
|
||
| <ItemGroup> | ||
| <InternalsVisibleTo Include="NexusMods.Networking.ModUpdates.Tests"/> | ||
| </ItemGroup> | ||
| </Project> |
155 changes: 155 additions & 0 deletions
155
src/Networking/NexusMods.Networking.ModUpdates/PerFeedCacheUpdater.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,155 @@ | ||
| using System.Diagnostics; | ||
| using NexusMods.Networking.ModUpdates.Private; | ||
| using NexusMods.Networking.ModUpdates.Structures; | ||
| using NexusMods.Networking.ModUpdates.Traits; | ||
| namespace NexusMods.Networking.ModUpdates; | ||
|
|
||
| /// <summary> | ||
| /// This is a helper struct which helps us make use of the 'most recently updated mods for game' | ||
| /// API endpoint to use as a cache. (Where 'game' is a 'feed') | ||
| /// For full info on internals, see the NexusMods.App project documentation. | ||
| /// | ||
| /// This API consists of the following: | ||
| /// | ||
| /// 1. Input [Constructor]: A set of items with a 'last update time' (see <see cref="ICanGetLastUpdatedTimestamp"/>) | ||
| /// and a 'unique id' (see <see cref="ICanGetUid"/>) that are relevant to the current 'feed' (game). | ||
| /// | ||
| /// 2. Update [Method]: Submit results from API endpoint returning 'most recently updated mods for game'. | ||
| /// This updates the internal state of the <see cref="MultiFeedCacheUpdater{TUpdateableItem}"/>. | ||
| /// | ||
| /// 3. Output [Info]: The <see cref="MultiFeedCacheUpdater{TUpdateableItem}"/> outputs items with 2 categories: | ||
| /// - Up-to-date mods. These should have their timestamp updated. | ||
| /// - Out of date mods. These require re-querying the data from external source. | ||
| /// </summary> | ||
| /// <remarks> | ||
| /// Within the Nexus Mods App: | ||
| /// | ||
| /// - 'Input' is our set of locally cached mod pages. | ||
| /// - 'Update' is our results of `updated.json` for a given game domain. | ||
| /// - 'Output' are the pages we need to update. | ||
| /// | ||
| /// The 'Feed' in the context of the Nexus App is the individual game's 'updated.json' endpoint; | ||
| /// i.e. a 'Game Mod Feed' | ||
| /// </remarks> | ||
| public class PerFeedCacheUpdater<TUpdateableItem> where TUpdateableItem : ICanGetLastUpdatedTimestamp, ICanGetUid | ||
| { | ||
| private readonly TUpdateableItem[] _items; | ||
| private readonly Dictionary<ModId, int> _itemToIndex; | ||
| private readonly CacheUpdaterAction[] _actions; | ||
|
|
||
| /// <summary> | ||
| /// Creates a (per-feed) cache updater from a given list of items for which | ||
| /// we want to check for updates. | ||
| /// </summary> | ||
| /// <param name="items">The items to check for updates.</param> | ||
| /// <param name="expiry"> | ||
| /// Maximum age before an item has to be re-checked for updates. | ||
| /// The max for Nexus Mods API is 1 month. | ||
| /// </param> | ||
| /// <remarks> | ||
| /// In order to ensure accuracy, the age field should include the lifespan of | ||
| /// the <see cref="PerFeedCacheUpdater{TUpdateableItem}"/> as well as the cache | ||
| /// time on the server's end. That should prevent any technical possible | ||
| /// eventual consistency errors due to race conditions. Although | ||
| /// </remarks> | ||
| public PerFeedCacheUpdater(TUpdateableItem[] items, TimeSpan expiry) | ||
| { | ||
| _items = items; | ||
| DebugVerifyAllItemsAreFromSameGame(); | ||
|
|
||
| _actions = new CacheUpdaterAction[items.Length]; | ||
| _itemToIndex = new Dictionary<ModId, int>(items.Length); | ||
| for (var x = 0; x < _items.Length; x++) | ||
| _itemToIndex[_items[x].GetUniqueId().ModId] = x; | ||
|
|
||
| // Set the action to refresh cache for any mods which exceed max age. | ||
| var utcNow = DateTime.UtcNow; | ||
| var minCachedDate = utcNow - expiry; | ||
| for (var x = 0; x < _items.Length; x++) | ||
| { | ||
| if (_items[x].GetLastUpdatedDate() < minCachedDate) | ||
| _actions[x] = CacheUpdaterAction.NeedsUpdate; | ||
| } | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Updates the internal state of the <see cref="PerFeedCacheUpdater{TUpdateableItem}"/> | ||
| /// provided the results of the 'most recently updated mods for game' endpoint. | ||
| /// </summary> | ||
| /// <param name="items"> | ||
| /// The items returned by the 'most recently updated mods for game' endpoint. | ||
| /// Wrap elements in a struct that implements <see cref="ICanGetLastUpdatedTimestamp"/> | ||
| /// and <see cref="ICanGetUid"/> if necessary. | ||
| /// </param> | ||
| public void Update<T>(IEnumerable<T> items) where T : ICanGetLastUpdatedTimestamp, ICanGetUid | ||
| { | ||
| foreach (var item in items) | ||
| UpdateSingleItem(item); | ||
| } | ||
|
|
||
| internal void UpdateSingleItem<T>(T item) where T : ICanGetLastUpdatedTimestamp, ICanGetUid | ||
| { | ||
| // Try to get index of the item. | ||
| // Not all the items from the update feed are locally stored, thus we need to | ||
| // make sure we actually have this item. | ||
| if (!_itemToIndex.TryGetValue(item.GetUniqueId().ModId, out var index)) | ||
| return; | ||
|
|
||
| var existingItem = _items[index]; | ||
|
|
||
| // If the file timestamp is newer than our cached copy, the item needs updating. | ||
| if (item.GetLastUpdatedDate() > existingItem.GetLastUpdatedDate()) | ||
| _actions[index] = CacheUpdaterAction.NeedsUpdate; | ||
| else | ||
| _actions[index] = CacheUpdaterAction.UpdateLastCheckedTimestamp; | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Determines the actions needed to taken on the items in the <see cref="PerFeedCacheUpdater{TUpdateableItem}"/>; | ||
| /// returning the items whose actions have to be taken grouped by the action that needs performed. | ||
| /// </summary> | ||
| public PerFeedCacheUpdaterResult<TUpdateableItem> Build() | ||
| { | ||
| // We now have files in 3 categories: | ||
| // - Up-to-date mods. (Determined in `Update` method) a.k.a. CacheUpdaterAction.UpdateLastCheckedTimestamp | ||
| // - Out of date mods. (Determined in `Update` method and constructor) a.k.a. CacheUpdaterAction.NeedsUpdate | ||
| // - Undetermined Mods. (Mods with CacheUpdaterAction.Default) | ||
| // - This is the case for mods that are not in the `updated.json` payload | ||
| // which for some reason are in our cache and are not out of date (not greater than expiry). | ||
| // - Alternatively, they are in our cache but not in the `updated.json` payload. | ||
| // - This can be the case if the expiry field is changed between calls, or the maxAge of the | ||
| // request whose results we feed to `Update` is inconsistent due to programmer error. | ||
| // - We return these in a separate category, but the consumer should treat them as 'Out of Date' | ||
| var result = new PerFeedCacheUpdaterResult<TUpdateableItem>(); | ||
| for (var x = 0; x < _actions.Length; x++) | ||
| { | ||
| switch (_actions[x]) | ||
| { | ||
| case CacheUpdaterAction.Default: | ||
| result.UndeterminedItems.Add(_items[x]); | ||
| break; | ||
| case CacheUpdaterAction.NeedsUpdate: | ||
| result.OutOfDateItems.Add(_items[x]); | ||
| break; | ||
| case CacheUpdaterAction.UpdateLastCheckedTimestamp: | ||
| result.UpToDateItems.Add(_items[x]); | ||
| break; | ||
| default: | ||
| throw new ArgumentOutOfRangeException(); | ||
| } | ||
| } | ||
|
|
||
| return result; | ||
| } | ||
|
|
||
| [Conditional("DEBUG")] | ||
| private void DebugVerifyAllItemsAreFromSameGame() | ||
| { | ||
| if (_items.Length == 0) return; | ||
|
|
||
| var firstGameId = _items[0].GetUniqueId().GameId; | ||
| var allSame = _items.All(x => x.GetUniqueId().GameId == firstGameId); | ||
| if (!allSame) | ||
| throw new ArgumentException("All items must have the same game id", nameof(_items)); | ||
| } | ||
| } |
44 changes: 44 additions & 0 deletions
44
src/Networking/NexusMods.Networking.ModUpdates/PerFeedCacheUpdaterResult.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,44 @@ | ||
| using NexusMods.Networking.ModUpdates.Traits; | ||
| namespace NexusMods.Networking.ModUpdates; | ||
|
|
||
| /// <summary> | ||
| /// Stores the result of updating the 'mod page' cache for a given feed or sets | ||
| /// of feeds. | ||
| /// </summary> | ||
| /// <typeparam name="TUpdateableItem">Wrapper for item supported by the cache updater.</typeparam> | ||
| public class PerFeedCacheUpdaterResult<TUpdateableItem> where TUpdateableItem : ICanGetLastUpdatedTimestamp, ICanGetUid | ||
| { | ||
| /// <summary> | ||
| /// This is a list of items that is 'out of date'. | ||
| /// For these items, we need to fetch updated info from the Nexus Servers and update the timestamp. | ||
| /// </summary> | ||
| public List<TUpdateableItem> OutOfDateItems { get; init; } = new(); | ||
|
|
||
| /// <summary> | ||
| /// These are the items that are 'up-to-date'. | ||
| /// Just update the timestamp on these items and you're good. | ||
| /// </summary> | ||
| public List<TUpdateableItem> UpToDateItems { get; init; } = new(); | ||
|
|
||
| /// <summary> | ||
| /// These are the items that are 'undetermined'. | ||
| /// | ||
| /// These should be treated the same as the items in <see cref="OutOfDateItems"/>; | ||
| /// however having items here is indicative of a possible programmer error. | ||
| /// (Due to inconsistent expiry parameter between Nexus API call and cache updater). | ||
| /// | ||
| /// Consider logging these items. | ||
| /// </summary> | ||
| public List<TUpdateableItem> UndeterminedItems { get; init; } = new(); | ||
|
|
||
| /// <summary> | ||
| /// Adds items from another <see cref="PerFeedCacheUpdaterResult{TUpdateableItem}"/> | ||
| /// into the current one. | ||
| /// </summary> | ||
| public void AddFrom(PerFeedCacheUpdaterResult<TUpdateableItem> other) | ||
| { | ||
| OutOfDateItems.AddRange(other.OutOfDateItems); | ||
| UpToDateItems.AddRange(other.UpToDateItems); | ||
| UndeterminedItems.AddRange(other.UndeterminedItems); | ||
| } | ||
| } |
24 changes: 24 additions & 0 deletions
24
src/Networking/NexusMods.Networking.ModUpdates/Private/CacheUpdaterAction.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,24 @@ | ||
| namespace NexusMods.Networking.ModUpdates.Private; | ||
|
|
||
| /// <summary> | ||
| /// Defines the actions that need to be taken on all elements submitted to the <see cref="PerFeedCacheUpdater{TUpdateableItem}"/>. | ||
| /// </summary> | ||
| internal enum CacheUpdaterAction : byte | ||
| { | ||
| /// <summary> | ||
| /// This defaults to <see cref="UpdateLastCheckedTimestamp"/>. | ||
| /// Either the entry is missing from the remote, or a programmer error has occurred. | ||
| /// </summary> | ||
| Default = 0, | ||
|
|
||
| /// <summary> | ||
| /// The item needs to be updated in the local cache. | ||
| /// </summary> | ||
| NeedsUpdate = 1, | ||
|
|
||
| /// <summary> | ||
| /// The item's 'last checked timestamp' needs to be updated. | ||
| /// (The item is already up to date) | ||
| /// </summary> | ||
| UpdateLastCheckedTimestamp = 2, | ||
| } |
8 changes: 8 additions & 0 deletions
8
src/Networking/NexusMods.Networking.ModUpdates/Structures/GameId.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,8 @@ | ||
| using TransparentValueObjects; | ||
| namespace NexusMods.Networking.ModUpdates.Structures; | ||
|
|
||
| /// <summary> | ||
| /// Represents the unique ID of an individual game in the Nexus Backend. | ||
| /// </summary> | ||
| [ValueObject<uint>] // Do not modify. Unsafe code relies on this. | ||
| public readonly partial struct GameId { } |
Oops, something went wrong.