diff --git a/src/SMAPI.Web/BackgroundService.cs b/src/SMAPI.Web/BackgroundService.cs index 09854fde6..ebcc17a74 100644 --- a/src/SMAPI.Web/BackgroundService.cs +++ b/src/SMAPI.Web/BackgroundService.cs @@ -3,14 +3,16 @@ using System.Threading; using System.Threading.Tasks; using Hangfire; +using Hangfire.Console; +using Hangfire.Server; +using Humanizer; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Options; using StardewModdingAPI.Toolkit; using StardewModdingAPI.Toolkit.Framework.Clients.CurseForgeExport; -using StardewModdingAPI.Toolkit.Framework.Clients.CurseForgeExport.ResponseModels; using StardewModdingAPI.Toolkit.Framework.Clients.NexusExport; -using StardewModdingAPI.Toolkit.Framework.Clients.NexusExport.ResponseModels; using StardewModdingAPI.Toolkit.Framework.Clients.Wiki; +using StardewModdingAPI.Web.Framework.Caching; using StardewModdingAPI.Web.Framework.Caching.CurseForgeExport; using StardewModdingAPI.Web.Framework.Caching.Mods; using StardewModdingAPI.Web.Framework.Caching.NexusExport; @@ -108,19 +110,19 @@ public Task StartAsync(CancellationToken cancellationToken) bool enableNexusExport = BackgroundService.NexusExportApiClient is not DisabledNexusExportApiClient; // set startup tasks - BackgroundJob.Enqueue(() => BackgroundService.UpdateWikiAsync()); + BackgroundJob.Enqueue(() => BackgroundService.UpdateWikiAsync(null)); if (enableCurseForgeExport) - BackgroundJob.Enqueue(() => BackgroundService.UpdateCurseForgeExportAsync()); + BackgroundJob.Enqueue(() => BackgroundService.UpdateCurseForgeExportAsync(null)); if (enableNexusExport) - BackgroundJob.Enqueue(() => BackgroundService.UpdateNexusExportAsync()); + BackgroundJob.Enqueue(() => BackgroundService.UpdateNexusExportAsync(null)); BackgroundJob.Enqueue(() => BackgroundService.RemoveStaleModsAsync()); // set recurring tasks - RecurringJob.AddOrUpdate("update wiki data", () => BackgroundService.UpdateWikiAsync(), "*/10 * * * *"); // every 10 minutes + RecurringJob.AddOrUpdate("update wiki data", () => BackgroundService.UpdateWikiAsync(null), "*/10 * * * *"); // every 10 minutes if (enableCurseForgeExport) - RecurringJob.AddOrUpdate("update CurseForge export", () => BackgroundService.UpdateCurseForgeExportAsync(), "*/10 * * * *"); + RecurringJob.AddOrUpdate("update CurseForge export", () => BackgroundService.UpdateCurseForgeExportAsync(null), "*/10 * * * *"); if (enableNexusExport) - RecurringJob.AddOrUpdate("update Nexus export", () => BackgroundService.UpdateNexusExportAsync(), "*/10 * * * *"); + RecurringJob.AddOrUpdate("update Nexus export", () => BackgroundService.UpdateNexusExportAsync(null), "*/10 * * * *"); RecurringJob.AddOrUpdate("remove stale mods", () => BackgroundService.RemoveStaleModsAsync(), "2/10 * * * *"); // offset by 2 minutes so it runs after updates (e.g. 00:02, 00:12, etc) BackgroundService.IsStarted = true; @@ -150,54 +152,48 @@ public void Dispose() ** Tasks ****/ /// Update the cached wiki metadata. - [AutomaticRetry(Attempts = 3, DelaysInSeconds = new[] { 30, 60, 120 })] - public static async Task UpdateWikiAsync() + /// Information about the context in which the job is performed. This is injected automatically by Hangfire. + [AutomaticRetry(Attempts = 3, DelaysInSeconds = [30, 60, 120])] + public static async Task UpdateWikiAsync(PerformContext? context) { if (!BackgroundService.IsStarted) throw new InvalidOperationException($"Must call {nameof(BackgroundService.StartAsync)} before scheduling tasks."); + context.WriteLine("Fetching data from wiki..."); WikiModList wikiCompatList = await new ModToolkit().GetWikiCompatibilityListAsync(); + + context.WriteLine("Saving data..."); BackgroundService.WikiCache.SaveWikiData(wikiCompatList.StableVersion, wikiCompatList.BetaVersion, wikiCompatList.Mods); + + context.WriteLine("Done!"); } /// Update the cached CurseForge mod dump. - [AutomaticRetry(Attempts = 3, DelaysInSeconds = new[] { 30, 60, 120 })] - public static async Task UpdateCurseForgeExportAsync() + /// Information about the context in which the job is performed. This is injected automatically by Hangfire. + [AutomaticRetry(Attempts = 3, DelaysInSeconds = [30, 60, 120])] + public static async Task UpdateCurseForgeExportAsync(PerformContext? context) { - if (!BackgroundService.IsStarted) - throw new InvalidOperationException($"Must call {nameof(BackgroundService.StartAsync)} before scheduling tasks."); - - var cache = BackgroundService.CurseForgeExportCache; - var client = BackgroundService.CurseForgeExportApiClient; - - if (await cache.CanRefreshFromAsync(client, BackgroundService.ExportStaleAge)) - { - CurseForgeFullExport data = await client.FetchExportAsync(); - cache.SetData(data); - } - - if (cache.IsStale(BackgroundService.ExportStaleAge)) - cache.SetData(null); // if the export is too old, fetch fresh mod data from the API instead + await UpdateExportAsync( + context, + BackgroundService.CurseForgeExportCache!, + BackgroundService.CurseForgeExportApiClient!, + client => client.FetchLastModifiedDateAsync(), + async (cache, client) => cache.SetData(await client.FetchExportAsync()) + ); } /// Update the cached Nexus mod dump. - [AutomaticRetry(Attempts = 3, DelaysInSeconds = new[] { 30, 60, 120 })] - public static async Task UpdateNexusExportAsync() + /// Information about the context in which the job is performed. This is injected automatically by Hangfire. + [AutomaticRetry(Attempts = 3, DelaysInSeconds = [30, 60, 120])] + public static async Task UpdateNexusExportAsync(PerformContext? context) { - if (!BackgroundService.IsStarted) - throw new InvalidOperationException($"Must call {nameof(BackgroundService.StartAsync)} before scheduling tasks."); - - var cache = BackgroundService.NexusExportCache; - var client = BackgroundService.NexusExportApiClient; - - if (await cache.CanRefreshFromAsync(client, BackgroundService.ExportStaleAge)) - { - NexusFullExport data = await client.FetchExportAsync(); - cache.SetData(data); - } - - if (cache.IsStale(BackgroundService.ExportStaleAge)) - cache.SetData(null); // if the export is too old, fetch fresh mod data from the site/API instead + await UpdateExportAsync( + context, + BackgroundService.NexusExportCache!, + BackgroundService.NexusExportApiClient!, + client => client.FetchLastModifiedDateAsync(), + async (cache, client) => cache.SetData(await client.FetchExportAsync()) + ); } /// Remove mods which haven't been requested in over 48 hours. @@ -209,10 +205,6 @@ public static Task RemoveStaleModsAsync() // remove mods in mod cache BackgroundService.ModCache.RemoveStaleMods(TimeSpan.FromHours(48)); - // remove stale export cache - if (BackgroundService.NexusExportCache.IsStale(BackgroundService.ExportStaleAge)) - BackgroundService.NexusExportCache.SetData(null); - return Task.CompletedTask; } @@ -229,5 +221,70 @@ private void TryInit() BackgroundService.JobServer = new BackgroundJobServer(); } + + /// Update the cached mods export for a site. + /// The export cache repository type. + /// The export API client. + /// Information about the context in which the job is performed. This is injected automatically by Hangfire. + /// The export cache to update. + /// The export API with which to fetch data from the remote API. + /// Fetch the date when the export on the server was last modified. + /// Fetch the latest export file from the Nexus Mods export API. + /// The method wasn't called before running this task. + private static async Task UpdateExportAsync(PerformContext? context, TCacheRepository cache, TExportApiClient client, Func> fetchLastModifiedDateAsync, Func fetchDataAsync) + where TCacheRepository : IExportCacheRepository + { + if (!BackgroundService.IsStarted) + throw new InvalidOperationException($"Must call {nameof(BackgroundService.StartAsync)} before scheduling tasks."); + + // refresh data + context.WriteLine("Checking if we can refresh the data..."); + if (BackgroundService.CanRefreshFromExportApi(await fetchLastModifiedDateAsync(client), cache, out string? failReason)) + { + context.WriteLine("Fetching data..."); + + await fetchDataAsync(cache, client); + + DateTimeOffset lastModified = cache.GetLastModified(); + context.WriteLine($"Cache updated. The data was last modified {lastModified:O} (age: {(DateTimeOffset.UtcNow - lastModified).Humanize()})."); + } + else + context.WriteLine($"Skipped data fetch: {failReason}."); + + // clear if stale + if (cache.IsStale(BackgroundService.ExportStaleAge)) + { + context.WriteLine("The cached data is stale, clearing cache..."); + cache.Clear(); + } + + context.WriteLine("Done!"); + } + + /// Get whether newer non-stale data can be fetched from the server. + /// The last-modified data from the remote API. + /// The repository to update. + /// The reason to log if we can't fetch data. + private static bool CanRefreshFromExportApi(DateTimeOffset serverModified, IExportCacheRepository repository, [NotNullWhen(false)] out string? failReason) + { + if (repository.IsStale(serverModified, BackgroundService.ExportStaleAge)) + { + failReason = $"server was last modified [{serverModified}], which already exceeds the {BackgroundService.ExportStaleAge}-minute-stale limit"; + return false; + } + + if (repository.IsLoaded()) + { + DateTimeOffset localModified = repository.GetLastModified(); + if (localModified >= serverModified) + { + failReason = $"server was last modified {serverModified:O} (age: {(DateTimeOffset.UtcNow - serverModified).Humanize()}), which {(serverModified == localModified ? "matches our cached data" : $"is older than our cached {localModified:O}")}"; + return false; + } + } + + failReason = null; + return true; + } } } diff --git a/src/SMAPI.Web/Framework/Caching/BaseCacheRepository.cs b/src/SMAPI.Web/Framework/Caching/BaseCacheRepository.cs index f5354b939..a2ae1bbab 100644 --- a/src/SMAPI.Web/Framework/Caching/BaseCacheRepository.cs +++ b/src/SMAPI.Web/Framework/Caching/BaseCacheRepository.cs @@ -3,14 +3,12 @@ namespace StardewModdingAPI.Web.Framework.Caching { /// The base logic for a cache repository. - internal abstract class BaseCacheRepository + internal abstract class BaseCacheRepository : ICacheRepository { /********* ** Public methods *********/ - /// Whether cached data is stale. - /// The date when the data was updated. - /// The age in minutes before data is considered stale. + /// public bool IsStale(DateTimeOffset lastUpdated, int staleMinutes) { return lastUpdated < DateTimeOffset.UtcNow.AddMinutes(-staleMinutes); diff --git a/src/SMAPI.Web/Framework/Caching/BaseExportCacheRepository.cs b/src/SMAPI.Web/Framework/Caching/BaseExportCacheRepository.cs new file mode 100644 index 000000000..22de31e36 --- /dev/null +++ b/src/SMAPI.Web/Framework/Caching/BaseExportCacheRepository.cs @@ -0,0 +1,26 @@ +using System; + +namespace StardewModdingAPI.Web.Framework.Caching +{ + /// The base logic for an export cache repository. + internal abstract class BaseExportCacheRepository : BaseCacheRepository, IExportCacheRepository + { + /********* + ** Public methods + *********/ + /// + public abstract bool IsLoaded(); + + /// + public abstract DateTimeOffset GetLastModified(); + + /// + public bool IsStale(int staleMinutes) + { + return this.IsStale(this.GetLastModified(), staleMinutes); + } + + /// + public abstract void Clear(); + } +} diff --git a/src/SMAPI.Web/Framework/Caching/CurseForgeExport/CurseForgeExportCacheMemoryRepository.cs b/src/SMAPI.Web/Framework/Caching/CurseForgeExport/CurseForgeExportCacheMemoryRepository.cs index 6fe80425a..da7046ec3 100644 --- a/src/SMAPI.Web/Framework/Caching/CurseForgeExport/CurseForgeExportCacheMemoryRepository.cs +++ b/src/SMAPI.Web/Framework/Caching/CurseForgeExport/CurseForgeExportCacheMemoryRepository.cs @@ -1,13 +1,11 @@ using System; using System.Diagnostics.CodeAnalysis; -using System.Threading.Tasks; -using StardewModdingAPI.Toolkit.Framework.Clients.CurseForgeExport; using StardewModdingAPI.Toolkit.Framework.Clients.CurseForgeExport.ResponseModels; namespace StardewModdingAPI.Web.Framework.Caching.CurseForgeExport { /// Manages cached mod data from the CurseForge export API in-memory. - internal class CurseForgeExportCacheMemoryRepository : BaseCacheRepository, ICurseForgeExportCacheRepository + internal class CurseForgeExportCacheMemoryRepository : BaseExportCacheRepository, ICurseForgeExportCacheRepository { /********* ** Fields @@ -21,22 +19,21 @@ internal class CurseForgeExportCacheMemoryRepository : BaseCacheRepository, ICur *********/ /// [MemberNotNullWhen(true, nameof(CurseForgeExportCacheMemoryRepository.Data))] - public bool IsLoaded() + public override bool IsLoaded() { return this.Data?.Mods.Count > 0; } /// - public async Task CanRefreshFromAsync(ICurseForgeExportApiClient client, int staleMinutes) + public override DateTimeOffset GetLastModified() { - DateTimeOffset serverLastModified = await client.FetchLastModifiedDateAsync(); + return this.Data?.LastModified ?? DateTimeOffset.MinValue; + } - return - !this.IsStale(serverLastModified, staleMinutes) - && ( - !this.IsLoaded() - || this.Data.LastModified < serverLastModified - ); + /// + public override void Clear() + { + this.SetData(null); } /// @@ -58,13 +55,5 @@ public void SetData(CurseForgeFullExport? export) { this.Data = export; } - - /// - public bool IsStale(int staleMinutes) - { - return - this.Data is null - || this.IsStale(this.Data.LastModified, staleMinutes); - } } } diff --git a/src/SMAPI.Web/Framework/Caching/CurseForgeExport/ICurseForgeExportCacheRepository.cs b/src/SMAPI.Web/Framework/Caching/CurseForgeExport/ICurseForgeExportCacheRepository.cs index 5027217ec..b3a46ec80 100644 --- a/src/SMAPI.Web/Framework/Caching/CurseForgeExport/ICurseForgeExportCacheRepository.cs +++ b/src/SMAPI.Web/Framework/Caching/CurseForgeExport/ICurseForgeExportCacheRepository.cs @@ -1,24 +1,14 @@ using System.Diagnostics.CodeAnalysis; -using System.Threading.Tasks; -using StardewModdingAPI.Toolkit.Framework.Clients.CurseForgeExport; using StardewModdingAPI.Toolkit.Framework.Clients.CurseForgeExport.ResponseModels; namespace StardewModdingAPI.Web.Framework.Caching.CurseForgeExport { /// Manages cached mod data from the CurseForge export API. - internal interface ICurseForgeExportCacheRepository : ICacheRepository + internal interface ICurseForgeExportCacheRepository : IExportCacheRepository { /********* ** Methods *********/ - /// Get whether the export data is currently available. - bool IsLoaded(); - - /// Get whether newer non-stale data can be fetched from the server. - /// The CurseForge API client. - /// The age in minutes before data is considered stale. - Task CanRefreshFromAsync(ICurseForgeExportApiClient client, int staleMinutes); - /// Get the cached data for a mod, if it exists in the export. /// The CurseForge mod ID. /// The fetched metadata. @@ -27,9 +17,5 @@ internal interface ICurseForgeExportCacheRepository : ICacheRepository /// Set the cached data to use. /// The export received from the CurseForge Mods API, or null to remove it. void SetData(CurseForgeFullExport? export); - - /// Get whether the cached data is stale. - /// The age in minutes before data is considered stale. - bool IsStale(int staleMinutes); } } diff --git a/src/SMAPI.Web/Framework/Caching/IExportCacheRepository.cs b/src/SMAPI.Web/Framework/Caching/IExportCacheRepository.cs new file mode 100644 index 000000000..e61fe331c --- /dev/null +++ b/src/SMAPI.Web/Framework/Caching/IExportCacheRepository.cs @@ -0,0 +1,24 @@ +using System; + +namespace StardewModdingAPI.Web.Framework.Caching +{ + /// Encapsulates logic for accessing data in a cached mod export from a remote API. + internal interface IExportCacheRepository : ICacheRepository + { + /********* + ** Methods + *********/ + /// Get whether the export data is currently available. + bool IsLoaded(); + + /// Get the date when the cached data was last modified. + DateTimeOffset GetLastModified(); + + /// Get whether the cached data is stale. + /// The age in minutes before data is considered stale. + bool IsStale(int staleMinutes); + + /// Clear all data in the cache. + void Clear(); + } +} diff --git a/src/SMAPI.Web/Framework/Caching/NexusExport/INexusExportCacheRepository.cs b/src/SMAPI.Web/Framework/Caching/NexusExport/INexusExportCacheRepository.cs index 5b12dad66..08d7568d7 100644 --- a/src/SMAPI.Web/Framework/Caching/NexusExport/INexusExportCacheRepository.cs +++ b/src/SMAPI.Web/Framework/Caching/NexusExport/INexusExportCacheRepository.cs @@ -1,24 +1,14 @@ using System.Diagnostics.CodeAnalysis; -using System.Threading.Tasks; -using StardewModdingAPI.Toolkit.Framework.Clients.NexusExport; using StardewModdingAPI.Toolkit.Framework.Clients.NexusExport.ResponseModels; namespace StardewModdingAPI.Web.Framework.Caching.NexusExport { /// Manages cached mod data from the Nexus export API. - internal interface INexusExportCacheRepository : ICacheRepository + internal interface INexusExportCacheRepository : IExportCacheRepository { /********* ** Methods *********/ - /// Get whether the export data is currently available. - bool IsLoaded(); - - /// Get whether newer non-stale data can be fetched from the server. - /// The Nexus API client. - /// The age in minutes before data is considered stale. - Task CanRefreshFromAsync(INexusExportApiClient client, int staleMinutes); - /// Get the cached data for a mod, if it exists in the export. /// The Nexus mod ID. /// The fetched metadata. @@ -27,9 +17,5 @@ internal interface INexusExportCacheRepository : ICacheRepository /// Set the cached data to use. /// The export received from the Nexus Mods API, or null to remove it. void SetData(NexusFullExport? export); - - /// Get whether the cached data is stale. - /// The age in minutes before data is considered stale. - bool IsStale(int staleMinutes); } } diff --git a/src/SMAPI.Web/Framework/Caching/NexusExport/NexusExportCacheMemoryRepository.cs b/src/SMAPI.Web/Framework/Caching/NexusExport/NexusExportCacheMemoryRepository.cs index d80f48204..194882b55 100644 --- a/src/SMAPI.Web/Framework/Caching/NexusExport/NexusExportCacheMemoryRepository.cs +++ b/src/SMAPI.Web/Framework/Caching/NexusExport/NexusExportCacheMemoryRepository.cs @@ -1,13 +1,11 @@ using System; using System.Diagnostics.CodeAnalysis; -using System.Threading.Tasks; -using StardewModdingAPI.Toolkit.Framework.Clients.NexusExport; using StardewModdingAPI.Toolkit.Framework.Clients.NexusExport.ResponseModels; namespace StardewModdingAPI.Web.Framework.Caching.NexusExport { /// Manages cached mod data from the Nexus export API in-memory. - internal class NexusExportCacheMemoryRepository : BaseCacheRepository, INexusExportCacheRepository + internal class NexusExportCacheMemoryRepository : BaseExportCacheRepository, INexusExportCacheRepository { /********* ** Fields @@ -21,22 +19,21 @@ internal class NexusExportCacheMemoryRepository : BaseCacheRepository, INexusExp *********/ /// [MemberNotNullWhen(true, nameof(NexusExportCacheMemoryRepository.Data))] - public bool IsLoaded() + public override bool IsLoaded() { return this.Data?.Data.Count > 0; } /// - public async Task CanRefreshFromAsync(INexusExportApiClient client, int staleMinutes) + public override DateTimeOffset GetLastModified() { - DateTimeOffset serverLastModified = await client.FetchLastModifiedDateAsync(); + return this.Data?.LastUpdated ?? DateTimeOffset.MinValue; + } - return - !this.IsStale(serverLastModified, staleMinutes) - && ( - !this.IsLoaded() - || this.Data.LastUpdated < serverLastModified - ); + /// + public override void Clear() + { + this.SetData(null); } /// @@ -58,12 +55,5 @@ public void SetData(NexusFullExport? export) { this.Data = export; } - - /// - public bool IsStale(int staleMinutes) - { - DateTimeOffset? lastUpdated = this.Data?.LastUpdated; - return lastUpdated.HasValue && this.IsStale(lastUpdated.Value, staleMinutes); - } } } diff --git a/src/SMAPI.Web/SMAPI.Web.csproj b/src/SMAPI.Web/SMAPI.Web.csproj index b151e9dc1..00c833237 100644 --- a/src/SMAPI.Web/SMAPI.Web.csproj +++ b/src/SMAPI.Web/SMAPI.Web.csproj @@ -17,6 +17,7 @@ + diff --git a/src/SMAPI.Web/Startup.cs b/src/SMAPI.Web/Startup.cs index e7674c3ce..a66fef153 100644 --- a/src/SMAPI.Web/Startup.cs +++ b/src/SMAPI.Web/Startup.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Net; using Hangfire; +using Hangfire.Console; using Hangfire.MemoryStorage; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; @@ -95,7 +96,8 @@ public void ConfigureServices(IServiceCollection services) .SetDataCompatibilityLevel(CompatibilityLevel.Version_170) .UseSimpleAssemblyNameTypeSerializer() .UseRecommendedSerializerSettings() - .UseMemoryStorage(); + .UseMemoryStorage() + .UseConsole(); }); // init background service