From a916590a991631e215067dc22eef510fc08de126 Mon Sep 17 00:00:00 2001 From: halgari Date: Tue, 25 Jun 2024 21:48:55 -0600 Subject: [PATCH] Reworked most of the download code --- .../DownloadService.cs | 16 ++--- .../Interfaces/IDownloadTask.cs | 6 +- .../Tasks/ADownloadTask.cs | 46 +++++++-------- .../Tasks/HttpDownloadTask.cs | 13 +++- .../Tasks/NxmDownloadTask.cs | 42 ++++++++----- .../Tasks/State/CompletedDownloadState.cs | 30 +--------- .../Tasks/State/DownloaderState.cs | 13 ++-- .../Tasks/State/HttpDownloadState.cs | 4 +- .../Tasks/State/NxmDownloadState.cs | 59 ++----------------- 9 files changed, 86 insertions(+), 143 deletions(-) diff --git a/src/Networking/NexusMods.Networking.Downloaders/DownloadService.cs b/src/Networking/NexusMods.Networking.Downloaders/DownloadService.cs index 80ea280edc..2a42470564 100644 --- a/src/Networking/NexusMods.Networking.Downloaders/DownloadService.cs +++ b/src/Networking/NexusMods.Networking.Downloaders/DownloadService.cs @@ -46,7 +46,7 @@ internal IEnumerable GetItemsToResume() var db = _conn.Db; var tasks = db.Find(DownloaderState.Status) - .Select(x => db.Get(x)) + .Select(x => DownloaderState.Load(db, x)) .Where(x => x.Status != DownloadTaskStatus.Completed && x.Status != DownloadTaskStatus.Cancelled) .Select(GetTaskFromState) @@ -55,7 +55,7 @@ internal IEnumerable GetItemsToResume() return tasks; } - internal IDownloadTask? GetTaskFromState(DownloaderState.Model state) + internal IDownloadTask? GetTaskFromState(DownloaderState.ReadOnly state) { if (state.Contains(HttpDownloadState.Uri)) { @@ -151,10 +151,10 @@ public Task StartAsync(CancellationToken cancellationToken) { var found = e.Lookup(id); if (found.HasValue) - found.Value.ResetState(db); + found.Value.RefreshState(); else { - var task = GetTaskFromState(db.Get(id)); + var task = GetTaskFromState(DownloaderState.Load(db, id)); if (task == null) return; e.AddOrUpdate(task); @@ -168,14 +168,14 @@ public Task StartAsync(CancellationToken cancellationToken) .Subscribe() .DisposeWith(_disposables); + var db = _conn.Db; // Cancel any orphaned downloads - foreach (var task in _conn.Db.FindIndexed((byte)DownloadTaskStatus.Downloading, DownloaderState.Status)) - { + foreach (var task in DownloaderState.FindByStatus(db, DownloadTaskStatus.Downloading)) + { try { _logger.LogInformation("Cancelling orphaned download task {Task}", task); - var state = _conn.Db.Get(task); - var downloadTask = GetTaskFromState(state); + var downloadTask = GetTaskFromState(task); downloadTask?.Cancel(); } catch (Exception ex) diff --git a/src/Networking/NexusMods.Networking.Downloaders/Interfaces/IDownloadTask.cs b/src/Networking/NexusMods.Networking.Downloaders/Interfaces/IDownloadTask.cs index ea8bd7b8f5..3a48c19854 100644 --- a/src/Networking/NexusMods.Networking.Downloaders/Interfaces/IDownloadTask.cs +++ b/src/Networking/NexusMods.Networking.Downloaders/Interfaces/IDownloadTask.cs @@ -13,7 +13,7 @@ public interface IDownloadTask /// /// The DownloaderState of the task. /// - DownloaderState.Model PersistentState { get; } + DownloaderState.ReadOnly PersistentState { get; } /// /// The download location of the task. @@ -66,10 +66,10 @@ public interface IDownloadTask void SetIsHidden(bool isHidden, ITransaction tx); /// - /// Reset (reload) the persistent state of the task from the database. + /// Refresh (reload) the persistent state of the task from the database. /// /// - void ResetState(IDb db); + void RefreshState(); } /// diff --git a/src/Networking/NexusMods.Networking.Downloaders/Tasks/ADownloadTask.cs b/src/Networking/NexusMods.Networking.Downloaders/Tasks/ADownloadTask.cs index e81a9741f1..e6dd8e0fdf 100644 --- a/src/Networking/NexusMods.Networking.Downloaders/Tasks/ADownloadTask.cs +++ b/src/Networking/NexusMods.Networking.Downloaders/Tasks/ADownloadTask.cs @@ -37,7 +37,7 @@ public abstract class ADownloadTask : ReactiveObject, IDownloadTask protected TemporaryPath _downloadLocation = default!; protected IFileSystem FileSystem; protected IFileOriginRegistry FileOriginRegistry; - private DownloaderState.Model _persistentState = null!; + private DownloaderState.ReadOnly _persistentState; protected ADownloadTask(IServiceProvider provider) { @@ -51,16 +51,21 @@ protected ADownloadTask(IServiceProvider provider) FileSystem = provider.GetRequiredService(); FileOriginRegistry = provider.GetRequiredService(); } - - - - public void Init(DownloaderState.Model state) + + public void Init(DownloaderState.ReadOnly state) { PersistentState = state; Downloaded = state.Downloaded; _downloadLocation = new TemporaryPath(FileSystem, FileSystem.FromUnsanitizedFullPath(state.DownloadPath), false); } - + + /// + /// Reloads the state of the download task from the database. + /// + public void RefreshState() + { + PersistentState = PersistentState.Rebase(); + } /// /// Sets up the inital state of the download task, creates the persistent state @@ -69,8 +74,9 @@ public void Init(DownloaderState.Model state) protected EntityId Create(ITransaction tx) { _downloadLocation = TemporaryFileManager.CreateFile(); - var state = new DownloaderState.Model(tx) + var state = new DownloaderState.New(tx) { + FriendlyName = "", Status = DownloadTaskStatus.Idle, Downloaded = Size.Zero, DownloadPath = DownloadLocation.ToString(), @@ -85,7 +91,7 @@ protected EntityId Create(ITransaction tx) protected async Task Init(ITransaction tx, EntityId id) { var result = await tx.Commit(); - PersistentState = result.Db.Get(result[id]); + PersistentState = DownloaderState.Load(result.Db, result[id]); } protected async Task<(string Name, Size Size)> GetNameAndSizeAsync(Uri uri) @@ -114,7 +120,7 @@ protected async Task Init(ITransaction tx, EntityId id) protected async Task SetStatus(DownloadTaskStatus status) { using var tx = Connection.BeginTransaction(); - tx.Add(PersistentState.Id, DownloaderState.Status, (byte)status); + tx.Add(PersistentState.Id, DownloaderState.Status, status); if (TransientState != null) { @@ -129,21 +135,21 @@ protected async Task SetStatus(DownloadTaskStatus status) } } - var result = await tx.Commit(); - PersistentState = result.Remap(PersistentState); + await tx.Commit(); + RefreshState(); } protected async Task MarkComplete() { using var tx = Connection.BeginTransaction(); - tx.Add(PersistentState.Id, DownloaderState.Status, (byte)DownloadTaskStatus.Completed); + tx.Add(PersistentState.Id, DownloaderState.Status, DownloadTaskStatus.Completed); tx.Add(PersistentState.Id, CompletedDownloadState.CompletedDateTime, DateTime.Now); - var result = await tx.Commit(); - PersistentState = result.Remap(PersistentState); + await tx.Commit(); + RefreshState(); } [Reactive] - public DownloaderState.Model PersistentState { get; set; } = null!; + public DownloaderState.ReadOnly PersistentState { get; protected set; } public AbsolutePath DownloadLocation => _downloadLocation; @@ -229,7 +235,7 @@ private void UpdateActivity() if (report is { Current.HasValue: true }) { Downloaded = report.Current.Value; - if (PersistentState.TryGet(DownloaderState.Size, out var size) && size != Size.Zero) + if (DownloaderState.Size.TryGet(PersistentState, out var size) && size != Size.Zero) Progress = Percent.CreateClamped((long)Downloaded.Value, (long)size.Value); if (report.Throughput.HasValue) Bandwidth = Bandwidth.From(report.Throughput.Value.Value); @@ -247,13 +253,7 @@ public void SetIsHidden(bool isHidden, ITransaction tx) if (PersistentState.Status != DownloadTaskStatus.Completed) return; tx.Add(PersistentState.Id, CompletedDownloadState.Hidden, isHidden); } - - /// - public void ResetState(IDb db) - { - PersistentState = db.Get(PersistentState.Id); - } - + /// /// Begin the process of downloading a file to the specified destination, should /// terminate when the download is complete or cancelled. The destination may have diff --git a/src/Networking/NexusMods.Networking.Downloaders/Tasks/HttpDownloadTask.cs b/src/Networking/NexusMods.Networking.Downloaders/Tasks/HttpDownloadTask.cs index cc99e8fea4..374f50975e 100644 --- a/src/Networking/NexusMods.Networking.Downloaders/Tasks/HttpDownloadTask.cs +++ b/src/Networking/NexusMods.Networking.Downloaders/Tasks/HttpDownloadTask.cs @@ -33,8 +33,15 @@ public async Task Create(Uri uri) protected override async Task Download(AbsolutePath destination, CancellationToken token) { - var url = PersistentState.Get(HttpDownloadState.Uri); - var size = PersistentState.Get(DownloaderState.Size); - await HttpDownloader.DownloadAsync([url], destination, size, TransientState, token); + if (PersistentState.TryGetAsHttpDownloadState(out var httpState)) + { + await HttpDownloader.DownloadAsync([httpState.Uri], destination, PersistentState.Size, TransientState, token); + } + else + { + throw new InvalidOperationException("Download task is not a HTTP download task."); + } + + } } diff --git a/src/Networking/NexusMods.Networking.Downloaders/Tasks/NxmDownloadTask.cs b/src/Networking/NexusMods.Networking.Downloaders/Tasks/NxmDownloadTask.cs index 9bb6851d10..29aa358e59 100644 --- a/src/Networking/NexusMods.Networking.Downloaders/Tasks/NxmDownloadTask.cs +++ b/src/Networking/NexusMods.Networking.Downloaders/Tasks/NxmDownloadTask.cs @@ -21,6 +21,16 @@ public class NxmDownloadTask : ADownloadTask { private readonly INexusApiClient _nexusApiClient; + private NxmDownloadState.ReadOnly NxPersistentState + { + get + { + if (!PersistentState.TryGetAsNxmDownloadState(out var nxState)) + throw new InvalidOperationException("Download task is not a NXM download task."); + return nxState; + } + } + public NxmDownloadTask(IServiceProvider provider) : base(provider) { _nexusApiClient = provider.GetRequiredService(); @@ -84,7 +94,8 @@ private async Task UpdateMetadata(CancellationToken token) { try { - var nxState = PersistentState.Db.Get(PersistentState.Id); + var nxState = NxPersistentState; + var fileInfos = await _nexusApiClient.ModFilesAsync(nxState.Game, nxState.ModId, token); var file = fileInfos.Data.Files.FirstOrDefault(f => f.FileId == nxState.FileId); @@ -96,15 +107,15 @@ private async Task UpdateMetadata(CancellationToken token) if (file is { SizeInBytes: not null }) { using var tx = Connection.BeginTransaction(); - if (info.Data.Name is not null) + if (!string.IsNullOrWhiteSpace(info.Data.Name)) tx.Add(eid, DownloaderState.FriendlyName, info.Data.Name); else tx.Add(eid, DownloaderState.FriendlyName, file.FileName); tx.Add(eid, DownloaderState.Size, Size.FromLong(file.SizeInBytes!.Value)); tx.Add(eid, DownloaderState.Version, file.Version); - var result = await tx.Commit(); - PersistentState = result.Db.Get(eid); + await tx.Commit(); + RefreshState(); return true; } } @@ -124,25 +135,26 @@ private async Task UpdateSizeAndName(HttpRequestMessage[] message) using var tx = Connection.BeginTransaction(); tx.Add(PersistentState.Id, DownloaderState.Size, size); tx.Add(PersistentState.Id, DownloaderState.FriendlyName, name); - var nxState = PersistentState.Db.Get(PersistentState.Id); Logger.LogDebug("Updated size and name for {Name} to {Size}", name, size); - var result = await tx.Commit(); - PersistentState = result.Db.Get(PersistentState.Id); + await tx.Commit(); + RefreshState(); } - + private async Task InitDownloadLinks(CancellationToken token) { Response links; - var state = PersistentState.Db.Get(PersistentState.Id); - - if (!PersistentState.TryGet(NxmDownloadState.NxmKey, out var key)) - links = await _nexusApiClient.DownloadLinksAsync(state.Game, state.ModId, state.FileId, token); - else - links = await _nexusApiClient.DownloadLinksAsync(state.Game, state.ModId, state.FileId, NXMKey.From(state.NxmKey), - state.ValidUntil, token); + var nxState = NxPersistentState; + if (!NxmDownloadState.NxmKey.TryGet(nxState, out var nxmKey)) + { + links = await _nexusApiClient.DownloadLinksAsync(nxState.Game, nxState.ModId, nxState.FileId, token); + } + else + { + links = await _nexusApiClient.DownloadLinksAsync(nxState.Game, nxState.ModId, nxState.FileId, NXMKey.From(nxState.NxmKey), nxState.ValidUntil, token); + } return links.Data.Select(u => new HttpRequestMessage(HttpMethod.Get, u.Uri)).ToArray(); } diff --git a/src/Networking/NexusMods.Networking.Downloaders/Tasks/State/CompletedDownloadState.cs b/src/Networking/NexusMods.Networking.Downloaders/Tasks/State/CompletedDownloadState.cs index ca83088c04..140cdb5334 100644 --- a/src/Networking/NexusMods.Networking.Downloaders/Tasks/State/CompletedDownloadState.cs +++ b/src/Networking/NexusMods.Networking.Downloaders/Tasks/State/CompletedDownloadState.cs @@ -1,12 +1,14 @@ using NexusMods.Abstractions.MnemonicDB.Attributes; using NexusMods.MnemonicDB.Abstractions; +using NexusMods.MnemonicDB.Abstractions.Models; namespace NexusMods.Networking.Downloaders.Tasks.State; /// /// Additional state for a that is completed /// -public static class CompletedDownloadState +[Include] +public partial class CompletedDownloadState : IModelDefinition { private const string Namespace = "NexusMods.Networking.Downloaders.Tasks.DownloaderState"; @@ -19,30 +21,4 @@ public static class CompletedDownloadState /// Whether the download is hidden (clear action) in the UI /// public static readonly BooleanAttribute Hidden = new(Namespace, nameof(Hidden)); - - /// - /// Model for reading and writing CompletedDownloadStates - /// - /// - public class Model(ITransaction tx) : DownloaderState.Model(tx) - { - - /// - /// The timestamp the download was completed at - /// - public DateTime CompletedAt - { - get => CompletedDateTime.Get(this, default(DateTime)); - set => CompletedDateTime.Add(this, value); - } - - /// - /// Whether the download is hidden (clear action) in the UI - /// - public bool IsHidden - { - get => Hidden.Get(this, false); - set => Hidden.Add(this, value); - } - } } diff --git a/src/Networking/NexusMods.Networking.Downloaders/Tasks/State/DownloaderState.cs b/src/Networking/NexusMods.Networking.Downloaders/Tasks/State/DownloaderState.cs index fc750dbaee..16f1dc8fa2 100644 --- a/src/Networking/NexusMods.Networking.Downloaders/Tasks/State/DownloaderState.cs +++ b/src/Networking/NexusMods.Networking.Downloaders/Tasks/State/DownloaderState.cs @@ -1,11 +1,8 @@ -using NexusMods.Abstractions.Games.DTO; using NexusMods.Abstractions.MnemonicDB.Attributes; -using NexusMods.MnemonicDB.Abstractions; using NexusMods.MnemonicDB.Abstractions.Attributes; using NexusMods.MnemonicDB.Abstractions.Models; using NexusMods.Networking.Downloaders.Interfaces; using NexusMods.Paths; -using Entity = NexusMods.MnemonicDB.Abstractions.Models.Entity; namespace NexusMods.Networking.Downloaders.Tasks.State; @@ -28,7 +25,7 @@ public partial class DownloaderState : IModelDefinition /// /// Status of the task associated with this state. /// - public static readonly ByteAttribute Status = new(Namespace, nameof(Status)) { IsIndexed = true, NoHistory = true }; + public static readonly EnumByteAttribute Status = new(Namespace, nameof(Status)) { IsIndexed = true, NoHistory = true }; /// /// Path to the temporary file being downloaded. @@ -48,20 +45,20 @@ public partial class DownloaderState : IModelDefinition /// /// Amount of already downloaded bytes. /// - public static readonly SizeAttribute Downloaded = new(Namespace, nameof(Downloaded)); + public static readonly SizeAttribute Downloaded = new(Namespace, nameof(Downloaded)) { IsOptional = true }; /// /// Amount of already downloaded bytes. /// - public static readonly SizeAttribute Size = new(Namespace, nameof(Size)); + public static readonly SizeAttribute Size = new(Namespace, nameof(Size)) { IsOptional = true }; /// /// Domain of the game the mod will be installed to. /// - public static readonly GameDomainAttribute GameDomain = new(Namespace, nameof(GameDomain)) { IsIndexed = true}; + public static readonly GameDomainAttribute GameDomain = new(Namespace, nameof(GameDomain)) { IsIndexed = true, IsOptional = true}; /// /// Version of the mod; can sometimes be arbitrary and not follow SemVer or any standard. /// - public static readonly StringAttribute Version = new(Namespace, nameof(Version)); + public static readonly StringAttribute Version = new(Namespace, nameof(Version)) { IsOptional = true }; } diff --git a/src/Networking/NexusMods.Networking.Downloaders/Tasks/State/HttpDownloadState.cs b/src/Networking/NexusMods.Networking.Downloaders/Tasks/State/HttpDownloadState.cs index ae49923ede..70fb5e6534 100644 --- a/src/Networking/NexusMods.Networking.Downloaders/Tasks/State/HttpDownloadState.cs +++ b/src/Networking/NexusMods.Networking.Downloaders/Tasks/State/HttpDownloadState.cs @@ -1,4 +1,5 @@ using NexusMods.Abstractions.MnemonicDB.Attributes; +using NexusMods.MnemonicDB.Abstractions.Models; namespace NexusMods.Networking.Downloaders.Tasks.State; @@ -6,7 +7,8 @@ namespace NexusMods.Networking.Downloaders.Tasks.State; /// State specific to suspend. /// // ReSharper disable once PartialTypeWithSinglePart -public static class HttpDownloadState +[Include] +public partial class HttpDownloadState : IModelDefinition { private const string Namespace = "NexusMods.Networking.Downloaders.Tasks.State.HttpDownloadState"; diff --git a/src/Networking/NexusMods.Networking.Downloaders/Tasks/State/NxmDownloadState.cs b/src/Networking/NexusMods.Networking.Downloaders/Tasks/State/NxmDownloadState.cs index e24518d43f..8f174d4e04 100644 --- a/src/Networking/NexusMods.Networking.Downloaders/Tasks/State/NxmDownloadState.cs +++ b/src/Networking/NexusMods.Networking.Downloaders/Tasks/State/NxmDownloadState.cs @@ -1,10 +1,12 @@ using NexusMods.Abstractions.NexusWebApi.Types; using NexusMods.MnemonicDB.Abstractions; using NexusMods.MnemonicDB.Abstractions.Attributes; +using NexusMods.MnemonicDB.Abstractions.Models; namespace NexusMods.Networking.Downloaders.Tasks.State; -public static class NxmDownloadState +[Include] +public partial class NxmDownloadState : IModelDefinition { private const string Namespace = "NexusMods.Networking.Downloaders.Tasks.State.NxmDownloadState"; @@ -34,58 +36,5 @@ public static class NxmDownloadState /// on the website /// public static readonly StringAttribute NxmKey = new(Namespace, nameof(NXMKey)); - - - /// - /// Model for reading and writing NXMDownloadStates - /// - public class Model(ITransaction tx) : DownloaderState.Model(tx) - { - - /// - /// ModId of the download task - /// - public ModId ModId - { - get => NxmDownloadState.ModId.Get(this); - set => NxmDownloadState.ModId.Add(this, value); - } - - /// - /// FileId of the download task - /// - public FileId FileId - { - get => NxmDownloadState.FileId.Get(this); - set => NxmDownloadState.FileId.Add(this, value); - } - - /// - /// Game domain of the download task - /// - public string Game - { - get => NxmDownloadState.Game.Get(this); - set => NxmDownloadState.Game.Add(this, value); - } - - /// - /// Expiry date of the download key - /// - public DateTime ValidUntil - { - get => NxmDownloadState.ValidUntil.Get(this); - set => NxmDownloadState.ValidUntil.Add(this, value); - } - - /// - /// The NXM key of the download task, used for free users and clicking "Download with manager" - /// - public string NxmKey - { - get => NxmDownloadState.NxmKey.Get(this); - set => NxmDownloadState.NxmKey.Add(this, value); - } - } - + }