Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Reworked most of the download code
Browse files Browse the repository at this point in the history
halgari committed Jun 26, 2024
1 parent 1224072 commit a916590
Showing 9 changed files with 86 additions and 143 deletions.
Original file line number Diff line number Diff line change
@@ -46,7 +46,7 @@ internal IEnumerable<IDownloadTask> GetItemsToResume()
var db = _conn.Db;

var tasks = db.Find(DownloaderState.Status)
.Select(x => db.Get<DownloaderState.Model>(x))
.Select(x => DownloaderState.Load(db, x))
.Where(x => x.Status != DownloadTaskStatus.Completed &&
x.Status != DownloadTaskStatus.Cancelled)
.Select(GetTaskFromState)
@@ -55,7 +55,7 @@ internal IEnumerable<IDownloadTask> GetItemsToResume()
return tasks;
}

internal IDownloadTask? GetTaskFromState(DownloaderState.Model state)
internal IDownloadTask? GetTaskFromState(DownloaderState.ReadOnly state)
{
if (state.Contains(HttpDownloadState.Uri))
{
@@ -151,10 +151,10 @@ public Task StartAsync(CancellationToken cancellationToken)
{
var found = e.Lookup(id);
if (found.HasValue)
found.Value.ResetState(db);
found.Value.RefreshState();
else
{
var task = GetTaskFromState(db.Get<DownloaderState.Model>(id));
var task = GetTaskFromState(DownloaderState.Load(db, id));
if (task == null)
return;
e.AddOrUpdate(task);
@@ -168,14 +168,14 @@ public Task StartAsync(CancellationToken cancellationToken)
.Subscribe()
.DisposeWith(_disposables);

var db = _conn.Db;
// Cancel any orphaned downloads
foreach (var task in _conn.Db.FindIndexed((byte)DownloadTaskStatus.Downloading, DownloaderState.Status))
{
foreach (var task in DownloaderState.FindByStatus(db, DownloadTaskStatus.Downloading))
{
try
{
_logger.LogInformation("Cancelling orphaned download task {Task}", task);
var state = _conn.Db.Get<DownloaderState.Model>(task);
var downloadTask = GetTaskFromState(state);
var downloadTask = GetTaskFromState(task);
downloadTask?.Cancel();
}
catch (Exception ex)
Original file line number Diff line number Diff line change
@@ -13,7 +13,7 @@ public interface IDownloadTask
/// <summary>
/// The DownloaderState of the task.
/// </summary>
DownloaderState.Model PersistentState { get; }
DownloaderState.ReadOnly PersistentState { get; }

/// <summary>
/// The download location of the task.
@@ -66,10 +66,10 @@ public interface IDownloadTask
void SetIsHidden(bool isHidden, ITransaction tx);

/// <summary>
/// Reset (reload) the persistent state of the task from the database.
/// Refresh (reload) the persistent state of the task from the database.
/// </summary>
/// <param name="db"></param>
void ResetState(IDb db);
void RefreshState();
}

/// <summary>
Original file line number Diff line number Diff line change
@@ -37,7 +37,7 @@ public abstract class ADownloadTask : ReactiveObject, IDownloadTask
protected TemporaryPath _downloadLocation = default!;
protected IFileSystem FileSystem;
protected IFileOriginRegistry FileOriginRegistry;
private DownloaderState.Model _persistentState = null!;
private DownloaderState.ReadOnly _persistentState;

protected ADownloadTask(IServiceProvider provider)
{
@@ -51,16 +51,21 @@ protected ADownloadTask(IServiceProvider provider)
FileSystem = provider.GetRequiredService<IFileSystem>();
FileOriginRegistry = provider.GetRequiredService<IFileOriginRegistry>();
}



public void Init(DownloaderState.Model state)

public void Init(DownloaderState.ReadOnly state)
{
PersistentState = state;
Downloaded = state.Downloaded;
_downloadLocation = new TemporaryPath(FileSystem, FileSystem.FromUnsanitizedFullPath(state.DownloadPath), false);
}


/// <summary>
/// Reloads the state of the download task from the database.
/// </summary>
public void RefreshState()
{
PersistentState = PersistentState.Rebase();
}

/// <summary>
/// Sets up the inital state of the download task, creates the persistent state
@@ -69,8 +74,9 @@ public void Init(DownloaderState.Model state)
protected EntityId Create(ITransaction tx)
{
_downloadLocation = TemporaryFileManager.CreateFile();
var state = new DownloaderState.Model(tx)
var state = new DownloaderState.New(tx)
{
FriendlyName = "<Unknown>",
Status = DownloadTaskStatus.Idle,
Downloaded = Size.Zero,
DownloadPath = DownloadLocation.ToString(),
@@ -85,7 +91,7 @@ protected EntityId Create(ITransaction tx)
protected async Task Init(ITransaction tx, EntityId id)
{
var result = await tx.Commit();
PersistentState = result.Db.Get<DownloaderState.Model>(result[id]);
PersistentState = DownloaderState.Load(result.Db, result[id]);
}

protected async Task<(string Name, Size Size)> GetNameAndSizeAsync(Uri uri)
@@ -114,7 +120,7 @@ protected async Task Init(ITransaction tx, EntityId id)
protected async Task SetStatus(DownloadTaskStatus status)
{
using var tx = Connection.BeginTransaction();
tx.Add(PersistentState.Id, DownloaderState.Status, (byte)status);
tx.Add(PersistentState.Id, DownloaderState.Status, status);

if (TransientState != null)
{
@@ -129,21 +135,21 @@ protected async Task SetStatus(DownloadTaskStatus status)
}
}

var result = await tx.Commit();
PersistentState = result.Remap(PersistentState);
await tx.Commit();
RefreshState();
}

protected async Task MarkComplete()
{
using var tx = Connection.BeginTransaction();
tx.Add(PersistentState.Id, DownloaderState.Status, (byte)DownloadTaskStatus.Completed);
tx.Add(PersistentState.Id, DownloaderState.Status, DownloadTaskStatus.Completed);
tx.Add(PersistentState.Id, CompletedDownloadState.CompletedDateTime, DateTime.Now);
var result = await tx.Commit();
PersistentState = result.Remap(PersistentState);
await tx.Commit();
RefreshState();
}

[Reactive]
public DownloaderState.Model PersistentState { get; set; } = null!;
public DownloaderState.ReadOnly PersistentState { get; protected set; }

public AbsolutePath DownloadLocation => _downloadLocation;

@@ -229,7 +235,7 @@ private void UpdateActivity()
if (report is { Current.HasValue: true })
{
Downloaded = report.Current.Value;
if (PersistentState.TryGet(DownloaderState.Size, out var size) && size != Size.Zero)
if (DownloaderState.Size.TryGet(PersistentState, out var size) && size != Size.Zero)
Progress = Percent.CreateClamped((long)Downloaded.Value, (long)size.Value);
if (report.Throughput.HasValue)
Bandwidth = Bandwidth.From(report.Throughput.Value.Value);
@@ -247,13 +253,7 @@ public void SetIsHidden(bool isHidden, ITransaction tx)
if (PersistentState.Status != DownloadTaskStatus.Completed) return;
tx.Add(PersistentState.Id, CompletedDownloadState.Hidden, isHidden);
}

/// <inheritdoc />
public void ResetState(IDb db)
{
PersistentState = db.Get<DownloaderState.Model>(PersistentState.Id);
}


/// <summary>
/// Begin the process of downloading a file to the specified destination, should
/// terminate when the download is complete or cancelled. The destination may have
Original file line number Diff line number Diff line change
@@ -33,8 +33,15 @@ public async Task Create(Uri uri)

protected override async Task Download(AbsolutePath destination, CancellationToken token)
{
var url = PersistentState.Get(HttpDownloadState.Uri);
var size = PersistentState.Get(DownloaderState.Size);
await HttpDownloader.DownloadAsync([url], destination, size, TransientState, token);
if (PersistentState.TryGetAsHttpDownloadState(out var httpState))
{
await HttpDownloader.DownloadAsync([httpState.Uri], destination, PersistentState.Size, TransientState, token);
}
else
{
throw new InvalidOperationException("Download task is not a HTTP download task.");
}


}
}
Original file line number Diff line number Diff line change
@@ -21,6 +21,16 @@ public class NxmDownloadTask : ADownloadTask
{
private readonly INexusApiClient _nexusApiClient;

private NxmDownloadState.ReadOnly NxPersistentState
{
get
{
if (!PersistentState.TryGetAsNxmDownloadState(out var nxState))
throw new InvalidOperationException("Download task is not a NXM download task.");
return nxState;
}
}

public NxmDownloadTask(IServiceProvider provider) : base(provider)
{
_nexusApiClient = provider.GetRequiredService<INexusApiClient>();
@@ -84,7 +94,8 @@ private async Task<bool> UpdateMetadata(CancellationToken token)
{
try
{
var nxState = PersistentState.Db.Get<NxmDownloadState.Model>(PersistentState.Id);
var nxState = NxPersistentState;

var fileInfos = await _nexusApiClient.ModFilesAsync(nxState.Game, nxState.ModId, token);

var file = fileInfos.Data.Files.FirstOrDefault(f => f.FileId == nxState.FileId);
@@ -96,15 +107,15 @@ private async Task<bool> UpdateMetadata(CancellationToken token)
if (file is { SizeInBytes: not null })
{
using var tx = Connection.BeginTransaction();
if (info.Data.Name is not null)
if (!string.IsNullOrWhiteSpace(info.Data.Name))
tx.Add(eid, DownloaderState.FriendlyName, info.Data.Name);
else
tx.Add(eid, DownloaderState.FriendlyName, file.FileName);

tx.Add(eid, DownloaderState.Size, Size.FromLong(file.SizeInBytes!.Value));
tx.Add(eid, DownloaderState.Version, file.Version);
var result = await tx.Commit();
PersistentState = result.Db.Get<NxmDownloadState.Model>(eid);
await tx.Commit();
RefreshState();
return true;
}
}
@@ -124,25 +135,26 @@ private async Task UpdateSizeAndName(HttpRequestMessage[] message)
using var tx = Connection.BeginTransaction();
tx.Add(PersistentState.Id, DownloaderState.Size, size);
tx.Add(PersistentState.Id, DownloaderState.FriendlyName, name);
var nxState = PersistentState.Db.Get<NxmDownloadState.Model>(PersistentState.Id);

Logger.LogDebug("Updated size and name for {Name} to {Size}", name, size);
var result = await tx.Commit();
PersistentState = result.Db.Get<NxmDownloadState.Model>(PersistentState.Id);
await tx.Commit();
RefreshState();
}

private async Task<HttpRequestMessage[]> InitDownloadLinks(CancellationToken token)
{
Response<DownloadLink[]> links;

var state = PersistentState.Db.Get<NxmDownloadState.Model>(PersistentState.Id);

if (!PersistentState.TryGet(NxmDownloadState.NxmKey, out var key))
links = await _nexusApiClient.DownloadLinksAsync(state.Game, state.ModId, state.FileId, token);
else
links = await _nexusApiClient.DownloadLinksAsync(state.Game, state.ModId, state.FileId, NXMKey.From(state.NxmKey),
state.ValidUntil, token);
var nxState = NxPersistentState;

if (!NxmDownloadState.NxmKey.TryGet(nxState, out var nxmKey))
{
links = await _nexusApiClient.DownloadLinksAsync(nxState.Game, nxState.ModId, nxState.FileId, token);
}
else
{
links = await _nexusApiClient.DownloadLinksAsync(nxState.Game, nxState.ModId, nxState.FileId, NXMKey.From(nxState.NxmKey), nxState.ValidUntil, token);
}
return links.Data.Select(u => new HttpRequestMessage(HttpMethod.Get, u.Uri)).ToArray();
}

Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
using NexusMods.Abstractions.MnemonicDB.Attributes;
using NexusMods.MnemonicDB.Abstractions;
using NexusMods.MnemonicDB.Abstractions.Models;

namespace NexusMods.Networking.Downloaders.Tasks.State;

/// <summary>
/// Additional state for a <see cref="DownloaderState"/> that is completed
/// </summary>
public static class CompletedDownloadState
[Include<DownloaderState>]
public partial class CompletedDownloadState : IModelDefinition
{
private const string Namespace = "NexusMods.Networking.Downloaders.Tasks.DownloaderState";

@@ -19,30 +21,4 @@ public static class CompletedDownloadState
/// Whether the download is hidden (clear action) in the UI
/// </summary>
public static readonly BooleanAttribute Hidden = new(Namespace, nameof(Hidden));

/// <summary>
/// Model for reading and writing CompletedDownloadStates
/// </summary>
/// <param name="tx"></param>
public class Model(ITransaction tx) : DownloaderState.Model(tx)
{

/// <summary>
/// The timestamp the download was completed at
/// </summary>
public DateTime CompletedAt
{
get => CompletedDateTime.Get(this, default(DateTime));
set => CompletedDateTime.Add(this, value);
}

/// <summary>
/// Whether the download is hidden (clear action) in the UI
/// </summary>
public bool IsHidden
{
get => Hidden.Get(this, false);
set => Hidden.Add(this, value);
}
}
}
Original file line number Diff line number Diff line change
@@ -1,11 +1,8 @@
using NexusMods.Abstractions.Games.DTO;
using NexusMods.Abstractions.MnemonicDB.Attributes;
using NexusMods.MnemonicDB.Abstractions;
using NexusMods.MnemonicDB.Abstractions.Attributes;
using NexusMods.MnemonicDB.Abstractions.Models;
using NexusMods.Networking.Downloaders.Interfaces;
using NexusMods.Paths;
using Entity = NexusMods.MnemonicDB.Abstractions.Models.Entity;

namespace NexusMods.Networking.Downloaders.Tasks.State;

@@ -28,7 +25,7 @@ public partial class DownloaderState : IModelDefinition
/// <summary>
/// Status of the task associated with this state.
/// </summary>
public static readonly ByteAttribute Status = new(Namespace, nameof(Status)) { IsIndexed = true, NoHistory = true };
public static readonly EnumByteAttribute<DownloadTaskStatus> Status = new(Namespace, nameof(Status)) { IsIndexed = true, NoHistory = true };

/// <summary>
/// Path to the temporary file being downloaded.
@@ -48,20 +45,20 @@ public partial class DownloaderState : IModelDefinition
/// <summary>
/// Amount of already downloaded bytes.
/// </summary>
public static readonly SizeAttribute Downloaded = new(Namespace, nameof(Downloaded));
public static readonly SizeAttribute Downloaded = new(Namespace, nameof(Downloaded)) { IsOptional = true };

/// <summary>
/// Amount of already downloaded bytes.
/// </summary>
public static readonly SizeAttribute Size = new(Namespace, nameof(Size));
public static readonly SizeAttribute Size = new(Namespace, nameof(Size)) { IsOptional = true };

/// <summary>
/// Domain of the game the mod will be installed to.
/// </summary>
public static readonly GameDomainAttribute GameDomain = new(Namespace, nameof(GameDomain)) { IsIndexed = true};
public static readonly GameDomainAttribute GameDomain = new(Namespace, nameof(GameDomain)) { IsIndexed = true, IsOptional = true};

/// <summary>
/// Version of the mod; can sometimes be arbitrary and not follow SemVer or any standard.
/// </summary>
public static readonly StringAttribute Version = new(Namespace, nameof(Version));
public static readonly StringAttribute Version = new(Namespace, nameof(Version)) { IsOptional = true };
}
Loading

0 comments on commit a916590

Please sign in to comment.