Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Source generator read models #1626

Merged
merged 70 commits into from
Jun 26, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
70 commits
Select commit Hold shift + click to select a range
1ff1526
Filestore and attributes mostly done
halgari Jun 4, 2024
107ac8b
WIP, slowly porting over Loadouts
halgari Jun 5, 2024
fb7b613
Loadouts project compiles
halgari Jun 5, 2024
bcea443
Work on syncronizer code
halgari Jun 5, 2024
bf6739d
WIP
halgari Jun 10, 2024
904ab0f
Update to new IndexSegment code from MnemonicDB
halgari Jun 10, 2024
4a5eaeb
Merge branch 'main' into source-generator-read-models
halgari Jun 10, 2024
7d5d62d
Abstraction tests pass
halgari Jun 10, 2024
64a9113
WIP
halgari Jun 10, 2024
c8b08f5
Datamodel almost passes
halgari Jun 10, 2024
0a9f757
Datamodel tests compile
halgari Jun 10, 2024
063447a
Apply service tests pass
halgari Jun 11, 2024
c94227d
All datamodel tests (except one) pass. Waiting on Github actions to c…
halgari Jun 11, 2024
4f7ef2e
Tons of code rework
halgari Jun 11, 2024
f4b3332
Nexus API compiles
halgari Jun 12, 2024
c899adf
Bannerlord compiles
halgari Jun 12, 2024
1ba9c3e
SDV compiles
halgari Jun 12, 2024
452c36b
Only the UI left
halgari Jun 12, 2024
6fbd93d
32 errors left, but it's almost midnight
halgari Jun 12, 2024
ee992ba
Everything builds, ship it, I'm sure it'll be fine
halgari Jun 12, 2024
f929ce0
Downloader tests pass
halgari Jun 12, 2024
5dc8cc2
Fix NexusWebApi tests
halgari Jun 12, 2024
cb4f21d
CLI tests pass
halgari Jun 12, 2024
f21c9ad
Bethesda tests pass
halgari Jun 12, 2024
7b8921a
Cyberpunk tests pass, UI tests pass
halgari Jun 12, 2024
3dc4b9e
Fix loadout deletion
halgari Jun 12, 2024
92cdcef
Merge branch 'main' into source-generator-read-models
halgari Jun 13, 2024
c8e2d5d
Wow tests actually pass if run in smaller groups
halgari Jun 13, 2024
1d92ce1
Fix a few bugs, delete end-to-end tests that we no longer want
halgari Jun 13, 2024
7156c53
Fix several exceptions that popped up during testing
halgari Jun 17, 2024
abd1b3f
Fix a bunch of remaining exceptions with observables
halgari Jun 17, 2024
48624d7
Can log in and out of the Nexus
halgari Jun 17, 2024
8c58766
Merge branch 'main' into source-generator-read-models
halgari Jun 18, 2024
08f372b
Merge main
halgari Jun 18, 2024
5a28436
Update MnemonicDB to handle the new feedback from reviewing MnemonicDB
halgari Jun 23, 2024
fe7977a
Merge branch 'main' into source-generator-read-models
halgari Jun 23, 2024
d75fb58
Merge main
halgari Jun 23, 2024
1b0a798
Handle feedback in Loadout.cs
halgari Jun 24, 2024
774d536
Update src/Abstractions/NexusMods.Abstractions.Loadouts/Files/Metadat…
halgari Jun 24, 2024
92e40ce
Update src/Abstractions/NexusMods.Abstractions.Loadouts/Files/File.cs
halgari Jun 24, 2024
0696741
Update src/Abstractions/NexusMods.Abstractions.Loadouts.Synchronizers…
halgari Jun 24, 2024
118dc82
Update src/Abstractions/NexusMods.Abstractions.Loadouts.Synchronizers…
halgari Jun 24, 2024
78c2b31
Update src/Games/NexusMods.Games.StardewValley/Models/SMAPIModDatabas…
halgari Jun 24, 2024
b3b0051
PR feedback
halgari Jun 24, 2024
b6fba4d
More PR feedback
halgari Jun 24, 2024
5959e03
Merge remote-tracking branch 'origin/source-generator-read-models' in…
halgari Jun 24, 2024
e2b743a
Update src/Games/NexusMods.Games.StardewValley/Models/SMAPIManifestMe…
halgari Jun 25, 2024
89b7191
Update src/Abstractions/NexusMods.Abstractions.Loadouts.Synchronizers…
halgari Jun 25, 2024
eaa8ca3
Update src/NexusMods.App/TelemetryProvider.cs
halgari Jun 25, 2024
d08333e
Update src/NexusMods.DataModel/GameRegistry/Registry.cs
halgari Jun 25, 2024
40280eb
Update src/NexusMods.DataModel/ArchiveInstaller.cs
halgari Jun 25, 2024
db56969
Update src/NexusMods.App/TelemetryProvider.cs
halgari Jun 25, 2024
01145ab
Fix compiler issues, handle some of the feedback
halgari Jun 25, 2024
fe68569
Merge branch 'main' into source-generator-read-models
halgari Jun 25, 2024
c6e6e75
Merge main
halgari Jun 25, 2024
9650a19
Handle some feedback
halgari Jun 25, 2024
2517289
Handle more feedback
halgari Jun 25, 2024
f14b169
remove `!hasValue`
halgari Jun 25, 2024
233d313
Handle final feedback
halgari Jun 25, 2024
0106d50
All works except the file downloading handler
halgari Jun 25, 2024
f0c1ffd
Fix the broken downloader tests
halgari Jun 25, 2024
466b054
Handle exceptions
halgari Jun 25, 2024
a0c6dfa
Revert changes to the download service so I can re-do the work so I b…
halgari Jun 25, 2024
6fcd267
Compiles again
halgari Jun 26, 2024
132e15c
Fix the `ObserveDatoms` bug in DownloadService.cs
halgari Jun 26, 2024
68640ce
Fix a bug with ModEnabledViewModel.cs
halgari Jun 26, 2024
6eff368
Fix the `Version` exception
halgari Jun 26, 2024
9570c94
Fix completed items not showing up in the download window
halgari Jun 26, 2024
1224072
WIP
halgari Jun 26, 2024
a916590
Reworked most of the download code
halgari Jun 26, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 6 additions & 5 deletions Directory.Packages.props
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,12 @@
<PackageVersion Include="LinqGen" Version="0.3.1" />
<PackageVersion Include="Nerdbank.FullDuplexStream" Version="1.1.12" />
<PackageVersion Include="Nerdbank.Streams" Version="2.11.74" />
<PackageVersion Include="NexusMods.MnemonicDB" Version="0.9.29" />
<PackageVersion Include="NexusMods.MnemonicDB.Abstractions" Version="0.9.29" />
<PackageVersion Include="NexusMods.MnemonicDB.Storage" Version="0.9.29" />
<PackageVersion Include="NexusMods.Paths" Version="0.9.4" />
<PackageVersion Include="NexusMods.MnemonicDB" Version="0.9.52" />
<PackageVersion Include="NexusMods.MnemonicDB.Abstractions" Version="0.9.52" />
<PackageVersion Include="NexusMods.MnemonicDB.Storage" Version="0.9.52" />
<PackageVersion Include="NexusMods.Paths" Version="0.9.5" />
<PackageVersion Include="NexusMods.Hashing.xxHash64" Version="2.0.1" />
<PackageVersion Include="NexusMods.Paths.TestingHelpers" Version="0.9.4" />
<PackageVersion Include="NexusMods.Paths.TestingHelpers" Version="0.9.5" />
<PackageVersion Include="NexusMods.Archives.Nx" Version="0.4.0" />
<PackageVersion Include="NexusMods.Telemetry.OpenTelemetry" Version="1.0.0" />
<PackageVersion Include="FomodInstaller.Interface" Version="1.2.0" />
Expand Down Expand Up @@ -113,6 +113,7 @@
<PackageVersion Include="Humanizer" Version="2.14.1" />
<PackageVersion Include="ini-parser-netstandard" Version="2.5.2" />
<PackageVersion Include="Mutagen.Bethesda.Skyrim" Version="0.44.0" />
<PackageVersion Include="NexusMods.MnemonicDB.SourceGenerator" Version="0.9.52" />
<PackageVersion Include="NLog.Extensions.Logging" Version="5.3.11" />
<PackageVersion Include="OneOf" Version="3.0.271" />
<PackageVersion Include="ReactiveUI.Fody" Version="19.5.41" />
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ public class ABenchmarkDatamodel(IServiceProvider provider) : ADataModelTest<ABe

public new IGame Game => base.Game;
public new GameInstallation Install => base.Install;
public new Loadout.Model BaseLoadout => base.BaseLoadout;
public new Loadout.ReadOnly BaseLoadout => base.BaseLoadout;

public static ABenchmarkDatamodel WithMod(IServiceProvider provider, string modName, IEnumerable<string> files)
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ protected void Init(string baseModName, string fileList)
if (_defaultSynchronizer == null)
throw new Exception($"Can't cast synchronizer to {typeof(DefaultSynchronizer)}. Did the test StubbedGame code change?");

_installation = _datamodel.BaseLoadout.Installation;
_installation = _datamodel.BaseLoadout.InstallationInstance;
_diskStateRegistry = _serviceProvider.GetRequiredService<IDiskStateRegistry>();
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
using NexusMods.Abstractions.DiskState;
using NexusMods.Abstractions.GameLocators;
using NexusMods.Abstractions.Games.DTO;
using NexusMods.Abstractions.Loadouts.Files;
using NexusMods.Benchmarks.Benchmarks.Loadouts.Harness;
using NexusMods.Benchmarks.Interfaces;
using NexusMods.Hashing.xxHash64;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,16 +1,13 @@
using NexusMods.Abstractions.MnemonicDB.Attributes;
using NexusMods.Hashing.xxHash64;
using NexusMods.MnemonicDB.Abstractions;
using NexusMods.MnemonicDB.Abstractions.Attributes;
using NexusMods.MnemonicDB.Abstractions.Models;
using NexusMods.Paths;

namespace NexusMods.Abstractions.DiskState;

/// <summary>
/// Attributes for each entry in the HashCache
/// </summary>
public static class HashCacheEntry
public partial class HashCacheEntry : IModelDefinition
{
private const string Namespace = "NexusMods.Abstractions.DiskState.HashCacheEntry";

Expand All @@ -33,45 +30,4 @@ public static class HashCacheEntry
/// The size of the file
/// </summary>
public static readonly SizeAttribute Size = new(Namespace, nameof(Size)) { NoHistory = true };


public class Model(ITransaction tx) : Entity(tx, (byte)IdPartitions.HashCache)
{
/// <summary>
/// The xxHash64 of the name
/// </summary>
public Hash NameHash
{
get => HashCacheEntry.NameHash.Get(this);
set => HashCacheEntry.NameHash.Add(this, value);
}

/// <summary>
/// The xxHash64 hash of the file
/// </summary>
public Hash Hash
{
get => HashCacheEntry.Hash.Get(this);
set => HashCacheEntry.Hash.Add(this, value);
}

/// <summary>
/// Last time the file was modified
/// </summary>
public DateTime LastModified
{
get => HashCacheEntry.LastModified.Get(this);
set => HashCacheEntry.LastModified.Add(this, value);
}

/// <summary>
/// The size of the file
/// </summary>
public Size Size
{
get => HashCacheEntry.Size.Get(this);
set => HashCacheEntry.Size.Add(this, value);
}
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ public interface IFileHashCache
/// you should call <see cref="IFileHashCache.IndexFileAsync" />; which will update the underlying
/// cached item.
/// </remarks>
bool TryGetCached(AbsolutePath path, out HashCacheEntry.Model entry);
bool TryGetCached(AbsolutePath path, out HashCacheEntry.ReadOnly entry);

/// <summary>
/// Asynchronously indexes the folder specified by <paramref name="path" />; putting it in the cache.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
<ItemGroup>
<ProjectReference Include="..\NexusMods.Abstractions.GameLocators\NexusMods.Abstractions.GameLocators.csproj" />
<ProjectReference Include="..\NexusMods.Abstractions.Loadouts\NexusMods.Abstractions.Loadouts.csproj" />
<PackageReference Include="NexusMods.MnemonicDB.SourceGenerator" PrivateAssets="all" OutputItemType="Analyzer" ReferenceOutputAssembly="false" />
</ItemGroup>

<ItemGroup>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
using NexusMods.Hashing.xxHash64;
using NexusMods.MnemonicDB.Abstractions;
using NexusMods.MnemonicDB.Abstractions.Attributes;
using NexusMods.MnemonicDB.Abstractions.BuiltInEntities;
using NexusMods.MnemonicDB.Abstractions.IndexSegments;
using NexusMods.MnemonicDB.Abstractions.Models;
using NexusMods.MnemonicDB.Storage;
Expand All @@ -15,7 +16,7 @@ namespace NexusMods.Abstractions.FileStore.Downloads;
/// <summary>
/// Attributes for the analysis data for a downloaded file
/// </summary>
public static class DownloadAnalysis
public partial class DownloadAnalysis : IModelDefinition
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Our download models are a complete mess, but I think StreamBasedFileOriginMetadata should include DownloadAnalysis.

CC @Al12rs.

Copy link
Contributor

@Al12rs Al12rs Jun 24, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we keep both yeah. It might be worth making a graph of these and decide what we actually need.
I'm not sure whether DownloadAnalysis still needs to be same entity as the DownloadTask .

Refactoring of download related entities could happen in a separate PRs if necessary.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I agree, we should do a pass on the analysis data this sprint

{
private const string Namespace = "NexusMods.Abstractions.FileStore.Downloads.DownloadAnalysis";
/// <summary>
Expand Down Expand Up @@ -46,82 +47,30 @@ public static class DownloadAnalysis
/// for example "the_awesome_mod_v1.0.zip"
/// </summary>
public static readonly StringAttribute SuggestedName = new(Namespace, nameof(SuggestedName));


public class Model(ITransaction tx) : Entity(tx)
{

/// <summary>
/// The hash of the downloaded archive from which this download is sourced from.
/// </summary>
public Hash Hash
{
get => DownloadAnalysis.Hash.Get(this);
set => DownloadAnalysis.Hash.Add(this, value);
}

/// <summary>
/// Size of the downloaded archive from which this download is sourced from.
/// </summary>
public Size Size
{
get => DownloadAnalysis.Size.Get(this);
set => DownloadAnalysis.Size.Add(this, value);
}


/// <summary>
/// The number of entries in the download
/// </summary>
public ulong Count
{
get => NumberOfEntries.Get(this);
set => NumberOfEntries.Add(this, value);
}

/// <summary>
/// The human-readable, friendly name for this download
/// </summary>
public string SuggestedName
{
get => DownloadAnalysis.SuggestedName.Get(this, "<Unknown>");
set => DownloadAnalysis.SuggestedName.Add(this, value);
}
/// <summary>
/// The contents of the download
/// </summary>
public static readonly BackReferenceAttribute<DownloadContentEntry> Contents = new(DownloadContentEntry.DownloadAnalysis);

/// <summary>
/// The contents of the download
/// </summary>
public Entities<EntityIds, DownloadContentEntry.Model> Contents
=> GetReverse<DownloadContentEntry.Model>(DownloadContentEntry.DownloadAnalysis);



public partial struct ReadOnly
{
/// <summary>
/// Get a file tree of the download's contents
/// </summary>
public ModFileTreeNode GetFileTree(IFileStore? fs = null)
{
return TreeCreator.Create(Contents, fs);
}

/// <summary>
/// The timestamp of the creation of this <see cref="DownloadAnalysis"/> entity
/// Download start time or first manual installation time
/// </summary>
public DateTime GetCreatedAt()
{
// Get the lowest transaction id, then get the timestamp of that transaction
var t = this.Select(d => d.T).Min();
var txEntity = Db.Get<Entity>(EntityId.From(t.Value));
return BuiltInAttributes.TxTimestanp.Get(txEntity);
}
}
}


/// <summary>
/// A single entry in the download analysis, this is a file that is contained in the download
/// </summary>
public static class DownloadContentEntry
public partial class DownloadContentEntry : IModelDefinition
{
private const string Namespace = "NexusMods.Abstractions.FileStore.Downloads.DownloadContentEntry";

Expand All @@ -143,46 +92,13 @@ public static class DownloadContentEntry
/// <summary>
/// The DownloadAnalysis that this entry is a part of
/// </summary>
public static readonly ReferenceAttribute DownloadAnalysis = new(Namespace, nameof(DownloadAnalysis));

public class Model(ITransaction tx) : Entity(tx, (byte)IdPartitions.DownloadAnalysis), TreeCreator.ITreeCreatorNode
public static readonly ReferenceAttribute<DownloadAnalysis> DownloadAnalysis = new(Namespace, nameof(DownloadAnalysis));

/// <inheritdoc />
public partial struct ReadOnly : TreeCreator.ITreeCreatorNode
{

/// <summary>
/// Hash of the file
/// </summary>
public Hash Hash
{
get => DownloadContentEntry.Hash.Get(this);
set => DownloadContentEntry.Hash.Add(this, value);
}

/// <summary>
/// Size of the file
/// </summary>
public Size Size
{
get => DownloadContentEntry.Size.Get(this);
set => DownloadContentEntry.Size.Add(this, value);
}

/// <summary>
/// Path of the file
/// </summary>
public RelativePath Path
{
get => DownloadContentEntry.Path.Get(this);
set => DownloadContentEntry.Path.Add(this, value);
}

public DownloadId DownloadAnalysisId
{
get => DownloadId.From(DownloadAnalysis.Get(this));
set => DownloadAnalysis.Add(this, value.Value);
}

}

}

/// <summary>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,19 +49,20 @@ public ValueTask<DownloadId> RegisterDownload(AbsolutePath path, string modName,
/// Indexes an already extracted download, returns a download id that can be used to retrieve the download later.
/// </summary>
public ValueTask<DownloadId> RegisterFolder(AbsolutePath path, MetadataFn metaDataFn, string modName, CancellationToken token = default);

/// <summary>
/// Get the analysis of a download
/// </summary>
public DownloadAnalysis.Model Get(DownloadId id);
public DownloadAnalysis.ReadOnly Get(DownloadId id);

/// <summary>
/// Get the analysis of all downloads
/// </summary>
public IEnumerable<DownloadAnalysis.Model> GetAll();
public IEnumerable<DownloadAnalysis.ReadOnly> GetAll();

/// <summary>
/// Finds all downloads that have the given hash
/// </summary>
public IEnumerable<DownloadAnalysis.Model> GetBy(Hash hash);
public IEnumerable<DownloadAnalysis.ReadOnly> GetBy(Hash hash);

}
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
<ProjectReference Include="..\NexusMods.Abstractions.Serialization\NexusMods.Abstractions.Serialization.csproj" />
<PackageReference Include="NexusMods.MnemonicDB.Storage" />
<PackageReference Include="TransparentValueObjects" PrivateAssets="all" OutputItemType="Analyzer" ReferenceOutputAssembly="false" />
<PackageReference Include="NexusMods.MnemonicDB.SourceGenerator" PrivateAssets="all" OutputItemType="Analyzer" ReferenceOutputAssembly="false" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="NexusMods.Paths" />
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,5 +16,5 @@ public interface ILoadoutDiagnosticEmitter : IDiagnosticEmitter
/// <summary>
/// Diagnoses a loadout and creates instances of <see cref="Diagnostic"/>.
/// </summary>
IAsyncEnumerable<Diagnostic> Diagnose(Loadout.Model loadout, CancellationToken cancellationToken);
IAsyncEnumerable<Diagnostic> Diagnose(Loadout.ReadOnly loadout, CancellationToken cancellationToken);
}
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,12 @@ public interface IDataReference
/// Resolves the data at <see cref="TxId"/> to an <see cref="Entity"/>.
/// </summary>
/// <returns><c>null</c> if the value doesn't exist in the data store.</returns>
Entity? ResolveData(IServiceProvider serviceProvider, TxId dataStore);
IReadOnlyModel? ResolveData(IServiceProvider serviceProvider, TxId dataStore);

/// <summary>
/// Converts the data from <see cref="ResolveData"/> to a string representation.
/// </summary>
string ToStringRepresentation(Entity data);
string ToStringRepresentation(IReadOnlyModel data);
}

/// <summary>
Expand All @@ -37,19 +37,19 @@ public interface IDataReference
/// <seealso cref="IDataReference"/>
[PublicAPI]
public interface IDataReference<out TDataId, TData> : IDataReference
where TData : Entity
where TData : IReadOnlyModel
{
/// <summary>
/// Gets the ID of the referenced data.
/// </summary>
TDataId DataId { get; }

/// <inheritdoc/>
Entity? IDataReference.ResolveData(IServiceProvider serviceProvider, TxId dataStore)
IReadOnlyModel? IDataReference.ResolveData(IServiceProvider serviceProvider, TxId dataStore)
=> ResolveData(serviceProvider, dataStore);

/// <inheritdoc/>
string IDataReference.ToStringRepresentation(Entity data)
string IDataReference.ToStringRepresentation(IReadOnlyModel data)
{
if (data is not TData actualData)
throw new ArgumentException($"Argument is not of type '{typeof(TData)}' but '{data.GetType()}'", nameof(data));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ namespace NexusMods.Abstractions.Diagnostics.References;
/// A reference to a <see cref="Loadout"/>.
/// </summary>
[PublicAPI]
public record LoadoutReference : IDataReference<LoadoutId, Loadout.Model>
public record LoadoutReference : IDataReference<LoadoutId, Loadout.ReadOnly>
{
/// <inheritdoc/>
public required TxId TxId { get; init; }
Expand All @@ -18,12 +18,12 @@ public record LoadoutReference : IDataReference<LoadoutId, Loadout.Model>
public required LoadoutId DataId { get; init; }

/// <inheritdoc/>
public Loadout.Model? ResolveData(IServiceProvider serviceProvider, IConnection dataStore)
public Loadout.ReadOnly ResolveData(IServiceProvider serviceProvider, IConnection dataStore)
{
var db = dataStore.AsOf(TxId);
return db.Get<Loadout.Model>(DataId.Value);
return Loadout.Load(db, DataId.Value);
}

/// <inheritdoc/>
public string ToStringRepresentation(Loadout.Model data) => data.Name;
public string ToStringRepresentation(Loadout.ReadOnly data) => data.Name;
}
Loading
Loading