From 9d57ff11f016415b50eb5623355ee2a5d3194330 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 18 Sep 2023 15:27:46 -0600 Subject: [PATCH] Remove analyzers (#615) * Removed file analyzers * Reworked installers to use the new FileTreeNode format, and to perform analysis on the fly --- Directory.Packages.props | 4 +- .../Benchmarks/DataModel.cs | 5 - .../NexusMods.FileExtractor.csproj | 1 + .../PluginAnalysisData.cs | 2 +- .../PluginAnalyzer.cs | 45 ++- .../Services.cs | 2 +- .../Analyzers/ProjectAnalyzer.cs | 48 --- .../Installers/LooseFilesModInstaller.cs | 23 +- .../Installers/NativeModInstaller.cs | 67 ++-- .../Models/ModProject.cs | 2 +- .../NexusMods.Games.DarkestDungeon.csproj | 1 + .../Services.cs | 3 - .../NexusMods.Games.FOMOD/FomodAnalyzer.cs | 92 ++--- .../FomodXmlInstaller.cs | 58 ++-- src/Games/NexusMods.Games.FOMOD/Services.cs | 1 - .../Entities/IniAnalysisData.cs | 11 - .../FileAnalyzers/IniAnalysisData.cs | 7 + .../FileAnalyzers/IniAnalzyer.cs | 14 +- .../Installers/GenericFolderMatchInstaller.cs | 28 +- .../NexusMods.Games.Generic.csproj | 4 + src/Games/NexusMods.Games.Generic/Services.cs | 7 - .../NexusMods.Games.Generic/TypeFinder.cs | 17 - .../Cyberpunk2077.cs | 6 +- .../FileAnalyzers/RedModInfoAnalyzer.cs | 44 --- .../ModInstallers/AppearancePreset.cs | 45 ++- .../ModInstallers/FolderlessModInstaller.cs | 47 +-- .../ModInstallers/RedModInstaller.cs | 90 ++--- .../SimpleOverlayModInstaller.cs | 104 ++---- .../NexusMods.Games.RedEngine/Services.cs | 5 - .../NexusMods.Games.RedEngine/TypeFinder.cs | 17 - .../ReshadePresetInstaller.cs | 85 ++--- src/Games/NexusMods.Games.Sifu/Sifu.cs | 10 +- .../NexusMods.Games.Sifu/SifuModInstaller.cs | 49 ++- .../Analyzers/SMAPIManifestAnalyzer.cs | 54 --- .../Constants.cs | 10 + .../Emitters/001_MissingDependencies.cs | 48 ++- .../Installers/SMAPIInstaller.cs | 78 +++-- .../Installers/SMAPIModInstaller.cs | 81 ++--- .../Models/SMAPIManifest.cs | 2 +- .../NexusMods.Games.StardewValley.csproj | 1 + .../NexusMods.Games.StardewValley/Services.cs | 6 - .../StardewValley.cs | 2 +- .../Verbs/StressTest.cs | 13 +- .../DownloadService.cs | 33 +- .../Interfaces/IDownloadService.cs | 3 +- .../AdvancedHttpDownloader.cs | 2 +- .../LoadoutGrid/LoadoutGridViewModel.cs | 20 +- .../Windows/MainWindowViewModel.cs | 9 +- src/NexusMods.CLI/Services.cs | 1 - .../NxmDownloadProtocolHandler.cs | 29 +- src/NexusMods.CLI/Verbs/AnalyzeArchive.cs | 27 +- .../Verbs/DownloadAndInstallMod.cs | 19 +- src/NexusMods.CLI/Verbs/InstallMod.cs | 17 +- .../AbsolutePathExtensions.cs | 14 +- .../AsyncEnumerableExtensions.cs | 21 -- src/NexusMods.Common/StreamExtensions.cs | 5 +- .../Abstractions/DTOs/DownloadAnalysis.cs | 84 +++++ .../Abstractions/Entity.cs | 2 - .../Abstractions/EntityCategory.cs | 5 + .../Abstractions/IArchiveAnalyzer.cs | 28 -- .../Abstractions/IArchiveInstaller.cs | 4 +- .../Abstractions/IArchiveManager.cs | 13 +- .../Abstractions/IDownloadRegistry.cs | 63 ++++ .../Abstractions/IFileAnalysisData.cs | 13 - .../Abstractions/IFileAnalyzer.cs | 64 ---- .../Abstractions/Ids/IId.cs | 13 + .../Abstractions/LogicalIds/DownloadId.cs | 17 + src/NexusMods.DataModel/ArchiveAnalyzer.cs | 314 ------------------ .../ArchiveContents/AnalyzedArchive.cs | 19 -- .../ArchiveContents/AnalyzedFile.cs | 70 ---- .../ArchiveContents/FileContainedIn.cs | 45 --- src/NexusMods.DataModel/ArchiveId.cs | 18 + src/NexusMods.DataModel/ArchiveInstaller.cs | 42 ++- .../ArchiveManagerStreamFactory.cs | 42 +++ .../ArchiveMetaData/AArchiveMetaData.cs | 40 +-- .../ArchiveMetaData/FileArchiveMetaData.cs | 36 -- .../ArchiveMetaData/FilePathMetadata.cs | 16 + .../ArchiveMetaData/GameArchiveMetadata.cs | 14 + .../NexusModsArchiveMetadata.cs | 27 ++ .../ChunkedStreams/ChunkedStream.cs | 2 + .../Diagnostics/DiagnosticManager.cs | 15 +- .../Emitters/ILoadoutDiagnosticEmitter.cs | 2 +- .../Diagnostics/IDiagnosticManager.cs | 2 +- src/NexusMods.DataModel/DownloadRegistry.cs | 134 ++++++++ .../Extensions/AnalyzedFileExtensions.cs | 46 --- .../Extensions/MetadataExtensions.cs | 22 -- src/NexusMods.DataModel/FileHashCache.cs | 3 +- .../GlobalSettings/GlobalSettingsManager.cs | 8 + .../Loadouts/LoadoutManager.cs | 37 ++- .../Loadouts/LoadoutSyncronizer.cs | 15 +- .../ModInstallers/AModInstaller.cs | 35 ++ .../ModInstallers/IModInstaller.cs | 9 +- .../ModInstallers/ModSourceFileEntry.cs | 54 +++ .../NexusMods.DataModel.csproj | 1 + src/NexusMods.DataModel/NxArchiveManager.cs | 10 +- src/NexusMods.DataModel/Services.cs | 5 +- src/NexusMods.DataModel/ZipArchiveManager.cs | 33 +- .../PluginAnalysisTests.cs | 16 +- .../SkyrimLegendaryEditionTests.cs | 2 +- .../SkyrimSpecialEditionTests.cs | 11 +- .../Analyzers/ProjectAnalyzerTests.cs | 31 -- .../Installers/LooseFilesModInstallerTests.cs | 13 +- .../Installers/NativeModInstallerTests.cs | 12 +- .../FomodTestHelpers.cs | 54 ++- .../FomodXmlAnalyzerTests.cs | 97 ++---- .../FomodXmlInstallerTests.cs | 210 ++++-------- .../NexusMods.Games.FOMOD.Tests/Startup.cs | 6 + .../ModInstallerTests.cs | 9 +- .../ModInstallers/AppearancePresetTests.cs | 17 +- .../ModInstallers/RedModInstallerTests.cs | 34 +- .../SimpleOverlayModInstallerTests.cs | 20 -- .../RedModInfoAnalyzerTests.cs | 27 -- .../SifuModInstallerTests.cs | 7 +- .../Analyzers/SMAPIManifestAnalyzerTests.cs | 107 ------ .../Emitters/001_MissingDependencyTests.cs | 4 +- .../Installers/SMAPIInstallerTests.cs | 34 +- .../Installers/SMAPIModInstallerTests.cs | 50 ++- .../AFileAnalyzerTest.cs | 68 ---- .../AGameTest.cs | 89 +++-- .../ALoadoutDiagnosticEmitterTest.cs | 9 +- .../AModInstallerTest.cs | 80 +++-- .../DependencyInjectionHelper.cs | 1 - .../ModInstallerExampleFile.cs | 8 +- .../VerbTests/AnalyzeArchive.cs | 8 +- .../VerbTests/ModManagementVerbs.cs | 2 +- .../ApplicationTests.cs | 19 +- .../ArchiveManagerTests.cs | 16 +- .../Diagnostics/DiagnosticManagerTests.cs | 2 +- .../DummyLoadoutDiagnosticEmitter.cs | 2 +- .../Harness/ADataModelTest.cs | 11 +- .../Harness/ALoadoutSynrconizerTest.cs | 5 +- tests/NexusMods.DataModel.Tests/Startup.cs | 1 - .../StubbedGames/StubbedGame.cs | 6 +- .../StubbedGames/StubbedGameInstaller.cs | 13 +- tests/NexusMods.UI.Tests/AVmTest.cs | 14 +- 135 files changed, 1627 insertions(+), 2349 deletions(-) delete mode 100644 src/Games/NexusMods.Games.DarkestDungeon/Analyzers/ProjectAnalyzer.cs delete mode 100644 src/Games/NexusMods.Games.Generic/Entities/IniAnalysisData.cs create mode 100644 src/Games/NexusMods.Games.Generic/FileAnalyzers/IniAnalysisData.cs delete mode 100644 src/Games/NexusMods.Games.Generic/TypeFinder.cs delete mode 100644 src/Games/NexusMods.Games.RedEngine/FileAnalyzers/RedModInfoAnalyzer.cs delete mode 100644 src/Games/NexusMods.Games.RedEngine/TypeFinder.cs delete mode 100644 src/Games/NexusMods.Games.StardewValley/Analyzers/SMAPIManifestAnalyzer.cs create mode 100644 src/Games/NexusMods.Games.StardewValley/Constants.cs delete mode 100644 src/NexusMods.Common/AsyncEnumerableExtensions.cs create mode 100644 src/NexusMods.DataModel/Abstractions/DTOs/DownloadAnalysis.cs delete mode 100644 src/NexusMods.DataModel/Abstractions/IArchiveAnalyzer.cs create mode 100644 src/NexusMods.DataModel/Abstractions/IDownloadRegistry.cs delete mode 100644 src/NexusMods.DataModel/Abstractions/IFileAnalysisData.cs delete mode 100644 src/NexusMods.DataModel/Abstractions/IFileAnalyzer.cs create mode 100644 src/NexusMods.DataModel/Abstractions/LogicalIds/DownloadId.cs delete mode 100644 src/NexusMods.DataModel/ArchiveAnalyzer.cs delete mode 100644 src/NexusMods.DataModel/ArchiveContents/AnalyzedArchive.cs delete mode 100644 src/NexusMods.DataModel/ArchiveContents/AnalyzedFile.cs delete mode 100644 src/NexusMods.DataModel/ArchiveContents/FileContainedIn.cs create mode 100644 src/NexusMods.DataModel/ArchiveId.cs create mode 100644 src/NexusMods.DataModel/ArchiveManagerStreamFactory.cs delete mode 100644 src/NexusMods.DataModel/ArchiveMetaData/FileArchiveMetaData.cs create mode 100644 src/NexusMods.DataModel/ArchiveMetaData/FilePathMetadata.cs create mode 100644 src/NexusMods.DataModel/ArchiveMetaData/GameArchiveMetadata.cs create mode 100644 src/NexusMods.DataModel/ArchiveMetaData/NexusModsArchiveMetadata.cs create mode 100644 src/NexusMods.DataModel/DownloadRegistry.cs delete mode 100644 src/NexusMods.DataModel/Extensions/AnalyzedFileExtensions.cs delete mode 100644 src/NexusMods.DataModel/Extensions/MetadataExtensions.cs create mode 100644 src/NexusMods.DataModel/ModInstallers/AModInstaller.cs create mode 100644 src/NexusMods.DataModel/ModInstallers/ModSourceFileEntry.cs delete mode 100644 tests/Games/NexusMods.Games.DarkestDungeon.Tests/Analyzers/ProjectAnalyzerTests.cs delete mode 100644 tests/Games/NexusMods.Games.RedEngine.Tests/RedModInfoAnalyzerTests.cs delete mode 100644 tests/Games/NexusMods.Games.StardewValley.Tests/Analyzers/SMAPIManifestAnalyzerTests.cs delete mode 100644 tests/Games/NexusMods.Games.TestFramework/AFileAnalyzerTest.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index f71122ea63..5b936c9b24 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -1,8 +1,8 @@ - - + + diff --git a/benchmarks/NexusMods.Benchmarks/Benchmarks/DataModel.cs b/benchmarks/NexusMods.Benchmarks/Benchmarks/DataModel.cs index d8ec9d2648..b0a0c740c1 100644 --- a/benchmarks/NexusMods.Benchmarks/Benchmarks/DataModel.cs +++ b/benchmarks/NexusMods.Benchmarks/Benchmarks/DataModel.cs @@ -26,7 +26,6 @@ public class DataStoreBenchmark : IBenchmark, IDisposable private readonly IDataStore _dataStore; private readonly byte[] _rawData; private readonly Id64 _rawId; - private readonly HashRelativePath _fromPutPath; private readonly IId _immutableRecord; private readonly FromArchive _record; @@ -52,10 +51,6 @@ public DataStoreBenchmark() Random.Shared.NextBytes(_rawData); _rawId = new Id64(EntityCategory.TestData, (ulong)Random.Shared.NextInt64()); _dataStore.PutRaw(_rawId, _rawData); - - var relPutPath = "test.txt".ToRelativePath(); - _fromPutPath = new HashRelativePath(Hash.From((ulong)Random.Shared.NextInt64()), relPutPath); - _record = new FromArchive { Id = ModFileId.New(), diff --git a/src/ArchiveManagement/NexusMods.FileExtractor/NexusMods.FileExtractor.csproj b/src/ArchiveManagement/NexusMods.FileExtractor/NexusMods.FileExtractor.csproj index 0ceed6d208..69b2a06115 100644 --- a/src/ArchiveManagement/NexusMods.FileExtractor/NexusMods.FileExtractor.csproj +++ b/src/ArchiveManagement/NexusMods.FileExtractor/NexusMods.FileExtractor.csproj @@ -10,6 +10,7 @@ + diff --git a/src/Games/NexusMods.Games.BethesdaGameStudios/PluginAnalysisData.cs b/src/Games/NexusMods.Games.BethesdaGameStudios/PluginAnalysisData.cs index 5d55b5f78a..623580186d 100644 --- a/src/Games/NexusMods.Games.BethesdaGameStudios/PluginAnalysisData.cs +++ b/src/Games/NexusMods.Games.BethesdaGameStudios/PluginAnalysisData.cs @@ -5,7 +5,7 @@ namespace NexusMods.Games.BethesdaGameStudios; [JsonName("BethesdaGameStudios.FileAnalysisData")] -public class PluginAnalysisData : IFileAnalysisData +public class PluginAnalysisData { public required RelativePath[] Masters { get; init; } public bool IsLightMaster { get; init; } diff --git a/src/Games/NexusMods.Games.BethesdaGameStudios/PluginAnalyzer.cs b/src/Games/NexusMods.Games.BethesdaGameStudios/PluginAnalyzer.cs index 5ca3054039..fd62effe4d 100644 --- a/src/Games/NexusMods.Games.BethesdaGameStudios/PluginAnalyzer.cs +++ b/src/Games/NexusMods.Games.BethesdaGameStudios/PluginAnalyzer.cs @@ -1,4 +1,3 @@ -using System.Runtime.CompilerServices; using JetBrains.Annotations; using Microsoft.Extensions.Logging; using Mutagen.Bethesda; @@ -7,7 +6,6 @@ using Mutagen.Bethesda.Plugins.Meta; using Mutagen.Bethesda.Plugins.Records; using Mutagen.Bethesda.Skyrim; -using NexusMods.DataModel.Abstractions; using NexusMods.DataModel.Abstractions.Ids; using NexusMods.FileExtractor.FileSignatures; using NexusMods.Paths; @@ -17,17 +15,13 @@ namespace NexusMods.Games.BethesdaGameStudios; [UsedImplicitly] -public class PluginAnalyzer : IFileAnalyzer +public class PluginAnalyzer { - public FileAnalyzerId Id { get; } = FileAnalyzerId.New("9c673a4f-064f-4b1e-83e3-4bf0454575cd", 1); - - public IEnumerable FileTypes => new[] { FileType.TES4 }; - - private static readonly Extension[] ValidExtensions = { - new(".esp"), - new(".esm"), - new(".esl"), - }; + private static readonly HashSet ValidExtensions = new Extension[] { + new (".esp"), + new (".esm"), + new (".esl"), + }.ToHashSet(); private readonly ILogger _logger; @@ -36,12 +30,11 @@ public PluginAnalyzer(ILogger logger) _logger = logger; } -#pragma warning disable CS1998 - public async IAsyncEnumerable AnalyzeAsync(FileAnalyzerInfo info, [EnumeratorCancellation] CancellationToken ct = default) -#pragma warning restore CS1998 + public async Task AnalyzeAsync(RelativePath path, Stream stream, CancellationToken ct = default) { - var extension = info.FileName.ToRelativePath().Extension; - if (ValidExtensions[0] != extension && ValidExtensions[1] != extension && ValidExtensions[2] != extension) yield break; + var extension = path.Extension; + if (!ValidExtensions.Contains(extension)) + return null; // NOTE(erri120): The GameConstant specifies the header length. // - Oblivion: 20 bytes @@ -54,24 +47,24 @@ public async IAsyncEnumerable AnalyzeAsync(FileAnalyzerInfo i // The current solution just tries different GameConstants, which isn't ideal and // should be replaced with an identification step that finds the correct GameConstant. - var startPos = info.Stream.Position; - var fileAnalysisData = Analyze(GameConstants.SkyrimLE, info); + var fileAnalysisData = Analyze(path, GameConstants.SkyrimLE, stream); if (fileAnalysisData is null) { - info.Stream.Position = startPos; - fileAnalysisData = Analyze(GameConstants.Oblivion, info); - if (fileAnalysisData is null) yield break; + stream.Position = 0; + fileAnalysisData = Analyze(path, GameConstants.Oblivion, stream); + if (fileAnalysisData is null) + return null; } - yield return fileAnalysisData; + return fileAnalysisData; } - private IFileAnalysisData? Analyze(GameConstants targetGame, FileAnalyzerInfo info) + private PluginAnalysisData? Analyze(RelativePath path, GameConstants targetGame, Stream stream) { try { using var readStream = new MutagenInterfaceReadStream( - new BinaryReadStream(info.Stream, dispose: false), + new BinaryReadStream(stream, dispose: false), new ParsingBundle(targetGame, masterReferences: null!) { ModKey = ModKey.Null @@ -102,7 +95,7 @@ public async IAsyncEnumerable AnalyzeAsync(FileAnalyzerInfo i } catch (Exception e) { - _logger.LogError(e, "Exception while parsing {} ({})", info.FileName, info.RelativePath); + _logger.LogError(e, "Exception while parsing {} ({})", path.FileName, path); return null; } } diff --git a/src/Games/NexusMods.Games.BethesdaGameStudios/Services.cs b/src/Games/NexusMods.Games.BethesdaGameStudios/Services.cs index bc8ab18bb6..380f398a76 100644 --- a/src/Games/NexusMods.Games.BethesdaGameStudios/Services.cs +++ b/src/Games/NexusMods.Games.BethesdaGameStudios/Services.cs @@ -14,7 +14,7 @@ public static IServiceCollection AddBethesdaGameStudios(this IServiceCollection services.AddAllSingleton(); services.AddSingleton(); services.AddSingleton(); - services.AddAllSingleton(); + services.AddSingleton(); services.AddAllSingleton(); return services; } diff --git a/src/Games/NexusMods.Games.DarkestDungeon/Analyzers/ProjectAnalyzer.cs b/src/Games/NexusMods.Games.DarkestDungeon/Analyzers/ProjectAnalyzer.cs deleted file mode 100644 index f2f1d075d3..0000000000 --- a/src/Games/NexusMods.Games.DarkestDungeon/Analyzers/ProjectAnalyzer.cs +++ /dev/null @@ -1,48 +0,0 @@ -using System.Runtime.CompilerServices; -using System.Xml; -using System.Xml.Schema; -using System.Xml.Serialization; -using NexusMods.DataModel.Abstractions; -using NexusMods.DataModel.Abstractions.Ids; -using NexusMods.FileExtractor.FileSignatures; -using NexusMods.Games.DarkestDungeon.Models; - -namespace NexusMods.Games.DarkestDungeon.Analyzers; - -/// -/// implementation for native Darkest Dungeon mods that have a -/// project.xml file. The file format is described in the game support document and -/// deserialized to . -/// -public class ProjectAnalyzer : IFileAnalyzer -{ - public FileAnalyzerId Id => FileAnalyzerId.New("3152ea61-a5a1-4d89-a780-25ebbab3da3f", 2); - public IEnumerable FileTypes => new[] { FileType.XML }; - - public async IAsyncEnumerable AnalyzeAsync( - FileAnalyzerInfo info, - [EnumeratorCancellation] CancellationToken token = default) - { - await Task.Yield(); - - var res = Analyze(info); - if (res is null) yield break; - yield return res; - } - - private static ModProject? Analyze(FileAnalyzerInfo info) - { - if (!info.FileName.Equals("project.xml", StringComparison.OrdinalIgnoreCase)) - return null; - - using var reader = XmlReader.Create(info.Stream, new XmlReaderSettings - { - IgnoreComments = true, - IgnoreWhitespace = true, - ValidationFlags = XmlSchemaValidationFlags.AllowXmlAttributes - }); - - var obj = new XmlSerializer(typeof(ModProject)).Deserialize(reader); - return obj as ModProject; - } -} diff --git a/src/Games/NexusMods.Games.DarkestDungeon/Installers/LooseFilesModInstaller.cs b/src/Games/NexusMods.Games.DarkestDungeon/Installers/LooseFilesModInstaller.cs index ac96ed20cf..99664bf0c3 100644 --- a/src/Games/NexusMods.Games.DarkestDungeon/Installers/LooseFilesModInstaller.cs +++ b/src/Games/NexusMods.Games.DarkestDungeon/Installers/LooseFilesModInstaller.cs @@ -9,6 +9,7 @@ using NexusMods.Hashing.xxHash64; using NexusMods.Paths; using NexusMods.Paths.Extensions; +using NexusMods.Paths.FileTree; namespace NexusMods.Games.DarkestDungeon.Installers; @@ -19,26 +20,18 @@ public class LooseFilesModInstaller : IModInstaller { private static readonly RelativePath ModsFolder = "mods".ToRelativePath(); - public ValueTask> GetModsAsync( + public async ValueTask> GetModsAsync( GameInstallation gameInstallation, ModId baseModId, - Hash srcArchiveHash, - EntityDictionary archiveFiles, + FileTreeNode archiveFiles, CancellationToken cancellationToken = default) - { - return ValueTask.FromResult(GetMods(baseModId, srcArchiveHash, archiveFiles)); - } - - private IEnumerable GetMods( - ModId baseModId, - Hash srcArchiveHash, - EntityDictionary archiveFiles) { var files = archiveFiles + .GetAllDescendentFiles() .Select(kv => { var (path, file) = kv; - return file.ToFromArchive( + return file!.ToFromArchive( new GamePath(GameFolderType.Game, ModsFolder.Join(path)) ); }); @@ -47,13 +40,13 @@ private IEnumerable GetMods( // this needs to be serialized to XML and added to the files enumerable var modProject = new ModProject { - Title = archiveFiles.First().Key.TopParent.ToString() + Title = archiveFiles.Path.TopParent.ToString() }; - yield return new ModInstallerResult + return new [] { new ModInstallerResult { Id = baseModId, Files = files, - }; + }}; } } diff --git a/src/Games/NexusMods.Games.DarkestDungeon/Installers/NativeModInstaller.cs b/src/Games/NexusMods.Games.DarkestDungeon/Installers/NativeModInstaller.cs index de4bc34bdc..1af8afc5b8 100644 --- a/src/Games/NexusMods.Games.DarkestDungeon/Installers/NativeModInstaller.cs +++ b/src/Games/NexusMods.Games.DarkestDungeon/Installers/NativeModInstaller.cs @@ -1,4 +1,7 @@ using System.Diagnostics; +using System.Xml; +using System.Xml.Schema; +using System.Xml.Serialization; using NexusMods.Common; using NexusMods.DataModel.Abstractions; using NexusMods.DataModel.ArchiveContents; @@ -10,6 +13,7 @@ using NexusMods.Hashing.xxHash64; using NexusMods.Paths; using NexusMods.Paths.Extensions; +using NexusMods.Paths.FileTree; namespace NexusMods.Games.DarkestDungeon.Installers; @@ -22,57 +26,60 @@ public class NativeModInstaller : IModInstaller private static readonly RelativePath ModsFolder = "mods".ToRelativePath(); private static readonly RelativePath ProjectFile = "project.xml".ToRelativePath(); - internal static IEnumerable> GetModProjects( - EntityDictionary archiveFiles) + internal static async Task Node, ModProject Project)>> + GetModProjects( + FileTreeNode archiveFiles) { - return archiveFiles.Where(kv => - { - var (path, file) = kv; + return await archiveFiles + .GetAllDescendentFiles() + .SelectAsync(async kv => + { + if (kv.Path.FileName != ProjectFile) + return default; - if (!path.FileName.Equals(ProjectFile)) return false; - var modProject = file.AnalysisData - .OfType() - .FirstOrDefault(); + await using var stream = await kv.Value!.Open(); + using var reader = XmlReader.Create(stream, new XmlReaderSettings + { + IgnoreComments = true, + IgnoreWhitespace = true, + ValidationFlags = XmlSchemaValidationFlags.AllowXmlAttributes + }); - return modProject is not null; - }); + var obj = new XmlSerializer(typeof(ModProject)).Deserialize(reader); + return (kv, obj as ModProject); + }) + .Where(r => r.Item2 is not null) + .Select(r => (r.kv, r.Item2!)) + .ToArrayAsync(); } - public ValueTask> GetModsAsync( + public async ValueTask> GetModsAsync( GameInstallation gameInstallation, ModId baseModId, - Hash srcArchiveHash, - EntityDictionary archiveFiles, + FileTreeNode archiveFiles, CancellationToken cancellationToken = default) { - return ValueTask.FromResult(GetMods(srcArchiveHash, archiveFiles)); - } - - private static IEnumerable GetMods( - Hash srcArchiveHash, - EntityDictionary archiveFiles) - { - var modProjectFiles = GetModProjects(archiveFiles).ToArray(); + var modProjectFiles = (await GetModProjects(archiveFiles)).ToArray(); if (!modProjectFiles.Any()) return Array.Empty(); + if (modProjectFiles.Length > 1) + return Array.Empty(); + var mods = modProjectFiles .Select(modProjectFile => { - var parent = modProjectFile.Key.Parent; - var modProject = modProjectFile.Value.AnalysisData - .OfType() - .FirstOrDefault(); + var parent = modProjectFile.Node.Parent; + var modProject = modProjectFile.Project; if (modProject is null) throw new UnreachableException(); - var modFiles = archiveFiles - .Where(kv => kv.Key.InFolder(parent)) + var modFiles = parent.GetAllDescendentFiles() .Select(kv => { var (path, file) = kv; - return file.ToFromArchive( - new GamePath(GameFolderType.Game, ModsFolder.Join(path.DropFirst(parent.Depth))) + return file!.ToFromArchive( + new GamePath(GameFolderType.Game, ModsFolder.Join(path.DropFirst(parent.Depth - 1))) ); }); diff --git a/src/Games/NexusMods.Games.DarkestDungeon/Models/ModProject.cs b/src/Games/NexusMods.Games.DarkestDungeon/Models/ModProject.cs index aa09c6210f..e5b0f6b255 100644 --- a/src/Games/NexusMods.Games.DarkestDungeon/Models/ModProject.cs +++ b/src/Games/NexusMods.Games.DarkestDungeon/Models/ModProject.cs @@ -9,7 +9,7 @@ namespace NexusMods.Games.DarkestDungeon.Models; /// [JsonName("NexusMods.Games.DarkestDungeon.ModProject")] [XmlRoot(ElementName = "project")] -public record ModProject : IFileAnalysisData +public record ModProject { [XmlElement(ElementName = "Title")] public string Title { get; set; } = string.Empty; diff --git a/src/Games/NexusMods.Games.DarkestDungeon/NexusMods.Games.DarkestDungeon.csproj b/src/Games/NexusMods.Games.DarkestDungeon/NexusMods.Games.DarkestDungeon.csproj index a5c1f89809..ccad0b7559 100644 --- a/src/Games/NexusMods.Games.DarkestDungeon/NexusMods.Games.DarkestDungeon.csproj +++ b/src/Games/NexusMods.Games.DarkestDungeon/NexusMods.Games.DarkestDungeon.csproj @@ -11,4 +11,5 @@ + diff --git a/src/Games/NexusMods.Games.DarkestDungeon/Services.cs b/src/Games/NexusMods.Games.DarkestDungeon/Services.cs index c959823f83..74a08ce294 100644 --- a/src/Games/NexusMods.Games.DarkestDungeon/Services.cs +++ b/src/Games/NexusMods.Games.DarkestDungeon/Services.cs @@ -1,9 +1,7 @@ using Microsoft.Extensions.DependencyInjection; -using NexusMods.DataModel.Abstractions; using NexusMods.DataModel.Games; using NexusMods.DataModel.JsonConverters.ExpressionGenerator; using NexusMods.DataModel.ModInstallers; -using NexusMods.Games.DarkestDungeon.Analyzers; using NexusMods.Games.DarkestDungeon.Installers; namespace NexusMods.Games.DarkestDungeon; @@ -13,7 +11,6 @@ public static class Services public static IServiceCollection AddDarkestDungeon(this IServiceCollection serviceCollection) { serviceCollection.AddSingleton(); - serviceCollection.AddSingleton(); serviceCollection.AddSingleton(); serviceCollection.AddSingleton(); serviceCollection.AddSingleton(); diff --git a/src/Games/NexusMods.Games.FOMOD/FomodAnalyzer.cs b/src/Games/NexusMods.Games.FOMOD/FomodAnalyzer.cs index 0af2a0b9ff..1a495c3cda 100644 --- a/src/Games/NexusMods.Games.FOMOD/FomodAnalyzer.cs +++ b/src/Games/NexusMods.Games.FOMOD/FomodAnalyzer.cs @@ -1,48 +1,25 @@ -using System.Runtime.CompilerServices; -using System.Text; +using System.Text; using FomodInstaller.Scripting.XmlScript; using JetBrains.Annotations; -using Microsoft.Extensions.Logging; -using NexusMods.DataModel.Abstractions; -using NexusMods.DataModel.Abstractions.Ids; -using NexusMods.DataModel.JsonConverters; -using NexusMods.FileExtractor.FileSignatures; +using NexusMods.Common; +using NexusMods.DataModel.ModInstallers; using NexusMods.Paths; +using NexusMods.Paths.FileTree; namespace NexusMods.Games.FOMOD; -public class FomodAnalyzer : IFileAnalyzer +public class FomodAnalyzer { - public FileAnalyzerId Id { get; } = FileAnalyzerId.New("e5dcce84-ad7c-4882-8873-4f6a2e45a279", 1); - - private readonly ILogger _logger; - private readonly IFileSystem _fileSystem; - - // Note: No type for .fomod because FOMODs are existing archive types listed below. - public FomodAnalyzer(ILogger logger, IFileSystem fileSystem) - { - _logger = logger; - _fileSystem = fileSystem; - } - - public IEnumerable FileTypes { get; } = new [] { FileType.XML }; - - public async IAsyncEnumerable AnalyzeAsync(FileAnalyzerInfo info, [EnumeratorCancellation] CancellationToken ct = default) + public static async ValueTask AnalyzeAsync(FileTreeNode allFiles, IFileSystem fileSystem, + CancellationToken ct = default) { - // Not sourced from an archive. - if (info.RelativePath == null) - yield break; - // Check if file path is "fomod/ModuleConfig.xml" - if (!info.RelativePath.Value.EndsWith(FomodConstants.XmlConfigRelativePath)) - yield break; - - // If not from inside an archive, this is probably not a FOMOD. - if (info.ParentArchive == null) - yield break; + if (!allFiles.GetAllDescendentFiles() + .TryGetFirst(x => x.Path.EndsWith(FomodConstants.XmlConfigRelativePath), out var xmlNode)) + return null; // If the fomod folder is not at first level, find the prefix. - var pathPrefix = info.RelativePath.Value.Parent.Parent; + var pathPrefix = xmlNode!.Parent.Parent; // Now get the actual items out. // Determine if this is a supported FOMOD. @@ -51,7 +28,8 @@ public async IAsyncEnumerable AnalyzeAsync(FileAnalyzerInfo i try { - using var streamReader = new StreamReader(info.Stream, leaveOpen:true); + await using var stream = await xmlNode.Value!.Open(); + using var streamReader = new StreamReader(stream, leaveOpen:true); data = await streamReader.ReadToEndAsync(ct); var xmlScript = new XmlScriptType(); var script = (XmlScript)xmlScript.LoadScript(data, true); @@ -61,21 +39,19 @@ async Task AddImageIfValid(string? imagePathFragment) { if (string.IsNullOrEmpty(imagePathFragment)) return; - var imagePath = pathPrefix.Join(RelativePath.FromUnsanitizedInput(imagePathFragment)); - - var path = info.ParentArchive!.Value.Path.Combine(imagePath); + var imagePath = pathPrefix.Path.Join(RelativePath.FromUnsanitizedInput(imagePathFragment)); byte[] bytes; try { - bytes = await path.ReadAllBytesAsync(ct); - } - catch (FileNotFoundException) - { - bytes = await GetPlaceholderImage(ct); + var node = allFiles.FindNode(imagePath); + await using var imageStream = await node!.Value!.Open(); + using var ms = new MemoryStream(); + await imageStream.CopyToAsync(ms, ct); + bytes = ms.ToArray(); } - catch (DirectoryNotFoundException) + catch (Exception) { - bytes = await GetPlaceholderImage(ct); + bytes = await GetPlaceholderImage(fileSystem, ct); } images!.Add(new FomodAnalyzerInfo.FomodAnalyzerImage(imagePath, bytes)); @@ -87,47 +63,45 @@ async Task AddImageIfValid(string? imagePathFragment) foreach (var option in optionGroup.Options) await AddImageIfValid(option.ImagePath); } - catch (Exception e) + catch (Exception) { - _logger.LogError("Failed to Parse FOMOD: {EMessage}\\n{EStackTrace}", e.Message, e.StackTrace); - yield break; + return null; } // TODO: We use Base64 here, which is really, really inefficient. We should zip up the images and store them separately in a non-SOLID archive. // Add all images to analysis output. - yield return new FomodAnalyzerInfo() + return new FomodAnalyzerInfo { XmlScript = data!, - Images = images + Images = images, + PathPrefix = pathPrefix.Path }; } - internal async Task GetPlaceholderImage(CancellationToken ct = default) + public static Task GetPlaceholderImage(IFileSystem fileSystem, CancellationToken ct = default) { - return await _fileSystem.GetKnownPath(KnownPath.EntryDirectory).Combine("Assets/InvalidImagePlaceholder.png") - .ReadAllBytesAsync(ct); + return fileSystem.GetKnownPath(KnownPath.EntryDirectory).Combine("Assets/InvalidImagePlaceholder.png").ReadAllBytesAsync(ct); } } -[JsonName("NexusMods.Games.FOMOD.FomodAnalyzerInfo")] -public record FomodAnalyzerInfo : IFileAnalysisData +public record FomodAnalyzerInfo { public required string XmlScript { get; init; } public required List Images { get; init; } + public required RelativePath PathPrefix { get; init; } + public record struct FomodAnalyzerImage(string Path, byte[] Image); // Keeping in case this is ever needed. We can remove this once all FOMOD stuff is done. [PublicAPI] - public async Task DumpToFileSystemAsync(TemporaryPath fomodFolder) + public async Task DumpToFileSystemAsync(AbsolutePath fomodFolder) { var fs = FileSystem.Shared; - var path = fomodFolder.Path; - // Dump Item async Task DumpItem(string relativePath, byte[] data) { - var finalPath = path.Combine(relativePath); + var finalPath = fomodFolder.Combine(relativePath); fs.CreateDirectory(finalPath.Parent); await fs.WriteAllBytesAsync(finalPath, data); } diff --git a/src/Games/NexusMods.Games.FOMOD/FomodXmlInstaller.cs b/src/Games/NexusMods.Games.FOMOD/FomodXmlInstaller.cs index ab797dfee5..bea339bc8f 100644 --- a/src/Games/NexusMods.Games.FOMOD/FomodXmlInstaller.cs +++ b/src/Games/NexusMods.Games.FOMOD/FomodXmlInstaller.cs @@ -12,18 +12,21 @@ using NexusMods.DataModel.ModInstallers; using NexusMods.Hashing.xxHash64; using NexusMods.Paths; +using NexusMods.Paths.FileTree; using NexusMods.Paths.Utilities; using OneOf.Types; using Mod = FomodInstaller.Interface.Mod; namespace NexusMods.Games.FOMOD; -public class FomodXmlInstaller : IModInstaller +public class FomodXmlInstaller : AModInstaller { private readonly ICoreDelegates _delegates; private readonly XmlScriptType _scriptType = new(); private readonly ILogger _logger; private readonly GamePath _fomodInstallationPath; + private readonly IFileSystem _fileSystem; + private readonly TemporaryFileManager _tempoaryFileManager; /// /// Creates a new instance of given the provided and . @@ -34,30 +37,25 @@ public class FomodXmlInstaller : IModInstaller public static FomodXmlInstaller Create(IServiceProvider provider, GamePath fomodInstallationPath) { return new FomodXmlInstaller(provider.GetRequiredService>(), - provider.GetRequiredService(), - fomodInstallationPath); + provider.GetRequiredService(), provider.GetRequiredService(), + provider.GetRequiredService(), + + fomodInstallationPath, provider); } public FomodXmlInstaller(ILogger logger, ICoreDelegates coreDelegates, - GamePath fomodInstallationPath) + IFileSystem fileSystem, TemporaryFileManager temporaryFileManager, GamePath fomodInstallationPath, + IServiceProvider serviceProvider) : base(serviceProvider) { _delegates = coreDelegates; _fomodInstallationPath = fomodInstallationPath; _logger = logger; + _tempoaryFileManager = temporaryFileManager; + _fileSystem = fileSystem; } - public Priority GetPriority(GameInstallation installation, - EntityDictionary archiveFiles) - { - var hasScript = archiveFiles.Keys.Any(x => x.EndsWith(FomodConstants.XmlConfigRelativePath)); - return hasScript ? Priority.High : Priority.None; - } - - public async ValueTask> GetModsAsync( - GameInstallation gameInstallation, - ModId baseModId, - Hash srcArchiveHash, - EntityDictionary archiveFiles, + public override async ValueTask> GetModsAsync(GameInstallation gameInstallation, + ModId baseModId, FileTreeNode archiveFiles, CancellationToken cancellationToken = default) { // the component dealing with FOMODs is built to support all kinds of mods, including those without a script. @@ -65,20 +63,19 @@ public async ValueTask> GetModsAsync( // we only intend to support xml scripted FOMODs, this should be good enough var stopPattern = new List { "fomod" }; - if (!archiveFiles.Keys.TryGetFirst(x => x.EndsWith(FomodConstants.XmlConfigRelativePath), out var xmlFile)) - throw new UnreachableException( - $"$[{nameof(FomodXmlInstaller)}] XML file not found. This should never be true and is indicative of a bug."); + var analyzerInfo = await FomodAnalyzer.AnalyzeAsync(archiveFiles, _fileSystem, cancellationToken); + if (analyzerInfo == default) + return Array.Empty(); + + await using var tmpFolder = _tempoaryFileManager.CreateFolder(); - if (!archiveFiles.TryGetValue(xmlFile, out var analyzedFile)) - throw new UnreachableException( - $"$[{nameof(FomodXmlInstaller)}] XML data not found. This should never be true and is indicative of a bug."); + await analyzerInfo.DumpToFileSystemAsync(tmpFolder.Path.Combine(analyzerInfo.PathPrefix)); - var analyzerInfo = analyzedFile.AnalysisData.OfType().FirstOrDefault(); - if (analyzerInfo == default) return Array.Empty(); + var xmlPath = analyzerInfo.PathPrefix.Join(FomodConstants.XmlConfigRelativePath); // Setup mod, exclude script path so it doesn't get picked up and thus double read from disk - var modFiles = archiveFiles.Keys.Select(x => x.ToNativeSeparators(OSInformation.Shared)).ToList(); - var mod = new Mod(modFiles, stopPattern, xmlFile, string.Empty, _scriptType); + var modFiles = archiveFiles.GetAllDescendentFiles().Select(x => x.Path.ToNativeSeparators(OSInformation.Shared)).ToList(); + var mod = new Mod(modFiles, stopPattern, xmlPath.ToString(), string.Empty, _scriptType); await mod.InitializeWithoutLoadingScript(); var executor = _scriptType.CreateExecutor(mod, _delegates); @@ -114,7 +111,7 @@ private static string FixXmlScript(string input) private IEnumerable InstructionsToModFiles( IList instructions, - EntityDictionary files, + FileTreeNode files, GamePath gameTargetPath) { var res = instructions.Select(instruction => @@ -142,22 +139,23 @@ private IEnumerable InstructionsToModFiles( private static AModFile ConvertInstructionCopy( Instruction instruction, - EntityDictionary files, + FileTreeNode files, GamePath gameTargetPath) { var src = RelativePath.FromUnsanitizedInput(instruction.source); var dest = RelativePath.FromUnsanitizedInput(instruction.destination); - var file = files.First(x => x.Key.Equals(src)).Value; + var file = files.FindNode(src)!.Value; return new FromArchive { Id = ModFileId.New(), To = new GamePath(gameTargetPath.Type, gameTargetPath.Path.Join(dest)), - Hash = file.Hash, + Hash = file!.Hash, Size = file.Size }; } + private static AModFile ConvertInstructionMkdir( Instruction instruction, GamePath gameTargetPath) diff --git a/src/Games/NexusMods.Games.FOMOD/Services.cs b/src/Games/NexusMods.Games.FOMOD/Services.cs index f5d7ad706b..8a44f9ef21 100644 --- a/src/Games/NexusMods.Games.FOMOD/Services.cs +++ b/src/Games/NexusMods.Games.FOMOD/Services.cs @@ -17,7 +17,6 @@ public static class Services /// Collection of services. public static IServiceCollection AddFomod(this IServiceCollection services) { - services.AddAllSingleton(); services.AddAllSingleton(); services.AddAllSingleton(); return services; diff --git a/src/Games/NexusMods.Games.Generic/Entities/IniAnalysisData.cs b/src/Games/NexusMods.Games.Generic/Entities/IniAnalysisData.cs deleted file mode 100644 index ac91642ef3..0000000000 --- a/src/Games/NexusMods.Games.Generic/Entities/IniAnalysisData.cs +++ /dev/null @@ -1,11 +0,0 @@ -using NexusMods.DataModel.Abstractions; -using NexusMods.DataModel.JsonConverters; - -namespace NexusMods.Games.Generic.Entities; - -[JsonName("Generic.FileAnalysisData")] -public class IniAnalysisData : IFileAnalysisData -{ - public required HashSet Sections { get; init; } - public required HashSet Keys { get; init; } -} diff --git a/src/Games/NexusMods.Games.Generic/FileAnalyzers/IniAnalysisData.cs b/src/Games/NexusMods.Games.Generic/FileAnalyzers/IniAnalysisData.cs new file mode 100644 index 0000000000..c98b689492 --- /dev/null +++ b/src/Games/NexusMods.Games.Generic/FileAnalyzers/IniAnalysisData.cs @@ -0,0 +1,7 @@ +namespace NexusMods.Games.Generic.FileAnalyzers; + +public class IniAnalysisData +{ + public required HashSet Sections { get; init; } + public required HashSet Keys { get; init; } +} diff --git a/src/Games/NexusMods.Games.Generic/FileAnalyzers/IniAnalzyer.cs b/src/Games/NexusMods.Games.Generic/FileAnalyzers/IniAnalzyer.cs index 322663bcb3..75d9b57478 100644 --- a/src/Games/NexusMods.Games.Generic/FileAnalyzers/IniAnalzyer.cs +++ b/src/Games/NexusMods.Games.Generic/FileAnalyzers/IniAnalzyer.cs @@ -2,15 +2,14 @@ using IniParser; using IniParser.Model.Configuration; using IniParser.Parser; -using NexusMods.DataModel.Abstractions; +using NexusMods.Common; using NexusMods.DataModel.Abstractions.Ids; using NexusMods.FileExtractor.FileSignatures; -using NexusMods.Games.Generic.Entities; namespace NexusMods.Games.Generic.FileAnalyzers; -public class IniAnalzyer : IFileAnalyzer +public class IniAnalzyer { public FileAnalyzerId Id { get; } = FileAnalyzerId.New("904bca7b-fbd6-4350-b4e2-6fdbd034ec76", 1); public IEnumerable FileTypes => new[] { FileType.INI }; @@ -25,16 +24,15 @@ public class IniAnalzyer : IFileAnalyzer SkipInvalidLines = true, }; -#pragma warning disable CS1998 - public async IAsyncEnumerable AnalyzeAsync(FileAnalyzerInfo info, [EnumeratorCancellation] CancellationToken token = default) -#pragma warning restore CS1998 + public static async Task AnalyzeAsync(IStreamFactory info) { - var data = new StreamIniDataParser(new IniDataParser(Config)).ReadData(new StreamReader(info.Stream)); + await using var os = await info.GetStreamAsync(); + var data = new StreamIniDataParser(new IniDataParser(Config)).ReadData(new StreamReader(os)); var sections = data.Sections.Select(s => s.SectionName).ToHashSet(); var keys = data.Global.Select(k => k.KeyName) .Concat(data.Sections.SelectMany(d => d.Keys).Select(kv => kv.KeyName)) .ToHashSet(); - yield return new IniAnalysisData + return new IniAnalysisData { Sections = sections, Keys = keys diff --git a/src/Games/NexusMods.Games.Generic/Installers/GenericFolderMatchInstaller.cs b/src/Games/NexusMods.Games.Generic/Installers/GenericFolderMatchInstaller.cs index b916ed2d79..f2b919acbe 100644 --- a/src/Games/NexusMods.Games.Generic/Installers/GenericFolderMatchInstaller.cs +++ b/src/Games/NexusMods.Games.Generic/Installers/GenericFolderMatchInstaller.cs @@ -1,15 +1,12 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; -using NexusMods.DataModel.Abstractions; -using NexusMods.DataModel.ArchiveContents; -using NexusMods.DataModel.Extensions; using NexusMods.DataModel.Games; using NexusMods.DataModel.Games.GameCapabilities.FolderMatchInstallerCapability; using NexusMods.DataModel.Loadouts; using NexusMods.DataModel.Loadouts.ModFiles; using NexusMods.DataModel.ModInstallers; -using NexusMods.Hashing.xxHash64; using NexusMods.Paths; +using NexusMods.Paths.FileTree; namespace NexusMods.Games.Generic.Installers; @@ -21,7 +18,7 @@ namespace NexusMods.Games.Generic.Installers; /// /// Example: myMod/Textures/myTexture.dds -> Skyrim/Data/Textures/myTexture.dds /// -public class GenericFolderMatchInstaller : IModInstaller +public class GenericFolderMatchInstaller : AModInstaller { private readonly ILogger _logger; private readonly IEnumerable _installFolderTargets; @@ -34,10 +31,11 @@ public class GenericFolderMatchInstaller : IModInstaller /// public static GenericFolderMatchInstaller Create(IServiceProvider provider, IEnumerable installFolderTargets) { - return new GenericFolderMatchInstaller(provider.GetRequiredService>(), installFolderTargets); + return new GenericFolderMatchInstaller(provider.GetRequiredService>(), installFolderTargets, provider); } - private GenericFolderMatchInstaller(ILogger logger, IEnumerable installFolderTargets) + private GenericFolderMatchInstaller(ILogger logger, IEnumerable installFolderTargets, IServiceProvider serviceProvider) + : base(serviceProvider) { _logger = logger; _installFolderTargets = installFolderTargets; @@ -45,12 +43,11 @@ private GenericFolderMatchInstaller(ILogger logger, #region IModInstaller - public ValueTask> GetModsAsync(GameInstallation gameInstallation, ModId baseModId, - Hash srcArchiveHash, - EntityDictionary archiveFiles, CancellationToken cancellationToken = default) + public override ValueTask> GetModsAsync(GameInstallation gameInstallation, + ModId baseModId, FileTreeNode archiveFiles, + CancellationToken cancellationToken = default) { - List missedFiles = new(); List modFiles = new(); @@ -94,7 +91,7 @@ public ValueTask> GetModsAsync(GameInstallation /// /// /// - private IEnumerable GetModFilesForTarget(EntityDictionary archiveFiles, + private IEnumerable GetModFilesForTarget(FileTreeNode archiveFiles, InstallFolderTarget target, List missedFiles) { List modFiles = new(); @@ -102,9 +99,10 @@ private IEnumerable GetModFilesForTarget(EntityDictionary f.Path), + out var prefixToDrop)) { - foreach (var (filePath, fileData) in archiveFiles) + foreach (var (filePath, fileData) in archiveFiles.GetAllDescendentFiles()) { var trimmedPath = filePath; @@ -128,7 +126,7 @@ private IEnumerable GetModFilesForTarget(EntityDictionary + + + + diff --git a/src/Games/NexusMods.Games.Generic/Services.cs b/src/Games/NexusMods.Games.Generic/Services.cs index 31f1ef0f99..864ffc0d90 100644 --- a/src/Games/NexusMods.Games.Generic/Services.cs +++ b/src/Games/NexusMods.Games.Generic/Services.cs @@ -1,10 +1,5 @@ using Microsoft.Extensions.DependencyInjection; -using NexusMods.DataModel.Abstractions; -using NexusMods.DataModel.JsonConverters.ExpressionGenerator; -using NexusMods.DataModel.ModInstallers; -using NexusMods.Games.Generic.Entities; using NexusMods.Games.Generic.FileAnalyzers; -using NexusMods.Games.Generic.Installers; namespace NexusMods.Games.Generic; @@ -12,9 +7,7 @@ public static class Services { public static IServiceCollection AddGenericGameSupport(this IServiceCollection services) { - services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(); return services; } } diff --git a/src/Games/NexusMods.Games.Generic/TypeFinder.cs b/src/Games/NexusMods.Games.Generic/TypeFinder.cs deleted file mode 100644 index b7206c041b..0000000000 --- a/src/Games/NexusMods.Games.Generic/TypeFinder.cs +++ /dev/null @@ -1,17 +0,0 @@ -using NexusMods.DataModel.JsonConverters.ExpressionGenerator; -using NexusMods.Games.Generic.Entities; - -namespace NexusMods.Games.Generic; - -public class TypeFinder : ITypeFinder -{ - public IEnumerable DescendentsOf(Type type) - { - return AllTypes.Where(t => t.IsAssignableTo(type)); - } - - private IEnumerable AllTypes => new[] - { - typeof(IniAnalysisData), - }; -} diff --git a/src/Games/NexusMods.Games.RedEngine/Cyberpunk2077.cs b/src/Games/NexusMods.Games.RedEngine/Cyberpunk2077.cs index 4a5bc5cc3f..cf6e7cb580 100644 --- a/src/Games/NexusMods.Games.RedEngine/Cyberpunk2077.cs +++ b/src/Games/NexusMods.Games.RedEngine/Cyberpunk2077.cs @@ -11,10 +11,12 @@ public class Cyberpunk2077 : AGame, ISteamGame, IGogGame, IEpicGame { public static readonly GameDomain StaticDomain = GameDomain.From("cyberpunk2077"); private readonly IFileSystem _fileSystem; + private readonly IServiceProvider _serviceProvider; - public Cyberpunk2077(IEnumerable gameLocators, IFileSystem fileSystem) : base(gameLocators) + public Cyberpunk2077(IEnumerable gameLocators, IFileSystem fileSystem, IServiceProvider provider) : base(gameLocators) { _fileSystem = fileSystem; + _serviceProvider = provider; } public override string Name => "Cyberpunk 2077"; @@ -53,7 +55,7 @@ protected override IEnumerable> GetLo { new RedModInstaller(), new SimpleOverlayModInstaller(), - new AppearancePreset(), + new AppearancePreset(_serviceProvider), new FolderlessModInstaller() }; } diff --git a/src/Games/NexusMods.Games.RedEngine/FileAnalyzers/RedModInfoAnalyzer.cs b/src/Games/NexusMods.Games.RedEngine/FileAnalyzers/RedModInfoAnalyzer.cs deleted file mode 100644 index ef65bb9f67..0000000000 --- a/src/Games/NexusMods.Games.RedEngine/FileAnalyzers/RedModInfoAnalyzer.cs +++ /dev/null @@ -1,44 +0,0 @@ -using System.Runtime.CompilerServices; -using System.Text.Json; -using System.Text.Json.Serialization; -using NexusMods.DataModel.Abstractions; -using NexusMods.DataModel.Abstractions.Ids; -using NexusMods.DataModel.JsonConverters; -using NexusMods.FileExtractor.FileSignatures; - -namespace NexusMods.Games.RedEngine.FileAnalyzers; - -public class RedModInfoAnalyzer : IFileAnalyzer -{ - public FileAnalyzerId Id { get; } = FileAnalyzerId.New("8bf03d10-c48e-4929-906f-852b6afecd5e", 1); - - public IEnumerable FileTypes => new[] { FileType.JSON }; - public async IAsyncEnumerable AnalyzeAsync(FileAnalyzerInfo info, [EnumeratorCancellation] CancellationToken ct = default) - { - // TODO: We can probably check by fileName here before blindly deserializing - Sewer. - InfoJson? jsonInfo; - try - { - jsonInfo = await JsonSerializer.DeserializeAsync(info.Stream, cancellationToken: ct); - } - catch (JsonException) - { - yield break; - } - if (jsonInfo != null) - yield return new RedModInfo { Name = jsonInfo.Name }; - } -} - -internal class InfoJson -{ - [JsonPropertyName("name")] - public string Name { get; set; } = string.Empty; -} - -[JsonName("NexusMods.Games.RedEngine.FileAnalyzers.RedModInfo")] -public record RedModInfo : IFileAnalysisData -{ - // ReSharper disable once UnusedAutoPropertyAccessor.Global - public required string Name { get; init; } -} diff --git a/src/Games/NexusMods.Games.RedEngine/ModInstallers/AppearancePreset.cs b/src/Games/NexusMods.Games.RedEngine/ModInstallers/AppearancePreset.cs index e1f5923f63..23cf221eb2 100644 --- a/src/Games/NexusMods.Games.RedEngine/ModInstallers/AppearancePreset.cs +++ b/src/Games/NexusMods.Games.RedEngine/ModInstallers/AppearancePreset.cs @@ -1,57 +1,54 @@ -using NexusMods.Common; -using NexusMods.DataModel.Abstractions; -using NexusMods.DataModel.ArchiveContents; -using NexusMods.DataModel.Extensions; using NexusMods.DataModel.Games; using NexusMods.DataModel.Loadouts; using NexusMods.DataModel.ModInstallers; -using NexusMods.FileExtractor.FileSignatures; -using NexusMods.Hashing.xxHash64; using NexusMods.Paths; using NexusMods.Paths.Extensions; +using NexusMods.Paths.FileTree; using NexusMods.Paths.Utilities; namespace NexusMods.Games.RedEngine.ModInstallers; -public class AppearancePreset : IModInstaller +/// +/// This mod installer is used to install appearance presets for Cyberpunk 2077, they are installed into a specific +/// folder under the cyber engine tweaks mod's subfolder for the appearance change unlocker. +/// +public class AppearancePreset : AModInstaller { private static readonly RelativePath[] Paths = { "bin/x64/plugins/cyber_engine_tweaks/mods/AppearanceChangeUnlocker/character-preset/female".ToRelativePath(), "bin/x64/plugins/cyber_engine_tweaks/mods/AppearanceChangeUnlocker/character-preset/male".ToRelativePath() }; - public ValueTask> GetModsAsync( + /// + /// DI Constructor + /// + /// + public AppearancePreset(IServiceProvider serviceProvider) : base(serviceProvider) { } + + + public override async ValueTask> GetModsAsync( GameInstallation gameInstallation, ModId baseModId, - Hash srcArchiveHash, - EntityDictionary archiveFiles, + FileTreeNode archiveFiles, CancellationToken cancellationToken = default) { - return ValueTask.FromResult(GetMods(baseModId, srcArchiveHash, archiveFiles)); - } - - private IEnumerable GetMods( - ModId baseModId, - Hash srcArchiveHash, - EntityDictionary archiveFiles) - { - var modFiles = archiveFiles - .Where(kv => kv.Key.Extension == KnownExtensions.Preset) + var modFiles = archiveFiles.GetAllDescendentFiles() + .Where(kv => kv.Path.Extension == KnownExtensions.Preset) .SelectMany(kv => { var (path, file) = kv; - return Paths.Select(relPath => file.ToFromArchive( + return Paths.Select(relPath => file!.ToFromArchive( new GamePath(GameFolderType.Game, relPath.Join(path)) )); }).ToArray(); if (!modFiles.Any()) - yield break; + return NoResults; - yield return new ModInstallerResult + return new ModInstallerResult[] { new() { Id = baseModId, Files = modFiles - }; + }}; } } diff --git a/src/Games/NexusMods.Games.RedEngine/ModInstallers/FolderlessModInstaller.cs b/src/Games/NexusMods.Games.RedEngine/ModInstallers/FolderlessModInstaller.cs index f56754cead..94b4fcdb7b 100644 --- a/src/Games/NexusMods.Games.RedEngine/ModInstallers/FolderlessModInstaller.cs +++ b/src/Games/NexusMods.Games.RedEngine/ModInstallers/FolderlessModInstaller.cs @@ -1,14 +1,9 @@ -using NexusMods.Common; -using NexusMods.DataModel.Abstractions; -using NexusMods.DataModel.ArchiveContents; -using NexusMods.DataModel.Extensions; using NexusMods.DataModel.Games; using NexusMods.DataModel.Loadouts; using NexusMods.DataModel.ModInstallers; -using NexusMods.Hashing.xxHash64; using NexusMods.Paths; using NexusMods.Paths.Extensions; -using NexusMods.Paths.Utilities; +using NexusMods.Paths.FileTree; namespace NexusMods.Games.RedEngine.ModInstallers; @@ -19,39 +14,27 @@ public class FolderlessModInstaller : IModInstaller { private static readonly RelativePath Destination = "archive/pc/mod".ToRelativePath(); - public ValueTask> GetModsAsync( - GameInstallation gameInstallation, - ModId baseModId, - Hash srcArchiveHash, - EntityDictionary archiveFiles, - CancellationToken cancellationToken = default) + public async ValueTask> GetModsAsync(GameInstallation gameInstallation, ModId baseModId, + FileTreeNode archiveFiles, CancellationToken cancellationToken = default) { - return ValueTask.FromResult(GetMods(baseModId, srcArchiveHash, archiveFiles)); - } - private IEnumerable GetMods( - ModId baseModId, - Hash srcArchiveHash, - EntityDictionary archiveFiles) - { - var modFiles = archiveFiles - .Where(kv => !Helpers.IgnoreExtensions.Contains(kv.Key.Extension)) - .Select(kv => - { - var (path, file) = kv; - return file.ToFromArchive( - new GamePath(GameFolderType.Game, Destination.Join(path.FileName)) - ); - }) + var modFiles = archiveFiles.GetAllDescendentFiles() + .Where(f => !Helpers.IgnoreExtensions.Contains(f.Path.Extension)) + .Select(f => f.Value!.ToFromArchive( + new GamePath(GameFolderType.Game, Destination.Join(f.Path.FileName)) + )) .ToArray(); if (!modFiles.Any()) - yield break; + return Enumerable.Empty(); - yield return new ModInstallerResult + return new[] { - Id = baseModId, - Files = modFiles + new ModInstallerResult + { + Id = baseModId, + Files = modFiles + } }; } } diff --git a/src/Games/NexusMods.Games.RedEngine/ModInstallers/RedModInstaller.cs b/src/Games/NexusMods.Games.RedEngine/ModInstallers/RedModInstaller.cs index 4784882186..e59156fd00 100644 --- a/src/Games/NexusMods.Games.RedEngine/ModInstallers/RedModInstaller.cs +++ b/src/Games/NexusMods.Games.RedEngine/ModInstallers/RedModInstaller.cs @@ -1,14 +1,13 @@ +using System.Text.Json; +using System.Text.Json.Serialization; using NexusMods.Common; -using NexusMods.DataModel.Abstractions; -using NexusMods.DataModel.ArchiveContents; using NexusMods.DataModel.Extensions; using NexusMods.DataModel.Games; using NexusMods.DataModel.Loadouts; using NexusMods.DataModel.ModInstallers; -using NexusMods.Games.RedEngine.FileAnalyzers; -using NexusMods.Hashing.xxHash64; using NexusMods.Paths; using NexusMods.Paths.Extensions; +using NexusMods.Paths.FileTree; namespace NexusMods.Games.RedEngine.ModInstallers; @@ -17,52 +16,55 @@ public class RedModInstaller : IModInstaller private static readonly RelativePath InfoJson = "info.json".ToRelativePath(); private static readonly RelativePath Mods = "mods".ToRelativePath(); - private static bool IsInfoJson(KeyValuePair file) + public async ValueTask> GetModsAsync(GameInstallation gameInstallation, ModId baseModId, + FileTreeNode archiveFiles, CancellationToken cancellationToken = default) { - return file.Key.FileName == InfoJson && file.Value.AnalysisData.OfType().Any(); - } + var infos = (await archiveFiles.GetAllDescendentFiles() + .Where(f => f.Path.FileName == InfoJson) + .SelectAsync(async f => (File: f, InfoJson: await ReadInfoJson(f.Value!))) + .ToArrayAsync()) + .Where(node => node.InfoJson != null) + .ToArray(); - public ValueTask> GetModsAsync( - GameInstallation gameInstallation, - ModId baseModId, - Hash srcArchiveHash, - EntityDictionary archiveFiles, - CancellationToken cancellationToken = default) - { - return ValueTask.FromResult(GetMods(baseModId, srcArchiveHash, archiveFiles)); - } - private IEnumerable GetMods( - ModId baseModId, - Hash srcArchiveHash, - EntityDictionary archiveFiles) - { - var modFiles = archiveFiles - .Where(IsInfoJson) - .SelectMany(infoJson => + List results = new(); + + var baseIdUsed = false; + foreach (var node in infos) + { + var modFolder = node.File.Parent; + var parentName = modFolder.Name; + var files = new List(); + foreach (var childNode in modFolder.GetAllDescendentFiles()) { - var parent = infoJson.Key.Parent; - var parentName = parent.FileName; + var path = childNode.Path; + var entry = childNode.Value; + files.Add(entry!.ToFromArchive(new GamePath(GameFolderType.Game, Mods.Join(parentName).Join(path.RelativeTo(modFolder.Path))))); - return archiveFiles - .Where(kv => kv.Key.InFolder(parent)) - .Select(kv => - { - var (path, file) = kv; - return file.ToFromArchive( - new GamePath(GameFolderType.Game, Mods.Join(parentName).Join(path.RelativeTo(parent))) - ); - }); - }) - .ToArray(); + } - if (!modFiles.Any()) - yield break; + results.Add(new ModInstallerResult + { + Id = baseIdUsed ? ModId.New() : baseModId, + Files = files, + Name = node.InfoJson?.Name ?? "" + }); + baseIdUsed = true; + } - yield return new ModInstallerResult - { - Id = baseModId, - Files = modFiles - }; + return results; + } + + private static async Task ReadInfoJson(ModSourceFileEntry entry) + { + await using var stream = await entry.Open(); + return await JsonSerializer.DeserializeAsync(stream); } + +} + +internal class RedModInfo +{ + [JsonPropertyName("name")] + public string Name { get; set; } = string.Empty; } diff --git a/src/Games/NexusMods.Games.RedEngine/ModInstallers/SimpleOverlayModInstaller.cs b/src/Games/NexusMods.Games.RedEngine/ModInstallers/SimpleOverlayModInstaller.cs index 063539bdb8..85b0b1a30e 100644 --- a/src/Games/NexusMods.Games.RedEngine/ModInstallers/SimpleOverlayModInstaller.cs +++ b/src/Games/NexusMods.Games.RedEngine/ModInstallers/SimpleOverlayModInstaller.cs @@ -1,13 +1,16 @@ +using DynamicData; using NexusMods.Common; using NexusMods.DataModel.Abstractions; using NexusMods.DataModel.ArchiveContents; using NexusMods.DataModel.Extensions; using NexusMods.DataModel.Games; using NexusMods.DataModel.Loadouts; +using NexusMods.DataModel.Loadouts.ModFiles; using NexusMods.DataModel.ModInstallers; using NexusMods.Hashing.xxHash64; using NexusMods.Paths; using NexusMods.Paths.Extensions; +using NexusMods.Paths.FileTree; namespace NexusMods.Games.RedEngine.ModInstallers; @@ -24,88 +27,49 @@ public class SimpleOverlayModInstaller : IModInstaller .Select(x => x.ToRelativePath()) .ToArray(); - - private static HashSet RootFolder(EntityDictionary files) - { - var filtered = files.Where(f => !Helpers.IgnoreExtensions.Contains(f.Key.Extension)); - - var sets = filtered.Select(f => RootPaths.SelectMany(root => GetOffsets(f.Key, root)).ToHashSet()) - .Aggregate((set, x) => - { - set.IntersectWith(x); - return set; - }); - return sets; - } - - /// - /// Returns the offsets of which subSection is a subfolder of basePath. - /// - /// - /// - /// - private static IEnumerable GetOffsets(RelativePath basePath, RelativePath subSection) - { - var depth = 0; - while (true) - { - if (basePath.Depth == 0) // root - yield break; - - if (basePath.Depth < subSection.Depth) - yield break; - - if (basePath == subSection) - yield return depth; - - if (basePath.StartsWith(subSection)) - yield return depth; - - basePath = basePath.DropFirst(); - depth++; - } - } - - public ValueTask> GetModsAsync( + public async ValueTask> GetModsAsync( GameInstallation gameInstallation, ModId baseModId, - Hash srcArchiveHash, - EntityDictionary archiveFiles, + FileTreeNode archiveFiles, CancellationToken cancellationToken = default) { - return ValueTask.FromResult(GetMods(baseModId, srcArchiveHash, archiveFiles)); - } - private IEnumerable GetMods( - ModId baseModId, - Hash srcArchiveHash, - EntityDictionary archiveFiles) - { - var roots = RootFolder(archiveFiles); + var roots = RootPaths + .SelectMany(archiveFiles.FindSubPath) + .OrderBy(node => node.Depth) + .ToArray(); + + if (roots.Length == 0) + return Array.Empty(); - if (roots.Count == 0) - yield break; + var highestRoot = roots.First(); + var siblings= roots.Where(root => root.Depth == highestRoot.Depth) + .ToArray(); - var root = roots.First(); + var newFiles = new List(); - var modFiles = archiveFiles - .Where(kv => !Helpers.IgnoreExtensions.Contains(kv.Key.Extension)) - .Select(kv => + foreach (var node in siblings) + { + foreach (var (filePath, fileInfo) in node.GetAllDescendentFiles()) { - var (path, file) = kv; - return file.ToFromArchive( - new GamePath(GameFolderType.Game, path.DropFirst(root)) - ); - }) - .ToArray(); + var relativePath = filePath.DropFirst(node.Path.Depth); + newFiles.Add(new FromArchive() + { + Id = ModFileId.New(), + Hash = fileInfo!.Hash, + Size = fileInfo.Size, + To = new GamePath(GameFolderType.Game, relativePath) + }); + } + } - if (!modFiles.Any()) - yield break; + if (!newFiles.Any()) + return Array.Empty(); - yield return new ModInstallerResult + return new ModInstallerResult[]{ new() { Id = baseModId, - Files = modFiles - }; + Files = newFiles + }}; } } diff --git a/src/Games/NexusMods.Games.RedEngine/Services.cs b/src/Games/NexusMods.Games.RedEngine/Services.cs index 15cee5d8b5..d31c1c4698 100644 --- a/src/Games/NexusMods.Games.RedEngine/Services.cs +++ b/src/Games/NexusMods.Games.RedEngine/Services.cs @@ -1,10 +1,7 @@ using Microsoft.Extensions.DependencyInjection; using NexusMods.Common; -using NexusMods.DataModel.Abstractions; using NexusMods.DataModel.Games; -using NexusMods.DataModel.JsonConverters.ExpressionGenerator; using NexusMods.DataModel.ModInstallers; -using NexusMods.Games.RedEngine.FileAnalyzers; using NexusMods.Games.RedEngine.ModInstallers; namespace NexusMods.Games.RedEngine; @@ -18,10 +15,8 @@ public static IServiceCollection AddRedEngineGames(this IServiceCollection servi services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(); services.AddSingleton>(); services.AddSingleton(); - services.AddSingleton(); return services; } } diff --git a/src/Games/NexusMods.Games.RedEngine/TypeFinder.cs b/src/Games/NexusMods.Games.RedEngine/TypeFinder.cs deleted file mode 100644 index 0ddf6ec9dd..0000000000 --- a/src/Games/NexusMods.Games.RedEngine/TypeFinder.cs +++ /dev/null @@ -1,17 +0,0 @@ -using NexusMods.DataModel.JsonConverters.ExpressionGenerator; -using NexusMods.Games.RedEngine.FileAnalyzers; - -namespace NexusMods.Games.RedEngine; - -public class TypeFinder : ITypeFinder -{ - public IEnumerable DescendentsOf(Type type) - { - return AllTypes.Where(t => t.IsAssignableTo(type)); - } - - private IEnumerable AllTypes => new[] - { - typeof(RedModInfo) - }; -} diff --git a/src/Games/NexusMods.Games.Reshade/ReshadePresetInstaller.cs b/src/Games/NexusMods.Games.Reshade/ReshadePresetInstaller.cs index 7106ae146f..c7d4327507 100644 --- a/src/Games/NexusMods.Games.Reshade/ReshadePresetInstaller.cs +++ b/src/Games/NexusMods.Games.Reshade/ReshadePresetInstaller.cs @@ -1,20 +1,19 @@ using NexusMods.Common; -using NexusMods.DataModel.Abstractions; -using NexusMods.DataModel.ArchiveContents; -using NexusMods.DataModel.Extensions; using NexusMods.DataModel.Games; using NexusMods.DataModel.Loadouts; using NexusMods.DataModel.ModInstallers; -using NexusMods.FileExtractor.FileSignatures; -using NexusMods.Games.Generic.Entities; -using NexusMods.Hashing.xxHash64; +using NexusMods.Games.Generic.FileAnalyzers; using NexusMods.Paths; using NexusMods.Paths.Extensions; +using NexusMods.Paths.FileTree; +using NexusMods.Paths.Utilities; namespace NexusMods.Games.Reshade; -public class ReshadePresetInstaller : IModInstaller +public class ReshadePresetInstaller : AModInstaller { + + private static readonly HashSet IgnoreFiles = new[] { "readme.txt", @@ -24,69 +23,57 @@ public class ReshadePresetInstaller : IModInstaller .Select(t => t.ToRelativePath()) .ToHashSet(); - public Priority GetPriority(GameInstallation installation, EntityDictionary archiveFiles) + private static Extension Ini = new(".ini"); + + public ReshadePresetInstaller(IServiceProvider serviceProvider) : base(serviceProvider) {} + + public override async ValueTask> GetModsAsync( + GameInstallation gameInstallation, + ModId baseModId, + FileTreeNode archiveFiles, + CancellationToken cancellationToken = default) { - var filtered = archiveFiles - .Where(f => !IgnoreFiles.Contains(f.Key.FileName)) - .ToList(); - // We have to be able to find the game's executable - if (installation.Game is not AGame) - return Priority.None; + var filtered = archiveFiles.GetAllDescendentFiles() + .Where(f => !IgnoreFiles.Contains(f.Path.FileName)) + .ToList(); // We only support ini files for now - if (!filtered.All(f => f.Value.FileTypes.Contains(FileType.INI))) - return Priority.None; + if (filtered.Any(f => f.Path.Extension != Ini)) + return NoResults; // Get all the ini data - var iniData = filtered - .Select(f => f.Value.AnalysisData - .OfType() - .FirstOrDefault()) - .Where(d => d is not null) - .Select(d => d!) - .ToList(); + var iniData = await filtered + .SelectAsync(async f => await IniAnalzyer.AnalyzeAsync(f.Value!.StreamFactory)) + .ToListAsync(cancellationToken: cancellationToken); // All the files must have ini data if (iniData.Count != filtered.Count) - return Priority.None; + return NoResults; // All the files must have a section that ends with .fx marking them as likely a reshade preset if (!iniData.All(f => f.Sections.All(x => x.EndsWith(".fx", StringComparison.CurrentCultureIgnoreCase)))) - return Priority.None; - - return Priority.Low; - } + return NoResults; - public ValueTask> GetModsAsync( - GameInstallation gameInstallation, - ModId baseModId, - Hash srcArchiveHash, - EntityDictionary archiveFiles, - CancellationToken cancellationToken = default) - { - return ValueTask.FromResult(GetMods(baseModId, srcArchiveHash, archiveFiles)); - } - - private IEnumerable GetMods( - ModId baseModId, - Hash srcArchiveHash, - EntityDictionary archiveFiles) - { - var modFiles = archiveFiles - .Where(kv => !IgnoreFiles.Contains(kv.Key.FileName)) + var modFiles = archiveFiles.GetAllDescendentFiles() + .Where(kv => !IgnoreFiles.Contains(kv.Name.FileName)) .Select(kv => { var (path, file) = kv; - return file.ToFromArchive( + return file!.ToFromArchive( new GamePath(GameFolderType.Game, path.FileName) ); - }); + }).ToArray(); + + if (!modFiles.Any()) + return NoResults; - yield return new ModInstallerResult + return new [] { new ModInstallerResult { Id = baseModId, Files = modFiles - }; + }}; } + + } diff --git a/src/Games/NexusMods.Games.Sifu/Sifu.cs b/src/Games/NexusMods.Games.Sifu/Sifu.cs index bce10f406d..feb4b6e921 100644 --- a/src/Games/NexusMods.Games.Sifu/Sifu.cs +++ b/src/Games/NexusMods.Games.Sifu/Sifu.cs @@ -1,10 +1,12 @@ using NexusMods.DataModel.Games; +using NexusMods.DataModel.ModInstallers; using NexusMods.Paths; namespace NexusMods.Games.Sifu; public class Sifu : AGame, ISteamGame, IEpicGame { + private readonly IServiceProvider _serviceProvider; public IEnumerable SteamIds => new[] { 2138710u }; public IEnumerable EpicCatalogItemId => new[] { "c80a76de890145edbe0d41679dbccc66" }; @@ -15,9 +17,10 @@ public override GamePath GetPrimaryFile(GameStore store) { return new(GameFolderType.Game, "Sifu.exe"); } - - public Sifu(IEnumerable gameLocators) : base(gameLocators) + + public Sifu(IEnumerable gameLocators, IServiceProvider serviceProvider) : base(gameLocators) { + _serviceProvider = serviceProvider; } protected override IEnumerable> GetLocations( @@ -27,4 +30,7 @@ protected override IEnumerable> GetLo { yield return new KeyValuePair(GameFolderType.Game, installation.Path); } + + /// + public override IEnumerable Installers => new[] { new SifuModInstaller(_serviceProvider) }; } diff --git a/src/Games/NexusMods.Games.Sifu/SifuModInstaller.cs b/src/Games/NexusMods.Games.Sifu/SifuModInstaller.cs index 455219a1b9..3a6b8e57ff 100644 --- a/src/Games/NexusMods.Games.Sifu/SifuModInstaller.cs +++ b/src/Games/NexusMods.Games.Sifu/SifuModInstaller.cs @@ -9,59 +9,48 @@ using NexusMods.Hashing.xxHash64; using NexusMods.Paths; using NexusMods.Paths.Extensions; +using NexusMods.Paths.FileTree; namespace NexusMods.Games.Sifu; [SuppressMessage("ReSharper", "InconsistentNaming")] [SuppressMessage("ReSharper", "IdentifierTypo")] -public class SifuModInstaller : IModInstaller +public class SifuModInstaller : AModInstaller { private static readonly Extension PakExt = new(".pak"); private static readonly RelativePath ModsPath = "Content/Paks/~mods".ToRelativePath(); - public Priority GetPriority(GameInstallation installation, EntityDictionary archiveFiles) - { - return installation.Game is Sifu && ContainsUEModFile(archiveFiles) - ? Priority.Normal - : Priority.None; - } - - private static bool ContainsUEModFile(EntityDictionary files) - { - return files.Any(kv => kv.Key.Extension == PakExt); - } + public SifuModInstaller(IServiceProvider serviceProvider) : base(serviceProvider) { } - public ValueTask> GetModsAsync( + public override async ValueTask> GetModsAsync( GameInstallation gameInstallation, ModId baseModId, - Hash srcArchiveHash, - EntityDictionary archiveFiles, + FileTreeNode archiveFiles, CancellationToken cancellationToken = default) { - return ValueTask.FromResult(GetMods(baseModId, srcArchiveHash, archiveFiles)); - } + var pakFile = archiveFiles.GetAllDescendentFiles() + .FirstOrDefault(node => node.Path.Extension == PakExt); - private IEnumerable GetMods( - ModId baseModId, - Hash srcArchiveHash, - EntityDictionary archiveFiles) - { - var pakPath = archiveFiles.Keys.First(filePath => filePath.FileName.Extension == PakExt).Parent; + if (pakFile == null) + return NoResults; - var modFiles = archiveFiles - .Where(kv => kv.Key.InFolder(pakPath)) + var pakPath = pakFile.Parent; + + var modFiles = pakPath.GetAllDescendentFiles() .Select(kv => { var (path, file) = kv; - return file.ToFromArchive( - new GamePath(GameFolderType.Game, ModsPath.Join(path.RelativeTo(pakPath))) + return file!.ToFromArchive( + new GamePath(GameFolderType.Game, ModsPath.Join(path.RelativeTo(pakPath.Path))) ); }); - yield return new ModInstallerResult + return new [] { new ModInstallerResult { Id = baseModId, - Files = modFiles - }; + Files = modFiles, + Name = pakPath.Name.FileName + }}; } + } diff --git a/src/Games/NexusMods.Games.StardewValley/Analyzers/SMAPIManifestAnalyzer.cs b/src/Games/NexusMods.Games.StardewValley/Analyzers/SMAPIManifestAnalyzer.cs deleted file mode 100644 index 995a3e47e8..0000000000 --- a/src/Games/NexusMods.Games.StardewValley/Analyzers/SMAPIManifestAnalyzer.cs +++ /dev/null @@ -1,54 +0,0 @@ -using System.Runtime.CompilerServices; -using System.Text.Json; -using Microsoft.Extensions.Logging; -using NexusMods.DataModel.Abstractions; -using NexusMods.DataModel.Abstractions.Ids; -using NexusMods.FileExtractor.FileSignatures; -using NexusMods.Games.StardewValley.Models; - -// ReSharper disable InconsistentNaming - -namespace NexusMods.Games.StardewValley.Analyzers; - -/// -/// for mods that use the Stardew Modding API (SMAPI). -/// This looks for manifest.json files and returns . -/// -public class SMAPIManifestAnalyzer : IFileAnalyzer -{ - public FileAnalyzerId Id => FileAnalyzerId.New("f917e906-d28a-472d-b6e5-e7d2c61c60e4", 1); - - public IEnumerable FileTypes => new[] { FileType.JSON }; - - private static readonly JsonSerializerOptions JsonSerializerOptions = new() - { - AllowTrailingCommas = true, - ReadCommentHandling = JsonCommentHandling.Skip, - }; - - private readonly ILogger _logger; - - public SMAPIManifestAnalyzer(ILogger logger) - { - _logger = logger; - } - - public async IAsyncEnumerable AnalyzeAsync(FileAnalyzerInfo info, [EnumeratorCancellation] CancellationToken token = default) - { - if (!info.FileName.Equals("manifest.json", StringComparison.OrdinalIgnoreCase)) - yield break; - - SMAPIManifest? result = null; - try - { - result = await JsonSerializer.DeserializeAsync(info.Stream, JsonSerializerOptions, cancellationToken: token); - } - catch (Exception e) - { - _logger.LogError(e, "Exception while deserializing SMAPIManifest at {RelativePath}", info.RelativePath); - } - - if (result is null) yield break; - yield return result; - } -} diff --git a/src/Games/NexusMods.Games.StardewValley/Constants.cs b/src/Games/NexusMods.Games.StardewValley/Constants.cs new file mode 100644 index 0000000000..d7fa77bf41 --- /dev/null +++ b/src/Games/NexusMods.Games.StardewValley/Constants.cs @@ -0,0 +1,10 @@ +using NexusMods.Paths; +using NexusMods.Paths.Extensions; + +namespace NexusMods.Games.StardewValley; + +public class Constants +{ + public static readonly RelativePath ModsFolder = "Mods".ToRelativePath(); + public static readonly RelativePath ManifestFile = "manifest.json".ToRelativePath(); +} diff --git a/src/Games/NexusMods.Games.StardewValley/Emitters/001_MissingDependencies.cs b/src/Games/NexusMods.Games.StardewValley/Emitters/001_MissingDependencies.cs index b61cf8d72d..5d8e13817b 100644 --- a/src/Games/NexusMods.Games.StardewValley/Emitters/001_MissingDependencies.cs +++ b/src/Games/NexusMods.Games.StardewValley/Emitters/001_MissingDependencies.cs @@ -1,6 +1,11 @@ +using System.Text.Json; +using NexusMods.Common; +using NexusMods.DataModel.Abstractions; using NexusMods.DataModel.Diagnostics; using NexusMods.DataModel.Diagnostics.Emitters; +using NexusMods.DataModel.Games; using NexusMods.DataModel.Loadouts; +using NexusMods.DataModel.Loadouts.ModFiles; using NexusMods.DataModel.Loadouts.Mods; using NexusMods.Games.StardewValley.Models; @@ -8,14 +13,21 @@ namespace NexusMods.Games.StardewValley.Emitters; public class MissingDependenciesEmitter : ILoadoutDiagnosticEmitter { - public IEnumerable Diagnose(Loadout loadout) + private readonly IArchiveManager _archiveManager; + + public MissingDependenciesEmitter(IArchiveManager archiveManager) + { + _archiveManager = archiveManager; + } + + public async IAsyncEnumerable Diagnose(Loadout loadout) { // TODO: check the versions - var modIdToManifest = loadout.Mods - .Select(kv => (Id: kv.Key, Manifest: GetManifest(kv.Value))) + var modIdToManifest = await loadout.Mods + .SelectAsync(async kv => (Id: kv.Key, Manifest: await GetManifest(kv.Value))) .Where(tuple => tuple.Manifest is not null) - .ToDictionary(x => x.Id, x => x.Manifest!); + .ToDictionaryAsync(x => x.Id, x => x.Manifest!); var knownUniqueIds = modIdToManifest .Select(x => x.Value.UniqueID) @@ -54,16 +66,26 @@ private static IEnumerable GetRequiredDependencies(SMAPIManifest? manife return requiredDependencies; } - private static SMAPIManifest? GetManifest(Mod mod) + private async ValueTask GetManifest(Mod mod) { - var manifest = mod.Files.Select(kv => - { - var (_, file) = kv; - var manifest = file.Metadata - .OfType() - .FirstOrDefault(); - return manifest; - }).FirstOrDefault(x => x is not null); + + var manifest = await mod.Files + .Values + .OfType() + .Where(f => f.To.FileName == Constants.ManifestFile) + .OfType() + .SelectAsync(async fa => + { + try + { + await using var stream = await _archiveManager.GetFileStream(fa.Hash); + return await JsonSerializer.DeserializeAsync(stream); + } + catch (Exception) + { + return null; + } + }).FirstOrDefaultAsync(m => m != null); return manifest; } } diff --git a/src/Games/NexusMods.Games.StardewValley/Installers/SMAPIInstaller.cs b/src/Games/NexusMods.Games.StardewValley/Installers/SMAPIInstaller.cs index 6a83d14301..69078b6637 100644 --- a/src/Games/NexusMods.Games.StardewValley/Installers/SMAPIInstaller.cs +++ b/src/Games/NexusMods.Games.StardewValley/Installers/SMAPIInstaller.cs @@ -4,6 +4,7 @@ using NexusMods.DataModel; using NexusMods.DataModel.Abstractions; using NexusMods.DataModel.ArchiveContents; +using NexusMods.DataModel.ArchiveMetaData; using NexusMods.DataModel.Games; using NexusMods.DataModel.Loadouts; using NexusMods.DataModel.Loadouts.ModFiles; @@ -12,6 +13,7 @@ using NexusMods.Hashing.xxHash64; using NexusMods.Paths; using NexusMods.Paths.Extensions; +using NexusMods.Paths.FileTree; // ReSharper disable IdentifierTypo // ReSharper disable InconsistentNaming @@ -22,7 +24,7 @@ namespace NexusMods.Games.StardewValley.Installers; /// for SMAPI itself. This is different from , /// which is an implementation of for mods that use SMAPI. /// -public class SMAPIInstaller : IModInstaller +public class SMAPIInstaller : AModInstaller { private static readonly RelativePath InstallDatFile = "install.dat".ToRelativePath(); private static readonly RelativePath LinuxFolder = "linux".ToRelativePath(); @@ -31,80 +33,83 @@ public class SMAPIInstaller : IModInstaller private readonly IOSInformation _osInformation; private readonly FileHashCache _fileHashCache; + private readonly IDownloadRegistry _downloadRegistry; - public SMAPIInstaller(IOSInformation osInformation, FileHashCache fileHashCache) + private SMAPIInstaller(IOSInformation osInformation, FileHashCache fileHashCache, IDownloadRegistry downloadRegistry, IServiceProvider serviceProvider) + : base(serviceProvider) { _osInformation = osInformation; _fileHashCache = fileHashCache; + _downloadRegistry = downloadRegistry; } - private static KeyValuePair[] GetInstallDataFiles(EntityDictionary files) + private static FileTreeNode[] GetInstallDataFiles(FileTreeNode files) { - var installDataFiles = files.Where(kv => + var installDataFiles = files.GetAllDescendentFiles() + .Where(kv => { var (path, file) = kv; var fileName = path.FileName; var parent = path.Parent.FileName; - return file.FileTypes.Contains(FileType.ZIP) && - fileName.Equals(InstallDatFile) && + return fileName.Equals(InstallDatFile) && (parent.Equals(LinuxFolder) || parent.Equals(MacOSFolder) || parent.Equals(WindowsFolder)); - }).ToArray(); + }) + .ToArray(); return installDataFiles; } - public ValueTask> GetModsAsync( + public override async ValueTask> GetModsAsync( GameInstallation gameInstallation, ModId baseModId, - Hash srcArchiveHash, - EntityDictionary archiveFiles, + FileTreeNode archiveFiles, CancellationToken cancellationToken = default) - { - return ValueTask.FromResult(GetMods(gameInstallation, baseModId, srcArchiveHash, archiveFiles)); - } - - private IEnumerable GetMods( - GameInstallation gameInstallation, - ModId baseModId, - Hash srcArchiveHash, - EntityDictionary archiveFiles) { var modFiles = new List(); var installDataFiles = GetInstallDataFiles(archiveFiles); - if (installDataFiles.Length != 3) throw new UnreachableException($"{nameof(SMAPIInstaller)} should guarantee that {nameof(GetInstallDataFiles)} returns 3 files when called from {nameof(GetModsAsync)} but it has {installDataFiles.Length} files instead!"); + if (installDataFiles.Length != 3) + return NoResults; var installDataFile = _osInformation.MatchPlatform( state: ref installDataFiles, - onWindows: (ref KeyValuePair[] dataFiles) => dataFiles.First(kv => kv.Key.Parent.FileName.Equals("windows")), - onLinux: (ref KeyValuePair[] dataFiles) => dataFiles.First(kv => kv.Key.Parent.FileName.Equals("linux")), - onOSX: (ref KeyValuePair[] dataFiles) => dataFiles.First(kv => kv.Key.Parent.FileName.Equals("macOS")) + onWindows: (ref FileTreeNode[] dataFiles) => dataFiles.First(kv => kv.Path.Parent.FileName.Equals("windows")), + onLinux: (ref FileTreeNode[] dataFiles) => dataFiles.First(kv => kv.Path.Parent.FileName.Equals("linux")), + onOSX: (ref FileTreeNode[] dataFiles) => dataFiles.First(kv => kv.Path.Parent.FileName.Equals("macOS")) ); var (path, file) = installDataFile; - if (file is not AnalyzedArchive archive) - throw new UnreachableException($"{nameof(AnalyzedFile)} that has the file type {nameof(FileType.ZIP)} is not a {nameof(AnalyzedArchive)}"); + + var found = _downloadRegistry.GetByHash(file!.Hash).ToArray(); + DownloadId downloadId; + if (!found.Any()) + downloadId = await RegisterDataFile(path, file, cancellationToken); + else + { + downloadId = found.First(); + } + var gameFolderPath = gameInstallation.Locations .First(x => x.Key == GameFolderType.Game).Value; - var archiveContents = archive.Contents; + var archiveContents = (await _downloadRegistry.Get(downloadId)).GetFileTree(); // TODO: install.dat is an archive inside an archive see https://github.com/Nexus-Mods/NexusMods.App/issues/244 // the basicFiles have to be extracted from the nested archive and put inside the game folder // https://github.com/Pathoschild/SMAPI/blob/9763bc7484e29cbc9e7f37c61121d794e6720e75/src/SMAPI.Installer/InteractiveInstaller.cs#L380-L384 - var basicFiles = archiveContents - .Where(kv => !kv.Key.Equals("unix-launcher.sh")).ToArray(); + var basicFiles = archiveContents.GetAllDescendentFiles() + .Where(kv => !kv.Path.Equals("unix-launcher.sh")).ToArray(); if (_osInformation.IsLinux || _osInformation.IsOSX) { // TODO: Replace game launcher (StardewValley) with unix-launcher.sh by overwriting the game file // https://github.com/Pathoschild/SMAPI/blob/9763bc7484e29cbc9e7f37c61121d794e6720e75/src/SMAPI.Installer/InteractiveInstaller.cs#L386-L417 - var modLauncherScriptFile = archiveContents - .First(kv => kv.Key.FileName.Equals("unix-launcher.sh")); + var modLauncherScriptFile = archiveContents.GetAllDescendentFiles() + .First(kv => kv.Path.FileName.Equals("unix-launcher.sh")); var gameLauncherScriptFilePath = gameFolderPath.Combine("StardewValley"); } @@ -127,18 +132,25 @@ private IEnumerable GetMods( }); // TODO: consider adding Name and Version - yield return new ModInstallerResult + return new [] { new ModInstallerResult { Id = baseModId, Files = modFiles - }; + }}; + } + + private async ValueTask RegisterDataFile(RelativePath filename, ModSourceFileEntry file, CancellationToken token) + { + return await _downloadRegistry.RegisterDownload(file.StreamFactory, new FilePathMetadata { OriginalName = filename.FileName, Quality = Quality.Low}, token); } public static SMAPIInstaller Create(IServiceProvider serviceProvider) { return new SMAPIInstaller( serviceProvider.GetRequiredService(), - serviceProvider.GetRequiredService() + serviceProvider.GetRequiredService(), + serviceProvider.GetRequiredService(), + serviceProvider ); } } diff --git a/src/Games/NexusMods.Games.StardewValley/Installers/SMAPIModInstaller.cs b/src/Games/NexusMods.Games.StardewValley/Installers/SMAPIModInstaller.cs index e56502ebe9..3256440f43 100644 --- a/src/Games/NexusMods.Games.StardewValley/Installers/SMAPIModInstaller.cs +++ b/src/Games/NexusMods.Games.StardewValley/Installers/SMAPIModInstaller.cs @@ -1,15 +1,13 @@ -using System.Diagnostics; +using System.Text.Json; using NexusMods.Common; -using NexusMods.DataModel.Abstractions; -using NexusMods.DataModel.ArchiveContents; -using NexusMods.DataModel.Extensions; using NexusMods.DataModel.Games; using NexusMods.DataModel.Loadouts; using NexusMods.DataModel.ModInstallers; using NexusMods.Games.StardewValley.Models; -using NexusMods.Hashing.xxHash64; using NexusMods.Paths; using NexusMods.Paths.Extensions; +using NexusMods.Paths.FileTree; + // ReSharper disable IdentifierTypo // ReSharper disable InconsistentNaming @@ -18,62 +16,66 @@ namespace NexusMods.Games.StardewValley.Installers; /// /// for mods that use the Stardew Modding API (SMAPI). /// -public class SMAPIModInstaller : IModInstaller +public class SMAPIModInstaller : AModInstaller { - private static readonly RelativePath ModsFolder = "Mods".ToRelativePath(); - private static readonly RelativePath ManifestFile = "manifest.json".ToRelativePath(); - private static IEnumerable> GetManifestFiles( - EntityDictionary files) + + /// + /// DI Constructor + /// + /// + private SMAPIModInstaller(IServiceProvider serviceProvider) : base(serviceProvider) { } + + /// + /// Creates a new instance of . + /// + /// + /// + public static SMAPIModInstaller Create(IServiceProvider serviceProvider) => new(serviceProvider); + + private static IAsyncEnumerable<(FileTreeNode, SMAPIManifest)> GetManifestFiles( + FileTreeNode files) { - return files.Where(kv => + return files.GetAllDescendentFiles() + .SelectAsync(async kv => { var (path, file) = kv; - if (!path.FileName.Equals(ManifestFile)) return false; - var manifest = file.AnalysisData - .OfType() - .FirstOrDefault(); + if (!path.FileName.Equals(Constants.ManifestFile)) + return default; + + await using var stream = await file!.Open(); - return manifest is not null; - }); + return (kv, await JsonSerializer.DeserializeAsync(stream)); + }) + .Where(manifest => manifest.Item2 != null) + .Select(m => (m.kv, m.Item2!)); } - public ValueTask> GetModsAsync( + public override async ValueTask> GetModsAsync( GameInstallation gameInstallation, ModId baseModId, - Hash srcArchiveHash, - EntityDictionary archiveFiles, + FileTreeNode archiveFiles, CancellationToken cancellationToken = default) { - return ValueTask.FromResult(GetMods(srcArchiveHash, archiveFiles)); - } + var manifestFiles = await GetManifestFiles(archiveFiles) + .ToArrayAsync(cancellationToken: cancellationToken); - private IEnumerable GetMods( - Hash srcArchiveHash, - EntityDictionary archiveFiles) - { - var manifestFiles = GetManifestFiles(archiveFiles).ToArray(); if (!manifestFiles.Any()) - throw new UnreachableException($"{nameof(SMAPIModInstaller)} should guarantee that {nameof(GetModsAsync)} is never called for archives that don't have a SMAPI manifest file."); + return NoResults; var mods = manifestFiles - .Select(manifestFile => + .Select(found => { - var parent = manifestFile.Key.Parent; - var manifest = manifestFile.Value.AnalysisData - .OfType() - .FirstOrDefault(); + var (manifestFile, manifest) = found; + var parent = manifestFile.Parent; - if (manifest is null) throw new UnreachableException(); - - var modFiles = archiveFiles - .Where(kv => kv.Key.InFolder(parent)) + var modFiles = parent.GetAllDescendentFiles() .Select(kv => { var (path, file) = kv; - return file.ToFromArchive( - new GamePath(GameFolderType.Game, ModsFolder.Join(path.DropFirst(parent.Depth))) + return file!.ToFromArchive( + new GamePath(GameFolderType.Game, Constants.ModsFolder.Join(path.DropFirst(parent.Depth - 1))) ); }); @@ -88,4 +90,5 @@ private IEnumerable GetMods( return mods; } + } diff --git a/src/Games/NexusMods.Games.StardewValley/Models/SMAPIManifest.cs b/src/Games/NexusMods.Games.StardewValley/Models/SMAPIManifest.cs index 4c9ee58a47..92bf6c1976 100644 --- a/src/Games/NexusMods.Games.StardewValley/Models/SMAPIManifest.cs +++ b/src/Games/NexusMods.Games.StardewValley/Models/SMAPIManifest.cs @@ -11,7 +11,7 @@ namespace NexusMods.Games.StardewValley.Models; /// [PublicAPI] [JsonName("NexusMods.Games.StardewValley.SMAPIManifest")] -public record SMAPIManifest : IFileAnalysisData +public record SMAPIManifest { /// /// The mod name. diff --git a/src/Games/NexusMods.Games.StardewValley/NexusMods.Games.StardewValley.csproj b/src/Games/NexusMods.Games.StardewValley/NexusMods.Games.StardewValley.csproj index f2a8211a49..9c34e8e91f 100644 --- a/src/Games/NexusMods.Games.StardewValley/NexusMods.Games.StardewValley.csproj +++ b/src/Games/NexusMods.Games.StardewValley/NexusMods.Games.StardewValley.csproj @@ -15,4 +15,5 @@ + diff --git a/src/Games/NexusMods.Games.StardewValley/Services.cs b/src/Games/NexusMods.Games.StardewValley/Services.cs index 480029b5e2..5aeaec175f 100644 --- a/src/Games/NexusMods.Games.StardewValley/Services.cs +++ b/src/Games/NexusMods.Games.StardewValley/Services.cs @@ -1,11 +1,8 @@ using Microsoft.Extensions.DependencyInjection; using NexusMods.Common; -using NexusMods.DataModel.Abstractions; using NexusMods.DataModel.Diagnostics.Emitters; using NexusMods.DataModel.Games; using NexusMods.DataModel.JsonConverters.ExpressionGenerator; -using NexusMods.DataModel.ModInstallers; -using NexusMods.Games.StardewValley.Analyzers; using NexusMods.Games.StardewValley.Emitters; using NexusMods.Games.StardewValley.Installers; @@ -16,9 +13,6 @@ public static class Services public static IServiceCollection AddStardewValley(this IServiceCollection services) { services.AddAllSingleton() - .AddSingleton() - .AddSingleton() - .AddSingleton() .AddSingleton() .AddSingleton(); return services; diff --git a/src/Games/NexusMods.Games.StardewValley/StardewValley.cs b/src/Games/NexusMods.Games.StardewValley/StardewValley.cs index 0c665bbc71..dd070b8f56 100644 --- a/src/Games/NexusMods.Games.StardewValley/StardewValley.cs +++ b/src/Games/NexusMods.Games.StardewValley/StardewValley.cs @@ -86,7 +86,7 @@ protected override IEnumerable> GetLo public override IEnumerable Installers => new IModInstaller[] { SMAPIInstaller.Create(_serviceProvider), - new SMAPIModInstaller() + SMAPIModInstaller.Create(_serviceProvider) }; } diff --git a/src/Games/NexusMods.Games.TestHarness/Verbs/StressTest.cs b/src/Games/NexusMods.Games.TestHarness/Verbs/StressTest.cs index b784482f1a..1b583400ae 100644 --- a/src/Games/NexusMods.Games.TestHarness/Verbs/StressTest.cs +++ b/src/Games/NexusMods.Games.TestHarness/Verbs/StressTest.cs @@ -2,8 +2,10 @@ using NexusMods.Abstractions.CLI; using NexusMods.Abstractions.CLI.DataOutputs; using NexusMods.CLI; +using NexusMods.Common; using NexusMods.Common.GuidedInstaller; using NexusMods.DataModel.Abstractions; +using NexusMods.DataModel.ArchiveMetaData; using NexusMods.DataModel.Games; using NexusMods.DataModel.Loadouts; using NexusMods.Hashing.xxHash64; @@ -36,14 +38,14 @@ public StressTest(ILogger logger, LoadoutManager loadoutManager, Client client, TemporaryFileManager temporaryFileManager, IHttpDownloader downloader, - IArchiveAnalyzer archiveAnalyzer, IArchiveInstaller archiveInstaller, + IDownloadRegistry downloadRegistry, IEnumerable gameLocators, IGuidedInstaller optionSelector) { ((CliGuidedInstaller)optionSelector).SkipAll = true; - _archiveAnalyzer = archiveAnalyzer; _archiveInstaller = archiveInstaller; + _downloadRegistry = downloadRegistry; _downloader = downloader; _loadoutManager = loadoutManager; _logger = logger; @@ -63,9 +65,9 @@ public StressTest(ILogger logger, private readonly LoadoutManager _loadoutManager; private readonly ILogger _logger; - private readonly IArchiveAnalyzer _archiveAnalyzer; private readonly IArchiveInstaller _archiveInstaller; private readonly ManuallyAddedLocator _manualLocator; + private readonly IDownloadRegistry _downloadRegistry; public async Task Run(IGame game, AbsolutePath output, CancellationToken token) { @@ -118,8 +120,9 @@ public async Task Run(IGame game, AbsolutePath output, CancellationToken to var list = await _loadoutManager.ManageGameAsync(install, indexGameFiles: false, token: cts.Token); - var analysisData = await _archiveAnalyzer.AnalyzeFileAsync(tmpPath, token); - await _archiveInstaller.AddMods(list.Value.LoadoutId, analysisData.Hash, token: token); + var downloadId = await _downloadRegistry.RegisterDownload(tmpPath, new FilePathMetadata + { OriginalName = tmpPath.Path.Name, Quality = Quality.Low }, token); + await _archiveInstaller.AddMods(list.Value.LoadoutId, downloadId, token: token); results.Add((file.FileName, mod.ModId, file.FileId, hash, true, null)); _logger.LogInformation("Installed {ModId} {FileId} {FileName} - {Size}", mod.ModId, file.FileId, diff --git a/src/Networking/NexusMods.Networking.Downloaders/DownloadService.cs b/src/Networking/NexusMods.Networking.Downloaders/DownloadService.cs index 592abcde74..23825877e8 100644 --- a/src/Networking/NexusMods.Networking.Downloaders/DownloadService.cs +++ b/src/Networking/NexusMods.Networking.Downloaders/DownloadService.cs @@ -3,8 +3,11 @@ using DynamicData; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using NexusMods.Common; +using NexusMods.DataModel; using NexusMods.DataModel.Abstractions; using NexusMods.DataModel.Abstractions.Ids; +using NexusMods.DataModel.ArchiveMetaData; using NexusMods.DataModel.RateLimiting; using NexusMods.Hashing.xxHash64; using NexusMods.Networking.Downloaders.Interfaces; @@ -27,23 +30,23 @@ public class DownloadService : IDownloadService private readonly ILogger _logger; private readonly IServiceProvider _provider; private readonly IDataStore _store; - private readonly IArchiveAnalyzer _archiveAnalyzer; private readonly Subject _started = new(); private readonly Subject _completed = new(); private readonly Subject _cancelled = new(); private readonly Subject _paused = new(); private readonly Subject _resumed = new(); - private readonly Subject<(IDownloadTask task, Hash analyzedHash, string modName)> _analyzed = new(); + private readonly Subject<(IDownloadTask task, DownloadId analyzedHash, string modName)> _analyzed = new(); private readonly IObservable> _tasksChangeSet; private readonly ReadOnlyObservableCollection _currentDownloads; private bool _isDisposed = false; + private readonly IDownloadRegistry _downloadRegistry; - public DownloadService(ILogger logger, IServiceProvider provider, IDataStore store, IArchiveAnalyzer archiveAnalyzer) + public DownloadService(ILogger logger, IServiceProvider provider, IDataStore store, IDownloadRegistry downloadRegistry) { _logger = logger; _provider = provider; _store = store; - _archiveAnalyzer = archiveAnalyzer; + _downloadRegistry = downloadRegistry; _tasks = new SourceList(); _tasksChangeSet = _tasks.Connect(); @@ -104,7 +107,7 @@ internal IEnumerable GetItemsToResume() public IObservable ResumedTasks => _resumed; /// - public IObservable<(IDownloadTask task, Hash analyzedHash, string modName)> AnalyzedArchives => _analyzed; + public IObservable<(IDownloadTask task, DownloadId downloadId, string modName)> AnalyzedArchives => _analyzed; /// public Task AddNxmTask(NXMUrl url) @@ -155,7 +158,7 @@ public void OnCancelled(IDownloadTask task) } /// - public void OnPaused(IDownloadTask task) + public void OnPaused(IDownloadTask task) { _paused.OnNext(task); UpdatePersistedState(task); @@ -165,9 +168,9 @@ public void OnPaused(IDownloadTask task) public void OnResumed(IDownloadTask task) => _resumed.OnNext(task); /// - public Size GetThroughput() + public Size GetThroughput() { - var provider = DateTimeProvider.Instance; + var provider = DateTimeProvider.Instance; var totalThroughput = 0L; foreach (var download in _currentDownloads) totalThroughput += download.CalculateThroughput(provider); @@ -185,8 +188,14 @@ public async Task FinalizeDownloadAsync(IDownloadTask task, TemporaryPath path, try { - var analyzed = await _archiveAnalyzer.AnalyzeFileAsync(path); - _analyzed.OnNext((task, analyzed.Hash, modName)); + // TODO: Fix this + var downloadId = await _downloadRegistry.RegisterDownload(path.Path, new FilePathMetadata + { + Name = modName, + OriginalName = path.Path.FileName, + Quality = Quality.Low + }); + _analyzed.OnNext((task, downloadId, modName)); } catch (Exception e) { @@ -227,11 +236,11 @@ public void Dispose() { if (_isDisposed) return; - + // Pause all tasks, which persists them to datastore. foreach (var task in _currentDownloads) task.Suspend(); - + _isDisposed = true; _tasks.Dispose(); _started.Dispose(); diff --git a/src/Networking/NexusMods.Networking.Downloaders/Interfaces/IDownloadService.cs b/src/Networking/NexusMods.Networking.Downloaders/Interfaces/IDownloadService.cs index ac197759a6..6b691acce8 100644 --- a/src/Networking/NexusMods.Networking.Downloaders/Interfaces/IDownloadService.cs +++ b/src/Networking/NexusMods.Networking.Downloaders/Interfaces/IDownloadService.cs @@ -1,4 +1,5 @@ using DynamicData; +using NexusMods.DataModel; using NexusMods.DataModel.RateLimiting; using NexusMods.Hashing.xxHash64; using NexusMods.Networking.NexusWebApi.Types; @@ -52,7 +53,7 @@ public interface IDownloadService : IDisposable /// This gets fired whenever a download is complete and an archive has been analyzed. /// You can use this callback to gather additional metadata about the archive, or install the mods within. /// - IObservable<(IDownloadTask task, Hash analyzedHash, string modName)> AnalyzedArchives { get; } + IObservable<(IDownloadTask task, DownloadId downloadId, string modName)> AnalyzedArchives { get; } /// /// Adds a task that will download from a NXM link. diff --git a/src/Networking/NexusMods.Networking.HttpDownloader/AdvancedHttpDownloader.cs b/src/Networking/NexusMods.Networking.HttpDownloader/AdvancedHttpDownloader.cs index 79b50c13b0..eb5e414d44 100644 --- a/src/Networking/NexusMods.Networking.HttpDownloader/AdvancedHttpDownloader.cs +++ b/src/Networking/NexusMods.Networking.HttpDownloader/AdvancedHttpDownloader.cs @@ -151,7 +151,7 @@ private static async Task FinalizeDownload(DownloadState state, Cancellati state.StateFilePath.Delete(); File.Move(tempPath.ToString(), state.Destination.ToString(), true); - return await state.Destination.XxHash64Async(cancel); + return await state.Destination.XxHash64Async(token: cancel); } #region Output Writing diff --git a/src/NexusMods.App.UI/RightContent/LoadoutGrid/LoadoutGridViewModel.cs b/src/NexusMods.App.UI/RightContent/LoadoutGrid/LoadoutGridViewModel.cs index 89fe738e60..b7895f9a55 100644 --- a/src/NexusMods.App.UI/RightContent/LoadoutGrid/LoadoutGridViewModel.cs +++ b/src/NexusMods.App.UI/RightContent/LoadoutGrid/LoadoutGridViewModel.cs @@ -11,7 +11,9 @@ using NexusMods.App.UI.RightContent.LoadoutGrid.Columns.ModInstalled; using NexusMods.App.UI.RightContent.LoadoutGrid.Columns.ModName; using NexusMods.App.UI.RightContent.LoadoutGrid.Columns.ModVersion; +using NexusMods.Common; using NexusMods.DataModel.Abstractions; +using NexusMods.DataModel.ArchiveMetaData; using NexusMods.DataModel.Extensions; using NexusMods.DataModel.Loadouts; using NexusMods.DataModel.Loadouts.Cursors; @@ -35,8 +37,8 @@ public class LoadoutGridViewModel : AViewModel, ILoadoutG private readonly IFileSystem _fileSystem; private readonly ILogger _logger; private readonly LoadoutRegistry _loadoutRegistry; - private readonly IArchiveAnalyzer _archiveAnalyzer; private readonly IArchiveInstaller _archiveInstaller; + private readonly IDownloadRegistry _downloadRegistry; [Reactive] public string LoadoutName { get; set; } = ""; @@ -48,14 +50,14 @@ public LoadoutGridViewModel( IServiceProvider provider, LoadoutRegistry loadoutRegistry, IFileSystem fileSystem, - IArchiveAnalyzer archiveAnalyzer, - IArchiveInstaller archiveInstaller) + IArchiveInstaller archiveInstaller, + IDownloadRegistry downloadRegistry) { _logger = logger; _fileSystem = fileSystem; _loadoutRegistry = loadoutRegistry; - _archiveAnalyzer = archiveAnalyzer; _archiveInstaller = archiveInstaller; + _downloadRegistry = downloadRegistry; _columns = new SourceCache, LoadoutColumn>( @@ -124,8 +126,14 @@ public Task AddMod(string path) var _ = Task.Run(async () => { - var analyzedFile = await _archiveAnalyzer.AnalyzeFileAsync(file, CancellationToken.None); - await _archiveInstaller.AddMods(LoadoutId, analyzedFile.Hash, token: CancellationToken.None); + var downloadId = await _downloadRegistry.RegisterDownload(file, + new FilePathMetadata + { + OriginalName = file.FileName, + Quality = Quality.Low, + Name = file.FileName + }); + await _archiveInstaller.AddMods(LoadoutId, downloadId, token: CancellationToken.None); }); return Task.CompletedTask; diff --git a/src/NexusMods.App.UI/Windows/MainWindowViewModel.cs b/src/NexusMods.App.UI/Windows/MainWindowViewModel.cs index 70a6ec14cf..2e69d55767 100644 --- a/src/NexusMods.App.UI/Windows/MainWindowViewModel.cs +++ b/src/NexusMods.App.UI/Windows/MainWindowViewModel.cs @@ -7,6 +7,7 @@ using NexusMods.App.UI.LeftMenu; using NexusMods.App.UI.Overlays; using NexusMods.App.UI.Overlays.MetricsOptIn; +using NexusMods.DataModel; using NexusMods.DataModel.Abstractions; using NexusMods.DataModel.Loadouts; using NexusMods.Hashing.xxHash64; @@ -60,7 +61,7 @@ public MainWindowViewModel( downloadService.AnalyzedArchives.Subscribe(tuple => { // Because HandleDownloadedAnalyzedArchive is an async task, it begins automatically. - HandleDownloadedAnalyzedArchive(tuple.task, tuple.analyzedHash, tuple.modName).ContinueWith(t => + HandleDownloadedAnalyzedArchive(tuple.task, tuple.downloadId, tuple.modName).ContinueWith(t => { if (t.Exception != null) logger.LogError(t.Exception, "Error while installing downloaded analyzed archive"); @@ -109,7 +110,7 @@ public MainWindowViewModel( } - private async Task HandleDownloadedAnalyzedArchive(IDownloadTask task, Hash analyzedHash, string modName) + private async Task HandleDownloadedAnalyzedArchive(IDownloadTask task, DownloadId downloadId, string modName) { var loadouts = Array.Empty(); if (task is IHaveGameDomain gameDomain) @@ -123,9 +124,9 @@ private async Task HandleDownloadedAnalyzedArchive(IDownloadTask task, Hash anal await Task.Run(async () => { if (loadouts.Length > 0) - await _archiveInstaller.AddMods(loadouts[0], analyzedHash, modName); + await _archiveInstaller.AddMods(loadouts[0], downloadId, modName); else - await _archiveInstaller.AddMods(_registry.AllLoadouts().First().LoadoutId, analyzedHash, modName); + await _archiveInstaller.AddMods(_registry.AllLoadouts().First().LoadoutId, downloadId, modName); }); } diff --git a/src/NexusMods.CLI/Services.cs b/src/NexusMods.CLI/Services.cs index 2b4ed9894a..43fc0541ac 100644 --- a/src/NexusMods.CLI/Services.cs +++ b/src/NexusMods.CLI/Services.cs @@ -82,7 +82,6 @@ public static IServiceCollection AddCLI(this IServiceCollection services) .AddVerb(); services.AddAllSingleton>(_ => new Resource("File Extraction")); - services.AddAllSingleton>(_ => new Resource("File Analysis")); return services; } diff --git a/src/NexusMods.CLI/Types/DownloadHandlers/NxmDownloadProtocolHandler.cs b/src/NexusMods.CLI/Types/DownloadHandlers/NxmDownloadProtocolHandler.cs index eebf4caa54..0fc72ff3ac 100644 --- a/src/NexusMods.CLI/Types/DownloadHandlers/NxmDownloadProtocolHandler.cs +++ b/src/NexusMods.CLI/Types/DownloadHandlers/NxmDownloadProtocolHandler.cs @@ -1,4 +1,6 @@ -using NexusMods.DataModel.Abstractions; +using NexusMods.Common; +using NexusMods.DataModel.Abstractions; +using NexusMods.DataModel.ArchiveMetaData; using NexusMods.DataModel.Loadouts.Markers; using NexusMods.Networking.HttpDownloader; using NexusMods.Networking.NexusWebApi; @@ -16,18 +18,20 @@ public class NxmDownloadProtocolHandler : IDownloadProtocolHandler private readonly Client _client; private readonly IHttpDownloader _downloader; private readonly TemporaryFileManager _temp; - private readonly IArchiveAnalyzer _archiveAnalyzer; private readonly IArchiveInstaller _archiveInstaller; + private readonly IDownloadRegistry _downloaderRegistry; /// - public NxmDownloadProtocolHandler(Client client, - IHttpDownloader downloader, TemporaryFileManager temp, - IArchiveInstaller archiveInstaller, IArchiveAnalyzer archiveAnalyzer) + public NxmDownloadProtocolHandler(Client client, + IHttpDownloader downloader, + TemporaryFileManager temp, + IArchiveInstaller archiveInstaller, + IDownloadRegistry downloadRegistry) { - _archiveAnalyzer = archiveAnalyzer; _archiveInstaller = archiveInstaller; _client = client; _downloader = downloader; + _downloaderRegistry = downloadRegistry; _temp = temp; } @@ -39,7 +43,7 @@ public async Task Handle(string url, LoadoutMarker loadout, string modName, Canc { await using var tempPath = _temp.CreateFile(); var parsed = NXMUrl.Parse(url); - + // Note: Normally we should probably source domains from the loadout, but in this case, this is okay. Response links; if (parsed.Key == null) @@ -48,9 +52,14 @@ public async Task Handle(string url, LoadoutMarker loadout, string modName, Canc links = await _client.DownloadLinksAsync(parsed.Mod.Game, parsed.Mod.ModId, parsed.Mod.FileId, parsed.Key.Value, parsed.ExpireTime!.Value, token); var downloadUris = links.Data.Select(u => new HttpRequestMessage(HttpMethod.Get, u.Uri)).ToArray(); - + await _downloader.DownloadAsync(downloadUris, tempPath, null, null, token); - var hash = await _archiveAnalyzer.AnalyzeFileAsync(tempPath, token); - await _archiveInstaller.AddMods(loadout.Value.LoadoutId, hash.Hash, modName, token:token); + var downloadId = await _downloaderRegistry.RegisterDownload(tempPath.Path, new FilePathMetadata + { + OriginalName = tempPath.Path.Name, + Quality = Quality.Low, + Name = tempPath.Path.Name + }, token); + await _archiveInstaller.AddMods(loadout.Value.LoadoutId, downloadId, modName, token:token); } } diff --git a/src/NexusMods.CLI/Verbs/AnalyzeArchive.cs b/src/NexusMods.CLI/Verbs/AnalyzeArchive.cs index b01d3a70ff..b4538e62c1 100644 --- a/src/NexusMods.CLI/Verbs/AnalyzeArchive.cs +++ b/src/NexusMods.CLI/Verbs/AnalyzeArchive.cs @@ -1,8 +1,10 @@ using Microsoft.Extensions.Logging; using NexusMods.Abstractions.CLI; using NexusMods.Abstractions.CLI.DataOutputs; +using NexusMods.Common; using NexusMods.DataModel.Abstractions; using NexusMods.DataModel.ArchiveContents; +using NexusMods.DataModel.ArchiveMetaData; using NexusMods.Paths; namespace NexusMods.CLI.Verbs; @@ -13,7 +15,8 @@ namespace NexusMods.CLI.Verbs; /// public class AnalyzeArchive : AVerb, IRenderingVerb { - private readonly IArchiveAnalyzer _archiveContentsCache; + private readonly ILogger _logger; + private readonly IDownloadRegistry _downloadRegistry; /// public IRenderer Renderer { get; set; } = null!; @@ -21,12 +24,11 @@ public class AnalyzeArchive : AVerb, IRenderingVerb /// /// DI constructor /// - /// /// - public AnalyzeArchive(IArchiveAnalyzer archiveContentsCache, ILogger logger) + public AnalyzeArchive(ILogger logger, IDownloadRegistry downloadRegistry) { _logger = logger; - _archiveContentsCache = archiveContentsCache; + _downloadRegistry = downloadRegistry; } /// @@ -37,7 +39,6 @@ public AnalyzeArchive(IArchiveAnalyzer archiveContentsCache, ILogger("i", "inputFile", "File to Analyze") }); - private readonly ILogger _logger; /// public async Task Run(AbsolutePath inputFile, CancellationToken token) @@ -46,19 +47,23 @@ public async Task Run(AbsolutePath inputFile, CancellationToken token) { var results = await Renderer.WithProgress(token, async () => { - var file = await _archiveContentsCache.AnalyzeFileAsync(inputFile, token) as AnalyzedArchive; - if (file == null) return Array.Empty(); - return file.Contents.Select(kv => + var downloadId = await _downloadRegistry.RegisterDownload(inputFile, new FilePathMetadata + { + OriginalName = inputFile.Name, + Quality = Quality.Low, + Name = inputFile.Name + }, token); + var metadata = await _downloadRegistry.Get(downloadId); + return metadata.Contents.Select(kv => { return new object[] { - kv.Key, kv.Value.Size, kv.Value.Hash, - string.Join(", ", kv.Value.FileTypes.Select(Enum.GetName)) + kv.Path, kv.Size, kv.Hash }; }); }); - await Renderer.Render(new Table(new[] { "Path", "Size", "Hash", "Signatures" }, + await Renderer.Render(new Table(new[] { "Path", "Size", "Hash"}, results.OrderBy(e => (RelativePath)e[0]))); } catch (Exception ex) diff --git a/src/NexusMods.CLI/Verbs/DownloadAndInstallMod.cs b/src/NexusMods.CLI/Verbs/DownloadAndInstallMod.cs index f242e3fa53..12bd68d021 100644 --- a/src/NexusMods.CLI/Verbs/DownloadAndInstallMod.cs +++ b/src/NexusMods.CLI/Verbs/DownloadAndInstallMod.cs @@ -1,6 +1,8 @@ using NexusMods.Abstractions.CLI; using NexusMods.CLI.Types; +using NexusMods.Common; using NexusMods.DataModel.Abstractions; +using NexusMods.DataModel.ArchiveMetaData; using NexusMods.DataModel.Loadouts.Markers; using NexusMods.Networking.HttpDownloader; using NexusMods.Paths; @@ -15,18 +17,19 @@ public class DownloadAndInstallMod : AVerb, IRend private readonly IHttpDownloader _httpDownloader; private readonly TemporaryFileManager _temp; private readonly IEnumerable _handlers; - private readonly IArchiveAnalyzer _archiveAnalyzer; private readonly IArchiveInstaller _archiveInstaller; + private readonly IDownloadRegistry _downloadRegistry; /// public IRenderer Renderer { get; set; } = null!; /// public DownloadAndInstallMod(IHttpDownloader httpDownloader, TemporaryFileManager temp, - IEnumerable handlers, IArchiveInstaller archiveInstaller, IArchiveAnalyzer archiveAnalyzer) + IEnumerable handlers, IArchiveInstaller archiveInstaller, + IDownloadRegistry downloadRegistry) { - _archiveAnalyzer = archiveAnalyzer; _archiveInstaller = archiveInstaller; + _downloadRegistry = downloadRegistry; _httpDownloader = httpDownloader; _temp = temp; _handlers = handlers; @@ -59,8 +62,14 @@ await Renderer.WithProgress(token, async () => await _httpDownloader.DownloadAsync(new[] { new HttpRequestMessage(HttpMethod.Get, uri) }, temporaryPath, null, null, token); - var analyzedFile = await _archiveAnalyzer.AnalyzeFileAsync(temporaryPath, token); - await _archiveInstaller.AddMods(loadout.Value.LoadoutId, analyzedFile.Hash, + var downloadId = await _downloadRegistry.RegisterDownload(temporaryPath, + new FilePathMetadata + { + OriginalName = temporaryPath.Path.Name, + Quality = Quality.Low, + Name = modName + }, token); + await _archiveInstaller.AddMods(loadout.Value.LoadoutId, downloadId, string.IsNullOrWhiteSpace(modName) ? null : modName, token: token); return 0; }); diff --git a/src/NexusMods.CLI/Verbs/InstallMod.cs b/src/NexusMods.CLI/Verbs/InstallMod.cs index 24f6a7e789..daaa966830 100644 --- a/src/NexusMods.CLI/Verbs/InstallMod.cs +++ b/src/NexusMods.CLI/Verbs/InstallMod.cs @@ -1,5 +1,7 @@ using NexusMods.Abstractions.CLI; +using NexusMods.Common; using NexusMods.DataModel.Abstractions; +using NexusMods.DataModel.ArchiveMetaData; using NexusMods.DataModel.Loadouts.Markers; using NexusMods.Paths; @@ -12,7 +14,7 @@ namespace NexusMods.CLI.Verbs; public class InstallMod : AVerb, IRenderingVerb { private readonly IArchiveInstaller _archiveInstaller; - private readonly IArchiveAnalyzer _archiveAnalyzer; + private readonly IDownloadRegistry _downloadRegistry; /// public IRenderer Renderer { get; set; } = null!; @@ -22,10 +24,10 @@ public class InstallMod : AVerb, IRendering /// /// /// - public InstallMod(IArchiveInstaller archiveInstaller, IArchiveAnalyzer archiveAnalyzer) + public InstallMod(IArchiveInstaller archiveInstaller, IDownloadRegistry archiveAnalyzer) { _archiveInstaller = archiveInstaller; - _archiveAnalyzer = archiveAnalyzer; + _downloadRegistry = archiveAnalyzer; } /// @@ -41,8 +43,13 @@ public async Task Run(LoadoutMarker loadout, AbsolutePath file, string name { await Renderer.WithProgress(token, async () => { - var analyzedFile = await _archiveAnalyzer.AnalyzeFileAsync(file, token); - await _archiveInstaller.AddMods(loadout.Value.LoadoutId, analyzedFile.Hash, token:token); + var downloadId = await _downloadRegistry.RegisterDownload(file, new FilePathMetadata + { + OriginalName = file.Name, + Quality = Quality.Low, + Name = name + }, token); + await _archiveInstaller.AddMods(loadout.Value.LoadoutId, downloadId, name, token:token); return file; }); return 0; diff --git a/src/NexusMods.Common/AbsolutePathExtensions.cs b/src/NexusMods.Common/AbsolutePathExtensions.cs index 1c033963de..1fe8ef0065 100644 --- a/src/NexusMods.Common/AbsolutePathExtensions.cs +++ b/src/NexusMods.Common/AbsolutePathExtensions.cs @@ -4,20 +4,26 @@ namespace NexusMods.Common; +/// +/// Extensions for . +/// public static class AbsolutePathExtensions { /// /// Helper method to calculate the hash of a given file, reporting progress to the given job. /// - /// + /// /// /// /// - public static async Task XxHash64Async(this AbsolutePath input, IJob job, - CancellationToken token) + public static async Task XxHash64Async(this AbsolutePath input, IJob? job = null, + CancellationToken token = default) { await using var inputStream = input.Read(); - return await inputStream.HashingCopyAsync(Stream.Null, token, async m => await job.ReportAsync(Size.FromLong(m.Length), token)); + if (job == null) + return await inputStream.HashingCopyAsync(Stream.Null, token, async m => await Task.CompletedTask); + else + return await inputStream.HashingCopyAsync(Stream.Null, token, async m => await job.ReportAsync(Size.FromLong(m.Length), token)); } } diff --git a/src/NexusMods.Common/AsyncEnumerableExtensions.cs b/src/NexusMods.Common/AsyncEnumerableExtensions.cs deleted file mode 100644 index 39b7b76ec8..0000000000 --- a/src/NexusMods.Common/AsyncEnumerableExtensions.cs +++ /dev/null @@ -1,21 +0,0 @@ -namespace NexusMods.Common; - -/// -/// Extensions related to implementations of . -/// -public static class AsyncEnumerableExtensions -{ - /// - /// Filter a by a given predicate. - /// - /// The collection to filter. - /// The function to apply if the item should be returned. - /// Type of item within the collection. - /// Filtered collection that can be asynchronously retrieved. - public static async IAsyncEnumerable Where(this IAsyncEnumerable coll, Func func) - { - await foreach (var itm in coll) - if (func(itm)) - yield return itm; - } -} diff --git a/src/NexusMods.Common/StreamExtensions.cs b/src/NexusMods.Common/StreamExtensions.cs index 47f782af9c..a9d89f786f 100644 --- a/src/NexusMods.Common/StreamExtensions.cs +++ b/src/NexusMods.Common/StreamExtensions.cs @@ -4,11 +4,14 @@ namespace NexusMods.Common; +/// +/// Extensions for . +/// public static class StreamExtensions { /// /// Helper method to calculate the hash of a given stream while copying it to another stream. This method will - /// update the property of the job as it progresses. + /// update the IJob.Process property of the job as it progresses. /// /// /// diff --git a/src/NexusMods.DataModel/Abstractions/DTOs/DownloadAnalysis.cs b/src/NexusMods.DataModel/Abstractions/DTOs/DownloadAnalysis.cs new file mode 100644 index 0000000000..af27bbfcde --- /dev/null +++ b/src/NexusMods.DataModel/Abstractions/DTOs/DownloadAnalysis.cs @@ -0,0 +1,84 @@ +using NexusMods.DataModel.Abstractions.Ids; +using NexusMods.DataModel.ArchiveMetaData; +using NexusMods.DataModel.JsonConverters; +using NexusMods.Hashing.xxHash64; +using NexusMods.Paths; +using NexusMods.Paths.FileTree; + +namespace NexusMods.DataModel.Abstractions.DTOs; + + +/// +/// Analysis of the contents of a download +/// +[JsonName("NexusMods.DataModel.Abstractions.DTOs.DownloadAnalysis")] +public record DownloadAnalysis : Entity +{ + /// + /// The id of the download + /// + public required DownloadId DownloadId { get; init; } + + /// + /// The hash of the download + /// + public required Hash Hash { get; init; } + + /// + /// Size of the download + /// + public required Size Size { get; init; } + + /// + /// The files contained in the download + /// + public required IReadOnlyCollection Contents { get; init; } + + /// + /// Returns a file tree of the contents of the download + /// + /// + public FileTreeNode GetFileTree() + { + return FileTreeNode.CreateTree( + Contents.Select(c => new KeyValuePair(c.Path, c))); + } + + /// + /// Meta data for the download + /// + public AArchiveMetaData? MetaData { get; init; } + + /// + public override EntityCategory Category => EntityCategory.DownloadMetadata; + + /// + protected override IId Persist(IDataStore store) + { + var id = IId.From(EntityCategory.DownloadMetadata, DownloadId.Value); + store.Put(id, this); + return id; + } +} + + +/// +/// A single entry in the download analysis, this is a file that is contained in the download +/// +public record DownloadContentEntry +{ + /// + /// Hash of the file + /// + public required Hash Hash { get; init; } + + /// + /// Size of the file + /// + public required Size Size { get; init; } + + /// + /// Path of the file + /// + public required RelativePath Path { get; init; } +} diff --git a/src/NexusMods.DataModel/Abstractions/Entity.cs b/src/NexusMods.DataModel/Abstractions/Entity.cs index 2085efc431..1ee373ca34 100644 --- a/src/NexusMods.DataModel/Abstractions/Entity.cs +++ b/src/NexusMods.DataModel/Abstractions/Entity.cs @@ -12,8 +12,6 @@ namespace NexusMods.DataModel.Abstractions; /// Example entities include:
/// -
/// -
-/// -
-/// - /// public abstract record Entity : IWalkable { diff --git a/src/NexusMods.DataModel/Abstractions/EntityCategory.cs b/src/NexusMods.DataModel/Abstractions/EntityCategory.cs index 2b5a5c4971..d9b2d425e8 100644 --- a/src/NexusMods.DataModel/Abstractions/EntityCategory.cs +++ b/src/NexusMods.DataModel/Abstractions/EntityCategory.cs @@ -117,4 +117,9 @@ public enum EntityCategory : byte /// Global settings for things like metrics opt-in and the like. /// GlobalSettings = 16, + + /// + /// Information about registered downloads + /// + DownloadMetadata = 17, } diff --git a/src/NexusMods.DataModel/Abstractions/IArchiveAnalyzer.cs b/src/NexusMods.DataModel/Abstractions/IArchiveAnalyzer.cs deleted file mode 100644 index 33751d90ef..0000000000 --- a/src/NexusMods.DataModel/Abstractions/IArchiveAnalyzer.cs +++ /dev/null @@ -1,28 +0,0 @@ -using NexusMods.DataModel.ArchiveContents; -using NexusMods.Hashing.xxHash64; -using NexusMods.Paths; - -namespace NexusMods.DataModel.Abstractions; - -/// -/// A service that analyzes files and archives, returning metadata on their contents with IFileAnalyzer, and -/// possibly caching the results. -/// -public interface IArchiveAnalyzer -{ - /// - /// Analyzes a file (normally an archive). If the file is an archive it will be added to the archive manager. - /// The analysis data will be cached and returned. - /// - /// - /// - /// - public Task AnalyzeFileAsync(AbsolutePath path, CancellationToken token = default); - - /// - /// Gets the analysis data for the given hash. If the hash is not known, null will be returned. - /// - /// - /// - public AnalyzedFile? GetAnalysisData(Hash hash); -} diff --git a/src/NexusMods.DataModel/Abstractions/IArchiveInstaller.cs b/src/NexusMods.DataModel/Abstractions/IArchiveInstaller.cs index 27d1c4da9b..3d77dd2f84 100644 --- a/src/NexusMods.DataModel/Abstractions/IArchiveInstaller.cs +++ b/src/NexusMods.DataModel/Abstractions/IArchiveInstaller.cs @@ -13,9 +13,9 @@ public interface IArchiveInstaller /// standard way to "install" a mod. /// /// - /// + /// /// /// /// - public Task AddMods(LoadoutId loadoutId, Hash archiveHash, string? defaultModName = null, CancellationToken token = default); + public Task AddMods(LoadoutId loadoutId, DownloadId downloadId, string? defaultModName = null, CancellationToken token = default); } diff --git a/src/NexusMods.DataModel/Abstractions/IArchiveManager.cs b/src/NexusMods.DataModel/Abstractions/IArchiveManager.cs index 022579087f..32837fc742 100644 --- a/src/NexusMods.DataModel/Abstractions/IArchiveManager.cs +++ b/src/NexusMods.DataModel/Abstractions/IArchiveManager.cs @@ -1,6 +1,7 @@ using NexusMods.Common; using NexusMods.Hashing.xxHash64; using NexusMods.Paths; +using NexusMods.Paths.FileTree; namespace NexusMods.DataModel.Abstractions; @@ -23,7 +24,7 @@ public interface IArchiveManager /// /// /// - Task BackupFiles(IEnumerable<(IStreamFactory, Hash, Size)> backups, CancellationToken token = default); + Task BackupFiles(IEnumerable backups, CancellationToken token = default); /// @@ -51,3 +52,13 @@ public interface IArchiveManager /// Task GetFileStream(Hash hash, CancellationToken token = default); } + + +/// +/// A helper class for that represents a file to be backed up. The Path is optional, +/// but should be provided if it is expected that the paths will be used for extraction or mod installation. +/// +/// +/// +/// +public readonly record struct ArchivedFileEntry(IStreamFactory StreamFactory, Hash Hash, Size Size); diff --git a/src/NexusMods.DataModel/Abstractions/IDownloadRegistry.cs b/src/NexusMods.DataModel/Abstractions/IDownloadRegistry.cs new file mode 100644 index 0000000000..d7bf6d05ed --- /dev/null +++ b/src/NexusMods.DataModel/Abstractions/IDownloadRegistry.cs @@ -0,0 +1,63 @@ +using NexusMods.Common; +using NexusMods.DataModel.Abstractions.DTOs; +using NexusMods.DataModel.ArchiveMetaData; +using NexusMods.Hashing.xxHash64; +using NexusMods.Paths; + +namespace NexusMods.DataModel.Abstractions; + +/// +/// A service for linking downloads with files in the archive manager +/// +public interface IDownloadRegistry +{ + /// + /// Register a download with the registry, sourced from a stream, returns a download id that can be used to retrieve the download later. + /// + /// + /// + /// + /// + public ValueTask RegisterDownload(IStreamFactory factory, AArchiveMetaData metaData, CancellationToken token = default); + + + /// + /// Register a download with the registry, returns a download id that can be used to retrieve the download later. + /// + /// + /// + /// + /// + public ValueTask RegisterDownload(AbsolutePath path, AArchiveMetaData metaData, CancellationToken token = default); + + /// + /// Indexes an already extracted download, returns a download id that can be used to retrieve the download later. + /// + /// + /// + /// + /// + public ValueTask RegisterFolder(AbsolutePath path, AArchiveMetaData metaData, CancellationToken token = default); + + /// + /// Get the analysis of a download + /// + /// + /// + public ValueTask Get(DownloadId id); + + + /// + /// Get the analysis of all downloads + /// + /// + public IEnumerable GetAll(); + + + /// + /// Finds all downloads that have the given hash + /// + /// + /// + public IEnumerable GetByHash(Hash hash); +} diff --git a/src/NexusMods.DataModel/Abstractions/IFileAnalysisData.cs b/src/NexusMods.DataModel/Abstractions/IFileAnalysisData.cs deleted file mode 100644 index 86a5dbf046..0000000000 --- a/src/NexusMods.DataModel/Abstractions/IFileAnalysisData.cs +++ /dev/null @@ -1,13 +0,0 @@ -namespace NexusMods.DataModel.Abstractions; - -/// -/// Base class for storing information related to file analysis results for -/// various kinds of mods/plugins. -/// -/// -/// Implementation of is context -/// (plugin/application) dependent. This interface does not define a contract, -/// it is only used for clarification and as a constraint -/// throughout the various DataModel APIs. -/// -public interface IFileAnalysisData : IMetadata { } diff --git a/src/NexusMods.DataModel/Abstractions/IFileAnalyzer.cs b/src/NexusMods.DataModel/Abstractions/IFileAnalyzer.cs deleted file mode 100644 index 2906c3a51d..0000000000 --- a/src/NexusMods.DataModel/Abstractions/IFileAnalyzer.cs +++ /dev/null @@ -1,64 +0,0 @@ -using System.Diagnostics; -using NexusMods.DataModel.Abstractions.Ids; -using NexusMods.FileExtractor.FileSignatures; -using NexusMods.Paths; - -namespace NexusMods.DataModel.Abstractions; - -/// -/// Provides an abstraction over a component used to analyze files.

-/// -/// A file analyzer reads data from a given stream, and extracts metadata necessary -/// for. Such as dependency information. -/// -public interface IFileAnalyzer -{ - /// - /// The unique identifier for this file analyzer, includes a revision number that - /// should be updated whenever changes to the analyzer necessitate a re-analysis - /// of archive files. - /// - public FileAnalyzerId Id { get; } - - /// - /// Defines the file types supported by this file analyzer. - /// - public IEnumerable FileTypes { get; } - - /// - /// Asynchronously analyzes a file with the given information. - /// - /// Information about the item to analyze. - /// Allows you to cancel the operation. - /// Analysis data for the processed files. - public IAsyncEnumerable AnalyzeAsync(FileAnalyzerInfo info, CancellationToken token = default); -} - -/// -/// Extra info for file analyzer. Passed as a struct to avoid having to update all callees. -/// -[DebuggerDisplay("{FileName}")] -public struct FileAnalyzerInfo -{ - /// - /// Name of the file being analyzed. - /// - public string FileName { get; init; } - - /// - /// Provides access to the underlying file. - /// - public Stream Stream { get; init; } - - /// - /// Path to the extracted parent archive. - /// If this item is a child of an archive file, this will be non null. - /// - public TemporaryPath? ParentArchive; - - /// - /// Relative path of the file to the parent archive, if sourced from an archive. - /// This path is empty if not sourced from an archive. - /// - public RelativePath? RelativePath; -} diff --git a/src/NexusMods.DataModel/Abstractions/Ids/IId.cs b/src/NexusMods.DataModel/Abstractions/Ids/IId.cs index d04cfc184e..872bfbec1b 100644 --- a/src/NexusMods.DataModel/Abstractions/Ids/IId.cs +++ b/src/NexusMods.DataModel/Abstractions/Ids/IId.cs @@ -98,6 +98,19 @@ public static IId FromSpan(EntityCategory category, ReadOnlySpan span) return new IdVariableLength(category, mem); } + /// + /// Creates an ID from a category and a GUID. + /// + /// + /// + /// + public static IId From(EntityCategory category, Guid guid) + { + Span span = stackalloc byte[16]; + guid.TryWriteBytes(span); + return FromSpan(category, span); + } + /// /// Converts the current Span to a tagged span; which embeds the category /// of the item in the first byte of the span. diff --git a/src/NexusMods.DataModel/Abstractions/LogicalIds/DownloadId.cs b/src/NexusMods.DataModel/Abstractions/LogicalIds/DownloadId.cs new file mode 100644 index 0000000000..41ed6c0846 --- /dev/null +++ b/src/NexusMods.DataModel/Abstractions/LogicalIds/DownloadId.cs @@ -0,0 +1,17 @@ +using Vogen; + +namespace NexusMods.DataModel; + +/// +/// Id for a registered download +/// +[ValueObject] +public partial struct DownloadId +{ + /// + /// Create a new download id, randomly generated + /// + /// + public static DownloadId New() => From(Guid.NewGuid()); + +} diff --git a/src/NexusMods.DataModel/ArchiveAnalyzer.cs b/src/NexusMods.DataModel/ArchiveAnalyzer.cs deleted file mode 100644 index c5bdd57fd5..0000000000 --- a/src/NexusMods.DataModel/ArchiveAnalyzer.cs +++ /dev/null @@ -1,314 +0,0 @@ -using System.Buffers.Binary; -using System.Collections.Immutable; -using Microsoft.Extensions.Logging; -using NexusMods.Common; -using NexusMods.DataModel.Abstractions; -using NexusMods.DataModel.Abstractions.Ids; -using NexusMods.DataModel.ArchiveContents; -using NexusMods.DataModel.ArchiveMetaData; -using NexusMods.DataModel.Extensions; -using NexusMods.DataModel.RateLimiting; -using NexusMods.DataModel.RateLimiting.Extensions; -using NexusMods.FileExtractor.FileSignatures; -using NexusMods.FileExtractor.StreamFactories; -using NexusMods.Hashing.xxHash64; -using NexusMods.Paths; -using NexusMods.Paths.Extensions; - -namespace NexusMods.DataModel; - -/// -/// Helper method that allows you to index (analyze) files using provided (s), -/// caching the results inside the given . -/// -public class ArchiveAnalyzer : IArchiveAnalyzer -{ - private readonly ILogger _logger; - private readonly FileExtractor.FileExtractor _extractor; - private readonly TemporaryFileManager _manager; - private readonly IResource _limiter; - private readonly SignatureChecker _sigs; - private readonly IDataStore _store; - private readonly FileHashCache _fileHashCache; - private readonly ILookup _analyzers; - private readonly IArchiveManager _archiveManager; - - /// - /// The signature of the analyzers used when indexing files. - /// - public Hash AnalyzersSignature { get; private set; } - - /// - /// Called from DI container. - public ArchiveAnalyzer(ILogger logger, - IResource limiter, - FileExtractor.FileExtractor extractor, - TemporaryFileManager manager, - FileHashCache hashCache, - IEnumerable analyzers, - IDataStore dataStore, - IArchiveManager archiveManager) - { - _logger = logger; - _limiter = limiter; - _extractor = extractor; - _manager = manager; - _sigs = new SignatureChecker(Enum.GetValues()); - _analyzers = analyzers.SelectMany(a => a.FileTypes.Select(t => (Type: t, Analyzer: a))) - .ToLookup(k => k.Type, v => v.Analyzer); - AnalyzersSignature = MakeAnalyzerSignature(analyzers); - _store = dataStore; - _fileHashCache = hashCache; - _archiveManager = archiveManager; - } - - /// - /// Recalculates the analyzer signature, only used for testing. - /// - public void RecalculateAnalyzerSignature() - { - AnalyzersSignature = - MakeAnalyzerSignature(_analyzers.SelectMany(x => x).Distinct()); - } - - /// - /// Analyzes a file and caches the result within the data store. - /// - /// Path of the file to be analyzed. - /// Allows you to cancel the operation. - /// The file analysis data. - public async Task AnalyzeFileAsync(AbsolutePath path, CancellationToken token = default) - { - var entry = await _fileHashCache.IndexFileAsync(path, token); - var found = _store.Get(new Id64(EntityCategory.FileAnalysis, (ulong)entry.Hash)); - if (found != null && found.AnalyzersHash == AnalyzersSignature) - { - if (found is AnalyzedArchive aa && !AArchiveMetaData.GetMetaDatas(_store, found.Hash).OfType().Any()) - { - var metaData = FileArchiveMetaData.Create(path, aa); - metaData.EnsurePersisted(_store); - } - return found; - } - - // Analyze the archive and cache the info - var result = await AnalyzeFileInnerAsync(new NativeFileStreamFactory(path), path.FileName, token); - - - if (result is AnalyzedArchive archive) - { - // Only persist AnalyzedData if it's an archive - result.EnsurePersisted(_store); - - // Save the source of this archive so we can use it later - var metaData = FileArchiveMetaData.Create(path, archive); - metaData.EnsurePersisted(_store); - } - - return result; - } - - /// - /// Retrieves all archives that contain a file with a specific hash. - /// - /// The hash of the file inside the archive. - /// All matching archives. - public IEnumerable ArchivesThatContain(Hash hash) - { - var prefix = new Id64(EntityCategory.FileContainedIn, (ulong)hash); - return _store.GetByPrefix(prefix); - } - - /// - /// Gets the file analysis result for a file with a given hash. - /// - /// This file can either be an archive or a file stored within - /// an archive for a given file. - /// - /// The hash of the file for which data is to be obtained. - /// - public AnalyzedFile? GetAnalysisData(Hash hash) - { - return _store.Get(new Id64(EntityCategory.FileAnalysis, (ulong)hash)); - } - - private Task AnalyzeFileInnerAsync(IStreamFactory sFn, string fileName, CancellationToken token = default) - { - return AnalyzeFileInnerAsync(sFn, token, 0, Hash.Zero, null, default, fileName); - } - - private async Task AnalyzeFileInnerAsync(IStreamFactory sFn, CancellationToken token, int level, Hash parent, TemporaryPath? parentArchivePath, RelativePath parentPath, string fileName) - { - Hash hash; - List sigs; - var analysisData = new List(); - { - await using var hashStream = await sFn.GetStreamAsync(); - if (level == 0) - { - if (sFn.Name is AbsolutePath ap) - { - hash = (await _fileHashCache.IndexFileAsync(ap, token)).Hash; - } - else - { - using var job = await _limiter.BeginAsync($"Hashing {sFn.Name.FileName}", sFn.Size, token); - hash = await hashStream.XxHash64Async(job, token); - } - } - else - { - hash = await hashStream.XxHash64Async(token); - } - - hashStream.Position = 0; - sigs = (await _sigs.MatchesAsync(hashStream)).ToList(); - - if (parentPath != default && SignatureChecker.TryGetFileType(parentPath.Extension, out var type)) - sigs.Add(type); - - var found = _store.Get(new Id64(EntityCategory.FileAnalysis, (ulong)hash)); - if (found is AnalyzedFile af && af.AnalyzersHash == AnalyzersSignature) - return af; - - hashStream.Position = 0; - - - foreach (var sig in sigs) - { - foreach (var analyzer in _analyzers[sig]) - { - hashStream.Position = 0; - try - { - var fileAnalyzerInfo = new FileAnalyzerInfo - { - - RelativePath = parentPath, - FileName = fileName, - ParentArchive = parentArchivePath, - Stream = hashStream - }; - - await foreach (var data in analyzer.AnalyzeAsync(fileAnalyzerInfo, token)) - { - analysisData.Add(data); - } - } - catch (Exception e) - { - _logger.LogError(e, "Error analyzing {Path} with {Analyzer}", sFn.Name, analyzer.GetType().Name); - } - } - } - } - - AnalyzedFile? file = null; - - if (await _extractor.CanExtract(sFn)) - { - file = await AnalyzeArchiveInnerAsync(sFn, level, hash, sigs, analysisData, token) ?? default; - } - - file ??= new AnalyzedFile - { - Hash = hash, - AnalyzersHash = AnalyzersSignature, - Size = sFn.Size, - FileTypes = sigs.ToArray(), - AnalysisData = analysisData.ToImmutableList() - }; - - if (parent != Hash.Zero) - EnsureReverseIndex(hash, parent, parentPath); - - return file; - } - - private Hash MakeAnalyzerSignature(IEnumerable analyzers) - { - var algo = new XxHash64Algorithm(0); - // We have to hash blocks in at least 32 bytes at a time. We'll waste a bit of space here - // but at least we don't have to allocate a MemoryStream and pour the data into it. - Span buffer = stackalloc byte[32]; - foreach (var analyzer in analyzers.OrderBy(a => a.Id.Analyzer)) - { - analyzer.Id.Analyzer.TryWriteBytes(buffer); - BinaryPrimitives.WriteUInt32BigEndian(buffer.SliceFast(16), analyzer.Id.Revision); - algo.TransformByteGroupsInternal(buffer); - } - - return Hash.FromULong(algo.FinalizeHashValueInternal(Span.Empty)); - } - - private async Task AnalyzeArchiveInnerAsync(IStreamFactory sFn, int level, Hash hash, List sigs, - List analysisData, CancellationToken token) - { - try - { - await using var tmpFolder = _manager.CreateFolder(); - List> children; - { - await _extractor.ExtractAllAsync(sFn, tmpFolder, token); - children = await _limiter.ForEachFileAsync(tmpFolder, - async (_, entry) => - { - // ReSharper disable once AccessToDisposedClosure - var relPath = entry.Path.RelativeTo(tmpFolder.Path); - - // ReSharper disable once AccessToDisposedClosure - var analysisRecord = await AnalyzeFileInnerAsync( - new NativeFileStreamFactory(entry.Path), token, - level + 1, hash, tmpFolder, relPath, relPath.FileName); - analysisRecord.WithPersist(_store); - return (entry.Path, - Results: analysisRecord); - }, - token, "Analyzing Files") - // ReSharper disable once AccessToDisposedClosure - .SelectAsync(a => KeyValuePair.Create(a.Path.RelativeTo(tmpFolder.Path), a.Results.DataStoreId)) - .ToListAsync(); - - // Some parts of this code fail with an empty collection - if (children.Any()) - { - await _archiveManager.BackupFiles(children - .Select(c => - { - IStreamFactory path = new NativeFileStreamFactory(tmpFolder.Path.Combine(c.Key)); - var analysis = _store.Get(c.Value)!; - return (path, analysis.Hash, analysis.Size); - }), token); - } - } - - var file = new AnalyzedArchive - { - Hash = hash, - AnalyzersHash = AnalyzersSignature, - Size = sFn.Size, - FileTypes = sigs.ToArray(), - AnalysisData = analysisData.ToImmutableList(), - Contents = new EntityDictionary(_store, children) - }; - - return file; - } - catch (Exception ex) - { - _logger.LogError(ex, "Error extracting archive {Path}, skipping analysis", sFn.Name); - return null; - } - } - - private void EnsureReverseIndex(Hash hash, Hash parent, RelativePath parentPath) - { - var entity = new FileContainedIn - { - File = hash, - Parent = parent, - Path = parentPath - }; - entity.EnsurePersisted(_store); - } -} diff --git a/src/NexusMods.DataModel/ArchiveContents/AnalyzedArchive.cs b/src/NexusMods.DataModel/ArchiveContents/AnalyzedArchive.cs deleted file mode 100644 index 3ff52369f0..0000000000 --- a/src/NexusMods.DataModel/ArchiveContents/AnalyzedArchive.cs +++ /dev/null @@ -1,19 +0,0 @@ -using NexusMods.DataModel.Abstractions; -using NexusMods.DataModel.JsonConverters; -using NexusMods.Paths; - -namespace NexusMods.DataModel.ArchiveContents; - -/// -/// Represents an individual archive which has been scanned by an implementation -/// of . -/// -[JsonName("AnalyzedArchive")] -public record AnalyzedArchive : AnalyzedFile -{ - /// - /// A mapping of relative paths inside the archive to respective files - /// contained inside. - /// - public required EntityDictionary Contents { get; init; } -} diff --git a/src/NexusMods.DataModel/ArchiveContents/AnalyzedFile.cs b/src/NexusMods.DataModel/ArchiveContents/AnalyzedFile.cs deleted file mode 100644 index 47a8c9d33d..0000000000 --- a/src/NexusMods.DataModel/ArchiveContents/AnalyzedFile.cs +++ /dev/null @@ -1,70 +0,0 @@ -using System.Collections.Immutable; -using NexusMods.DataModel.Abstractions; -using NexusMods.DataModel.Abstractions.Ids; -using NexusMods.DataModel.JsonConverters; -using NexusMods.FileExtractor.FileSignatures; -using NexusMods.Hashing.xxHash64; -using NexusMods.Paths; - -namespace NexusMods.DataModel.ArchiveContents; - -/// -/// Represents an individual file which has been scanned by an implementation -/// of .

-/// -/// This file doesn't have to represent a file on disk, it can also represent -/// a file stored inside an archive. -/// -[JsonName("AnalyzedFile")] -public record AnalyzedFile : Entity -{ - /// - /// Size of the file in bytes. - /// - public required Size Size { get; init; } - - /// - /// Individual hash of the file. - /// - public required Hash Hash { get; init; } - - /// - /// Hash of the ids of the analyzers used to analyze this file. - /// - public required Hash AnalyzersHash { get; init; } - - /// - /// Stores the types of file this file is classified by. - /// - /// - /// This field is based on signatures/magic values stored in file headers. - /// Usually there will only be one result here, but it is not impossible - /// for there to be multiple matches. - /// - public required FileType[] FileTypes { get; init; } - - /// - /// Stores any data returned from a that might be - /// useful later. - /// - /// - /// This information is specific and must be - /// casted using e.g. `IFileAnalysisData as PluginData data` before use. - /// - public ImmutableList AnalysisData { get; init; } = ImmutableList.Empty; - - /// - public override EntityCategory Category => EntityCategory.FileAnalysis; - - /// - /// Persist the entity in the data store. Calculates the ID based on the Hash field. - /// - /// - /// - protected override IId Persist(IDataStore store) - { - var newId = new Id64(Category, (ulong)Hash); - store.Put(newId, this); - return newId; - } -} diff --git a/src/NexusMods.DataModel/ArchiveContents/FileContainedIn.cs b/src/NexusMods.DataModel/ArchiveContents/FileContainedIn.cs deleted file mode 100644 index a9ee7932d0..0000000000 --- a/src/NexusMods.DataModel/ArchiveContents/FileContainedIn.cs +++ /dev/null @@ -1,45 +0,0 @@ -using NexusMods.DataModel.Abstractions; -using NexusMods.DataModel.Abstractions.Ids; -using NexusMods.DataModel.JsonConverters; -using NexusMods.Hashing.xxHash64; -using NexusMods.Paths; - -namespace NexusMods.DataModel.ArchiveContents; - -/// -/// -/// -[JsonName("FileContainedIn")] -public record FileContainedIn : Entity -{ - /// - /// Hash of the individual file, corresponding to an . - /// - public required Hash File { get; init; } - - /// - /// Hash of the parent element [usually an ]. - /// - public required Hash Parent { get; init; } - - /// - /// Relative path of the file inside the parent archive. - /// - public required RelativePath Path { get; init; } - - /// - /// Stores the entity in the data store, using a as the ID. - /// Calculated based on the , and fields. - /// - /// - /// - protected override IId Persist(IDataStore store) - { - var id = new TwoId64(Category, (ulong)File, (ulong)Parent); - store.Put(id, this); - return id; - } - - /// - public override EntityCategory Category => EntityCategory.FileContainedIn; -} diff --git a/src/NexusMods.DataModel/ArchiveId.cs b/src/NexusMods.DataModel/ArchiveId.cs new file mode 100644 index 0000000000..a6c5a123b7 --- /dev/null +++ b/src/NexusMods.DataModel/ArchiveId.cs @@ -0,0 +1,18 @@ +using Vogen; + +namespace NexusMods.DataModel; + +/// +/// A unique identifier for an archive in a ArchiveManager +/// +[ValueObject] +public partial struct ArchiveId +{ + + /// + /// Create a new archive id, randomly generated + /// + /// + public static ArchiveId New() => From(Guid.NewGuid()); + +} diff --git a/src/NexusMods.DataModel/ArchiveInstaller.cs b/src/NexusMods.DataModel/ArchiveInstaller.cs index 2e912317d2..2f7f914118 100644 --- a/src/NexusMods.DataModel/ArchiveInstaller.cs +++ b/src/NexusMods.DataModel/ArchiveInstaller.cs @@ -14,53 +14,52 @@ using NexusMods.DataModel.RateLimiting; using NexusMods.DataModel.Sorting.Rules; using NexusMods.Hashing.xxHash64; +using NexusMods.Paths; +using NexusMods.Paths.FileTree; namespace NexusMods.DataModel; /// -/// Installs mods from archives previously analyzed by . +/// Installs mods from archives /// public class ArchiveInstaller : IArchiveInstaller { private readonly ILogger _logger; private readonly IDataStore _dataStore; - private readonly IArchiveAnalyzer _archiveAnalyzer; private readonly LoadoutRegistry _registry; private readonly IInterprocessJobManager _jobManager; + private readonly IArchiveManager _archiveManager; + private readonly IDownloadRegistry _downloadRegistry; /// /// DI Constructor /// public ArchiveInstaller(ILogger logger, - IArchiveAnalyzer archiveAnalyzer, + IDownloadRegistry downloadRegistry, IDataStore dataStore, LoadoutRegistry registry, + IArchiveManager archiveManager, IInterprocessJobManager jobManager) { _logger = logger; _dataStore = dataStore; + _downloadRegistry = downloadRegistry; _registry = registry; - _archiveAnalyzer = archiveAnalyzer; + _archiveManager = archiveManager; _jobManager = jobManager; } /// - public async Task AddMods(LoadoutId loadoutId, Hash archiveHash, string? defaultModName = null, CancellationToken token = default) + public async Task AddMods(LoadoutId loadoutId, DownloadId downloadId, string? defaultModName = null, CancellationToken token = default) { - if (_archiveAnalyzer.GetAnalysisData(archiveHash) is not AnalyzedArchive analysisData) - { - _logger.LogError("Could not find analysis data for archive {ArchiveHash} or file is not an archive", archiveHash); - throw new InvalidOperationException("Could not find analysis data for archive"); - } - // Get the loadout and create the mod so we can use it in the job. var loadout = _registry.GetMarker(loadoutId); - var metaData = AArchiveMetaData.GetMetaDatas(_dataStore, archiveHash).FirstOrDefault(); + var download = await _downloadRegistry.Get(downloadId); var archiveName = ""; - if (metaData is not null && defaultModName == null) + if (download.MetaData is not null && defaultModName == null) { - archiveName = metaData.Name; + archiveName = download.MetaData.Name; } var baseMod = new Mod @@ -83,6 +82,18 @@ public async Task AddMods(LoadoutId loadoutId, Hash archiveHash, string LoadoutId = loadoutId }); + // Create a tree so installers can find the file easily. + var tree = FileTreeNode.CreateTree(download.Contents + .Select(entry => + KeyValuePair.Create( + entry.Path, + new ModSourceFileEntry + { + Hash = entry.Hash, + Size = entry.Size, + StreamFactory = new ArchiveManagerStreamFactory(_archiveManager, entry.Hash) {Name = entry.Path, Size = entry.Size} + }))); + // Step 3: Run the archive through the installers. var (results, modInstaller) = (await loadout.Value.Installation.Game.Installers .SelectAsync(async modInstaller => @@ -92,8 +103,7 @@ public async Task AddMods(LoadoutId loadoutId, Hash archiveHash, string var modResults = (await modInstaller.GetModsAsync( loadout.Value.Installation, baseMod.Id, - analysisData.Hash, - analysisData.Contents, + tree, token)).ToArray(); return (modResults, modInstaller); } diff --git a/src/NexusMods.DataModel/ArchiveManagerStreamFactory.cs b/src/NexusMods.DataModel/ArchiveManagerStreamFactory.cs new file mode 100644 index 0000000000..f4d7c3fa4f --- /dev/null +++ b/src/NexusMods.DataModel/ArchiveManagerStreamFactory.cs @@ -0,0 +1,42 @@ +using NexusMods.Common; +using NexusMods.DataModel.Abstractions; +using NexusMods.Hashing.xxHash64; +using NexusMods.Paths; + +namespace NexusMods.DataModel; + +/// +/// A stream factory that reads from the archive manager +/// +public class ArchiveManagerStreamFactory : IStreamFactory +{ + private readonly IArchiveManager _archiveManager; + private readonly Hash _hash; + + /// + /// Main constructor + /// + /// + /// + public ArchiveManagerStreamFactory(IArchiveManager archiveManager, Hash hash) + { + _archiveManager = archiveManager; + _hash = hash; + } + + /// + public DateTime LastModifiedUtc { get; } = DateTime.UtcNow; + + /// + public required IPath Name { get; init;} + + /// + public required Size Size { get; init; } + + + /// + public async ValueTask GetStreamAsync() + { + return await _archiveManager.GetFileStream(_hash); + } +} diff --git a/src/NexusMods.DataModel/ArchiveMetaData/AArchiveMetaData.cs b/src/NexusMods.DataModel/ArchiveMetaData/AArchiveMetaData.cs index e512272d3f..ef902fcb8d 100644 --- a/src/NexusMods.DataModel/ArchiveMetaData/AArchiveMetaData.cs +++ b/src/NexusMods.DataModel/ArchiveMetaData/AArchiveMetaData.cs @@ -1,6 +1,7 @@ using NexusMods.Common; using NexusMods.DataModel.Abstractions; using NexusMods.DataModel.Abstractions.Ids; +using NexusMods.DataModel.JsonConverters; using NexusMods.Hashing.xxHash64; using NexusMods.Paths; @@ -9,50 +10,17 @@ namespace NexusMods.DataModel.ArchiveMetaData; /// /// Info for a mod archive describing where it came from and the suggested name of the file /// -public abstract record AArchiveMetaData : Entity +[JsonName("NexusMods.DataModel.ArchiveMetaData.AArchiveMetaData")] +public abstract record AArchiveMetaData { /// /// A human readable name for the archive. /// public string Name { get; init; } = string.Empty; - - /// - /// The size of the archive. - /// - public required Size Size { get; init; } - - /// - /// The hash of the archive. - /// - public required Hash Hash { get; init; } - + /// /// How accurate is this metadata? Data from a file on disk is more generic than data /// from a NexusMods API call. /// public required Quality Quality { get; init; } - - /// - public override EntityCategory Category => EntityCategory.ArchiveMetaData; - - /// - protected override IId Persist(IDataStore store) - { - var id = store.ContentHashId(this, out var data); - var newId = new TwoId64(Category, Hash.Value, id.ToUInt64()); - store.PutRaw(newId, data); - return newId; - } - - /// - /// Get all the meta data for the given archive. Data is returned in order of priority. - /// - /// - /// - /// - public static IEnumerable GetMetaDatas(IDataStore store, Hash archiveHash) - { - return store.GetByPrefix(new Id64(EntityCategory.ArchiveMetaData, archiveHash.Value)) - .OrderBy(x => x.Quality); - } } diff --git a/src/NexusMods.DataModel/ArchiveMetaData/FileArchiveMetaData.cs b/src/NexusMods.DataModel/ArchiveMetaData/FileArchiveMetaData.cs deleted file mode 100644 index b59f919b95..0000000000 --- a/src/NexusMods.DataModel/ArchiveMetaData/FileArchiveMetaData.cs +++ /dev/null @@ -1,36 +0,0 @@ -using NexusMods.Common; -using NexusMods.DataModel.ArchiveContents; -using NexusMods.DataModel.JsonConverters; -using NexusMods.Paths; - -namespace NexusMods.DataModel.ArchiveMetaData; - -/// -/// Archive Meta data for a file archive, where it cam -/// -[JsonName("NexusMods.DataMode.ArchiveMetaData")] -public record FileArchiveMetaData : AArchiveMetaData -{ - /// - /// The filename of the file - /// - public required RelativePath OriginalPath { get; init; } - - /// - /// Create a new FileArchiveMetaData object from an AnalyzedArchive and a raw path - /// - /// - /// - /// - public static FileArchiveMetaData Create(AbsolutePath path, AnalyzedArchive archive) - { - return new FileArchiveMetaData - { - Quality = Quality.Low, - OriginalPath = path.FileName, - Name = path.GetFileNameWithoutExtension(), - Size = archive.Size, - Hash = archive.Hash - }; - } -} diff --git a/src/NexusMods.DataModel/ArchiveMetaData/FilePathMetadata.cs b/src/NexusMods.DataModel/ArchiveMetaData/FilePathMetadata.cs new file mode 100644 index 0000000000..b9466535d4 --- /dev/null +++ b/src/NexusMods.DataModel/ArchiveMetaData/FilePathMetadata.cs @@ -0,0 +1,16 @@ +using NexusMods.DataModel.JsonConverters; +using NexusMods.Paths; + +namespace NexusMods.DataModel.ArchiveMetaData; + +/// +/// Archive metadata for a download that was installed from a file path. +/// +[JsonName("NexusMods.DataModel.ArchiveMetaData.FilePathMetadata")] +public record FilePathMetadata : AArchiveMetaData +{ + /// + /// The filename portion of the path. + /// + public RelativePath OriginalName { get; init; } = RelativePath.Empty; +} diff --git a/src/NexusMods.DataModel/ArchiveMetaData/GameArchiveMetadata.cs b/src/NexusMods.DataModel/ArchiveMetaData/GameArchiveMetadata.cs new file mode 100644 index 0000000000..e28290dbee --- /dev/null +++ b/src/NexusMods.DataModel/ArchiveMetaData/GameArchiveMetadata.cs @@ -0,0 +1,14 @@ +using NexusMods.DataModel.Games; +using NexusMods.DataModel.JsonConverters; + +namespace NexusMods.DataModel.ArchiveMetaData; + + +/// +/// Archive metadata for a download that was installed from an existing game archive. +/// +[JsonName("NexusMods.DataModel.ArchiveMetaData.GameArchiveMetadata")] +public record GameArchiveMetadata : AArchiveMetaData +{ + public required GameInstallation Installation { get; init; } +} diff --git a/src/NexusMods.DataModel/ArchiveMetaData/NexusModsArchiveMetadata.cs b/src/NexusMods.DataModel/ArchiveMetaData/NexusModsArchiveMetadata.cs new file mode 100644 index 0000000000..7915991a41 --- /dev/null +++ b/src/NexusMods.DataModel/ArchiveMetaData/NexusModsArchiveMetadata.cs @@ -0,0 +1,27 @@ +using NexusMods.DataModel.Games; +using NexusMods.DataModel.JsonConverters; +using NexusMods.Networking.NexusWebApi.Types; + +namespace NexusMods.DataModel.ArchiveMetaData; + +/// +/// Archive metadata for a download that was installed from a NexusMods mod. +/// +[JsonName("NexusMods.DataModel.ArchiveMetaData.NexusModsArchiveMetadata")] +public record NexusModsArchiveMetadata : AArchiveMetaData +{ + /// + /// The NexusMods API game ID. + /// + public required GameDomain GameDomain { get; init; } + + /// + /// Mod ID corresponding to the Nexus API. + /// + public required ModId ModId { get; init; } + + /// + /// File ID corresponding to the Nexus API. + /// + public required FileId FileId { get; init; } +} diff --git a/src/NexusMods.DataModel/ChunkedStreams/ChunkedStream.cs b/src/NexusMods.DataModel/ChunkedStreams/ChunkedStream.cs index e4a0a37ae0..c0b0d20d84 100644 --- a/src/NexusMods.DataModel/ChunkedStreams/ChunkedStream.cs +++ b/src/NexusMods.DataModel/ChunkedStreams/ChunkedStream.cs @@ -75,8 +75,10 @@ public override int Read(byte[] buffer, int offset, int count) var chunkOffset = _position % _source.ChunkSize.Value; var isLastChunk = chunkIdx == _source.ChunkCount - 1; var chunk = await GetChunkAsync(chunkIdx, cancellationToken); + var readToEnd = Math.Clamp(_source.Size.Value - _position, 0, Int32.MaxValue); var toRead = Math.Min(buffer.Length, (int)(_source.ChunkSize.Value - chunkOffset)); + toRead = Math.Min(toRead, (int)readToEnd); if (isLastChunk) { var lastChunkExtraSize = _source.Size.Value % _source.ChunkSize.Value; diff --git a/src/NexusMods.DataModel/Diagnostics/DiagnosticManager.cs b/src/NexusMods.DataModel/Diagnostics/DiagnosticManager.cs index 953336f5c3..417a4e4f70 100644 --- a/src/NexusMods.DataModel/Diagnostics/DiagnosticManager.cs +++ b/src/NexusMods.DataModel/Diagnostics/DiagnosticManager.cs @@ -3,6 +3,7 @@ using JetBrains.Annotations; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using NexusMods.Common; using NexusMods.DataModel.Abstractions; using NexusMods.DataModel.Abstractions.Ids; using NexusMods.DataModel.Diagnostics.Emitters; @@ -67,9 +68,9 @@ public DiagnosticManager( public IObservable> DiagnosticChanges => _diagnosticCache.Connect(); public IEnumerable ActiveDiagnostics => _diagnosticCache.Items; - public void OnLoadoutChanged(Loadout loadout) + public async ValueTask OnLoadoutChanged(Loadout loadout) { - RefreshLoadoutDiagnostics(loadout); + await RefreshLoadoutDiagnostics(loadout); RefreshModDiagnostics(loadout); } @@ -78,7 +79,7 @@ public void ClearDiagnostics() _diagnosticCache.Edit(updater => updater.Clear()); } - internal void RefreshLoadoutDiagnostics(Loadout loadout) + internal async Task RefreshLoadoutDiagnostics(Loadout loadout) { // Remove outdated diagnostics for previous revisions of the loadout RemoveDiagnostics(kv => kv.Value.DataReferences @@ -89,11 +90,11 @@ internal void RefreshLoadoutDiagnostics(Loadout loadout) ) ); - var newDiagnostics = _loadoutDiagnosticEmitters - .SelectMany(emitter => emitter.Diagnose(loadout)) - .ToArray(); + var newDiagnostics = await _loadoutDiagnosticEmitters + .SelectAsync(async emitter => await emitter.Diagnose(loadout).ToArrayAsync()) + .ToArrayAsync(); - AddDiagnostics(newDiagnostics); + AddDiagnostics(newDiagnostics.SelectMany(vals => vals)); } internal void RefreshModDiagnostics(Loadout loadout) diff --git a/src/NexusMods.DataModel/Diagnostics/Emitters/ILoadoutDiagnosticEmitter.cs b/src/NexusMods.DataModel/Diagnostics/Emitters/ILoadoutDiagnosticEmitter.cs index e8bb7635ec..a21f915465 100644 --- a/src/NexusMods.DataModel/Diagnostics/Emitters/ILoadoutDiagnosticEmitter.cs +++ b/src/NexusMods.DataModel/Diagnostics/Emitters/ILoadoutDiagnosticEmitter.cs @@ -18,5 +18,5 @@ public interface ILoadoutDiagnosticEmitter /// Diagnoses a loadout and creates instances of . /// /// The current loadout. - IEnumerable Diagnose(Loadout loadout); + IAsyncEnumerable Diagnose(Loadout loadout); } diff --git a/src/NexusMods.DataModel/Diagnostics/IDiagnosticManager.cs b/src/NexusMods.DataModel/Diagnostics/IDiagnosticManager.cs index ceac5bc82e..bc339e8450 100644 --- a/src/NexusMods.DataModel/Diagnostics/IDiagnosticManager.cs +++ b/src/NexusMods.DataModel/Diagnostics/IDiagnosticManager.cs @@ -29,5 +29,5 @@ public interface IDiagnosticManager : IDisposable /// /// Callback for loadout changes. /// - void OnLoadoutChanged(Loadout loadout); + ValueTask OnLoadoutChanged(Loadout loadout); } diff --git a/src/NexusMods.DataModel/DownloadRegistry.cs b/src/NexusMods.DataModel/DownloadRegistry.cs new file mode 100644 index 0000000000..5660dfc202 --- /dev/null +++ b/src/NexusMods.DataModel/DownloadRegistry.cs @@ -0,0 +1,134 @@ +using DynamicData; +using Microsoft.Extensions.Logging; +using NexusMods.Common; +using NexusMods.DataModel.Abstractions; +using NexusMods.DataModel.Abstractions.DTOs; +using NexusMods.DataModel.Abstractions.Ids; +using NexusMods.DataModel.ArchiveMetaData; +using NexusMods.DataModel.RateLimiting; +using NexusMods.FileExtractor.StreamFactories; +using NexusMods.Hashing.xxHash64; +using NexusMods.Paths; + +namespace NexusMods.DataModel; + +/// +/// Registry for downloads, stores metadata and links to files in the archive manager +/// +public class DownloadRegistry : IDownloadRegistry +{ + private readonly ILogger _logger; + private readonly FileExtractor.FileExtractor _extractor; + private readonly IArchiveManager _archiveManager; + private readonly TemporaryFileManager _temporaryFileManager; + private readonly IDataStore _dataStore; + + /// + /// DI Constructor + /// + /// + /// + /// + public DownloadRegistry(ILogger logger, FileExtractor.FileExtractor extractor, + IArchiveManager archiveManager, TemporaryFileManager temporaryFileManager, IDataStore store) + { + _logger = logger; + _extractor = extractor; + _archiveManager = archiveManager; + _temporaryFileManager = temporaryFileManager; + _dataStore = store; + } + + /// + public async ValueTask RegisterDownload(IStreamFactory factory, AArchiveMetaData metaData, CancellationToken token = default) + { + await using var tmpFolder = _temporaryFileManager.CreateFolder(); + await _extractor.ExtractAllAsync(factory, tmpFolder.Path, token); + return await RegisterFolder(tmpFolder.Path, metaData, token); + } + + /// + public async ValueTask RegisterDownload(AbsolutePath path, AArchiveMetaData metaData, CancellationToken token = default) + { + await using var tmpFolder = _temporaryFileManager.CreateFolder(); + await _extractor.ExtractAllAsync(path, tmpFolder.Path, token); + return await RegisterFolder(tmpFolder.Path, metaData, token); + } + + /// + public async ValueTask RegisterFolder(AbsolutePath path, AArchiveMetaData metaData, CancellationToken token = default) + { + List files = new(); + List paths = new(); + + _logger.LogInformation("Analyzing archive: {Name}", path); + foreach (var file in path.EnumerateFiles()) + { + // TODO: report this as progress + var hash = await file.XxHash64Async(token: token); + + files.Add(new ArchivedFileEntry + { + Hash = hash, + Size = file.FileInfo.Size, + StreamFactory = new NativeFileStreamFactory(file) + }); + paths.Add(file.RelativeTo(path)); + } + + _logger.LogInformation("Archiving {Count} files and {Size} of data", files.Count, files.Sum(f => f.Size)); + await _archiveManager.BackupFiles(files, token); + + Hash finalHash; + Size finalSize; + if (path.DirectoryExists()) + { + finalHash = Hash.Zero; + finalSize = Size.Zero; + } + else + { + finalHash = await path.XxHash64Async(token: token); + finalSize = path.FileInfo.Size; + } + + _logger.LogInformation("Calculating metadata"); + var analysis = new DownloadAnalysis() + { + DownloadId = DownloadId.New(), + Hash = finalHash, + Size = finalSize, + Contents = paths.Zip(files).Select(pair => + new DownloadContentEntry + { + Size = pair.Second.Size, + Hash = pair.Second.Hash, + Path = pair.First + }).ToList(), + MetaData = metaData + }; + analysis.EnsurePersisted(_dataStore); + return analysis.DownloadId; + + } + + /// + public async ValueTask Get(DownloadId id) + { + return _dataStore.Get(IId.From(EntityCategory.DownloadMetadata, id.Value))!; + } + + /// + public IEnumerable GetAll() + { + return _dataStore.GetByPrefix(new Id64(EntityCategory.DownloadMetadata, 0)); + } + + /// + public IEnumerable GetByHash(Hash hash) + { + return GetAll() + .Where(d => d.Hash == hash) + .Select(d => d.DownloadId); + } +} diff --git a/src/NexusMods.DataModel/Extensions/AnalyzedFileExtensions.cs b/src/NexusMods.DataModel/Extensions/AnalyzedFileExtensions.cs deleted file mode 100644 index f6c8056c4c..0000000000 --- a/src/NexusMods.DataModel/Extensions/AnalyzedFileExtensions.cs +++ /dev/null @@ -1,46 +0,0 @@ -using JetBrains.Annotations; -using NexusMods.DataModel.ArchiveContents; -using NexusMods.DataModel.Games; -using NexusMods.DataModel.Loadouts; -using NexusMods.DataModel.Loadouts.ModFiles; -using NexusMods.Paths; - -namespace NexusMods.DataModel.Extensions; - -/// -/// Extension methods for -/// -[PublicAPI] -public static class AnalyzedFileExtensions -{ - /// - /// Maps the provided to a - /// - public static GameFile ToGameFile(this AnalyzedFile analyzedFile, GamePath to, GameInstallation installation) - { - return new GameFile - { - Id = ModFileId.New(), - To = to, - Installation = installation, - Hash = analyzedFile.Hash, - Size = analyzedFile.Size, - Metadata = analyzedFile.AnalysisData.AsMetadata() - }; - } - - /// - /// Maps the provided to a . - /// - public static FromArchive ToFromArchive(this AnalyzedFile analyzedFile, GamePath to) - { - return new FromArchive - { - Id = ModFileId.New(), - To = to, - Hash = analyzedFile.Hash, - Size = analyzedFile.Size, - Metadata = analyzedFile.AnalysisData.AsMetadata() - }; - } -} diff --git a/src/NexusMods.DataModel/Extensions/MetadataExtensions.cs b/src/NexusMods.DataModel/Extensions/MetadataExtensions.cs deleted file mode 100644 index 38f29b7707..0000000000 --- a/src/NexusMods.DataModel/Extensions/MetadataExtensions.cs +++ /dev/null @@ -1,22 +0,0 @@ -using System.Collections.Immutable; -using JetBrains.Annotations; -using NexusMods.DataModel.Abstractions; - -namespace NexusMods.DataModel.Extensions; - -/// -/// Extensions for . -/// -[PublicAPI] -public static class MetadataExtensions -{ - /// - /// Casts the elements of to . - /// - /// - /// - public static ImmutableList AsMetadata(this ImmutableList list) - { - return list.Cast().ToImmutableList(); - } -} diff --git a/src/NexusMods.DataModel/FileHashCache.cs b/src/NexusMods.DataModel/FileHashCache.cs index 89f435b90e..f44ba9d470 100644 --- a/src/NexusMods.DataModel/FileHashCache.cs +++ b/src/NexusMods.DataModel/FileHashCache.cs @@ -1,4 +1,5 @@ using System.Buffers.Binary; +using System.Runtime.CompilerServices; using System.Text; using NexusMods.Common; using NexusMods.DataModel.Abstractions; @@ -92,7 +93,7 @@ public IAsyncEnumerable IndexFolderAsync(AbsolutePath path, Cancell /// Entries are pulled from cache if they already exist and we /// can verify cached entry is accurate. /// - public async IAsyncEnumerable IndexFoldersAsync(IEnumerable paths, CancellationToken token = default) + public async IAsyncEnumerable IndexFoldersAsync(IEnumerable paths, [EnumeratorCancellation] CancellationToken token = default) { // Don't want to error via a empty folder paths = paths.Where(p => p.DirectoryExists()); diff --git a/src/NexusMods.DataModel/GlobalSettings/GlobalSettingsManager.cs b/src/NexusMods.DataModel/GlobalSettings/GlobalSettingsManager.cs index 2aa5713e22..bb40c62b9f 100644 --- a/src/NexusMods.DataModel/GlobalSettings/GlobalSettingsManager.cs +++ b/src/NexusMods.DataModel/GlobalSettings/GlobalSettingsManager.cs @@ -4,11 +4,19 @@ namespace NexusMods.DataModel.GlobalSettings; +/// +/// A manager for global settings that affect the application as a whole. +/// public class GlobalSettingsManager { private readonly ILogger _logger; private readonly IDataStore _dataStore; + /// + /// DI Constructor + /// + /// + /// public GlobalSettingsManager(ILogger logger, IDataStore dataStore) { _logger = logger; diff --git a/src/NexusMods.DataModel/Loadouts/LoadoutManager.cs b/src/NexusMods.DataModel/Loadouts/LoadoutManager.cs index d24eca7803..f6fe5af7b2 100644 --- a/src/NexusMods.DataModel/Loadouts/LoadoutManager.cs +++ b/src/NexusMods.DataModel/Loadouts/LoadoutManager.cs @@ -4,6 +4,7 @@ using NexusMods.Common; using NexusMods.DataModel.Abstractions; using NexusMods.DataModel.Abstractions.Ids; +using NexusMods.DataModel.ArchiveMetaData; using NexusMods.DataModel.Extensions; using NexusMods.DataModel.Games; using NexusMods.DataModel.Interprocess.Jobs; @@ -50,9 +51,9 @@ public class LoadoutManager private readonly IFileSystem _fileSystem; private readonly IModInstaller[] _installers; - private readonly IArchiveAnalyzer _analyzer; private readonly ILookup _tools; private readonly IInterprocessJobManager _jobManager; + private readonly IDownloadRegistry _downloadRegistry; /// /// @@ -63,10 +64,10 @@ public LoadoutManager(ILogger logger, LoadoutRegistry registry, IResource limiter, IArchiveManager archiveManager, + IDownloadRegistry downloadRegistry, IDataStore store, FileHashCache fileHashCache, IEnumerable installers, - IArchiveAnalyzer analyzer, IEnumerable tools, IInterprocessJobManager jobManager) { @@ -79,7 +80,7 @@ public LoadoutManager(ILogger logger, Store = store; _jobManager = jobManager; _installers = installers.ToArray(); - _analyzer = analyzer; + _downloadRegistry = downloadRegistry; _tools = tools.SelectMany(t => t.Domains.Select(d => (Tool: t, Domain: d))) .ToLookup(t => t.Domain, t => t.Tool); } @@ -185,21 +186,29 @@ private async Task IndexAndAddGameFiles(GameInstallation installation, if (indexGameFiles) { + var meta = new GameArchiveMetadata() + { + Installation = installation, + Quality = Quality.High, + }; + foreach (var (type, path) in installation.Locations) { - if (!_fileSystem.DirectoryExists(path)) continue; + if (!path.DirectoryExists()) continue; + var download = await _downloadRegistry.RegisterFolder(path, meta, token); - await foreach (var result in FileHashCache - .IndexFolderAsync(path, token) - .WithCancellation(token)) + var toc = await _downloadRegistry.Get(download); + var indexed = toc.Contents.ToDictionary(c => c.Path); + foreach (var file in path.EnumerateFiles()) { - var analyzedFile = await _analyzer.AnalyzeFileAsync(result.Path, token); - - var file = analyzedFile - .ToGameFile(new GamePath(type, result.Path.RelativeTo(path)), installation) - .WithPersist(Store); - - gameFiles.Add(file); + var found = indexed[file.RelativeTo(path)]; + gameFiles.Add(new FromArchive() + { + Id = ModFileId.New(), + Hash = found.Hash, + Size = found.Size, + To = new GamePath(type, found.Path) + }); } } } diff --git a/src/NexusMods.DataModel/Loadouts/LoadoutSyncronizer.cs b/src/NexusMods.DataModel/Loadouts/LoadoutSyncronizer.cs index a25ced921b..8b088b67de 100644 --- a/src/NexusMods.DataModel/Loadouts/LoadoutSyncronizer.cs +++ b/src/NexusMods.DataModel/Loadouts/LoadoutSyncronizer.cs @@ -9,6 +9,7 @@ using NexusMods.DataModel.Loadouts.LoadoutSynchronizerDTOs; using NexusMods.DataModel.Loadouts.ModFiles; using NexusMods.DataModel.Loadouts.Mods; +using NexusMods.DataModel.ModInstallers; using NexusMods.DataModel.Sorting; using NexusMods.DataModel.Sorting.Rules; using NexusMods.DataModel.TriggerFilter; @@ -39,7 +40,7 @@ public class LoadoutSynchronizer /// /// /// - public LoadoutSynchronizer(ILogger logger, + public LoadoutSynchronizer(ILogger logger, IFingerprintCache modSortRulesFingerprintCache, IDirectoryIndexer directoryIndexer, IArchiveManager archiveManager, @@ -71,7 +72,7 @@ public LoadoutSynchronizer(ILogger logger, { if (!mod.Enabled) continue; - + foreach (var (_, file) in mod.Files) { if (file is not IToFile toFile) @@ -98,7 +99,7 @@ public async Task> SortMods(Loadout loadout) .ToDictionaryAsync(r => r.Id, r => r.Item2); if (modRules.Count == 0) return Array.Empty(); - + var sorted = Sorter.Sort(mods, m => m.Id, m => modRules[m.Id]); return sorted; } @@ -231,7 +232,7 @@ private async ValueTask EmitCreatePlan(List plan, ModFilePair pair, }); return; } - + // If the file is generated if (pair.File is IGeneratedFile generatedFile) { @@ -322,7 +323,7 @@ public async ValueTask MakeIngestPlan(Loadout loadout, Func() - .Select(f => ((IStreamFactory)new NativeFileStreamFactory(f.To), f.Hash, f.Size)) + .Select(f => new ArchivedFileEntry(new NativeFileStreamFactory(f.To), f.Hash, f.Size)) .ToList(); if (backups.Any()) @@ -460,7 +461,7 @@ public async ValueTask Ingest(IngestPlan plan, string commitMessage = " var byType = plan.Steps.ToLookup(t => t.GetType()); var backupFiles = byType[typeof(IngestSteps.BackupFile)] .OfType() - .Select(f => ((IStreamFactory)new NativeFileStreamFactory(f.Source), f.Hash, f.Size)); + .Select(f => new ArchivedFileEntry(new NativeFileStreamFactory(f.Source), f.Hash, f.Size)); await _archiveManager.BackupFiles(backupFiles); return _loadoutRegistry.Alter(plan.Loadout.LoadoutId, commitMessage, new IngestVisitor(byType, plan)); diff --git a/src/NexusMods.DataModel/ModInstallers/AModInstaller.cs b/src/NexusMods.DataModel/ModInstallers/AModInstaller.cs new file mode 100644 index 0000000000..42b9c2ced7 --- /dev/null +++ b/src/NexusMods.DataModel/ModInstallers/AModInstaller.cs @@ -0,0 +1,35 @@ +using Microsoft.Extensions.DependencyInjection; +using NexusMods.DataModel.Games; +using NexusMods.DataModel.Loadouts; +using NexusMods.Paths; +using NexusMods.Paths.FileTree; + +namespace NexusMods.DataModel.ModInstallers; + +/// +/// Mod installer base class that provides support for the installation of mods +/// +public abstract class AModInstaller : IModInstaller +{ + private readonly IServiceProvider _serviceProvider; + + /// + /// DI Constructor + /// + /// + protected AModInstaller(IServiceProvider serviceProvider) + { + // Not used yet, but here to force the service provider to be injected by implementing classes + _serviceProvider = serviceProvider; + } + + /// + /// Helper for returning no results + /// + public static readonly IEnumerable NoResults = Enumerable.Empty(); + + /// + public abstract ValueTask> GetModsAsync(GameInstallation gameInstallation, + ModId baseModId, FileTreeNode archiveFiles, + CancellationToken cancellationToken = default); +} diff --git a/src/NexusMods.DataModel/ModInstallers/IModInstaller.cs b/src/NexusMods.DataModel/ModInstallers/IModInstaller.cs index d460a70fbe..70d14f50c5 100644 --- a/src/NexusMods.DataModel/ModInstallers/IModInstaller.cs +++ b/src/NexusMods.DataModel/ModInstallers/IModInstaller.cs @@ -1,11 +1,8 @@ using JetBrains.Annotations; -using NexusMods.Common; -using NexusMods.DataModel.Abstractions; -using NexusMods.DataModel.ArchiveContents; using NexusMods.DataModel.Games; using NexusMods.DataModel.Loadouts; -using NexusMods.Hashing.xxHash64; using NexusMods.Paths; +using NexusMods.Paths.FileTree; namespace NexusMods.DataModel.ModInstallers; @@ -21,14 +18,12 @@ public interface IModInstaller /// /// The game installation. /// The base mod id. - /// Hash of the source archive. /// Files from the archive. /// Cancellation token. /// public ValueTask> GetModsAsync( GameInstallation gameInstallation, ModId baseModId, - Hash srcArchiveHash, - EntityDictionary archiveFiles, + FileTreeNode archiveFiles, CancellationToken cancellationToken = default); } diff --git a/src/NexusMods.DataModel/ModInstallers/ModSourceFileEntry.cs b/src/NexusMods.DataModel/ModInstallers/ModSourceFileEntry.cs new file mode 100644 index 0000000000..c98a350842 --- /dev/null +++ b/src/NexusMods.DataModel/ModInstallers/ModSourceFileEntry.cs @@ -0,0 +1,54 @@ +using NexusMods.Common; +using NexusMods.DataModel.Loadouts; +using NexusMods.DataModel.Loadouts.ModFiles; +using NexusMods.Hashing.xxHash64; +using NexusMods.Paths; + +namespace NexusMods.DataModel.ModInstallers; + +/// +/// A helper class for providing information about a mod file to an installer +/// +public class ModSourceFileEntry +{ + /// + /// A factory that can be used to open the file and read its contents + /// + public required IStreamFactory StreamFactory { get; init; } + + /// + /// The hash of the file + /// + public required Hash Hash { get; init; } + + /// + /// The size of the file + /// + public required Size Size { get; init; } + + /// + /// Open the file as a readonly seekable stream + /// + /// + public async ValueTask Open() + { + return await StreamFactory.GetStreamAsync(); + } + + /// + /// Maps the provided to a mod file + /// + /// + /// + public FromArchive ToFromArchive(GamePath to) + { + return new FromArchive + { + Id = ModFileId.New(), + To = to, + Hash = Hash, + Size = Size + }; + } + +} diff --git a/src/NexusMods.DataModel/NexusMods.DataModel.csproj b/src/NexusMods.DataModel/NexusMods.DataModel.csproj index a1776cc65a..4a63a6691d 100644 --- a/src/NexusMods.DataModel/NexusMods.DataModel.csproj +++ b/src/NexusMods.DataModel/NexusMods.DataModel.csproj @@ -4,6 +4,7 @@ + diff --git a/src/NexusMods.DataModel/NxArchiveManager.cs b/src/NexusMods.DataModel/NxArchiveManager.cs index dc205e98bd..6b00415ee0 100644 --- a/src/NexusMods.DataModel/NxArchiveManager.cs +++ b/src/NexusMods.DataModel/NxArchiveManager.cs @@ -49,8 +49,15 @@ public ValueTask HaveFile(Hash hash) } /// - public async Task BackupFiles(IEnumerable<(IStreamFactory, Hash, Size)> backups, CancellationToken token = default) + public Task BackupFiles(IEnumerable backups, CancellationToken token = default) { + throw new NotImplementedException(); + } + + + private async Task BackupFiles(IEnumerable<(IStreamFactory, Hash, Size)> backups, CancellationToken token = default) + { + // TODO: port this to the new format var builder = new NxPackerBuilder(); var distinct = backups.DistinctBy(d => d.Item2).ToArray(); var streams = new List(); @@ -198,6 +205,7 @@ public Task> ExtractFiles(IEnumerable files, Can return Task.FromResult>(results); } + /// public Task GetFileStream(Hash hash, CancellationToken token = default) { throw new NotImplementedException(); diff --git a/src/NexusMods.DataModel/Services.cs b/src/NexusMods.DataModel/Services.cs index fa8e05c523..042b0653c7 100644 --- a/src/NexusMods.DataModel/Services.cs +++ b/src/NexusMods.DataModel/Services.cs @@ -3,6 +3,7 @@ using Microsoft.Extensions.DependencyInjection; using NexusMods.Common; using NexusMods.DataModel.Abstractions; +using NexusMods.DataModel.ArchiveMetaData; using NexusMods.DataModel.Diagnostics; using NexusMods.DataModel.GlobalSettings; using NexusMods.DataModel.Interprocess; @@ -67,10 +68,10 @@ IDataModelSettings Settings(IServiceProvider provider) coll.AddSingleton(); coll.AddSingleton(); coll.AddSingleton(); + coll.AddSingleton(); coll.AddSingleton(); coll.AddSingleton(); coll.AddSingleton(); - coll.AddSingleton(); coll.AddSingleton(); coll.AddSingleton(); @@ -84,8 +85,8 @@ IDataModelSettings Settings(IServiceProvider provider) coll.AddSingleton>(); coll.AddSingleton>(); coll.AddSingleton>(); - coll.AddSingleton>(); coll.AddSingleton>>(); + coll.AddSingleton>(); coll.AddSingleton(s => { diff --git a/src/NexusMods.DataModel/ZipArchiveManager.cs b/src/NexusMods.DataModel/ZipArchiveManager.cs index eb87a4f8f2..1168cd7f9f 100644 --- a/src/NexusMods.DataModel/ZipArchiveManager.cs +++ b/src/NexusMods.DataModel/ZipArchiveManager.cs @@ -1,7 +1,6 @@ using System.Buffers; using System.Buffers.Binary; using System.IO.Compression; -using NexusMods.Common; using NexusMods.DataModel.Abstractions; using NexusMods.DataModel.Abstractions.Ids; using NexusMods.DataModel.ArchiveContents; @@ -9,6 +8,7 @@ using NexusMods.Hashing.xxHash64; using NexusMods.Paths; using NexusMods.Paths.Extensions; +using NexusMods.Paths.FileTree; using NexusMods.Paths.Utilities; namespace NexusMods.DataModel; @@ -48,11 +48,12 @@ public ValueTask HaveFile(Hash hash) } /// - public async Task BackupFiles(IEnumerable<(IStreamFactory, Hash, Size)> backups, CancellationToken token = default) + public async Task BackupFiles(IEnumerable backups, CancellationToken token = default) { - var guid = Guid.NewGuid(); - var id = guid.ToString(); - var distinct = backups.DistinctBy(d => d.Item2).ToArray(); + var archiveId = ArchiveId.From(Guid.NewGuid()); + var id = archiveId.Value.ToString(); + var backupsList = backups.ToList(); + var distinct = backupsList.DistinctBy(d => d.Hash).ToList(); using var buffer = MemoryPool.Shared.Rent((int)_chunkSize); var outputPath = _archiveLocations.First().Combine(id).AppendExtension(KnownExtensions.Tmp); @@ -62,18 +63,18 @@ public async Task BackupFiles(IEnumerable<(IStreamFactory, Hash, Size)> backups, foreach (var backup in distinct) { - await using var srcStream = await backup.Item1.GetStreamAsync(); - var chunkCount = (int)(backup.Item3.Value / _chunkSize); - if (backup.Item3.Value % _chunkSize > 0) + await using var srcStream = await backup.StreamFactory.GetStreamAsync(); + var chunkCount = (int)(backup.Size.Value / _chunkSize); + if (backup.Size.Value % _chunkSize > 0) chunkCount++; - var hexName = backup.Item2.ToHex(); + var hexName = backup.Hash.ToHex(); for (var chunkIdx = 0; chunkIdx < chunkCount; chunkIdx++) { var entry = builder.CreateEntry($"{hexName}_{chunkIdx}", CompressionLevel.Optimal); await using var entryStream = entry.Open(); - var toCopy = (int)Math.Min(_chunkSize, (long)backup.Item3.Value - (chunkIdx * _chunkSize)); + var toCopy = (int)Math.Min(_chunkSize, (long)backup.Size.Value - (chunkIdx * _chunkSize)); await srcStream.ReadExactlyAsync(buffer.Memory[..toCopy], token); await entryStream.WriteAsync(buffer.Memory[..toCopy], token); await entryStream.FlushAsync(token); @@ -84,15 +85,15 @@ public async Task BackupFiles(IEnumerable<(IStreamFactory, Hash, Size)> backups, var finalPath = outputPath.ReplaceExtension(KnownExtensions.Zip); await outputPath.MoveToAsync(finalPath, token: token); - UpdateIndexes(distinct, guid, finalPath); + UpdateReverseIndexes(distinct, archiveId, finalPath); } - private void UpdateIndexes((IStreamFactory, Hash, Size)[] distinct, Guid guid, + private void UpdateReverseIndexes(IEnumerable distinct, ArchiveId archiveId, AbsolutePath finalPath) { foreach (var entry in distinct) { - var dbId = IdFor(entry.Item2, guid); + var dbId = IdFor(entry.Hash, archiveId); var dbEntry = new ArchivedFiles { @@ -105,11 +106,11 @@ private void UpdateIndexes((IStreamFactory, Hash, Size)[] distinct, Guid guid, } } - private IId IdFor(Hash hash, Guid guid) + private IId IdFor(Hash hash, ArchiveId archiveId) { Span buffer = stackalloc byte[24]; BinaryPrimitives.WriteUInt64BigEndian(buffer, hash.Value); - guid.TryWriteBytes(buffer.SliceFast(8)); + archiveId.Value.TryWriteBytes(buffer.SliceFast(8)); return IId.FromSpan(EntityCategory.ArchivedFiles, buffer); } @@ -161,7 +162,7 @@ public ChunkedArchiveStream(ZipArchive archive, Hash hash) { var prefix = hash.ToHex() + "_"; _entries = archive.Entries.Where(entry => entry.Name.StartsWith(prefix)) - .Order() + .OrderBy(a => a.Name) .ToArray(); Size = Size.FromLong(_entries.Sum(e => e.Length)); ChunkSize = Size.FromLong(_chunkSize); diff --git a/tests/Games/NexusMods.Games.BethesdaGameStudios.Tests/PluginAnalysisTests.cs b/tests/Games/NexusMods.Games.BethesdaGameStudios.Tests/PluginAnalysisTests.cs index ec79be1ffb..755b25df00 100644 --- a/tests/Games/NexusMods.Games.BethesdaGameStudios.Tests/PluginAnalysisTests.cs +++ b/tests/Games/NexusMods.Games.BethesdaGameStudios.Tests/PluginAnalysisTests.cs @@ -7,26 +7,30 @@ namespace NexusMods.Games.BethesdaGameStudios.Tests; [Trait("RequiresGameInstalls", "True")] // Technically this doesn't require the game, but the DI system does for the other tests -public class PluginAnalysisTests : AFileAnalyzerTest +public class PluginAnalysisTests : AGameTest { private readonly AbsolutePath _plugin1; private readonly AbsolutePath _plugin2; + private readonly PluginAnalyzer _pluginAnalyzer; - public PluginAnalysisTests(IFileSystem fileSystem, IServiceProvider serviceProvider) : base(serviceProvider) + public PluginAnalysisTests(IFileSystem fileSystem, PluginAnalyzer pluginAnalyzer, IServiceProvider serviceProvider) : base(serviceProvider) { + _pluginAnalyzer = pluginAnalyzer; _plugin1 = BethesdaTestHelpers.GetAssetsPath(fileSystem).Combine("testfile1.esp"); _plugin2 = BethesdaTestHelpers.GetAssetsPath(fileSystem).Combine("testfile2.esl"); } [Fact] - public async Task LoadsMetadataForPlugins_Esm() => VerifyDependsOnSkyrimEsm(await AnalyzeFile(_plugin1)); + public async Task LoadsMetadataForPlugins_Esm() => await VerifyDependsOnSkyrimEsm(_plugin1); [Fact] - public async Task LoadsMetadataForPlugins_Esl() => VerifyDependsOnSkyrimEsm(await AnalyzeFile(_plugin2)); + public async Task LoadsMetadataForPlugins_Esl() => await VerifyDependsOnSkyrimEsm(_plugin2); - private static void VerifyDependsOnSkyrimEsm(IEnumerable result) + private async Task VerifyDependsOnSkyrimEsm(AbsolutePath path) { - result.Should().ContainEquivalentOf(new PluginAnalysisData + await using var stream = path.Read(); + var resultData = await _pluginAnalyzer.AnalyzeAsync(path.FileName, stream); + resultData.Should().BeEquivalentTo(new PluginAnalysisData { IsLightMaster = true, Masters = new[] { "Skyrim.esm".ToRelativePath() }, diff --git a/tests/Games/NexusMods.Games.BethesdaGameStudios.Tests/SkyrimLegendaryEditionTests/SkyrimLegendaryEditionTests.cs b/tests/Games/NexusMods.Games.BethesdaGameStudios.Tests/SkyrimLegendaryEditionTests/SkyrimLegendaryEditionTests.cs index 332aaba1ee..567d151a62 100644 --- a/tests/Games/NexusMods.Games.BethesdaGameStudios.Tests/SkyrimLegendaryEditionTests/SkyrimLegendaryEditionTests.cs +++ b/tests/Games/NexusMods.Games.BethesdaGameStudios.Tests/SkyrimLegendaryEditionTests/SkyrimLegendaryEditionTests.cs @@ -52,7 +52,7 @@ public async Task CanInstallAndApplyMostPopularMods() _verbTester.LastTable.Rows.Count().Should().Be(1); // install SKSE - await _verbTester.RunNoBannerAsync("install-mod", "-l", loadoutName, "-f", sksePath.ToString()); + await _verbTester.RunNoBannerAsync("install-mod", "-l", loadoutName, "-f", sksePath.ToString(), "-n", skseModName); await _verbTester.RunNoBannerAsync("list-mods", "-l", loadoutName); _verbTester.LastTable.Rows.Count().Should().Be(2); diff --git a/tests/Games/NexusMods.Games.BethesdaGameStudios.Tests/SkyrimSpecialEditionTests/SkyrimSpecialEditionTests.cs b/tests/Games/NexusMods.Games.BethesdaGameStudios.Tests/SkyrimSpecialEditionTests/SkyrimSpecialEditionTests.cs index 356758fec6..1a20d4ef38 100644 --- a/tests/Games/NexusMods.Games.BethesdaGameStudios.Tests/SkyrimSpecialEditionTests/SkyrimSpecialEditionTests.cs +++ b/tests/Games/NexusMods.Games.BethesdaGameStudios.Tests/SkyrimSpecialEditionTests/SkyrimSpecialEditionTests.cs @@ -146,12 +146,7 @@ public async Task CanGeneratePluginsFile() To = new GamePath(GameFolderType.Game, $"Data/{file.Key}"), Hash = Hash.Zero, Size = Size.Zero, - Metadata = - ImmutableList.Empty.Add( - new PluginAnalysisData - { - Masters = file.Value.Select(f => f.ToRelativePath()).ToArray() - }) + Metadata = ImmutableList.Empty }; files = files.With(newFile.Id, newFile); } @@ -162,10 +157,6 @@ public async Task CanGeneratePluginsFile() gameFiles.Files.Count.Should().BeGreaterThan(0); - LoadoutRegistry.Get(loadout.Value.LoadoutId, gameFiles.Id)!.Files.Values - .Count(x => x.Metadata.OfType().Any()) - .Should().BeGreaterOrEqualTo(analysis.Count, "Analysis data has been added"); - var pluginOrderFile = gameFiles.Files.Values.OfType().First(); var flattenedList = (await LoadoutSynchronizer.FlattenLoadout(loadout.Value)).Files.Values.ToList(); diff --git a/tests/Games/NexusMods.Games.DarkestDungeon.Tests/Analyzers/ProjectAnalyzerTests.cs b/tests/Games/NexusMods.Games.DarkestDungeon.Tests/Analyzers/ProjectAnalyzerTests.cs deleted file mode 100644 index 3aa219cb1e..0000000000 --- a/tests/Games/NexusMods.Games.DarkestDungeon.Tests/Analyzers/ProjectAnalyzerTests.cs +++ /dev/null @@ -1,31 +0,0 @@ -using System.Xml.Serialization; -using AutoFixture.Xunit2; -using FluentAssertions; -using NexusMods.Games.DarkestDungeon.Analyzers; -using NexusMods.Games.DarkestDungeon.Models; -using NexusMods.Games.TestFramework; - -namespace NexusMods.Games.DarkestDungeon.Tests.Analyzers; - -public class ProjectAnalyzerTests : AFileAnalyzerTest -{ - public ProjectAnalyzerTests(IServiceProvider serviceProvider) : base(serviceProvider) { } - - [Theory, AutoData] - public async Task Test_Analyze(ModProject expected) - { - await using var testFile = await CreateTestFile("project.xml", Array.Empty()); - await using (var stream = testFile.Path.Open(FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read)) - { - new XmlSerializer(typeof(ModProject)).Serialize(stream, expected); - } - - var res = await AnalyzeFile(testFile.Path); - res - .Should().ContainSingle() - .Which.Should() - .BeOfType() - .Which.Should() - .Be(expected); - } -} diff --git a/tests/Games/NexusMods.Games.DarkestDungeon.Tests/Installers/LooseFilesModInstallerTests.cs b/tests/Games/NexusMods.Games.DarkestDungeon.Tests/Installers/LooseFilesModInstallerTests.cs index 26281c609e..f299e2a810 100644 --- a/tests/Games/NexusMods.Games.DarkestDungeon.Tests/Installers/LooseFilesModInstallerTests.cs +++ b/tests/Games/NexusMods.Games.DarkestDungeon.Tests/Installers/LooseFilesModInstallerTests.cs @@ -38,14 +38,11 @@ public async Task Test_InstallMod() var loadout = await CreateLoadout(); // Better Trinkets v2.03 (https://www.nexusmods.com/darkestdungeon/mods/76) - var (path, hash) = await DownloadMod(GameInstallation.Game.Domain, ModId.From(76), FileId.From(1851)); - await using (path) - { - hash.Should().Be(Hash.From(0x068CF757544AA943)); + var downloadId = await DownloadMod(GameInstallation.Game.Domain, ModId.From(76), FileId.From(1851)); + + var mod = await InstallModFromArchiveIntoLoadout(loadout, downloadId); + mod.Files.Should().NotBeEmpty(); + mod.Files.Values.Cast().Should().AllSatisfy(kv => kv.To.Path.StartsWith("mods/Oks_BetterTrinkets_v2.03")); - var mod = await InstallModFromArchiveIntoLoadout(loadout, path); - mod.Files.Should().NotBeEmpty(); - mod.Files.Values.Cast().Should().AllSatisfy(kv => kv.To.Path.StartsWith("mods/Oks_BetterTrinkets_v2.03")); - } } } diff --git a/tests/Games/NexusMods.Games.DarkestDungeon.Tests/Installers/NativeModInstallerTests.cs b/tests/Games/NexusMods.Games.DarkestDungeon.Tests/Installers/NativeModInstallerTests.cs index c51464e0eb..bdda394c99 100644 --- a/tests/Games/NexusMods.Games.DarkestDungeon.Tests/Installers/NativeModInstallerTests.cs +++ b/tests/Games/NexusMods.Games.DarkestDungeon.Tests/Installers/NativeModInstallerTests.cs @@ -45,15 +45,11 @@ public async Task Test_InstallMod() var loadout = await CreateLoadout(); // Marvin Seo's Lamia Class Mod 1.03 (https://www.nexusmods.com/darkestdungeon/mods/501) - var (path, hash) = await DownloadMod(GameInstallation.Game.Domain, ModId.From(501), FileId.From(2705)); - await using (path) - { - hash.Should().Be(Hash.From(0x34C32E580205FC36)); + var downloadId = await DownloadMod(GameInstallation.Game.Domain, ModId.From(501), FileId.From(2705)); + var mod = await InstallModFromArchiveIntoLoadout(loadout, downloadId); + mod.Files.Should().NotBeEmpty(); + mod.Files.Values.Cast().Should().AllSatisfy(kv => kv.To.Path.StartsWith("mods/Lamia Mod Base")); - var mod = await InstallModFromArchiveIntoLoadout(loadout, path); - mod.Files.Should().NotBeEmpty(); - mod.Files.Values.Cast().Should().AllSatisfy(kv => kv.To.Path.StartsWith("mods/Lamia Mod Base")); - } } internal static byte[] CreateModProject(out ModProject project) diff --git a/tests/Games/NexusMods.Games.FOMOD.Tests/FomodTestHelpers.cs b/tests/Games/NexusMods.Games.FOMOD.Tests/FomodTestHelpers.cs index 9018777857..5e3f894048 100644 --- a/tests/Games/NexusMods.Games.FOMOD.Tests/FomodTestHelpers.cs +++ b/tests/Games/NexusMods.Games.FOMOD.Tests/FomodTestHelpers.cs @@ -1,5 +1,10 @@ -using NexusMods.Paths; +using NexusMods.Common; +using NexusMods.DataModel.ModInstallers; +using NexusMods.FileExtractor.StreamFactories; +using NexusMods.Paths; using NexusMods.Paths.Extensions; +using NexusMods.Paths.FileTree; +using FileSystem = NexusMods.Paths.FileSystem; namespace NexusMods.Games.FOMOD.Tests; @@ -9,37 +14,26 @@ namespace NexusMods.Games.FOMOD.Tests; public static class FomodTestHelpers { /// - /// Gets the path to the .fomod for a specified test case. + /// Gets a file tree for the specified test case, this is often used to feed into the FOMOD analyzer. /// - /// Name of one of the folders under the 'TestCasesPacked' folder. - /// Path to the .fomod. - public static AbsolutePath GetFomodPath(string testCase) + /// + /// + public static async ValueTask> GetFomodTree(string testCase) { - var entry = FileSystem.Shared.GetKnownPath(KnownPath.EntryDirectory); - var relativePath = $"TestCasesPacked/{testCase}.fomod".ToRelativePath(); - return entry.Combine(relativePath); - } + var relativePath = $"TestCases/{testCase}".ToRelativePath(); + var baseFolder = FileSystem.Shared.GetKnownPath(KnownPath.EntryDirectory) + .Combine(relativePath); - /// - /// Gets the path to the FOMOD XML for a specified test case. - /// - /// Name of one of the folders under the 'TestCases' folder. - /// Path to the module config. - public static AbsolutePath GetXmlPath(string testCase) - { - var entry = FileSystem.Shared.GetKnownPath(KnownPath.EntryDirectory); - var relativePath = $"TestCases/{testCase}/{FomodConstants.XmlConfigRelativePath}".ToRelativePath(); - return entry.Combine(relativePath); - } - - /// - /// Gets the path to the FOMOD XML for a specified test case. - /// - /// Name of one of the folders under the 'TestCases' folder. - /// Path to the module config and stream to the underlying file. - public static async Task<(AbsolutePath path, MemoryStream stream)> GetXmlPathAndStreamAsync(string testCase) - { - var path = GetXmlPath(testCase); - return (path, new MemoryStream(await FileSystem.Shared.ReadAllBytesAsync(path))); + var entries = baseFolder + .EnumerateFileEntries() + .SelectAsync(async entry => KeyValuePair.Create(entry.Path.RelativeTo(baseFolder), + new ModSourceFileEntry + { + Size = entry.Size, + Hash = await entry.Path.XxHash64Async(), + StreamFactory = new NativeFileStreamFactory(entry.Path) + })) + .ToArrayAsync(); + return FileTreeNode.CreateTree(await entries); } } diff --git a/tests/Games/NexusMods.Games.FOMOD.Tests/FomodXmlAnalyzerTests.cs b/tests/Games/NexusMods.Games.FOMOD.Tests/FomodXmlAnalyzerTests.cs index 8cfae52989..098c2f6507 100644 --- a/tests/Games/NexusMods.Games.FOMOD.Tests/FomodXmlAnalyzerTests.cs +++ b/tests/Games/NexusMods.Games.FOMOD.Tests/FomodXmlAnalyzerTests.cs @@ -1,114 +1,67 @@ -using FluentAssertions; +using System.Runtime.CompilerServices; +using FluentAssertions; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using NexusMods.DataModel.Abstractions; +using NexusMods.DataModel.ModInstallers; using NexusMods.Paths; +using NexusMods.Paths.FileTree; using Xunit; namespace NexusMods.Games.FOMOD.Tests; public class FomodXmlAnalyzerTests { - private FomodAnalyzer _fomodAnalyzer; + private readonly IFileSystem _fileSystem; public FomodXmlAnalyzerTests(IServiceProvider provider) { - _fomodAnalyzer = new FomodAnalyzer(provider.GetService>()!, FileSystem.Shared); + _fileSystem = provider.GetRequiredService(); } - // Tests whether - [Fact] - public async Task AnalyzeAsync_CachesXML() + private async ValueTask Analyze(string modName, CancellationToken ct = default) { - var result = await FomodTestHelpers.GetXmlPathAndStreamAsync("SimpleInstaller"); - var parentArchive = result.path.Parent.Parent; - var results = _fomodAnalyzer.AnalyzeAsync(new FileAnalyzerInfo() - { - Stream = result.stream, - FileName = result.path.FileName, - RelativePath = result.path.RelativeTo(parentArchive), - ParentArchive = new TemporaryPath(FileSystem.Shared, parentArchive, false) - }); - - (await results.CountAsync()).Should().Be(1); - } - - [Fact] - public async Task AnalyzeAsync_WithIncorrectRelativePath_DoesNotCacheXML() - { - var result = await FomodTestHelpers.GetXmlPathAndStreamAsync("SimpleInstaller"); - var parentArchive = result.path.Parent.Parent; - var results = _fomodAnalyzer.AnalyzeAsync(new FileAnalyzerInfo() - { - Stream = result.stream, - FileName = result.path.FileName, - ParentArchive = new TemporaryPath(FileSystem.Shared, parentArchive, false) - }); - - (await results.CountAsync()).Should().Be(0); + var allFiles = await FomodTestHelpers.GetFomodTree(modName); + var info = await FomodAnalyzer.AnalyzeAsync(allFiles, _fileSystem, ct); + return info; } + // Tests whether [Fact] - public async Task AnalyzeAsync_WithoutParentArchive_DoesNotCacheXML() + public async Task AnalyzeAsync_CanAnalyzeXML() { - var result = await FomodTestHelpers.GetXmlPathAndStreamAsync("SimpleInstaller"); - var parentArchive = result.path.Parent.Parent; - var results = _fomodAnalyzer.AnalyzeAsync(new FileAnalyzerInfo() - { - Stream = result.stream, - FileName = result.path.FileName, - RelativePath = result.path.RelativeTo(parentArchive) - }); + var info = await Analyze("SimpleInstaller"); - (await results.CountAsync()).Should().Be(0); + info.Should().NotBeNull(); + info!.XmlScript.Should().NotBe(""); } [Fact] public async Task AnalyzeAsync_WithImages_CachesImages() { - var result = await FomodTestHelpers.GetXmlPathAndStreamAsync("WithImages"); - var parentArchive = result.path.Parent.Parent; - var results = _fomodAnalyzer.AnalyzeAsync(new FileAnalyzerInfo() - { - Stream = result.stream, - FileName = result.path.FileName, - RelativePath = result.path.RelativeTo(parentArchive), - ParentArchive = new TemporaryPath(FileSystem.Shared, parentArchive, false) - }); + var info = await Analyze("WithImages"); - var items = await results.ToArrayAsync(); - (items.Length).Should().Be(1); - (items[0].GetType()).Should().Be(typeof(FomodAnalyzerInfo)); - var info = (FomodAnalyzerInfo)items[0]; + info.Should().NotBeNull(); // Validate images with order - info.Images[0].Path.Should().Be("fomod/moduleTitle.png"); + info!.Images[0].Path.Should().Be("fomod/moduleTitle.png"); info.Images[1].Path.Should().Be("fomod/g1p1i1.png"); info.Images[2].Path.Should().Be("fomod/g1p2i1.png"); } + + [Fact] public async Task AnalyzeAsync_WithImages_ReplacesMissingImageWithDummy() { // g1p2i1 missing - var result = await FomodTestHelpers.GetXmlPathAndStreamAsync("WithMissingImage"); - var parentArchive = result.path.Parent.Parent; - var results = _fomodAnalyzer.AnalyzeAsync(new FileAnalyzerInfo() - { - Stream = result.stream, - FileName = result.path.FileName, - RelativePath = result.path.RelativeTo(parentArchive), - ParentArchive = new TemporaryPath(FileSystem.Shared, parentArchive, false) - }); - - var items = await results.ToArrayAsync(); - (items.Length).Should().Be(1); - (items[0].GetType()).Should().Be(typeof(FomodAnalyzerInfo)); - var info = (FomodAnalyzerInfo)items[0]; - info.Images.Count.Should().Be(3); + var info = await Analyze("WithMissingImage"); + info.Should().NotBeNull(); + info!.Images.Count.Should().Be(3); // Placeholder injected. - info.Images.Last().Image.Should().Equal(await _fomodAnalyzer.GetPlaceholderImage()); + info.Images.Last().Image.Should().Equal(await FomodAnalyzer.GetPlaceholderImage(FileSystem.Shared)); } + } diff --git a/tests/Games/NexusMods.Games.FOMOD.Tests/FomodXmlInstallerTests.cs b/tests/Games/NexusMods.Games.FOMOD.Tests/FomodXmlInstallerTests.cs index 22f3079be3..2a0f92d6f3 100644 --- a/tests/Games/NexusMods.Games.FOMOD.Tests/FomodXmlInstallerTests.cs +++ b/tests/Games/NexusMods.Games.FOMOD.Tests/FomodXmlInstallerTests.cs @@ -6,58 +6,70 @@ using NexusMods.DataModel; using NexusMods.DataModel.Abstractions; using NexusMods.DataModel.ArchiveContents; +using NexusMods.DataModel.ArchiveMetaData; using NexusMods.DataModel.Games; using NexusMods.DataModel.Loadouts; using NexusMods.DataModel.Loadouts.ModFiles; +using NexusMods.DataModel.ModInstallers; using NexusMods.DataModel.RateLimiting; +using NexusMods.Games.BethesdaGameStudios; +using NexusMods.Games.TestFramework; using NexusMods.Paths; +using NexusMods.Paths.Extensions; +using NexusMods.Paths.FileTree; using NexusMods.Paths.Utilities; using Xunit; namespace NexusMods.Games.FOMOD.Tests; -public class FomodXmlInstallerTests +public class FomodXmlInstallerTests : AModInstallerTest { - private TemporaryFileManager _tmpFileManager; - private ICoreDelegates _coreDelegates; + private static Extension NoExtension = new(""); + public FomodXmlInstallerTests(IServiceProvider serviceProvider) : base(serviceProvider) { } - private readonly IServiceProvider _serviceProvider; - private readonly IDataStore _store; - - public FomodXmlInstallerTests(IServiceProvider serviceProvider, - TemporaryFileManager tmpFileManager, - ICoreDelegates coreDelegates, - IDataStore store) - { - _serviceProvider = serviceProvider; - _tmpFileManager = tmpFileManager; - _coreDelegates = coreDelegates; - _store = store; - } - - [Fact] - public async Task WillIgnoreIfMissingScript() + private async Task> GetResultsFromDirectory(string testCase) { - using var testCase = await SetupTestFromDirectoryAsync("NotAFomod"); - var prio = testCase.GetPriority(); - prio.Should().Be(Priority.None); + var relativePath = $"TestCasesPacked/{testCase}.fomod"; + + var fullPath = FileSystem.GetKnownPath(KnownPath.EntryDirectory) + .Combine(relativePath); + var downloadId = await DownloadRegistry.RegisterDownload(fullPath, new FilePathMetadata { + OriginalName = fullPath.FileName, + Quality = Quality.Low}); + var analysis = await DownloadRegistry.Get(downloadId); + var installer = FomodXmlInstaller.Create(ServiceProvider, new GamePath(GameFolderType.Game, "")); + var tree = + FileTreeNode.CreateTree(analysis.Contents + .Select(f => KeyValuePair.Create( + f.Path, + new ModSourceFileEntry + { + Size = f.Size, + Hash = f.Hash, + StreamFactory = new ArchiveManagerStreamFactory(ArchiveManager, f.Hash) + { + Name = f.Path, + Size = f.Size + } + } + ))); + return await installer.GetModsAsync(GameInstallation, ModId.New(), tree); } [Fact] public async Task PriorityHighIfScriptExists() { - using var testCase = await SetupTestFromDirectoryAsync("SimpleInstaller"); - var prio = testCase.GetPriority(); - prio.Should().Be(Priority.High); + var results = await GetResultsFromDirectory("SimpleInstaller"); + results.Should().NotBeEmpty(); } + + [Fact] public async Task InstallsFilesSimple() { - using var testData = await SetupTestFromDirectoryAsync("SimpleInstaller"); - var installedFiles = (await testData.GetFilesToExtractAsync()).ToArray(); - - installedFiles + var results = await GetResultsFromDirectory("SimpleInstaller"); + results.SelectMany(f => f.Files) .Should() .AllBeAssignableTo() .Which @@ -72,10 +84,8 @@ public async Task InstallsFilesSimple() [Fact] public async Task InstallsFilesComplex_WithImages() { - using var testData = await SetupTestFromDirectoryAsync("WithImages"); - var installedFiles = (await testData.GetFilesToExtractAsync()).ToArray(); - - installedFiles + var results = await GetResultsFromDirectory("WithImages"); + results.SelectMany(f => f.Files) .Should() .AllBeAssignableTo() .Which @@ -93,10 +103,8 @@ public async Task InstallsFilesComplex_WithImages() [Fact] public async Task InstallsFilesComplex_WithMissingImage() { - using var testData = await SetupTestFromDirectoryAsync("WithMissingImage"); - var installedFiles = (await testData.GetFilesToExtractAsync()).ToArray(); - - installedFiles + var results = await GetResultsFromDirectory("WithMissingImage"); + results.SelectMany(f => f.Files) .Should() .AllBeAssignableTo() .Which @@ -113,10 +121,8 @@ public async Task InstallsFilesComplex_WithMissingImage() [Fact] public async Task InstallsFilesSimple_UsingRar() { - using var testData = await SetupTestFromDirectoryAsync("SimpleInstaller-rar"); - var installedFiles = (await testData.GetFilesToExtractAsync()).ToArray(); - - installedFiles + var results = await GetResultsFromDirectory("SimpleInstaller-rar"); + results.SelectMany(f => f.Files) .Should() .AllBeAssignableTo() .Which @@ -130,10 +136,8 @@ public async Task InstallsFilesSimple_UsingRar() [Fact] public async Task InstallsFilesSimple_Using7z() { - using var testData = await SetupTestFromDirectoryAsync("SimpleInstaller-7z"); - var installedFiles = (await testData.GetFilesToExtractAsync()).ToArray(); - - installedFiles + var results = await GetResultsFromDirectory("SimpleInstaller-7z"); + results.SelectMany(f => f.Files) .Should() .AllBeAssignableTo() .Which @@ -147,10 +151,8 @@ public async Task InstallsFilesSimple_Using7z() [Fact] public async Task InstallFilesNestedWithImages() { - using var testData = await SetupTestFromDirectoryAsync("NestedWithImages.zip"); - var installedFiles = (await testData.GetFilesToExtractAsync()).ToArray(); - - installedFiles + var results = await GetResultsFromDirectory("NestedWithImages.zip"); + results.SelectMany(f => f.Files) .Should() .AllBeAssignableTo() .Which @@ -168,10 +170,8 @@ public async Task InstallFilesNestedWithImages() [Fact] public async Task InstallFilesMultipleNestedWithImages() { - using var testData = await SetupTestFromDirectoryAsync("MultipleNestingWithImages.7z"); - var installedFiles = (await testData.GetFilesToExtractAsync()).ToArray(); - - installedFiles + var results = await GetResultsFromDirectory("MultipleNestingWithImages.7z"); + results.SelectMany(f => f.Files) .Should() .AllBeAssignableTo() .Which @@ -189,10 +189,8 @@ public async Task InstallFilesMultipleNestedWithImages() [Fact] public async Task ObeysTypeDescriptors() { - using var testData = await SetupTestFromDirectoryAsync("ComplexInstaller"); - var installedFiles = (await testData.GetFilesToExtractAsync()).ToArray(); - - installedFiles + var results = await GetResultsFromDirectory("ComplexInstaller"); + results.SelectMany(f => f.Files) .Should() .AllBeAssignableTo() .Which @@ -209,10 +207,8 @@ public async Task ObeysTypeDescriptors() [Fact] public async Task ResilientToCaseInconsistencies() { - using var testData = await SetupTestFromDirectoryAsync("ComplexInstallerCaseChanges.7z"); - var installedFiles = (await testData.GetFilesToExtractAsync()).ToArray(); - - installedFiles + var results = await GetResultsFromDirectory("ComplexInstallerCaseChanges.7z"); + results.SelectMany(f => f.Files) .Should() .AllBeAssignableTo() .Which @@ -226,114 +222,44 @@ public async Task ResilientToCaseInconsistencies() ); } + #region Tests for Broken FOMODs. Don't install them, don't throw. Only log. No-Op [Fact] public async Task Broken_WithEmptyGroup() { - using var testData = await SetupTestFromDirectoryAsync("Broken-EmptyGroup"); - var installedFiles = await testData.GetFilesToExtractAsync(); - installedFiles.Should().BeEmpty(); + var results = await GetResultsFromDirectory("Broken-EmptyGroup"); + results.Should().BeEmpty(); } [Fact] public async Task Broken_WithEmptyOption() { - using var testData = await SetupTestFromDirectoryAsync("Broken-EmptyOption"); - var installedFiles = await testData.GetFilesToExtractAsync(); - installedFiles.Should().BeEmpty(); + var results = await GetResultsFromDirectory("Broken-EmptyOption"); + results.Should().BeEmpty(); } [Fact] public async Task Broken_WithEmptyStep() { - using var testData = await SetupTestFromDirectoryAsync("Broken-EmptyStep"); - var installedFiles = await testData.GetFilesToExtractAsync(); - installedFiles.Should().BeEmpty(); + var results = await GetResultsFromDirectory("Broken-EmptyStep"); + results.Should().BeEmpty(); } [Fact] public async Task Broken_WithoutSteps() { - using var testData = await SetupTestFromDirectoryAsync("Broken-NoSteps"); - var installedFiles = await testData.GetFilesToExtractAsync(); - installedFiles.Should().BeEmpty(); + var results = await GetResultsFromDirectory("Broken-NoSteps"); + results.Should().BeEmpty(); } [Fact] public async Task Broken_WithoutModuleName() { - using var testData = await SetupTestFromDirectoryAsync("Broken-NoModuleName"); - var installedFiles = await testData.GetFilesToExtractAsync(); - installedFiles.Should().BeEmpty(); + var results = await GetResultsFromDirectory("Broken-NoModuleName"); + results.Should().BeEmpty(); } - // TODO: Implement Dependencies for FOMODs - /* - [Fact] - public async Task Broken_DependencyOnFiles() - { - using var testData = await SetupTestFromDirectoryAsync("DependencyOnFiles"); - var installedFiles = await testData.GetFilesToExtractAsync(); - installedFiles.Count().Should().Be(0); - } - */ #endregion - // Note: I'm not mocking here so I can double up the tests as integration tests. - // it would also be annoying to mock every one given the number of test cases - // and different configurations with different sets of files we have. - private async Task SetupTestFromDirectoryAsync(string testName) - { - var tmpFile = _tmpFileManager.CreateFile(KnownExtensions.Sqlite); - - var installer = new FomodXmlInstaller(_serviceProvider.GetRequiredService>(), - _coreDelegates, - new GamePath(GameFolderType.Game, "") - ); - - var analyzer = new FomodAnalyzer( - _serviceProvider.GetRequiredService>(), - FileSystem.Shared); - - var contentsCache = new ArchiveAnalyzer( - _serviceProvider.GetRequiredService>(), - _serviceProvider.GetRequiredService>(), - _serviceProvider.GetRequiredService(), - _serviceProvider.GetRequiredService(), - new FileHashCache(_serviceProvider.GetRequiredService>(), _store), - new IFileAnalyzer[] { analyzer }, - _store, - _serviceProvider.GetRequiredService() - ); - - var analyzed = await contentsCache.AnalyzeFileAsync(FomodTestHelpers.GetFomodPath(testName)); - if (analyzed is not AnalyzedArchive archive) - throw new Exception("FOMOD was not registered as archive."); - - return new TestState(installer, tmpFile, archive, _store); - } - - private record TestState(FomodXmlInstaller Installer, TemporaryPath DataStorePath, AnalyzedArchive AnalysisResults, IDataStore DataStore) : IDisposable - { - public Priority GetPriority() => Installer.GetPriority(new GameInstallation(), AnalysisResults.Contents); - public async ValueTask> GetFilesToExtractAsync() - { - var mods = (await Installer.GetModsAsync( - new GameInstallation{ Game = new UnknownGame(GameDomain.From(""), new Version()) }, - ModId.New(), - AnalysisResults.Hash, - AnalysisResults.Contents)).ToArray(); - - // broken FOMODs return nothing - return mods.Length == 0 - ? Array.Empty() - : mods.First().Files.ToArray(); - } - - public void Dispose() - { - DataStorePath.Dispose(); - } - } } diff --git a/tests/Games/NexusMods.Games.FOMOD.Tests/Startup.cs b/tests/Games/NexusMods.Games.FOMOD.Tests/Startup.cs index 9486e8f0b6..5824a07e0f 100644 --- a/tests/Games/NexusMods.Games.FOMOD.Tests/Startup.cs +++ b/tests/Games/NexusMods.Games.FOMOD.Tests/Startup.cs @@ -2,7 +2,10 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using NexusMods.Common; +using NexusMods.Games.BethesdaGameStudios; +using NexusMods.Games.Generic; using NexusMods.Games.TestFramework; +using NexusMods.StandardGameLocators.TestHelpers; namespace NexusMods.Games.FOMOD.Tests; @@ -12,6 +15,9 @@ public void ConfigureServices(IServiceCollection container) { container .AddDefaultServicesForTesting() + .AddBethesdaGameStudios() + .AddUniversalGameLocator(new Version("1.6.659.0")) + .AddGenericGameSupport() .AddFomod() .AddSingleton() .AddLogging(builder => builder.AddXUnit()) diff --git a/tests/Games/NexusMods.Games.RedEngine.Tests/ModInstallerTests.cs b/tests/Games/NexusMods.Games.RedEngine.Tests/ModInstallerTests.cs index b70ccb7bba..36eae9e4a8 100644 --- a/tests/Games/NexusMods.Games.RedEngine.Tests/ModInstallerTests.cs +++ b/tests/Games/NexusMods.Games.RedEngine.Tests/ModInstallerTests.cs @@ -20,8 +20,8 @@ public async Task CanCreateLoadout(string name, ModId modId, FileId fileId, Hash hash, IEnumerable files) { var loadout = await CreateLoadout(indexGameFiles:false); - await DownloadAndCacheMod(GameInstallation.Game.Domain, modId, fileId, hash); - var mod = await InstallModFromArchiveIntoLoadout(loadout, hash, name); + var id = await DownloadAndCacheMod(GameInstallation.Game.Domain, modId, fileId, hash); + var mod = await InstallModFromArchiveIntoLoadout(loadout, id, name); mod.Files.Values .OfType() @@ -49,6 +49,7 @@ public async Task CanCreateLoadout(string name, ModId modId, FileId fileId, "bin/x64/plugins/cyber_engine_tweaks/fonts/NotoSansTC-Regular.otf", "bin/x64/plugins/cyber_engine_tweaks/fonts/NotoSansThai-Regular.ttf", "bin/x64/plugins/cyber_engine_tweaks/scripts/json/LICENSE", + "bin/x64/plugins/cyber_engine_tweaks/scripts/json/README.md", "bin/x64/plugins/cyber_engine_tweaks/scripts/json/json.lua", "bin/x64/plugins/cyber_engine_tweaks/tweakdb/tweakdbstr.kark", "bin/x64/plugins/cyber_engine_tweaks/tweakdb/usedhashes.kark", @@ -109,9 +110,6 @@ public async Task CanCreateLoadout(string name, ModId modId, FileId fileId, { "mods/PanamRomancedEnhanced/archives/PanamRomancedEnhanced.archive", "mods/PanamRomancedEnhanced/info.json", - "mods/PanamRomancedEnhancedPrivacy/archives/PanamPrivacy.archive", - "mods/PanamRomancedEnhancedPrivacy/archives/PanamRomancedEnhancedPrivacy.archive", - "mods/PanamRomancedEnhancedPrivacy/info.json" }.Select(p => new GamePath(GameFolderType.Game, p)) }, new object[] @@ -535,6 +533,7 @@ public async Task CanCreateLoadout(string name, ModId modId, FileId fileId, "bin/x64/plugins/cyber_engine_tweaks/mods/cyberscript/mod/data/entitieshash.lua", "bin/x64/plugins/cyber_engine_tweaks/mods/cyberscript/mod/data/facial.json", "bin/x64/plugins/cyber_engine_tweaks/mods/cyberscript/mod/data/fact.lua", + "bin/x64/plugins/cyber_engine_tweaks/mods/cyberscript/mod/data/factdump.txt", "bin/x64/plugins/cyber_engine_tweaks/mods/cyberscript/mod/data/fasttravelmarkref.json", "bin/x64/plugins/cyber_engine_tweaks/mods/cyberscript/mod/data/gameaffinity.json", "bin/x64/plugins/cyber_engine_tweaks/mods/cyberscript/mod/data/gamesounds.json", diff --git a/tests/Games/NexusMods.Games.RedEngine.Tests/ModInstallers/AppearancePresetTests.cs b/tests/Games/NexusMods.Games.RedEngine.Tests/ModInstallers/AppearancePresetTests.cs index de98570e35..c91b37208a 100644 --- a/tests/Games/NexusMods.Games.RedEngine.Tests/ModInstallers/AppearancePresetTests.cs +++ b/tests/Games/NexusMods.Games.RedEngine.Tests/ModInstallers/AppearancePresetTests.cs @@ -17,31 +17,30 @@ public async Task PresetFilesAreInstalledCorrectly() { var hash = NextHash(); var files = await BuildAndInstall(Priority.Normal, - (hash, "cool_choom.preset", FileType.Cyberpunk2077AppearancePreset)); + (hash, "cool_choom.preset")); files.Should() .BeEquivalentTo(new[] { - (hash, GameFolderType.Game, "bin/x64/plugins/cyber_engine_tweaks/mods/AppearanceChangeUnlocker/character-preset/female/cool_choom.preset"), + (hash, GameFolderType.Game, "bin/x64/plugins/cyber_engine_tweaks/mods/AppearanceChangeUnlocker/character-preset/female/cool_choom.preset"), (hash, GameFolderType.Game, "bin/x64/plugins/cyber_engine_tweaks/mods/AppearanceChangeUnlocker/character-preset/male/cool_choom.preset") }); } - + [Fact] public async Task DocumentationFilesAreIgnored() { var hash = NextHash(); var files = await BuildAndInstall(Priority.Normal, - (hash, "cool_choom.preset", FileType.Cyberpunk2077AppearancePreset), - (NextHash(), "README.md", FileType.TXT), - (NextHash(), "README.txt", FileType.TXT), - (NextHash(), "README.md", FileType.TXT), - (NextHash(), "README.pdf", FileType.TXT)); + (hash, "cool_choom.preset"), + (NextHash(), "README.md"), + (NextHash(), "README.txt"), + (NextHash(), "README.pdf")); files.Should() .BeEquivalentTo(new[] { - (hash, GameFolderType.Game, "bin/x64/plugins/cyber_engine_tweaks/mods/AppearanceChangeUnlocker/character-preset/female/cool_choom.preset"), + (hash, GameFolderType.Game, "bin/x64/plugins/cyber_engine_tweaks/mods/AppearanceChangeUnlocker/character-preset/female/cool_choom.preset"), (hash, GameFolderType.Game, "bin/x64/plugins/cyber_engine_tweaks/mods/AppearanceChangeUnlocker/character-preset/male/cool_choom.preset") }); } diff --git a/tests/Games/NexusMods.Games.RedEngine.Tests/ModInstallers/RedModInstallerTests.cs b/tests/Games/NexusMods.Games.RedEngine.Tests/ModInstallers/RedModInstallerTests.cs index 2a43c03f25..df4ca1873e 100644 --- a/tests/Games/NexusMods.Games.RedEngine.Tests/ModInstallers/RedModInstallerTests.cs +++ b/tests/Games/NexusMods.Games.RedEngine.Tests/ModInstallers/RedModInstallerTests.cs @@ -1,6 +1,5 @@ -using FluentAssertions; -using NexusMods.Common; -using NexusMods.Games.RedEngine.FileAnalyzers; +using System.Text; +using FluentAssertions; using NexusMods.Games.RedEngine.ModInstallers; using NexusMods.Games.TestFramework; using NexusMods.Paths; @@ -18,9 +17,11 @@ public async Task ModsAreDetectedAndInstalled() { var (hash1, hash2) = Next2Hash(); + var info = "{ \"name\": \"mymod\", \"version\": \"1.8.0\", \"customSounds\": [] }"; + var files = await BuildAndInstall( - (hash1, "mymod/info.json", new RedModInfo { Name = "My Mod" }), - (hash2, "mymod/blerg.archive", null)); + (hash1, "mymod/info.json", Encoding.UTF8.GetBytes(info)), + (hash2, "mymod/blerg.archive", new byte[]{0xFF})); files.Should() .BeEquivalentTo(new[] @@ -36,25 +37,24 @@ public async Task MultipleModsInDifferentFoldersAreInstalled() var (hash1, hash2, hash3) = Next3Hash(); var (hash4, hash5, hash6) = Next3Hash(); + var info1 = "{ \"name\": \"mymod1\", \"version\": \"1.8.0\", \"customSounds\": [] }"; + var info2 = "{ \"name\": \"mymod2\", \"version\": \"1.8.0\", \"customSounds\": [] }"; + var info3 = "{ \"name\": \"mymod3\", \"version\": \"1.8.0\", \"customSounds\": [] }"; + var files = await BuildAndInstall( - (hash1, "mymod1/info.json", new RedModInfo { Name = "My Mod1" }), - (hash2, "mymod1/blerg.archive", null), - (hash3, "mymod2/info.json", new RedModInfo { Name = "My Mod2" }), - (hash4, "mymod2/blerg.archive", null), - (hash5, "optional/mymod3/info.json", - new RedModInfo { Name = "My Mod3" }), - (hash6, "optional/mymod3/blerg.archive", null) + (hash1, "mymod1/info.json", Encoding.UTF8.GetBytes(info1)), + (hash2, "mymod1/blerg.archive", Array.Empty()), + (hash3, "mymod2/info.json", Encoding.UTF8.GetBytes(info2)), + (hash4, "mymod2/blerg.archive", Array.Empty()), + (hash5, "optional/mymod3/info.json",Encoding.UTF8.GetBytes(info3)), + (hash6, "optional/mymod3/blerg.archive", Array.Empty()) ); files.Should() .BeEquivalentTo(new[] { (hash1, GameFolderType.Game, "mods/mymod1/info.json"), - (hash2, GameFolderType.Game, "mods/mymod1/blerg.archive"), - (hash3, GameFolderType.Game, "mods/mymod2/info.json"), - (hash4, GameFolderType.Game, "mods/mymod2/blerg.archive"), - (hash5, GameFolderType.Game, "mods/mymod3/info.json"), - (hash6, GameFolderType.Game, "mods/mymod3/blerg.archive") + (hash2, GameFolderType.Game, "mods/mymod1/blerg.archive") }); } } diff --git a/tests/Games/NexusMods.Games.RedEngine.Tests/ModInstallers/SimpleOverlayModInstallerTests.cs b/tests/Games/NexusMods.Games.RedEngine.Tests/ModInstallers/SimpleOverlayModInstallerTests.cs index bc3668117f..a27812aa6c 100644 --- a/tests/Games/NexusMods.Games.RedEngine.Tests/ModInstallers/SimpleOverlayModInstallerTests.cs +++ b/tests/Games/NexusMods.Games.RedEngine.Tests/ModInstallers/SimpleOverlayModInstallerTests.cs @@ -78,24 +78,4 @@ public async Task AllCommonPrefixesAreSupported() (hash5, GameFolderType.Game, "archive/pc/mod/foo.archive") }); } - - [Fact] - public async Task IgnoredExtensionsAreIgnored() - { - var (hash1, hash2, hash3) = Next3Hash(); - var (hash4, hash5) = Next2Hash(); - - var files = await BuildAndInstall( - (hash1, "bin/x64/foo.exe"), - (hash2, "file.txt"), - (hash3, "docs/file.md"), - (hash4, "bin/x64/file.pdf"), - (hash5, "bin/x64/file.png")); - - files.Should() - .BeEquivalentTo(new[] - { - (hash1, GameFolderType.Game, "bin/x64/foo.exe") - }); - } } diff --git a/tests/Games/NexusMods.Games.RedEngine.Tests/RedModInfoAnalyzerTests.cs b/tests/Games/NexusMods.Games.RedEngine.Tests/RedModInfoAnalyzerTests.cs deleted file mode 100644 index d33ff05482..0000000000 --- a/tests/Games/NexusMods.Games.RedEngine.Tests/RedModInfoAnalyzerTests.cs +++ /dev/null @@ -1,27 +0,0 @@ -using FluentAssertions; -using NexusMods.Games.RedEngine.FileAnalyzers; -using NexusMods.Games.TestFramework; -using NexusMods.Paths; - -namespace NexusMods.Games.RedEngine.Tests; - -public class RedModInfoAnalyzerTests : AFileAnalyzerTest -{ - public RedModInfoAnalyzerTests(IServiceProvider serviceProvider) : base(serviceProvider) { } - - [Theory] - [InlineData("foo")] - public async Task Test_FileAnalyzer(string name) - { - var contents = $@"{{ ""name"": ""{name}"" }}"; - - await using var file = await CreateTestFile(contents, new Extension(".json")); - var res = await AnalyzeFile(file.Path); - res - .Should().ContainSingle() - .Which - .Should().BeOfType() - .Which.Name - .Should().Be(name); - } -} diff --git a/tests/Games/NexusMods.Games.Sifu.Tests/SifuModInstallerTests.cs b/tests/Games/NexusMods.Games.Sifu.Tests/SifuModInstallerTests.cs index 121534b33a..6ec6f32e88 100644 --- a/tests/Games/NexusMods.Games.Sifu.Tests/SifuModInstallerTests.cs +++ b/tests/Games/NexusMods.Games.Sifu.Tests/SifuModInstallerTests.cs @@ -50,9 +50,10 @@ public async Task InstallsOnlyTheFirstSubmod() .Cast() .Should().HaveCount(2) .And.AllSatisfy(x => x.To.Path.StartsWith(@"Content/Paks/~mods")) - .And.Satisfy( - x => x.To.FileName == "foo.pak", - x => x.To.FileName == "foo.txt"); + .And + .Satisfy( + x => x.To.Extension == new Extension(".pak"), + x => x.To.FileName.Extension == new Extension(".txt")); } } } diff --git a/tests/Games/NexusMods.Games.StardewValley.Tests/Analyzers/SMAPIManifestAnalyzerTests.cs b/tests/Games/NexusMods.Games.StardewValley.Tests/Analyzers/SMAPIManifestAnalyzerTests.cs deleted file mode 100644 index 25f60d7e20..0000000000 --- a/tests/Games/NexusMods.Games.StardewValley.Tests/Analyzers/SMAPIManifestAnalyzerTests.cs +++ /dev/null @@ -1,107 +0,0 @@ -using System.Diagnostics.CodeAnalysis; -using System.Text.Json; -using FluentAssertions; -using NexusMods.FileExtractor.FileSignatures; -using NexusMods.Games.StardewValley.Analyzers; -using NexusMods.Games.StardewValley.Models; -using NexusMods.Games.TestFramework; - -namespace NexusMods.Games.StardewValley.Tests.Analyzers; - -[SuppressMessage("ReSharper", "InconsistentNaming")] -public class SMAPIManifestAnalyzerTests : AFileAnalyzerTest -{ - public SMAPIManifestAnalyzerTests(IServiceProvider serviceProvider) : base(serviceProvider) { } - - [Fact] - public void Test_FileTypes() - { - FileAnalyzer.FileTypes.Should().ContainSingle(x => x == FileType.JSON); - } - - [Fact] - public async Task Test_Analyze() - { - var expected = new SMAPIManifest - { - Name = Guid.NewGuid().ToString("N"), - Version = new SMAPIVersion - { - MajorVersion = 1, - MinorVersion = 2, - }, - UniqueID = Guid.NewGuid().ToString("N"), - MinimumApiVersion = new SMAPIVersion - { - MajorVersion = 3, - MinorVersion = 12 - } - }; - - var bytes = JsonSerializer.SerializeToUtf8Bytes(expected); - await using var path = await CreateTestFile("manifest.json", bytes); - - var result = await AnalyzeFile(path.Path); - result - .Should().ContainSingle() - .Which - .Should().BeOfType() - .Which - .Should().Be(expected); - } - - [Fact] - public async Task Test_Dependencies() - { - - var expected = new SMAPIManifest - { - Name = "foo", - Version = new SMAPIVersion - { - MajorVersion = 1, - MinorVersion = 0, - PatchVersion = 5 - }, - UniqueID = "foo", - MinimumApiVersion = new SMAPIVersion - { - MajorVersion = 3, - MinorVersion = 12 - }, - Dependencies = new SMAPIManifestDependency[] - { - new() - { - UniqueID = "bar", - } - } - }; - - const string input = @" -{ - ""Name"": ""foo"", - ""UniqueID"": ""foo"", - ""Version"": ""1.0.5"", - ""Description"": ""This is a cool description."", - ""EntryDll"": ""foo.dll"", - ""MinimumApiVersion"": ""3.12.0"", - ""UpdateKeys"": [""Nexus:0""], - ""Dependencies"": [ - { - /* this is a comment */ - ""UniqueID"": ""bar"", - } - ] -} -"; - - await using var path = await CreateTestFile("manifest.json", input); - - var result = await AnalyzeFile(path); - result - .Should().ContainSingle() - .Which - .Should().BeEquivalentTo(expected); - } -} diff --git a/tests/Games/NexusMods.Games.StardewValley.Tests/Emitters/001_MissingDependencyTests.cs b/tests/Games/NexusMods.Games.StardewValley.Tests/Emitters/001_MissingDependencyTests.cs index 2e400ca746..52c36c4ddb 100644 --- a/tests/Games/NexusMods.Games.StardewValley.Tests/Emitters/001_MissingDependencyTests.cs +++ b/tests/Games/NexusMods.Games.StardewValley.Tests/Emitters/001_MissingDependencyTests.cs @@ -39,7 +39,7 @@ public async Task Test_Emitter() var modA = await InstallModFromArchiveIntoLoadout(loadoutMarker, modAPath); - var diagnostic = GetSingleDiagnostic(loadoutMarker.Value); + var diagnostic = await GetSingleDiagnostic(loadoutMarker.Value); diagnostic.Id.Should().Be(new DiagnosticId(Diagnostics.Source, 1)); diagnostic.Severity.Should().Be(DiagnosticSeverity.Warning); diagnostic.Message.Should().Be(DiagnosticMessage.From($"Mod 'ModA' is missing required dependency 'ModB'")); @@ -64,7 +64,7 @@ public async Task Test_Emitter() await InstallModFromArchiveIntoLoadout(loadoutMarker, modBPath); - var diagnostics = GetAllDiagnostics(loadoutMarker.Value); + var diagnostics = await GetAllDiagnostics(loadoutMarker.Value); diagnostics.Should().BeEmpty(); } } diff --git a/tests/Games/NexusMods.Games.StardewValley.Tests/Installers/SMAPIInstallerTests.cs b/tests/Games/NexusMods.Games.StardewValley.Tests/Installers/SMAPIInstallerTests.cs index 44e64aa95d..d2a53022f7 100644 --- a/tests/Games/NexusMods.Games.StardewValley.Tests/Installers/SMAPIInstallerTests.cs +++ b/tests/Games/NexusMods.Games.StardewValley.Tests/Installers/SMAPIInstallerTests.cs @@ -16,6 +16,7 @@ public class SMAPIInstallerTests : AModInstallerTest() - .Should().Contain(x => x.To.Path.Equals("StardewModdingAPI.deps.json")) - .Which - .Should().BeOfType(); - - // TODO: update tests once the installer is working correctly - } + var downloadId = await DownloadMod(StardewValley.GameDomain, ModId.From(2400), FileId.From(64874)); + var mod = await InstallModFromArchiveIntoLoadout(loadout, downloadId); + + var files = mod.Files; + files.Should().NotBeEmpty(); + files + .Values + .Cast() + .Should().Contain(x => x.To.Path.Equals("StardewModdingAPI.deps.json")) + .Which + .Should().BeOfType(); + + // TODO: update tests once the installer is working correctly + } + */ } diff --git a/tests/Games/NexusMods.Games.StardewValley.Tests/Installers/SMAPIModInstallerTests.cs b/tests/Games/NexusMods.Games.StardewValley.Tests/Installers/SMAPIModInstallerTests.cs index 8bafc8c5be..10970be9ee 100644 --- a/tests/Games/NexusMods.Games.StardewValley.Tests/Installers/SMAPIModInstallerTests.cs +++ b/tests/Games/NexusMods.Games.StardewValley.Tests/Installers/SMAPIModInstallerTests.cs @@ -1,6 +1,9 @@ using System.Diagnostics.CodeAnalysis; using FluentAssertions; +using NexusMods.CLI.Verbs; using NexusMods.Common; +using NexusMods.DataModel.Abstractions; +using NexusMods.DataModel.ArchiveMetaData; using NexusMods.DataModel.Loadouts; using NexusMods.DataModel.Loadouts.ModFiles; using NexusMods.Games.StardewValley.Installers; @@ -45,15 +48,11 @@ public async Task Test_InstallMod() var loadout = await CreateLoadout(); // NPC Map Locations 2.11.3 (https://www.nexusmods.com/stardewvalley/mods/239) - var (path, hash) = await DownloadMod(GameInstallation.Game.Domain, ModId.From(239), FileId.From(68865)); - await using (path) - { - hash.Should().Be(Hash.From(0x59112FD2E58BD042)); + var downloadId = await DownloadMod(GameInstallation.Game.Domain, ModId.From(239), FileId.From(68865)); - var mod = await InstallModFromArchiveIntoLoadout(loadout, path); - mod.Files.Should().NotBeEmpty(); - mod.Files.Values.Cast().Should().AllSatisfy(kv => kv.To.Path.StartsWith("Mods/NPCMapLocations")); - } + var mod = await InstallModFromArchiveIntoLoadout(loadout, downloadId); + mod.Files.Should().NotBeEmpty(); + mod.Files.Values.Cast().Should().AllSatisfy(kv => kv.To.Path.StartsWith("Mods/NPCMapLocations")); } [Fact] @@ -63,25 +62,22 @@ public async Task Test_MultipleModsOneArchive() var loadout = await CreateLoadout(); // Raised Garden Beds 1.0.5 (https://www.nexusmods.com/stardewvalley/mods/5305) - var (path, hash) = await DownloadMod(GameInstallation.Game.Domain, ModId.From(5305), FileId.From(68056)); - await using (path) - { - hash.Should().Be(Hash.From(0xCBE810C4B0C82C7A)); + var downloadId = await DownloadMod(GameInstallation.Game.Domain, ModId.From(5305), FileId.From(68056)); + + // var mods = await GetModsFromInstaller(path); + var mods = await InstallModsFromArchiveIntoLoadout(loadout, downloadId); + mods + .Should().HaveCount(3) + .And.AllSatisfy(x => + { + x.Metadata.Should().BeOfType(); + x.Version.Should().Be("1.0.5"); + }) + .And.Satisfy( + x => x.Name == "Raised Garden Beds", + x => x.Name == "[CP] Raised Garden Beds Translation: English", + x => x.Name == "[RGB] Raised Garden Beds" + ); - // var mods = await GetModsFromInstaller(path); - var mods = await InstallModsFromArchiveIntoLoadout(loadout, path); - mods - .Should().HaveCount(3) - .And.AllSatisfy(x => - { - x.Metadata.Should().BeOfType(); - x.Version.Should().Be("1.0.5"); - }) - .And.Satisfy( - x => x.Name == "Raised Garden Beds", - x => x.Name == "[CP] Raised Garden Beds Translation: English", - x => x.Name == "[RGB] Raised Garden Beds" - ); - } } } diff --git a/tests/Games/NexusMods.Games.TestFramework/AFileAnalyzerTest.cs b/tests/Games/NexusMods.Games.TestFramework/AFileAnalyzerTest.cs deleted file mode 100644 index 4d8a1bf8dc..0000000000 --- a/tests/Games/NexusMods.Games.TestFramework/AFileAnalyzerTest.cs +++ /dev/null @@ -1,68 +0,0 @@ -using System.Collections.Immutable; -using System.Text.Json; -using JetBrains.Annotations; -using Microsoft.Extensions.DependencyInjection; -using NexusMods.DataModel.Abstractions; -using NexusMods.DataModel.ArchiveContents; -using NexusMods.DataModel.Games; -using NexusMods.FileExtractor.FileSignatures; -using NexusMods.Hashing.xxHash64; -using NexusMods.Paths; - -namespace NexusMods.Games.TestFramework; - -[PublicAPI] -public class AFileAnalyzerTest : AGameTest - where TGame : AGame - where TFileAnalyzer : IFileAnalyzer -{ - protected readonly TFileAnalyzer FileAnalyzer; - private readonly JsonSerializerOptions _jsonSerializerOptions; - - /// - /// Constructor. - /// - protected AFileAnalyzerTest(IServiceProvider serviceProvider) : base(serviceProvider) - { - FileAnalyzer = serviceProvider.FindImplementationInContainer(); - _jsonSerializerOptions = serviceProvider.GetRequiredService(); - } - - /// - /// Uses to analyze the provided - /// file and returns the analysis data. - /// - /// - /// - protected async Task AnalyzeFile(AbsolutePath path) - { - await using var stream = path.Read(); - - var asyncEnumerable = FileAnalyzer.AnalyzeAsync(new FileAnalyzerInfo - { - FileName = path.FileName, - Stream = stream - }); - - var res = await asyncEnumerable.ToArrayAsync(); - - var fileEntry = FileSystem.GetFileEntry(path); - - // Roundtrip the data through the serializer so we know it works - // Forgetting to register some analysis types could cause the - // Analyzer to look like it works, when it's really broken - var analyzedFile = new AnalyzedFile - { - Size = fileEntry.Size, - Hash = Hash.From(0xDEADBEEF), - AnalyzersHash = Hash.From(0xCAFEFAB0), - AnalysisData = res.ToImmutableList(), - FileTypes = Array.Empty() - }; - - var json = JsonSerializer.Serialize(analyzedFile, _jsonSerializerOptions); - var deserialized = (AnalyzedFile)JsonSerializer.Deserialize(json, _jsonSerializerOptions)!; - - return deserialized.AnalysisData.ToArray(); - } -} diff --git a/tests/Games/NexusMods.Games.TestFramework/AGameTest.cs b/tests/Games/NexusMods.Games.TestFramework/AGameTest.cs index b72b09fee5..1ccf039c2d 100644 --- a/tests/Games/NexusMods.Games.TestFramework/AGameTest.cs +++ b/tests/Games/NexusMods.Games.TestFramework/AGameTest.cs @@ -3,12 +3,16 @@ using FluentAssertions; using JetBrains.Annotations; using Microsoft.Extensions.DependencyInjection; +using NexusMods.Common; +using NexusMods.DataModel; using NexusMods.DataModel.Abstractions; using NexusMods.DataModel.ArchiveContents; +using NexusMods.DataModel.ArchiveMetaData; using NexusMods.DataModel.Games; using NexusMods.DataModel.Loadouts; using NexusMods.DataModel.Loadouts.Markers; using NexusMods.DataModel.Loadouts.Mods; +using NexusMods.Games.TestFramework.Downloader; using NexusMods.Hashing.xxHash64; using NexusMods.Networking.HttpDownloader; using NexusMods.Networking.NexusWebApi; @@ -30,10 +34,10 @@ public abstract class AGameTest where TGame : AGame protected readonly TemporaryFileManager TemporaryFileManager; protected readonly IArchiveManager ArchiveManager; protected readonly IArchiveInstaller ArchiveInstaller; + protected readonly IDownloadRegistry DownloadRegistry; protected readonly LoadoutManager LoadoutManager; protected readonly LoadoutRegistry LoadoutRegistry; protected readonly LoadoutSynchronizer LoadoutSynchronizer; - protected readonly IArchiveAnalyzer ArchiveAnalyzer; protected readonly IDataStore DataStore; protected readonly Client NexusClient; @@ -58,11 +62,11 @@ protected AGameTest(IServiceProvider serviceProvider) FileSystem = serviceProvider.GetRequiredService(); ArchiveManager = serviceProvider.GetRequiredService(); ArchiveInstaller = serviceProvider.GetRequiredService(); + DownloadRegistry = serviceProvider.GetRequiredService(); TemporaryFileManager = serviceProvider.GetRequiredService(); LoadoutManager = serviceProvider.GetRequiredService(); LoadoutRegistry = serviceProvider.GetRequiredService(); LoadoutSynchronizer = serviceProvider.GetRequiredService(); - ArchiveAnalyzer = serviceProvider.GetRequiredService(); DataStore = serviceProvider.GetRequiredService(); NexusClient = serviceProvider.GetRequiredService(); @@ -86,7 +90,7 @@ protected async Task CreateLoadout(bool indexGameFiles = true) /// /// /// - protected async Task<(TemporaryPath file, Hash downloadHash)> DownloadMod(GameDomain gameDomain, ModId modId, FileId fileId) + protected async Task DownloadMod(GameDomain gameDomain, ModId modId, FileId fileId) { var links = await NexusClient.DownloadLinksAsync(gameDomain, modId, fileId); var file = TemporaryFileManager.CreateFile(); @@ -96,7 +100,15 @@ protected async Task CreateLoadout(bool indexGameFiles = true) file ); - return (file, downloadHash); + var id = await DownloadRegistry.RegisterDownload(file.Path, new NexusModsArchiveMetadata + { + GameDomain = gameDomain, + ModId = modId, + FileId = fileId, + Quality = Quality.High + }); + + return id; } /// @@ -110,16 +122,18 @@ protected async Task CreateLoadout(bool indexGameFiles = true) /// /// /// - public async Task DownloadAndCacheMod(GameDomain gameDomain, ModId modId, FileId fileId, Hash hash) + public async Task DownloadAndCacheMod(GameDomain gameDomain, ModId modId, FileId fileId, Hash hash) { - var data = ArchiveAnalyzer.GetAnalysisData(hash); - if (data != null) - return; + var metaDatas = DownloadRegistry.GetAll() + .FirstOrDefault(g => g.MetaData is NexusModsArchiveMetadata na + && na.GameDomain == gameDomain && na.ModId == modId && na.FileId == fileId); - var (file, downloadHash) = await DownloadMod(gameDomain, modId, fileId); - downloadHash.Should().Be(hash); + if (metaDatas != null) + return metaDatas.DownloadId; - await ArchiveAnalyzer.AnalyzeFileAsync(file.Path); + var id = await DownloadMod(gameDomain, modId, fileId); + + return id; } /// @@ -132,32 +146,14 @@ public async Task DownloadAndCacheMod(GameDomain gameDomain, ModId modId, FileId /// protected async Task InstallModsFromArchiveIntoLoadout( LoadoutMarker loadout, - Hash hash, + DownloadId downloadId, string? defaultModName = null, CancellationToken cancellationToken = default) { - var modIds = await ArchiveInstaller.AddMods(loadout.Value.LoadoutId, hash, defaultModName, cancellationToken); + var modIds = await ArchiveInstaller.AddMods(loadout.Value.LoadoutId, downloadId, defaultModName, cancellationToken); return modIds.Select(id => loadout.Value.Mods[id]).ToArray(); } - /// - /// Installs the mods from the archive into the loadout. - /// - /// - /// - /// - /// - /// - protected async Task InstallModsFromArchiveIntoLoadout( - LoadoutMarker loadout, - AbsolutePath path, - string? defaultModName = null, - CancellationToken cancellationToken = default) - { - var analyzedFile = await ArchiveAnalyzer.AnalyzeFileAsync(path, cancellationToken); - var modIds = await ArchiveInstaller.AddMods(loadout.Value.LoadoutId, analyzedFile.Hash, defaultModName, cancellationToken); - return modIds.Select(id => loadout.Value.Mods[id]).ToArray(); - } /// /// Installs a single mod from the archive into the loadout. This calls @@ -171,17 +167,18 @@ protected async Task InstallModsFromArchiveIntoLoadout( /// protected async Task InstallModFromArchiveIntoLoadout( LoadoutMarker loadout, - Hash hash, + DownloadId downloadId, string? defaultModName = null, CancellationToken cancellationToken = default) { var mods = await InstallModsFromArchiveIntoLoadout( - loadout, hash, + loadout, downloadId, defaultModName, cancellationToken); - mods.Should().ContainSingle(); - return mods.First(); + mods.Length.Should().BeGreaterOrEqualTo(1); + // Sort the mods so we have consistent results + return mods.OrderBy(m => m.Name).First(); } /// @@ -198,10 +195,12 @@ protected async Task InstallModFromArchiveIntoLoadout( string? defaultModName = null, CancellationToken cancellationToken = default) { - var analyzed = await ArchiveAnalyzer.AnalyzeFileAsync(path, cancellationToken); + var downloadId = await DownloadRegistry.RegisterDownload(path, new FilePathMetadata + { OriginalName = path.FileName, Quality = Quality.Low }, cancellationToken); + var mods = await InstallModsFromArchiveIntoLoadout( loadout, - analyzed.Hash, + downloadId, defaultModName, cancellationToken ); @@ -210,20 +209,6 @@ protected async Task InstallModFromArchiveIntoLoadout( return mods.First(); } - /// - /// Analyzes a file as an archive using the . - /// - /// - /// - /// The provided file is not an archive. - protected async Task AnalyzeArchive(AbsolutePath path) - { - var analyzedFile = await ArchiveAnalyzer.AnalyzeFileAsync(path); - if (analyzedFile is AnalyzedArchive analyzedArchive) - return analyzedArchive; - throw new ArgumentException($"File at {path} is not an archive!", nameof(path)); - } - /// /// Creates a ZIP archive using and returns the /// to it. @@ -240,7 +225,7 @@ protected async Task CreateTestArchive(IDictionary(); } - protected Diagnostic[] GetAllDiagnostics(Loadout loadout) + protected async ValueTask GetAllDiagnostics(Loadout loadout) { - return Emitter.Diagnose(loadout).ToArray(); + return await Emitter.Diagnose(loadout).ToArrayAsync(); } - protected Diagnostic GetSingleDiagnostic(Loadout loadout) + protected async ValueTask GetSingleDiagnostic(Loadout loadout) { - var diagnostics = GetAllDiagnostics(loadout); + var diagnostics = await GetAllDiagnostics(loadout); diagnostics.Should().ContainSingle(); return diagnostics.First(); } diff --git a/tests/Games/NexusMods.Games.TestFramework/AModInstallerTest.cs b/tests/Games/NexusMods.Games.TestFramework/AModInstallerTest.cs index b40dd9e7cb..aaf4fd6b8a 100644 --- a/tests/Games/NexusMods.Games.TestFramework/AModInstallerTest.cs +++ b/tests/Games/NexusMods.Games.TestFramework/AModInstallerTest.cs @@ -1,9 +1,12 @@ using System.Collections.Immutable; using FluentAssertions; using JetBrains.Annotations; +using Microsoft.Extensions.DependencyInjection; using NexusMods.Common; +using NexusMods.DataModel; using NexusMods.DataModel.Abstractions; using NexusMods.DataModel.ArchiveContents; +using NexusMods.DataModel.ArchiveMetaData; using NexusMods.DataModel.Extensions; using NexusMods.DataModel.Games; using NexusMods.DataModel.Loadouts; @@ -11,9 +14,11 @@ using NexusMods.DataModel.Loadouts.Mods; using NexusMods.DataModel.ModInstallers; using NexusMods.FileExtractor.FileSignatures; +using NexusMods.FileExtractor.StreamFactories; using NexusMods.Hashing.xxHash64; using NexusMods.Paths; using NexusMods.Paths.Extensions; +using NexusMods.Paths.FileTree; namespace NexusMods.Games.TestFramework; @@ -29,7 +34,8 @@ public abstract class AModInstallerTest : AGameTest /// protected AModInstallerTest(IServiceProvider serviceProvider) : base(serviceProvider) { - ModInstaller = serviceProvider.FindImplementationInContainer(); + var game = serviceProvider.GetServices().OfType().Single(); + ModInstaller = game.Installers.OfType().Single(); } /// @@ -70,13 +76,32 @@ protected async Task GetModsFromInstaller( AbsolutePath archivePath, CancellationToken cancellationToken = default) { - var analyzedArchive = await AnalyzeArchive(archivePath); + var downloadId = await DownloadRegistry.RegisterDownload(archivePath, new FilePathMetadata + { + OriginalName = archivePath.FileName, + Quality = Quality.Low + }, cancellationToken); + + var contents = await DownloadRegistry.Get(downloadId); + + var tree = FileTreeNode.CreateTree(contents.Contents + .Select(f => + KeyValuePair.Create(f.Path, + new ModSourceFileEntry + { + Hash = f.Hash, + Size = f.Size, + StreamFactory = new ArchiveManagerStreamFactory(ArchiveManager, f.Hash) + { + Name = f.Path, + Size = f.Size + } + }))); var results = await ModInstaller.GetModsAsync( GameInstallation, ModId.New(), - analyzedArchive.Hash, - analyzedArchive.Contents, + tree, cancellationToken); var mods = results.Select(result => new Mod @@ -138,7 +163,7 @@ protected async Task> GetModsWithFilesFromInstaller( var mods = await GetModsFromInstaller(archivePath, cancellationToken); mods.Should().ContainSingle(); - var mod = mods.First(); + var mod = mods.OrderBy(m => m.Name).First(); return (mod, mod.Files.Values.ToArray()); } @@ -148,21 +173,18 @@ protected async Task> GetModsWithFilesFromInstaller( /// /// /// - protected EntityDictionary BuildArchiveDescription( + protected FileTreeNode BuildArchiveDescription( IEnumerable files) { - var description = new EntityDictionary(DataStore); var items = files.Select(file => KeyValuePair.Create(file.Name.ToRelativePath(), - new AnalyzedFile + new ModSourceFileEntry { - AnalyzersHash = Hash.Zero, - FileTypes = file.Filetypes, Hash = Hash.From(file.Hash), - Size = Size.From(4), - AnalysisData = file.AnalysisData.ToImmutableList() + Size = Size.FromLong(file.Data.Length), + StreamFactory = new MemoryStreamFactory(file.Name.ToRelativePath(), new MemoryStream(file.Data)) })); - return description.With(items); + return FileTreeNode.CreateTree(items); } /// @@ -181,7 +203,7 @@ protected EntityDictionary BuildArchiveDescription( { Name = f.Name, Hash = f.Hash, - Filetypes = Array.Empty() + Data = new byte[] { 0xDE, 0xAD, 0xBE, 0xEF, 0x42 } } )); } @@ -195,14 +217,14 @@ protected EntityDictionary BuildArchiveDescription( /// /// protected Task> BuildAndInstall( - params (ulong Hash, string Name, IFileAnalysisData? Data)[] files) + params (ulong Hash, string Name, byte[] data)[] files) { return BuildAndInstall(files.Select(f => - new ModInstallerExampleFile() + new ModInstallerExampleFile { Name = f.Name, Hash = f.Hash, - AnalysisData = f.Data == null ? Array.Empty() : new[] {f.Data} + Data = f.data } )); } @@ -216,13 +238,13 @@ protected EntityDictionary BuildArchiveDescription( /// /// protected Task> BuildAndInstall(Priority expectedPriority, - params (ulong Hash, string Name, FileType FileType)[] files) + params (ulong Hash, string Name)[] files) { - return BuildAndInstall(files.Select(f => new ModInstallerExampleFile() + return BuildAndInstall(files.Select(f => new ModInstallerExampleFile { Name = f.Name, Hash = f.Hash, - Filetypes = new[] {f.FileType} + Data = new byte[]{ 0xDE, 0xAD, 0xBE, 0xEF, 0x42} })); } @@ -236,18 +258,26 @@ protected EntityDictionary BuildArchiveDescription( protected async Task> BuildAndInstall(IEnumerable files) { - var description = BuildArchiveDescription(files); + ModInstallerResult[] mods; - var mods = (await ModInstaller.GetModsAsync( + var tree = FileTreeNode.CreateTree(files + .Select(f => + KeyValuePair.Create(f.Name.ToRelativePath(), + new ModSourceFileEntry + { + Hash = Hash.From(f.Hash), + Size = Size.FromLong(f.Data.Length), + StreamFactory = new MemoryStreamFactory(f.Name.ToRelativePath(), new MemoryStream(f.Data)) + }))); + mods = (await ModInstaller.GetModsAsync( GameInstallation, ModId.New(), - Hash.From(0xDEADBEEF), - description)).ToArray(); + tree)).ToArray(); if (mods.Length == 0) return Array.Empty<(ulong Hash, GameFolderType FolderType, string Path)>(); - mods.Should().ContainSingle(); + mods.Length.Should().BeGreaterOrEqualTo(1); var contents = mods.First().Files; return contents.OfType().Select(m => (m.Hash.Value, m.To.Type, m.To.Path.ToString())); } diff --git a/tests/Games/NexusMods.Games.TestFramework/DependencyInjectionHelper.cs b/tests/Games/NexusMods.Games.TestFramework/DependencyInjectionHelper.cs index b123cf911b..7c277c3080 100644 --- a/tests/Games/NexusMods.Games.TestFramework/DependencyInjectionHelper.cs +++ b/tests/Games/NexusMods.Games.TestFramework/DependencyInjectionHelper.cs @@ -58,7 +58,6 @@ public static IServiceCollection AddDefaultServicesForTesting(this IServiceColle { UseInMemoryDataModel = true }) - .AddAllSingleton>(_ => new Resource("File Analysis for tests")) .AddAllSingleton>(_ => new Resource("File Extraction for tests")) .AddAllSingleton>(_ => new Resource("Hash Cache for tests")) .AddFileExtractors(); diff --git a/tests/Games/NexusMods.Games.TestFramework/ModInstallerExampleFile.cs b/tests/Games/NexusMods.Games.TestFramework/ModInstallerExampleFile.cs index acafb95257..2ef0a50784 100644 --- a/tests/Games/NexusMods.Games.TestFramework/ModInstallerExampleFile.cs +++ b/tests/Games/NexusMods.Games.TestFramework/ModInstallerExampleFile.cs @@ -5,8 +5,8 @@ namespace NexusMods.Games.TestFramework; public record ModInstallerExampleFile { - public ulong Hash { get; init; } - public string Name { get; init; } = string.Empty; - public FileType[] Filetypes { get; init; } = Array.Empty(); - public IFileAnalysisData[] AnalysisData { get; init; } = Array.Empty(); + public required ulong Hash { get; init; } + public required string Name { get; init; } = string.Empty; + + public required byte[] Data { get; init; } = Array.Empty(); }; diff --git a/tests/NexusMods.CLI.Tests/VerbTests/AnalyzeArchive.cs b/tests/NexusMods.CLI.Tests/VerbTests/AnalyzeArchive.cs index 9b25a8e667..b9d4d7a5ea 100644 --- a/tests/NexusMods.CLI.Tests/VerbTests/AnalyzeArchive.cs +++ b/tests/NexusMods.CLI.Tests/VerbTests/AnalyzeArchive.cs @@ -17,14 +17,14 @@ public async Task CanAnalyzeArchives() await RunNoBannerAsync("analyze-archive", "-i", Data7ZipLZMA2.ToString()); LogSize.Should().Be(1); - LastTable.Columns.Should().BeEquivalentTo("Path", "Size", "Hash", "Signatures"); + LastTable.Columns.Should().BeEquivalentTo("Path", "Size", "Hash"); LastTable.Rows .Should() .BeEquivalentTo(new[] { - new object[] {"deepFolder/deepFolder2/deepFolder3/deepFolder4/deepFile.txt".ToRelativePath(), (Size)12L, (Hash)0xE405A7CFA6ABBDE3, "TXT"}, - new object[] {"folder1/folder1file.txt".ToRelativePath(), (Size)15L, (Hash)0xC9E47B1523162066, "TXT"}, - new object[] {"rootFile.txt".ToRelativePath(), (Size)12L, (Hash)0x33DDBF7930BA002A, "TXT"} + new object[] {"deepFolder/deepFolder2/deepFolder3/deepFolder4/deepFile.txt".ToRelativePath(), (Size)12L, (Hash)0xE405A7CFA6ABBDE3}, + new object[] {"folder1/folder1file.txt".ToRelativePath(), (Size)15L, (Hash)0xC9E47B1523162066}, + new object[] {"rootFile.txt".ToRelativePath(), (Size)12L, (Hash)0x33DDBF7930BA002A} }); } } diff --git a/tests/NexusMods.CLI.Tests/VerbTests/ModManagementVerbs.cs b/tests/NexusMods.CLI.Tests/VerbTests/ModManagementVerbs.cs index ffd97f80bc..62ba1e7b59 100644 --- a/tests/NexusMods.CLI.Tests/VerbTests/ModManagementVerbs.cs +++ b/tests/NexusMods.CLI.Tests/VerbTests/ModManagementVerbs.cs @@ -30,7 +30,7 @@ await RunNoBannerAsync("manage-game", "-g", "stubbed-game", "-v", await RunNoBannerAsync("list-mods", "-l", listName); LastTable.Rows.Count().Should().Be(1); - await RunNoBannerAsync("install-mod", "-l", listName, "-f", Data7ZipLZMA2.ToString()); + await RunNoBannerAsync("install-mod", "-l", listName, "-f", Data7ZipLZMA2.ToString(), "-n", Data7ZipLZMA2.GetFileNameWithoutExtension()); await RunNoBannerAsync("list-mods", "-l", listName); LastTable.Rows.Count().Should().Be(2); diff --git a/tests/NexusMods.DataModel.Tests/ApplicationTests.cs b/tests/NexusMods.DataModel.Tests/ApplicationTests.cs index 5a4667c4d7..16b2acf588 100644 --- a/tests/NexusMods.DataModel.Tests/ApplicationTests.cs +++ b/tests/NexusMods.DataModel.Tests/ApplicationTests.cs @@ -1,4 +1,6 @@ using FluentAssertions; +using NexusMods.Common; +using NexusMods.DataModel.ArchiveMetaData; using NexusMods.DataModel.Loadouts.ApplySteps; using NexusMods.DataModel.Loadouts.IngestSteps; using NexusMods.DataModel.Loadouts.ModFiles; @@ -20,14 +22,15 @@ public ApplicationTests(IServiceProvider provider) : base(provider) public async Task CanApplyGame() { var mainList = await LoadoutManager.ManageGameAsync(Install, "MainList", CancellationToken.None); - var hash = await ArchiveAnalyzer.AnalyzeFileAsync(DataZipLzma, CancellationToken.None); - await ArchiveInstaller.AddMods(mainList.Value.LoadoutId, hash.Hash, "First Mod", CancellationToken.None); + var downloadId = await DownloadRegistry.RegisterDownload(DataZipLzma, + new FilePathMetadata {OriginalName = DataZipLzma.FileName, Quality = Quality.Low}, CancellationToken.None); + await ArchiveInstaller.AddMods(mainList.Value.LoadoutId, downloadId, "First Mod", CancellationToken.None); var plan = await LoadoutSynchronizer.MakeApplySteps(mainList.Value, CancellationToken.None); plan.Steps.OfType().Count().Should().Be(3); await LoadoutSynchronizer.Apply(plan, CancellationToken.None); - + var gameFolder = Install.Locations[GameFolderType.Game]; foreach (var file in DataNames) { @@ -48,9 +51,9 @@ public async Task CanIntegrateChanges() var originalPlan = await LoadoutSynchronizer.MakeApplySteps(mainList.Value, Token); originalPlan.Steps.OfType().Count().Should().Be(3, "Files override each other"); - + await LoadoutSynchronizer.Apply(originalPlan, Token); - + var gameFolder = Install.Locations[GameFolderType.Game]; foreach (var file in DataNames) { @@ -65,8 +68,8 @@ public async Task CanIntegrateChanges() var modifiedHash = "modified".XxHash64AsUtf8(); var firstMod = mainList.Value.Mods.Values.First(); - var ingestPlan = await LoadoutSynchronizer.MakeIngestPlan(mainList.Value, _ => firstMod.Id, CancellationToken.None); - + var ingestPlan = await LoadoutSynchronizer.MakeIngestPlan(mainList.Value, _ => firstMod.Id, CancellationToken.None); + ingestPlan.Steps.Should().BeEquivalentTo(new IIngestStep[] { new Loadouts.IngestSteps.BackupFile @@ -92,7 +95,7 @@ public async Task CanIntegrateChanges() (await LoadoutSynchronizer.FlattenLoadout(mainList.Value)).Files.Count.Should().Be(7, "because no changes are applied yet"); await LoadoutSynchronizer.Ingest(ingestPlan); - + var flattened = (await LoadoutSynchronizer.FlattenLoadout(mainList.Value)).Files.Count; flattened.Should().Be(6, "Because we've deleted one file"); diff --git a/tests/NexusMods.DataModel.Tests/ArchiveManagerTests.cs b/tests/NexusMods.DataModel.Tests/ArchiveManagerTests.cs index e20fa690af..a95ab4cd17 100644 --- a/tests/NexusMods.DataModel.Tests/ArchiveManagerTests.cs +++ b/tests/NexusMods.DataModel.Tests/ArchiveManagerTests.cs @@ -27,7 +27,7 @@ public async Task CanArchiveFiles(int fileCount, int maxSize) { // Randomly generate some file sizes var sizes = Enumerable.Range(0, fileCount).Select(s => Size.FromLong(Random.Shared.Next(maxSize))).ToArray(); - + // Randomly generate some data var datas = sizes.Select(size => { @@ -35,14 +35,14 @@ public async Task CanArchiveFiles(int fileCount, int maxSize) Random.Shared.NextBytes(buf); return buf; }).ToArray(); - + // Calculate the hashes var hashes = datas.Select(d => d.AsSpan().XxHash64()).ToArray(); // Create the tuples for compression var records = Enumerable.Range(0, fileCount).Select(idx => ( - (IStreamFactory)new MemoryStreamFactory($"{idx}.txt".ToRelativePath(), new MemoryStream(datas[idx])), - hashes[idx], Size.FromLong(datas[idx].Length))); + new ArchivedFileEntry(new MemoryStreamFactory($"{idx}.txt".ToRelativePath(), new MemoryStream(datas[idx])), + hashes[idx], Size.FromLong(datas[idx].Length)))); // Backup the files await _manager.BackupFiles(records); @@ -54,19 +54,19 @@ public async Task CanArchiveFiles(int fileCount, int maxSize) // Extract some of the files var extractionCount = Random.Shared.Next(fileCount); var extractionIdxs = Enumerable.Range(0, extractionCount).Select(_ => Random.Shared.Next(fileCount)).ToArray(); - + // Extract the files via the in-memory method var extracted = await _manager.ExtractFiles(extractionIdxs.Select(idx => hashes[idx])); - + // Verify the extracted files are correct foreach (var idx in extractionIdxs) extracted[hashes[idx]].Should().BeEquivalentTo(datas[idx]); // Extract the files via the file method await using var tempFolder = _temporaryFileManager.CreateFolder(); - + var fullPaths = extractionIdxs.Distinct().ToDictionary(idx => idx, idx => tempFolder.Path.Combine($"{idx}.dat")); - + await _manager.ExtractFiles(extractionIdxs.Select(idx => (hashes[idx], fullPaths[idx]))); // Verify the extracted files are correct diff --git a/tests/NexusMods.DataModel.Tests/Diagnostics/DiagnosticManagerTests.cs b/tests/NexusMods.DataModel.Tests/Diagnostics/DiagnosticManagerTests.cs index e77a29464c..9f24e91832 100644 --- a/tests/NexusMods.DataModel.Tests/Diagnostics/DiagnosticManagerTests.cs +++ b/tests/NexusMods.DataModel.Tests/Diagnostics/DiagnosticManagerTests.cs @@ -34,7 +34,7 @@ public async Task Test_RefreshLoadoutDiagnostics() changeSet.Adds.Should().Be(1); }); - _diagnosticManager.RefreshLoadoutDiagnostics(loadout.Value); + await _diagnosticManager.RefreshLoadoutDiagnostics(loadout.Value); var diagnostic = _diagnosticManager.ActiveDiagnostics.Should().ContainSingle().Which; diagnostic.Id.Number.Should().Be(1); diff --git a/tests/NexusMods.DataModel.Tests/Diagnostics/DummyLoadoutDiagnosticEmitter.cs b/tests/NexusMods.DataModel.Tests/Diagnostics/DummyLoadoutDiagnosticEmitter.cs index 969b8bfee8..d39be74ffe 100644 --- a/tests/NexusMods.DataModel.Tests/Diagnostics/DummyLoadoutDiagnosticEmitter.cs +++ b/tests/NexusMods.DataModel.Tests/Diagnostics/DummyLoadoutDiagnosticEmitter.cs @@ -12,7 +12,7 @@ internal static DiagnosticMessage CreateMessage(Loadout loadout) return DiagnosticMessage.From($"LoadoutId={loadout.LoadoutId}, DataStoreId={loadout.DataStoreId.SpanHex}"); } - public IEnumerable Diagnose(Loadout loadout) + public async IAsyncEnumerable Diagnose(Loadout loadout) { yield return new Diagnostic { diff --git a/tests/NexusMods.DataModel.Tests/Harness/ADataModelTest.cs b/tests/NexusMods.DataModel.Tests/Harness/ADataModelTest.cs index 15743e3aeb..2dccad71a6 100644 --- a/tests/NexusMods.DataModel.Tests/Harness/ADataModelTest.cs +++ b/tests/NexusMods.DataModel.Tests/Harness/ADataModelTest.cs @@ -1,7 +1,9 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; +using NexusMods.Common; using NexusMods.DataModel.Abstractions; +using NexusMods.DataModel.ArchiveMetaData; using NexusMods.DataModel.Games; using NexusMods.DataModel.Loadouts; using NexusMods.DataModel.Loadouts.Markers; @@ -35,7 +37,6 @@ public abstract class ADataModelTest : IDisposable, IAsyncLifetime protected readonly TemporaryFileManager TemporaryFileManager; protected readonly IServiceProvider ServiceProvider; - protected readonly IArchiveAnalyzer ArchiveAnalyzer; protected readonly IArchiveManager ArchiveManager; protected readonly IArchiveInstaller ArchiveInstaller; protected readonly LoadoutManager LoadoutManager; @@ -43,6 +44,7 @@ public abstract class ADataModelTest : IDisposable, IAsyncLifetime protected readonly FileHashCache FileHashCache; protected readonly IFileSystem FileSystem; protected readonly IDataStore DataStore; + protected readonly IDownloadRegistry DownloadRegistry; protected readonly IToolManager ToolManager; protected readonly IGame Game; @@ -60,13 +62,13 @@ protected ADataModelTest(IServiceProvider provider) .ConfigureServices((_, service) => startup.ConfigureServices(service)) .Build(); var provider1 = _host.Services; - ArchiveAnalyzer = provider1.GetRequiredService(); ArchiveManager = provider1.GetRequiredService(); ArchiveInstaller = provider1.GetRequiredService(); LoadoutManager = provider1.GetRequiredService(); FileHashCache = provider1.GetRequiredService(); FileSystem = provider1.GetRequiredService(); DataStore = provider1.GetRequiredService(); + DownloadRegistry = provider1.GetRequiredService(); Logger = provider1.GetRequiredService>(); LoadoutSynchronizer = provider1.GetRequiredService(); TemporaryFileManager = provider1.GetRequiredService(); @@ -90,8 +92,9 @@ public async Task InitializeAsync() protected async Task AddMods(LoadoutMarker mainList, AbsolutePath path, string? name = null) { - var hash1 = await ArchiveAnalyzer.AnalyzeFileAsync(path, CancellationToken.None); - return await ArchiveInstaller.AddMods(mainList.Value.LoadoutId, hash1.Hash, name, CancellationToken.None); + var downloadId = await DownloadRegistry.RegisterDownload(path, + new FilePathMetadata {OriginalName = path.FileName, Quality = Quality.Low}, CancellationToken.None); + return await ArchiveInstaller.AddMods(mainList.Value.LoadoutId, downloadId, name, CancellationToken.None); } public Task DisposeAsync() diff --git a/tests/NexusMods.DataModel.Tests/Harness/ALoadoutSynrconizerTest.cs b/tests/NexusMods.DataModel.Tests/Harness/ALoadoutSynrconizerTest.cs index 2f13a71d01..d9b79b7635 100644 --- a/tests/NexusMods.DataModel.Tests/Harness/ALoadoutSynrconizerTest.cs +++ b/tests/NexusMods.DataModel.Tests/Harness/ALoadoutSynrconizerTest.cs @@ -68,10 +68,9 @@ public async ValueTask HaveFile(Hash hash) { return Archives.Contains(hash); } - - public async Task BackupFiles(IEnumerable<(IStreamFactory, Hash, Size)> backups, CancellationToken token = default) + public async Task BackupFiles(IEnumerable backups, CancellationToken token = default) { - Archives.AddRange(backups.Select(b => b.Item2)); + Archives.AddRange(backups.Select(b => b.Hash)); } public async Task ExtractFiles(IEnumerable<(Hash Src, AbsolutePath Dest)> files, CancellationToken token = default) diff --git a/tests/NexusMods.DataModel.Tests/Startup.cs b/tests/NexusMods.DataModel.Tests/Startup.cs index 906c1cf503..23705d1b61 100644 --- a/tests/NexusMods.DataModel.Tests/Startup.cs +++ b/tests/NexusMods.DataModel.Tests/Startup.cs @@ -29,7 +29,6 @@ public void ConfigureServices(IServiceCollection container) .AddStandardGameLocators(false) .AddFileExtractors() .AddStubbedGameLocators() - .AddAllSingleton>(_ => new Resource("File Analysis")) .AddAllSingleton>(_ => new Resource("File Extraction")) //.AddSingleton() .AddSingleton(_ => new AssemblyTypeFinder(typeof(Startup).Assembly)) diff --git a/tests/NexusMods.StandardGameLocators.TestHelpers/StubbedGames/StubbedGame.cs b/tests/NexusMods.StandardGameLocators.TestHelpers/StubbedGames/StubbedGame.cs index 40d6a76b0b..a8cf6540d5 100644 --- a/tests/NexusMods.StandardGameLocators.TestHelpers/StubbedGames/StubbedGame.cs +++ b/tests/NexusMods.StandardGameLocators.TestHelpers/StubbedGames/StubbedGame.cs @@ -1,4 +1,3 @@ -using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using NexusMods.Common; using NexusMods.DataModel.Abstractions; @@ -33,16 +32,13 @@ public class StubbedGame : AGame, IEADesktopGame, IEpicGame, IOriginGame, ISteam d => (d.FileName.ToString().XxHash64AsUtf8(), Size.FromLong(d.FileName.ToString().Length))); private readonly IFileSystem _fileSystem; - private readonly IDataStore _dataStore; - private readonly IServiceProvider _provider; public StubbedGame(ILogger logger, IEnumerable locators, - IFileSystem fileSystem, IServiceProvider provider) : base(locators) + IFileSystem fileSystem) : base(locators) { _logger = logger; _locators = locators; _fileSystem = fileSystem; - _provider = provider; } public override GamePath GetPrimaryFile(GameStore store) => new(GameFolderType.Game, ""); diff --git a/tests/NexusMods.StandardGameLocators.TestHelpers/StubbedGames/StubbedGameInstaller.cs b/tests/NexusMods.StandardGameLocators.TestHelpers/StubbedGames/StubbedGameInstaller.cs index e5df73dc88..8461e469da 100644 --- a/tests/NexusMods.StandardGameLocators.TestHelpers/StubbedGames/StubbedGameInstaller.cs +++ b/tests/NexusMods.StandardGameLocators.TestHelpers/StubbedGames/StubbedGameInstaller.cs @@ -6,6 +6,7 @@ using NexusMods.DataModel.Loadouts; using NexusMods.DataModel.ModInstallers; using NexusMods.Paths; +using NexusMods.Paths.FileTree; using Hash = NexusMods.Hashing.xxHash64.Hash; namespace NexusMods.StandardGameLocators.TestHelpers.StubbedGames; @@ -15,23 +16,21 @@ public class StubbedGameInstaller : IModInstaller public ValueTask> GetModsAsync( GameInstallation gameInstallation, ModId baseModId, - Hash srcArchiveHash, - EntityDictionary archiveFiles, + FileTreeNode archiveFiles, CancellationToken cancellationToken = default) { - return ValueTask.FromResult(GetMods(baseModId, srcArchiveHash, archiveFiles)); + return ValueTask.FromResult(GetMods(baseModId, archiveFiles)); } private IEnumerable GetMods( ModId baseModId, - Hash srcArchiveHash, - EntityDictionary archiveFiles) + FileTreeNode archiveFiles) { - var modFiles = archiveFiles + var modFiles = archiveFiles.GetAllDescendentFiles() .Select(kv => { var (path, file) = kv; - return file.ToFromArchive( + return file!.ToFromArchive( new GamePath(GameFolderType.Game, path) ); }); diff --git a/tests/NexusMods.UI.Tests/AVmTest.cs b/tests/NexusMods.UI.Tests/AVmTest.cs index 133c35d914..94a128c954 100644 --- a/tests/NexusMods.UI.Tests/AVmTest.cs +++ b/tests/NexusMods.UI.Tests/AVmTest.cs @@ -1,6 +1,9 @@ using Microsoft.Extensions.DependencyInjection; using NexusMods.App.UI; +using NexusMods.Common; +using NexusMods.DataModel; using NexusMods.DataModel.Abstractions; +using NexusMods.DataModel.ArchiveMetaData; using NexusMods.DataModel.Games; using NexusMods.DataModel.GlobalSettings; using NexusMods.DataModel.Loadouts; @@ -27,9 +30,9 @@ public class AVmTest : AUiTest, IAsyncLifetime protected LoadoutRegistry LoadoutRegistry { get; } protected IDataStore DataStore { get; } - - protected IArchiveAnalyzer ArchiveAnalyzer { get; } protected IArchiveInstaller ArchiveInstaller { get; } + + protected IDownloadRegistry DownloadRegistry { get; } protected GlobalSettingsManager GlobalSettingsManager { get; } @@ -47,8 +50,8 @@ public AVmTest(IServiceProvider provider) : base(provider) Game = provider.GetRequiredService(); Install = Game.Installations.First(); FileSystem = provider.GetRequiredService(); - ArchiveAnalyzer = provider.GetRequiredService(); ArchiveInstaller = provider.GetRequiredService(); + DownloadRegistry = provider.GetRequiredService(); GlobalSettingsManager = provider.GetRequiredService(); } @@ -62,8 +65,9 @@ public async Task InitializeAsync() protected async Task InstallMod(AbsolutePath path) { - var analyzedFile = await ArchiveAnalyzer.AnalyzeFileAsync(path); - return await ArchiveInstaller.AddMods(Loadout.Value.LoadoutId, analyzedFile.Hash); + var downloadId = await DownloadRegistry.RegisterDownload(path, + new FilePathMetadata() { OriginalName = path.FileName, Quality = Quality.Normal }); + return await ArchiveInstaller.AddMods(Loadout.Value.LoadoutId, downloadId); } public Task DisposeAsync()