Remove analyzers (#615)

* Removed file analyzers * Reworked installers to use the new FileTreeNode format, and to perform analysis on the fly
Nexus-Mods · Sep 18, 2023 · 9d57ff1 · 9d57ff1
1 parent 8e53f4b
commit 9d57ff1
Show file tree

Hide file tree

Showing 135 changed files with 1,627 additions and 2,349 deletions.
diff --git a/Directory.Packages.props b/Directory.Packages.props
@@ -1,8 +1,8 @@
 <Project>
     <ItemGroup>
         <!-- Custom Packages -->
-        <PackageVersion Include="NexusMods.Hashing.xxHash64" Version="0.9.0" />
-        <PackageVersion Include="NexusMods.Paths" Version="0.1.4" />
+        <PackageVersion Include="NexusMods.Hashing.xxHash64" Version="1.0.1" />
+        <PackageVersion Include="NexusMods.Paths" Version="0.1.6" />
         <PackageVersion Include="NexusMods.Archives.Nx" Version="0.3.3-preview" />
         <PackageVersion Include="FomodInstaller.Interface" Version="1.2.0" />
         <PackageVersion Include="FomodInstaller.Scripting.XmlScript" Version="1.0.0" />

diff --git a/benchmarks/NexusMods.Benchmarks/Benchmarks/DataModel.cs b/benchmarks/NexusMods.Benchmarks/Benchmarks/DataModel.cs
@@ -26,7 +26,6 @@ public class DataStoreBenchmark : IBenchmark, IDisposable
     private readonly IDataStore _dataStore;
     private readonly byte[] _rawData;
     private readonly Id64 _rawId;
-    private readonly HashRelativePath _fromPutPath;
     private readonly IId _immutableRecord;
     private readonly FromArchive _record;
 
@@ -52,10 +51,6 @@ public DataStoreBenchmark()
         Random.Shared.NextBytes(_rawData);
         _rawId = new Id64(EntityCategory.TestData, (ulong)Random.Shared.NextInt64());
         _dataStore.PutRaw(_rawId, _rawData);
-
-        var relPutPath = "test.txt".ToRelativePath();
-        _fromPutPath = new HashRelativePath(Hash.From((ulong)Random.Shared.NextInt64()), relPutPath);
-
         _record = new FromArchive
         {
             Id = ModFileId.New(),

diff --git a/src/ArchiveManagement/NexusMods.FileExtractor/NexusMods.FileExtractor.csproj b/src/ArchiveManagement/NexusMods.FileExtractor/NexusMods.FileExtractor.csproj
@@ -10,6 +10,7 @@
     <ItemGroup>
         <PackageReference Include="CliWrap" />
         <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" />
+        <PackageReference Include="System.Linq.Async" />
     </ItemGroup>
 
     <ItemGroup>

diff --git a/src/Games/NexusMods.Games.BethesdaGameStudios/PluginAnalysisData.cs b/src/Games/NexusMods.Games.BethesdaGameStudios/PluginAnalysisData.cs
@@ -5,7 +5,7 @@
 namespace NexusMods.Games.BethesdaGameStudios;
 
 [JsonName("BethesdaGameStudios.FileAnalysisData")]
-public class PluginAnalysisData : IFileAnalysisData
+public class PluginAnalysisData
 {
     public required RelativePath[] Masters { get; init; }
     public bool IsLightMaster { get; init; }

diff --git a/src/Games/NexusMods.Games.BethesdaGameStudios/PluginAnalyzer.cs b/src/Games/NexusMods.Games.BethesdaGameStudios/PluginAnalyzer.cs
@@ -1,4 +1,3 @@
-using System.Runtime.CompilerServices;
 using JetBrains.Annotations;
 using Microsoft.Extensions.Logging;
 using Mutagen.Bethesda;
@@ -7,7 +6,6 @@
 using Mutagen.Bethesda.Plugins.Meta;
 using Mutagen.Bethesda.Plugins.Records;
 using Mutagen.Bethesda.Skyrim;
-using NexusMods.DataModel.Abstractions;
 using NexusMods.DataModel.Abstractions.Ids;
 using NexusMods.FileExtractor.FileSignatures;
 using NexusMods.Paths;
@@ -17,17 +15,13 @@
 namespace NexusMods.Games.BethesdaGameStudios;
 
 [UsedImplicitly]
-public class PluginAnalyzer : IFileAnalyzer
+public class PluginAnalyzer
 {
-    public FileAnalyzerId Id { get; } = FileAnalyzerId.New("9c673a4f-064f-4b1e-83e3-4bf0454575cd", 1);
-
-    public IEnumerable<FileType> FileTypes => new[] { FileType.TES4 };
-
-    private static readonly Extension[] ValidExtensions = {
-        new(".esp"),
-        new(".esm"),
-        new(".esl"),
-    };
+    private static readonly HashSet<Extension> ValidExtensions = new Extension[] {
+        new (".esp"),
+        new (".esm"),
+        new (".esl"),
+    }.ToHashSet();
 
     private readonly ILogger<PluginAnalyzer> _logger;
 
@@ -36,12 +30,11 @@ public PluginAnalyzer(ILogger<PluginAnalyzer> logger)
         _logger = logger;
     }
 
-#pragma warning disable CS1998
-    public async IAsyncEnumerable<IFileAnalysisData> AnalyzeAsync(FileAnalyzerInfo info, [EnumeratorCancellation] CancellationToken ct = default)
-#pragma warning restore CS1998
+    public async Task<PluginAnalysisData?> AnalyzeAsync(RelativePath path, Stream stream, CancellationToken ct = default)
     {
-        var extension = info.FileName.ToRelativePath().Extension;
-        if (ValidExtensions[0] != extension && ValidExtensions[1] != extension && ValidExtensions[2] != extension) yield break;
+        var extension = path.Extension;
+        if (!ValidExtensions.Contains(extension))
+            return null;
 
         // NOTE(erri120): The GameConstant specifies the header length.
         // - Oblivion: 20 bytes
@@ -54,24 +47,24 @@ public async IAsyncEnumerable<IFileAnalysisData> AnalyzeAsync(FileAnalyzerInfo i
         // The current solution just tries different GameConstants, which isn't ideal and
         // should be replaced with an identification step that finds the correct GameConstant.
 
-        var startPos = info.Stream.Position;
-        var fileAnalysisData = Analyze(GameConstants.SkyrimLE, info);
+        var fileAnalysisData = Analyze(path, GameConstants.SkyrimLE, stream);
         if (fileAnalysisData is null)
         {
-            info.Stream.Position = startPos;
-            fileAnalysisData = Analyze(GameConstants.Oblivion, info);
-            if (fileAnalysisData is null) yield break;
+            stream.Position = 0;
+            fileAnalysisData = Analyze(path, GameConstants.Oblivion, stream);
+            if (fileAnalysisData is null)
+                return null;
         }
 
-        yield return fileAnalysisData;
+        return fileAnalysisData;
     }
 
-    private IFileAnalysisData? Analyze(GameConstants targetGame, FileAnalyzerInfo info)
+    private PluginAnalysisData? Analyze(RelativePath path, GameConstants targetGame, Stream stream)
     {
         try
         {
             using var readStream = new MutagenInterfaceReadStream(
-                new BinaryReadStream(info.Stream, dispose: false),
+                new BinaryReadStream(stream, dispose: false),
                 new ParsingBundle(targetGame, masterReferences: null!)
                 {
                     ModKey = ModKey.Null
@@ -102,7 +95,7 @@ public async IAsyncEnumerable<IFileAnalysisData> AnalyzeAsync(FileAnalyzerInfo i
         }
         catch (Exception e)
         {
-            _logger.LogError(e, "Exception while parsing {} ({})", info.FileName, info.RelativePath);
+            _logger.LogError(e, "Exception while parsing {} ({})", path.FileName, path);
             return null;
         }
     }

diff --git a/src/Games/NexusMods.Games.BethesdaGameStudios/Services.cs b/src/Games/NexusMods.Games.BethesdaGameStudios/Services.cs
@@ -14,7 +14,7 @@ public static IServiceCollection AddBethesdaGameStudios(this IServiceCollection
         services.AddAllSingleton<IGame, SkyrimLegendaryEdition>();
         services.AddSingleton<ITool, SkyrimLegendaryEditionGameTool>();
         services.AddSingleton<ITool, SkyrimSpecialEditionGameTool>();
-        services.AddAllSingleton<IFileAnalyzer, PluginAnalyzer>();
+        services.AddSingleton<PluginAnalyzer>();
         services.AddAllSingleton<ITypeFinder, TypeFinder>();
         return services;
     }

diff --git a/src/Games/NexusMods.Games.DarkestDungeon/Analyzers/ProjectAnalyzer.cs b/src/Games/NexusMods.Games.DarkestDungeon/Analyzers/ProjectAnalyzer.cs
diff --git a/src/Games/NexusMods.Games.DarkestDungeon/Installers/LooseFilesModInstaller.cs b/src/Games/NexusMods.Games.DarkestDungeon/Installers/LooseFilesModInstaller.cs
@@ -9,6 +9,7 @@
 using NexusMods.Hashing.xxHash64;
 using NexusMods.Paths;
 using NexusMods.Paths.Extensions;
+using NexusMods.Paths.FileTree;
 
 namespace NexusMods.Games.DarkestDungeon.Installers;
 
@@ -19,26 +20,18 @@ public class LooseFilesModInstaller : IModInstaller
 {
     private static readonly RelativePath ModsFolder = "mods".ToRelativePath();
 
-    public ValueTask<IEnumerable<ModInstallerResult>> GetModsAsync(
+    public async ValueTask<IEnumerable<ModInstallerResult>> GetModsAsync(
         GameInstallation gameInstallation,
         ModId baseModId,
-        Hash srcArchiveHash,
-        EntityDictionary<RelativePath, AnalyzedFile> archiveFiles,
+        FileTreeNode<RelativePath, ModSourceFileEntry> archiveFiles,
         CancellationToken cancellationToken = default)
-    {
-        return ValueTask.FromResult(GetMods(baseModId, srcArchiveHash, archiveFiles));
-    }
-
-    private IEnumerable<ModInstallerResult> GetMods(
-        ModId baseModId,
-        Hash srcArchiveHash,
-        EntityDictionary<RelativePath, AnalyzedFile> archiveFiles)
     {
         var files = archiveFiles
+            .GetAllDescendentFiles()
             .Select(kv =>
             {
                 var (path, file) = kv;
-                return file.ToFromArchive(
+                return file!.ToFromArchive(
                     new GamePath(GameFolderType.Game, ModsFolder.Join(path))
                 );
             });
@@ -47,13 +40,13 @@ private IEnumerable<ModInstallerResult> GetMods(
         // this needs to be serialized to XML and added to the files enumerable
         var modProject = new ModProject
         {
-            Title = archiveFiles.First().Key.TopParent.ToString()
+            Title = archiveFiles.Path.TopParent.ToString()
         };
 
-        yield return new ModInstallerResult
+        return new [] { new ModInstallerResult
         {
             Id = baseModId,
             Files = files,
-        };
+        }};
     }
 }
diff --git a/src/Games/NexusMods.Games.DarkestDungeon/Installers/NativeModInstaller.cs b/src/Games/NexusMods.Games.DarkestDungeon/Installers/NativeModInstaller.cs
@@ -1,4 +1,7 @@
 using System.Diagnostics;
+using System.Xml;
+using System.Xml.Schema;
+using System.Xml.Serialization;
 using NexusMods.Common;
 using NexusMods.DataModel.Abstractions;
 using NexusMods.DataModel.ArchiveContents;
@@ -10,6 +13,7 @@
 using NexusMods.Hashing.xxHash64;
 using NexusMods.Paths;
 using NexusMods.Paths.Extensions;
+using NexusMods.Paths.FileTree;
 
 namespace NexusMods.Games.DarkestDungeon.Installers;
 
@@ -22,57 +26,60 @@ public class NativeModInstaller : IModInstaller
     private static readonly RelativePath ModsFolder = "mods".ToRelativePath();
     private static readonly RelativePath ProjectFile = "project.xml".ToRelativePath();
 
-    internal static IEnumerable<KeyValuePair<RelativePath, AnalyzedFile>> GetModProjects(
-        EntityDictionary<RelativePath, AnalyzedFile> archiveFiles)
+    internal static async Task<IEnumerable<(FileTreeNode<RelativePath, ModSourceFileEntry> Node, ModProject Project)>>
+        GetModProjects(
+        FileTreeNode<RelativePath, ModSourceFileEntry> archiveFiles)
     {
-        return archiveFiles.Where(kv =>
-        {
-            var (path, file) = kv;
+        return await archiveFiles
+            .GetAllDescendentFiles()
+            .SelectAsync(async kv =>
+            {
+                if (kv.Path.FileName != ProjectFile)
+                    return default;
 
-            if (!path.FileName.Equals(ProjectFile)) return false;
-            var modProject = file.AnalysisData
-                .OfType<ModProject>()
-                .FirstOrDefault();
+                await using var stream = await kv.Value!.Open();
+                using var reader = XmlReader.Create(stream, new XmlReaderSettings
+                {
+                    IgnoreComments = true,
+                    IgnoreWhitespace = true,
+                    ValidationFlags = XmlSchemaValidationFlags.AllowXmlAttributes
+                });
 
-            return modProject is not null;
-        });
+                var obj = new XmlSerializer(typeof(ModProject)).Deserialize(reader);
+                return (kv, obj as ModProject);
+            })
+            .Where(r => r.Item2 is not null)
+            .Select(r => (r.kv, r.Item2!))
+            .ToArrayAsync();
     }
 
-    public ValueTask<IEnumerable<ModInstallerResult>> GetModsAsync(
+    public async ValueTask<IEnumerable<ModInstallerResult>> GetModsAsync(
         GameInstallation gameInstallation,
         ModId baseModId,
-        Hash srcArchiveHash,
-        EntityDictionary<RelativePath, AnalyzedFile> archiveFiles,
+        FileTreeNode<RelativePath, ModSourceFileEntry> archiveFiles,
         CancellationToken cancellationToken = default)
     {
-        return ValueTask.FromResult(GetMods(srcArchiveHash, archiveFiles));
-    }
-
-    private static IEnumerable<ModInstallerResult> GetMods(
-        Hash srcArchiveHash,
-        EntityDictionary<RelativePath, AnalyzedFile> archiveFiles)
-    {
-        var modProjectFiles = GetModProjects(archiveFiles).ToArray();
+        var modProjectFiles = (await GetModProjects(archiveFiles)).ToArray();
         if (!modProjectFiles.Any())
             return Array.Empty<ModInstallerResult>();
 
+        if (modProjectFiles.Length > 1)
+            return Array.Empty<ModInstallerResult>();
+
         var mods = modProjectFiles
             .Select(modProjectFile =>
             {
-                var parent = modProjectFile.Key.Parent;
-                var modProject = modProjectFile.Value.AnalysisData
-                    .OfType<ModProject>()
-                    .FirstOrDefault();
+                var parent = modProjectFile.Node.Parent;
+                var modProject = modProjectFile.Project;
 
                 if (modProject is null) throw new UnreachableException();
 
-                var modFiles = archiveFiles
-                    .Where(kv => kv.Key.InFolder(parent))
+                var modFiles = parent.GetAllDescendentFiles()
                     .Select(kv =>
                     {
                         var (path, file) = kv;
-                        return file.ToFromArchive(
-                            new GamePath(GameFolderType.Game, ModsFolder.Join(path.DropFirst(parent.Depth)))
+                        return file!.ToFromArchive(
+                            new GamePath(GameFolderType.Game, ModsFolder.Join(path.DropFirst(parent.Depth - 1)))
                         );
                     });
 

diff --git a/src/Games/NexusMods.Games.DarkestDungeon/Models/ModProject.cs b/src/Games/NexusMods.Games.DarkestDungeon/Models/ModProject.cs
@@ -9,7 +9,7 @@ namespace NexusMods.Games.DarkestDungeon.Models;
 /// </summary>
 [JsonName("NexusMods.Games.DarkestDungeon.ModProject")]
 [XmlRoot(ElementName = "project")]
-public record ModProject : IFileAnalysisData
+public record ModProject
 {
     [XmlElement(ElementName = "Title")]
     public string Title { get; set; } = string.Empty;

diff --git a/src/Games/NexusMods.Games.DarkestDungeon/NexusMods.Games.DarkestDungeon.csproj b/src/Games/NexusMods.Games.DarkestDungeon/NexusMods.Games.DarkestDungeon.csproj
@@ -11,4 +11,5 @@
         <EmbeddedResource Include="Resources\DarkestDungeon\icon.png" />
     </ItemGroup>
 
+
 </Project>
diff --git a/src/Games/NexusMods.Games.DarkestDungeon/Services.cs b/src/Games/NexusMods.Games.DarkestDungeon/Services.cs
@@ -1,9 +1,7 @@
 using Microsoft.Extensions.DependencyInjection;
-using NexusMods.DataModel.Abstractions;
 using NexusMods.DataModel.Games;
 using NexusMods.DataModel.JsonConverters.ExpressionGenerator;
 using NexusMods.DataModel.ModInstallers;
-using NexusMods.Games.DarkestDungeon.Analyzers;
 using NexusMods.Games.DarkestDungeon.Installers;
 
 namespace NexusMods.Games.DarkestDungeon;
@@ -13,7 +11,6 @@ public static class Services
     public static IServiceCollection AddDarkestDungeon(this IServiceCollection serviceCollection)
     {
         serviceCollection.AddSingleton<IGame, DarkestDungeon>();
-        serviceCollection.AddSingleton<IFileAnalyzer, ProjectAnalyzer>();
         serviceCollection.AddSingleton<IModInstaller, NativeModInstaller>();
         serviceCollection.AddSingleton<IModInstaller, LooseFilesModInstaller>();
         serviceCollection.AddSingleton<ITypeFinder, TypeFinder>();
Original file line number	Diff line number	Diff line change
Expand Up		@@ -11,4 +11,5 @@
		<EmbeddedResource Include="Resources\DarkestDungeon\icon.png" />
		</ItemGroup>


		</Project>