Schema failsafes

.gitattributes

@@ -1,65 +1,53 @@
 # Auto detect text files and perform LF normalization
 * text=auto
-
 # Documents
 *.md        text diff=markdown
-
 # Graphics
 *.png       binary
 *.jpg       binary
 *.jpeg      binary
 *.gif       binary
 *.ico       binary
 *.svg       binary
-
 # Scripts (Unix)
 *.bash      text eol=lf
 *.sh        text eol=lf
 *.zsh       text eol=lf
-
 # Scripts (Windows)
 *.bat       text eol=crlf
 *.cmd       text eol=crlf
 *.ps1       text eol=crlf
-
 # Archives
 *.7z        binary
 *.gz        binary
 *.tar       binary
 *.tgz       binary
 *.zip       binary
-
 # Code files
 *.cs        text diff=csharp
-
 # Project files
 *.sln       text eol=crlf
 *.csproj    text eol=crlf
-
 *.targets   text eol=crlf
 *.filters   text eol=crlf
 *.filters   text eol=crlf
 *.vcxitems  text eol=crlf
-
 # Dynamic libraries
 *.so        binary
 *.dylib     binary
 *.dll       binary
-
 # Executables
 *.exe       binary
 *.out       binary
 *.app       binary
-
 # Text files where line endings should be preserved
 *.patch     -text
-
 # Exclude files from exporting
 .gitattributes  export-ignore
 .gitignore      export-ignore
 .gitkeep        export-ignore
-
 # Verify
 *.verified.txt text eol=lf working-tree-encoding=UTF-8
 *.verified.xml text eol=lf working-tree-encoding=UTF-8
 *.verified.json text eol=lf working-tree-encoding=UTF-8
+*.rocksdb.zip filter=lfs diff=lfs merge=lfs -text

.github/workflows/clean_environment_tests.yaml

@@ -30,7 +30,7 @@ jobs:
 
   build-and-test:
     if: github.event_name == 'push' || github.event.pull_request.draft == false
-    uses: Nexus-Mods/NexusMods.App.Meta/.github/workflows/dotnet-build-and-test-with-osx.yaml@9c4844439c53f8b8b9f64fb707c91b469238b15e
+    uses: Nexus-Mods/NexusMods.App.Meta/.github/workflows/dotnet-build-and-test-with-osx.yaml@ae64a3be780a74e94b59ee463a413083013c8b0c
     with:
       extra-test-args: "--blame-hang-timeout 20m"
       test-filter: "RequiresNetworking!=True&FlakeyTest!=True"

Directory.Packages.props

@@ -42,6 +42,7 @@
     <PackageVersion Include="OpenTelemetry.Extensions.Hosting" Version="1.8.1" />
     <PackageVersion Include="Polly.Core" Version="8.4.2" />
     <PackageVersion Include="Polly" Version="8.4.2" />
+    <PackageVersion Include="Verify" Version="26.6.0" />
     <PackageVersion Include="ZstdSharp.Port" Version="0.8.1" />
   </ItemGroup>
   <ItemGroup>
@@ -136,4 +137,4 @@
     <PackageVersion Include="Splat.Microsoft.Extensions.Logging" Version="15.2.22" />
     <PackageVersion Include="TransparentValueObjects" Version="1.0.1" />
   </ItemGroup>
-</Project>
+</Project>

NexusMods.App.sln

@@ -268,6 +268,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.Abstractions.Reso
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.Abstractions.Resources.Resilience", "src\Abstractions\NexusMods.Abstractions.Resources.Resilience\NexusMods.Abstractions.Resources.Resilience.csproj", "{04219A58-C99C-4C3B-A477-5E4B29D1F275}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.DataModel.SchemaVersions", "src\NexusMods.DataModel.SchemaVersions\NexusMods.DataModel.SchemaVersions.csproj", "{79E13AD1-187B-42F7-BDC3-EF8ABA308973}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.DataModel.SchemaVersions.Tests", "tests\NexusMods.DataModel.SchemaVersions.Tests\NexusMods.DataModel.SchemaVersions.Tests.csproj", "{A5A2932D-B3EF-480B-BEBC-793F6FC90EDE}"
+EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.Games.MountAndBlade2Bannerlord", "src\Games\NexusMods.Games.MountAndBlade2Bannerlord\NexusMods.Games.MountAndBlade2Bannerlord.csproj", "{8D7E82BB-2F8D-455A-AF12-C486D9EC3B77}"
 EndProject
 Global
@@ -700,6 +704,14 @@ Global
 		{04219A58-C99C-4C3B-A477-5E4B29D1F275}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{04219A58-C99C-4C3B-A477-5E4B29D1F275}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{04219A58-C99C-4C3B-A477-5E4B29D1F275}.Release|Any CPU.Build.0 = Release|Any CPU
+		{79E13AD1-187B-42F7-BDC3-EF8ABA308973}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{79E13AD1-187B-42F7-BDC3-EF8ABA308973}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{79E13AD1-187B-42F7-BDC3-EF8ABA308973}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{79E13AD1-187B-42F7-BDC3-EF8ABA308973}.Release|Any CPU.Build.0 = Release|Any CPU
+		{A5A2932D-B3EF-480B-BEBC-793F6FC90EDE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{A5A2932D-B3EF-480B-BEBC-793F6FC90EDE}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{A5A2932D-B3EF-480B-BEBC-793F6FC90EDE}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{A5A2932D-B3EF-480B-BEBC-793F6FC90EDE}.Release|Any CPU.Build.0 = Release|Any CPU
 		{8D7E82BB-2F8D-455A-AF12-C486D9EC3B77}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{8D7E82BB-2F8D-455A-AF12-C486D9EC3B77}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{8D7E82BB-2F8D-455A-AF12-C486D9EC3B77}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -828,6 +840,8 @@ Global
 		{BE8C17C4-E3B0-4D07-8CD0-0D15C3CCA9D5} = {0CB73565-1207-4A56-A79F-6A8E9BBD795C}
 		{D3BA5B5A-668A-443B-872C-3116CBB0BC0D} = {0CB73565-1207-4A56-A79F-6A8E9BBD795C}
 		{04219A58-C99C-4C3B-A477-5E4B29D1F275} = {0CB73565-1207-4A56-A79F-6A8E9BBD795C}
+		{79E13AD1-187B-42F7-BDC3-EF8ABA308973} = {E7BAE287-D505-4D6D-A090-665A64309B2D}
+		{A5A2932D-B3EF-480B-BEBC-793F6FC90EDE} = {52AF9D62-7D5B-4AD0-BA12-86F2AA67428B}
 		{8D7E82BB-2F8D-455A-AF12-C486D9EC3B77} = {70D38D24-79AE-4600-8E83-17F3C11BA81F}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution

src/NexusMods.App.UI/Pages/LoadoutPage/LoadoutPage.cs

@@ -48,7 +48,7 @@ public override ILoadoutViewModel CreateViewModel(LoadoutPageContext context)
     public override IEnumerable<PageDiscoveryDetails?> GetDiscoveryDetails(IWorkspaceContext workspaceContext)
     {
         if (workspaceContext is not LoadoutContext loadoutContext) yield break;
-
+        
         yield return new PageDiscoveryDetails
         {
             SectionName = "Mods",

src/NexusMods.App/Program.cs

@@ -13,6 +13,7 @@
 using NexusMods.CrossPlatform;
 using NexusMods.CrossPlatform.Process;
 using NexusMods.DataModel;
+using NexusMods.DataModel.Migrations;
 using NexusMods.Paths;
 using NexusMods.ProxyConsole;
 using NexusMods.Settings;
@@ -59,9 +60,15 @@ public static int Main(string[] args)
         );
         var services = host.Services;
 
+        // Run the migrations
+        var migration = services.GetRequiredService<MigrationService>();
+        migration.Run().Wait();
+
+
         // Okay to do wait here, as we are in the main process thread.
         host.StartAsync().Wait(timeout: TimeSpan.FromMinutes(5));
 
+
         // Start the CLI server if we are the main process.
         var cliServer = services.GetService<CliServer>();
         cliServer?.StartCliServerAsync().Wait(timeout: TimeSpan.FromSeconds(5));

src/NexusMods.DataModel.SchemaVersions/IMigration.cs

@@ -0,0 +1,36 @@
+using NexusMods.MnemonicDB.Abstractions;
+
+namespace NexusMods.DataModel.Migrations;
+
+/// <summary>
+/// A definition of a single data migration
+/// </summary>
+public interface IMigration
+{
+    /// <summary>
+    /// The name of the migration
+    /// </summary>
+    public string Name { get; }
+
+    /// <summary>
+    /// A long description of the migration
+    /// </summary>
+    public string Description { get; }
+
+    /// <summary>
+    /// A date for the migration's creation. Not used for anything other than sorting. Migrations
+    /// will be run in order of this date.
+    /// </summary>
+    public DateTimeOffset CreatedAt { get; }
+
+    /// <summary>
+    /// Returns true if the migration should run. This function should do any sort of querying and processing to make sure
+    /// data is in the format expected by the migration.
+    /// </summary>
+    public bool ShouldRun(IDb db);
+
+    /// <summary>
+    /// Runs the migration
+    /// </summary>
+    public void Migrate(IDb basis, ITransaction tx);
+}

src/NexusMods.DataModel.SchemaVersions/MigrationService.cs

@@ -0,0 +1,44 @@
+using Microsoft.Extensions.Logging;
+using NexusMods.MnemonicDB.Abstractions;
+
+namespace NexusMods.DataModel.Migrations;
+
+/// <summary>
+/// Updates the state of a database and provides hooks for migrating schemas
+/// and transforming data between versions.
+/// </summary>
+public class MigrationService
+{
+    private readonly ILogger<MigrationService> _logger;
+    private readonly IConnection _connection;
+    private readonly List<IMigration> _migrations;
+
+    public MigrationService(ILogger<MigrationService> logger, IConnection connection, IEnumerable<IMigration> migrations)
+    {
+        _logger = logger;
+        _connection = connection;
+        _migrations = migrations.OrderBy(m => m.CreatedAt).ToList();
+    }
+
+    public async Task Run()
+    {
+        // Run all migrations, for now this interface works by handing a transaction to each migration, in the future we'll need
+        // to add support for changing history of the datoms and not just the most recent state. But until we need such a migration
+        // we'll go with this approach as it's simpler.
+        foreach (var migration in _migrations)
+        {
+            var db = _connection.Db;
+            if (!migration.ShouldRun(db))
+            {
+                _logger.LogInformation("Migration {Name} skipped", migration.Name);
+                continue;
+            }
+
+            _logger.LogInformation("Running migration {Name}", migration.Name);
+            using var tx = _connection.BeginTransaction();
+            migration.Migrate(db, tx);
+            await tx.Commit();
+            _logger.LogInformation("Migration {Name} completed", migration.Name);
+        }
+    }
+}

src/NexusMods.DataModel.SchemaVersions/Migrations/UpsertFingerprint.cs

@@ -0,0 +1,42 @@
+using NexusMods.DataModel.SchemaVersions;
+using NexusMods.Hashing.xxHash3;
+using NexusMods.MnemonicDB.Abstractions;
+using NexusMods.MnemonicDB.Abstractions.ElementComparers;
+using NexusMods.MnemonicDB.Abstractions.ValueSerializers;
+
+namespace NexusMods.DataModel.Migrations.Migrations;
+
+public class UpsertFingerprint : IMigration
+{
+    public string Name => "Upsert Fingerprint";
+    public string Description => "Upserts the fingerprint of the database, creating it if it does not exist.";
+
+    /// <summary>
+    /// Max value so it always runs last
+    /// </summary>
+    public DateTimeOffset CreatedAt => DateTimeOffset.MaxValue;
+
+    public bool ShouldRun(IDb db)
+    {
+        if (!db.AttributeCache.Has(SchemaVersion.Fingerprint.Id))
+            return true;
+
+        var fingerprints = db.Datoms(SchemaVersion.Fingerprint);
+        // No fingerprint, we need to create it
+        if (fingerprints.Count == 0)
+            return true;
+
+        var currentFingerprint = SchemaFingerprint.GenerateFingerprint(db);
+        var dbFingerprint = Hash.From(UInt64Serializer.Read(fingerprints[0].ValueSpan));
+        // Is the fingerprint up to date?
+        return currentFingerprint != dbFingerprint;
+    }
+
+    public void Migrate(IDb basis, ITransaction tx)
+    {
+        var eid = basis.Datoms(SchemaVersion.Fingerprint).Select(d => d.E)
+            .FirstOrDefault(tx.TempId());
+
+        tx.Add(eid, SchemaVersion.Fingerprint, SchemaFingerprint.GenerateFingerprint(basis));
+    }
+}

src/NexusMods.DataModel.SchemaVersions/NexusMods.DataModel.SchemaVersions.csproj

@@ -0,0 +1,16 @@
+<Project Sdk="Microsoft.NET.Sdk">
+    <!-- NuGet Package Shared Details -->
+    <Import Project="$([MSBuild]::GetPathOfFileAbove('NuGet.Build.props', '$(MSBuildThisFileDirectory)../'))" />
+
+    <ItemGroup>
+      <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" />
+      <PackageReference Include="NexusMods.Hashing.xxHash3" />
+      <PackageReference Include="NexusMods.MnemonicDB.Abstractions" />
+      <PackageReference Include="NexusMods.MnemonicDB.SourceGenerator" PrivateAssets="all" OutputItemType="Analyzer" ReferenceOutputAssembly="false" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <ProjectReference Include="..\Abstractions\NexusMods.Abstractions.MnemonicDB.Attributes\NexusMods.Abstractions.MnemonicDB.Attributes.csproj" />
+    </ItemGroup>
+
+</Project>

src/NexusMods.DataModel.SchemaVersions/SchemaFingerprint.cs

@@ -0,0 +1,40 @@
+using System.Text;
+using NexusMods.Hashing.xxHash3;
+using NexusMods.MnemonicDB.Abstractions;
+
+namespace NexusMods.DataModel.SchemaVersions;
+
+/// <summary>
+/// Tools for generating a hash of all the attributes of a schema so that we can detect changes.
+/// </summary>
+public class SchemaFingerprint
+{
+    public static Hash GenerateFingerprint(IDb db)
+    {
+        StringBuilder sb = new();
+        var cache = db.AttributeCache;
+
+
+        void AppendLine(string s)
+        {
+            // We want platform independent newlines.
+            sb.Append(s);
+            sb.Append("\n");
+        }
+
+        foreach (var id in cache.AllAttributeIds.OrderBy(id => id.Id, StringComparer.Ordinal))
+        {
+            var aid = cache.GetAttributeId(id);
+            AppendLine(id.ToString());
+            AppendLine(cache.GetValueTag(aid).ToString());
+            AppendLine(cache.IsIndexed(aid).ToString());
+            AppendLine(cache.IsCardinalityMany(aid).ToString());
+            AppendLine(cache.IsNoHistory(aid).ToString());
+            AppendLine("--");
+        }
+        // Use ascii as the attribute names must be ascii and this makes data comparisons simpler.
+        var bytes = Encoding.ASCII.GetBytes(sb.ToString());
+        return bytes.xxHash3();
+    }
+
+}

src/NexusMods.DataModel.SchemaVersions/SchemaVersion.cs

@@ -0,0 +1,15 @@
+using NexusMods.Abstractions.MnemonicDB.Attributes;
+using NexusMods.MnemonicDB.Abstractions.Models;
+
+namespace NexusMods.DataModel.Migrations;
+
+public partial class SchemaVersion : IModelDefinition
+{
+    public const string Namespace = "NexusMods.DataModel.SchemaVersioning.SchemaVersionModel";
+
+    /// <summary>
+    /// The current fingerprint of the database. This is used to detect when schema updates do not need to be performend,
+    /// and the app can start without the rather expensive upgrade process.
+    /// </summary>
+    public static readonly HashAttribute Fingerprint = new(Namespace, "Fingerprint");
+}

src/NexusMods.DataModel.SchemaVersions/Services.cs

@@ -0,0 +1,17 @@
+using Microsoft.Extensions.DependencyInjection;
+using NexusMods.DataModel.Migrations;
+using NexusMods.DataModel.Migrations.Migrations;
+
+namespace NexusMods.DataModel.SchemaVersions;
+
+public static class Services
+{
+    public static IServiceCollection AddMigrations(this IServiceCollection services)
+    {
+        services.AddSchemaVersionModel();
+        services.AddSingleton<IMigration, UpsertFingerprint>();
+        services.AddSingleton<MigrationService>();
+        return services;
+    }
+
+}

src/NexusMods.DataModel/NexusMods.DataModel.csproj

@@ -17,6 +17,7 @@
         <ProjectReference Include="..\Extensions\NexusMods.Extensions.DynamicData\NexusMods.Extensions.DynamicData.csproj" />
         <ProjectReference Include="..\Extensions\NexusMods.Extensions.Hashing\NexusMods.Extensions.Hashing.csproj" />
         <ProjectReference Include="..\NexusMods.App.GarbageCollection.DataModel\NexusMods.App.GarbageCollection.DataModel.csproj" />
+        <ProjectReference Include="..\NexusMods.DataModel.SchemaVersions\NexusMods.DataModel.SchemaVersions.csproj" />
         <ProjectReference Include="..\NexusMods.ProxyConsole.Abstractions\NexusMods.ProxyConsole.Abstractions.csproj" />
     </ItemGroup>