diff --git a/.gitattributes b/.gitattributes index 49a746641..5f00feb74 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,9 +1,7 @@ # Auto detect text files and perform LF normalization * text=auto - # Documents *.md text diff=markdown - # Graphics *.png binary *.jpg binary @@ -11,55 +9,45 @@ *.gif binary *.ico binary *.svg binary - # Scripts (Unix) *.bash text eol=lf *.sh text eol=lf *.zsh text eol=lf - # Scripts (Windows) *.bat text eol=crlf *.cmd text eol=crlf *.ps1 text eol=crlf - # Archives *.7z binary *.gz binary *.tar binary *.tgz binary *.zip binary - # Code files *.cs text diff=csharp - # Project files *.sln text eol=crlf *.csproj text eol=crlf - *.targets text eol=crlf *.filters text eol=crlf *.filters text eol=crlf *.vcxitems text eol=crlf - # Dynamic libraries *.so binary *.dylib binary *.dll binary - # Executables *.exe binary *.out binary *.app binary - # Text files where line endings should be preserved *.patch -text - # Exclude files from exporting .gitattributes export-ignore .gitignore export-ignore .gitkeep export-ignore - # Verify *.verified.txt text eol=lf working-tree-encoding=UTF-8 *.verified.xml text eol=lf working-tree-encoding=UTF-8 *.verified.json text eol=lf working-tree-encoding=UTF-8 +*.rocksdb.zip filter=lfs diff=lfs merge=lfs -text diff --git a/.github/workflows/clean_environment_tests.yaml b/.github/workflows/clean_environment_tests.yaml index 417a4a6ab..34257cf42 100644 --- a/.github/workflows/clean_environment_tests.yaml +++ b/.github/workflows/clean_environment_tests.yaml @@ -30,7 +30,7 @@ jobs: build-and-test: if: github.event_name == 'push' || github.event.pull_request.draft == false - uses: Nexus-Mods/NexusMods.App.Meta/.github/workflows/dotnet-build-and-test-with-osx.yaml@9c4844439c53f8b8b9f64fb707c91b469238b15e + uses: Nexus-Mods/NexusMods.App.Meta/.github/workflows/dotnet-build-and-test-with-osx.yaml@ae64a3be780a74e94b59ee463a413083013c8b0c with: extra-test-args: "--blame-hang-timeout 20m" test-filter: "RequiresNetworking!=True&FlakeyTest!=True" diff --git a/Directory.Packages.props b/Directory.Packages.props index 3ddb28a86..383cf0fe8 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -42,6 +42,7 @@ + @@ -136,4 +137,4 @@ - + \ No newline at end of file diff --git a/NexusMods.App.sln b/NexusMods.App.sln index 21b19686a..440619ec6 100644 --- a/NexusMods.App.sln +++ b/NexusMods.App.sln @@ -268,6 +268,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.Abstractions.Reso EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.Abstractions.Resources.Resilience", "src\Abstractions\NexusMods.Abstractions.Resources.Resilience\NexusMods.Abstractions.Resources.Resilience.csproj", "{04219A58-C99C-4C3B-A477-5E4B29D1F275}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.DataModel.SchemaVersions", "src\NexusMods.DataModel.SchemaVersions\NexusMods.DataModel.SchemaVersions.csproj", "{79E13AD1-187B-42F7-BDC3-EF8ABA308973}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.DataModel.SchemaVersions.Tests", "tests\NexusMods.DataModel.SchemaVersions.Tests\NexusMods.DataModel.SchemaVersions.Tests.csproj", "{A5A2932D-B3EF-480B-BEBC-793F6FC90EDE}" +EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.Games.MountAndBlade2Bannerlord", "src\Games\NexusMods.Games.MountAndBlade2Bannerlord\NexusMods.Games.MountAndBlade2Bannerlord.csproj", "{8D7E82BB-2F8D-455A-AF12-C486D9EC3B77}" EndProject Global @@ -700,6 +704,14 @@ Global {04219A58-C99C-4C3B-A477-5E4B29D1F275}.Debug|Any CPU.Build.0 = Debug|Any CPU {04219A58-C99C-4C3B-A477-5E4B29D1F275}.Release|Any CPU.ActiveCfg = Release|Any CPU {04219A58-C99C-4C3B-A477-5E4B29D1F275}.Release|Any CPU.Build.0 = Release|Any CPU + {79E13AD1-187B-42F7-BDC3-EF8ABA308973}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {79E13AD1-187B-42F7-BDC3-EF8ABA308973}.Debug|Any CPU.Build.0 = Debug|Any CPU + {79E13AD1-187B-42F7-BDC3-EF8ABA308973}.Release|Any CPU.ActiveCfg = Release|Any CPU + {79E13AD1-187B-42F7-BDC3-EF8ABA308973}.Release|Any CPU.Build.0 = Release|Any CPU + {A5A2932D-B3EF-480B-BEBC-793F6FC90EDE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A5A2932D-B3EF-480B-BEBC-793F6FC90EDE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A5A2932D-B3EF-480B-BEBC-793F6FC90EDE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A5A2932D-B3EF-480B-BEBC-793F6FC90EDE}.Release|Any CPU.Build.0 = Release|Any CPU {8D7E82BB-2F8D-455A-AF12-C486D9EC3B77}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {8D7E82BB-2F8D-455A-AF12-C486D9EC3B77}.Debug|Any CPU.Build.0 = Debug|Any CPU {8D7E82BB-2F8D-455A-AF12-C486D9EC3B77}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -828,6 +840,8 @@ Global {BE8C17C4-E3B0-4D07-8CD0-0D15C3CCA9D5} = {0CB73565-1207-4A56-A79F-6A8E9BBD795C} {D3BA5B5A-668A-443B-872C-3116CBB0BC0D} = {0CB73565-1207-4A56-A79F-6A8E9BBD795C} {04219A58-C99C-4C3B-A477-5E4B29D1F275} = {0CB73565-1207-4A56-A79F-6A8E9BBD795C} + {79E13AD1-187B-42F7-BDC3-EF8ABA308973} = {E7BAE287-D505-4D6D-A090-665A64309B2D} + {A5A2932D-B3EF-480B-BEBC-793F6FC90EDE} = {52AF9D62-7D5B-4AD0-BA12-86F2AA67428B} {8D7E82BB-2F8D-455A-AF12-C486D9EC3B77} = {70D38D24-79AE-4600-8E83-17F3C11BA81F} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution diff --git a/src/NexusMods.App.UI/Pages/LoadoutPage/LoadoutPage.cs b/src/NexusMods.App.UI/Pages/LoadoutPage/LoadoutPage.cs index c77bdcc81..6b373faaf 100644 --- a/src/NexusMods.App.UI/Pages/LoadoutPage/LoadoutPage.cs +++ b/src/NexusMods.App.UI/Pages/LoadoutPage/LoadoutPage.cs @@ -48,7 +48,7 @@ public override ILoadoutViewModel CreateViewModel(LoadoutPageContext context) public override IEnumerable GetDiscoveryDetails(IWorkspaceContext workspaceContext) { if (workspaceContext is not LoadoutContext loadoutContext) yield break; - + yield return new PageDiscoveryDetails { SectionName = "Mods", diff --git a/src/NexusMods.App/Program.cs b/src/NexusMods.App/Program.cs index 71fc8a056..fbd26d4e1 100644 --- a/src/NexusMods.App/Program.cs +++ b/src/NexusMods.App/Program.cs @@ -13,6 +13,7 @@ using NexusMods.CrossPlatform; using NexusMods.CrossPlatform.Process; using NexusMods.DataModel; +using NexusMods.DataModel.Migrations; using NexusMods.Paths; using NexusMods.ProxyConsole; using NexusMods.Settings; @@ -59,9 +60,15 @@ public static int Main(string[] args) ); var services = host.Services; + // Run the migrations + var migration = services.GetRequiredService(); + migration.Run().Wait(); + + // Okay to do wait here, as we are in the main process thread. host.StartAsync().Wait(timeout: TimeSpan.FromMinutes(5)); + // Start the CLI server if we are the main process. var cliServer = services.GetService(); cliServer?.StartCliServerAsync().Wait(timeout: TimeSpan.FromSeconds(5)); diff --git a/src/NexusMods.DataModel.SchemaVersions/IMigration.cs b/src/NexusMods.DataModel.SchemaVersions/IMigration.cs new file mode 100644 index 000000000..84061cd22 --- /dev/null +++ b/src/NexusMods.DataModel.SchemaVersions/IMigration.cs @@ -0,0 +1,36 @@ +using NexusMods.MnemonicDB.Abstractions; + +namespace NexusMods.DataModel.Migrations; + +/// +/// A definition of a single data migration +/// +public interface IMigration +{ + /// + /// The name of the migration + /// + public string Name { get; } + + /// + /// A long description of the migration + /// + public string Description { get; } + + /// + /// A date for the migration's creation. Not used for anything other than sorting. Migrations + /// will be run in order of this date. + /// + public DateTimeOffset CreatedAt { get; } + + /// + /// Returns true if the migration should run. This function should do any sort of querying and processing to make sure + /// data is in the format expected by the migration. + /// + public bool ShouldRun(IDb db); + + /// + /// Runs the migration + /// + public void Migrate(IDb basis, ITransaction tx); +} diff --git a/src/NexusMods.DataModel.SchemaVersions/MigrationService.cs b/src/NexusMods.DataModel.SchemaVersions/MigrationService.cs new file mode 100644 index 000000000..6312d2c47 --- /dev/null +++ b/src/NexusMods.DataModel.SchemaVersions/MigrationService.cs @@ -0,0 +1,44 @@ +using Microsoft.Extensions.Logging; +using NexusMods.MnemonicDB.Abstractions; + +namespace NexusMods.DataModel.Migrations; + +/// +/// Updates the state of a database and provides hooks for migrating schemas +/// and transforming data between versions. +/// +public class MigrationService +{ + private readonly ILogger _logger; + private readonly IConnection _connection; + private readonly List _migrations; + + public MigrationService(ILogger logger, IConnection connection, IEnumerable migrations) + { + _logger = logger; + _connection = connection; + _migrations = migrations.OrderBy(m => m.CreatedAt).ToList(); + } + + public async Task Run() + { + // Run all migrations, for now this interface works by handing a transaction to each migration, in the future we'll need + // to add support for changing history of the datoms and not just the most recent state. But until we need such a migration + // we'll go with this approach as it's simpler. + foreach (var migration in _migrations) + { + var db = _connection.Db; + if (!migration.ShouldRun(db)) + { + _logger.LogInformation("Migration {Name} skipped", migration.Name); + continue; + } + + _logger.LogInformation("Running migration {Name}", migration.Name); + using var tx = _connection.BeginTransaction(); + migration.Migrate(db, tx); + await tx.Commit(); + _logger.LogInformation("Migration {Name} completed", migration.Name); + } + } +} diff --git a/src/NexusMods.DataModel.SchemaVersions/Migrations/UpsertFingerprint.cs b/src/NexusMods.DataModel.SchemaVersions/Migrations/UpsertFingerprint.cs new file mode 100644 index 000000000..02a2d4eea --- /dev/null +++ b/src/NexusMods.DataModel.SchemaVersions/Migrations/UpsertFingerprint.cs @@ -0,0 +1,42 @@ +using NexusMods.DataModel.SchemaVersions; +using NexusMods.Hashing.xxHash3; +using NexusMods.MnemonicDB.Abstractions; +using NexusMods.MnemonicDB.Abstractions.ElementComparers; +using NexusMods.MnemonicDB.Abstractions.ValueSerializers; + +namespace NexusMods.DataModel.Migrations.Migrations; + +public class UpsertFingerprint : IMigration +{ + public string Name => "Upsert Fingerprint"; + public string Description => "Upserts the fingerprint of the database, creating it if it does not exist."; + + /// + /// Max value so it always runs last + /// + public DateTimeOffset CreatedAt => DateTimeOffset.MaxValue; + + public bool ShouldRun(IDb db) + { + if (!db.AttributeCache.Has(SchemaVersion.Fingerprint.Id)) + return true; + + var fingerprints = db.Datoms(SchemaVersion.Fingerprint); + // No fingerprint, we need to create it + if (fingerprints.Count == 0) + return true; + + var currentFingerprint = SchemaFingerprint.GenerateFingerprint(db); + var dbFingerprint = Hash.From(UInt64Serializer.Read(fingerprints[0].ValueSpan)); + // Is the fingerprint up to date? + return currentFingerprint != dbFingerprint; + } + + public void Migrate(IDb basis, ITransaction tx) + { + var eid = basis.Datoms(SchemaVersion.Fingerprint).Select(d => d.E) + .FirstOrDefault(tx.TempId()); + + tx.Add(eid, SchemaVersion.Fingerprint, SchemaFingerprint.GenerateFingerprint(basis)); + } +} diff --git a/src/NexusMods.DataModel.SchemaVersions/NexusMods.DataModel.SchemaVersions.csproj b/src/NexusMods.DataModel.SchemaVersions/NexusMods.DataModel.SchemaVersions.csproj new file mode 100644 index 000000000..d75810675 --- /dev/null +++ b/src/NexusMods.DataModel.SchemaVersions/NexusMods.DataModel.SchemaVersions.csproj @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/src/NexusMods.DataModel.SchemaVersions/SchemaFingerprint.cs b/src/NexusMods.DataModel.SchemaVersions/SchemaFingerprint.cs new file mode 100644 index 000000000..189cc9a05 --- /dev/null +++ b/src/NexusMods.DataModel.SchemaVersions/SchemaFingerprint.cs @@ -0,0 +1,40 @@ +using System.Text; +using NexusMods.Hashing.xxHash3; +using NexusMods.MnemonicDB.Abstractions; + +namespace NexusMods.DataModel.SchemaVersions; + +/// +/// Tools for generating a hash of all the attributes of a schema so that we can detect changes. +/// +public class SchemaFingerprint +{ + public static Hash GenerateFingerprint(IDb db) + { + StringBuilder sb = new(); + var cache = db.AttributeCache; + + + void AppendLine(string s) + { + // We want platform independent newlines. + sb.Append(s); + sb.Append("\n"); + } + + foreach (var id in cache.AllAttributeIds.OrderBy(id => id.Id, StringComparer.Ordinal)) + { + var aid = cache.GetAttributeId(id); + AppendLine(id.ToString()); + AppendLine(cache.GetValueTag(aid).ToString()); + AppendLine(cache.IsIndexed(aid).ToString()); + AppendLine(cache.IsCardinalityMany(aid).ToString()); + AppendLine(cache.IsNoHistory(aid).ToString()); + AppendLine("--"); + } + // Use ascii as the attribute names must be ascii and this makes data comparisons simpler. + var bytes = Encoding.ASCII.GetBytes(sb.ToString()); + return bytes.xxHash3(); + } + +} diff --git a/src/NexusMods.DataModel.SchemaVersions/SchemaVersion.cs b/src/NexusMods.DataModel.SchemaVersions/SchemaVersion.cs new file mode 100644 index 000000000..4855a85b7 --- /dev/null +++ b/src/NexusMods.DataModel.SchemaVersions/SchemaVersion.cs @@ -0,0 +1,15 @@ +using NexusMods.Abstractions.MnemonicDB.Attributes; +using NexusMods.MnemonicDB.Abstractions.Models; + +namespace NexusMods.DataModel.Migrations; + +public partial class SchemaVersion : IModelDefinition +{ + public const string Namespace = "NexusMods.DataModel.SchemaVersioning.SchemaVersionModel"; + + /// + /// The current fingerprint of the database. This is used to detect when schema updates do not need to be performend, + /// and the app can start without the rather expensive upgrade process. + /// + public static readonly HashAttribute Fingerprint = new(Namespace, "Fingerprint"); +} diff --git a/src/NexusMods.DataModel.SchemaVersions/Services.cs b/src/NexusMods.DataModel.SchemaVersions/Services.cs new file mode 100644 index 000000000..e4874c27d --- /dev/null +++ b/src/NexusMods.DataModel.SchemaVersions/Services.cs @@ -0,0 +1,17 @@ +using Microsoft.Extensions.DependencyInjection; +using NexusMods.DataModel.Migrations; +using NexusMods.DataModel.Migrations.Migrations; + +namespace NexusMods.DataModel.SchemaVersions; + +public static class Services +{ + public static IServiceCollection AddMigrations(this IServiceCollection services) + { + services.AddSchemaVersionModel(); + services.AddSingleton(); + services.AddSingleton(); + return services; + } + +} diff --git a/src/NexusMods.DataModel/NexusMods.DataModel.csproj b/src/NexusMods.DataModel/NexusMods.DataModel.csproj index 7e3ae1edf..10a6f97c0 100644 --- a/src/NexusMods.DataModel/NexusMods.DataModel.csproj +++ b/src/NexusMods.DataModel/NexusMods.DataModel.csproj @@ -17,6 +17,7 @@ + diff --git a/src/NexusMods.DataModel/Services.cs b/src/NexusMods.DataModel/Services.cs index 40f6ca684..aa93a1a23 100644 --- a/src/NexusMods.DataModel/Services.cs +++ b/src/NexusMods.DataModel/Services.cs @@ -17,6 +17,8 @@ using NexusMods.DataModel.CommandLine.Verbs; using NexusMods.DataModel.Diagnostics; using NexusMods.DataModel.JsonConverters; +using NexusMods.DataModel.Migrations; +using NexusMods.DataModel.SchemaVersions; using NexusMods.DataModel.Settings; using NexusMods.DataModel.Sorting; using NexusMods.DataModel.Synchronizer; @@ -39,6 +41,7 @@ public static IServiceCollection AddDataModel(this IServiceCollection coll) { coll.AddMnemonicDB(); coll.AddAnalyzers(); + coll.AddMigrations(); // Settings coll.AddSettings(); diff --git a/tests/NexusMods.DataModel.SchemaVersions.Tests/LegacyDatabaseSupportTests.TestDatabase_name=SDV.4_11_2024.rocksdb.zip.verified.txt b/tests/NexusMods.DataModel.SchemaVersions.Tests/LegacyDatabaseSupportTests.TestDatabase_name=SDV.4_11_2024.rocksdb.zip.verified.txt new file mode 100644 index 000000000..93b9162da --- /dev/null +++ b/tests/NexusMods.DataModel.SchemaVersions.Tests/LegacyDatabaseSupportTests.TestDatabase_name=SDV.4_11_2024.rocksdb.zip.verified.txt @@ -0,0 +1,10 @@ +{ + Name: SDV.4_11_2024.rocksdb.zip, + OldFingerprint: None, + NewFingerprint: 0x1DDE8ED4581368B7, + Loadouts: 1, + LoadoutItemGroups: 201, + Files: 16729, + Collections: 2, + Created: 2024-11-04 17:45:27 +} \ No newline at end of file diff --git a/tests/NexusMods.DataModel.SchemaVersions.Tests/LegacyDatabaseSupportTests.cs b/tests/NexusMods.DataModel.SchemaVersions.Tests/LegacyDatabaseSupportTests.cs new file mode 100644 index 000000000..d44776ec0 --- /dev/null +++ b/tests/NexusMods.DataModel.SchemaVersions.Tests/LegacyDatabaseSupportTests.cs @@ -0,0 +1,110 @@ +using FluentAssertions; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using NexusMods.Abstractions.FileExtractor; +using NexusMods.Abstractions.Loadouts; +using NexusMods.Hashing.xxHash3; +using NexusMods.MnemonicDB; +using NexusMods.MnemonicDB.Abstractions; +using NexusMods.MnemonicDB.Abstractions.ElementComparers; +using NexusMods.MnemonicDB.Storage; +using NexusMods.MnemonicDB.Storage.RocksDbBackend; +using NexusMods.Paths; +using Xunit; + +namespace NexusMods.DataModel.Migrations.Tests; + +public class LegacyDatabaseSupportTests(IServiceProvider provider, TemporaryFileManager tempManager, IFileExtractor extractor) +{ + [Theory] + [MemberData(nameof(DatabaseNames))] + public async Task TestDatabase(string name) + { + var path = DatabaseFolder().Combine(name); + path.FileExists.Should().BeTrue(); + + await using var workingFolder = tempManager.CreateFolder(); + await extractor.ExtractAllAsync(path, workingFolder.Path); + + using var backend = new Backend(); + var settings = new DatomStoreSettings + { + Path = workingFolder.Path.Combine("MnemonicDB.rocksdb"), + }; + using var datomStore = new DatomStore(provider.GetRequiredService>(), settings, backend); + var connection = new Connection(provider.GetRequiredService>(), datomStore, provider, provider.GetServices()); + + var oldFingerprint = RecordedFingerprint(connection.Db); + + var migrationService = new MigrationService(provider.GetRequiredService>(), connection, provider.GetServices()); + await migrationService.Run(); + + await Verify(GetStatistics(connection.Db, name, oldFingerprint)).UseParameters(name); + } + + private Hash? RecordedFingerprint(IDb db) + { + var cache = db.AttributeCache; + if (!cache.Has(SchemaVersion.Fingerprint.Id)) + return null; + + var fingerprints = db.Datoms(SchemaVersion.Fingerprint); + if (fingerprints.Count == 0) + return null; + return Hash.From(ValueTag.UInt64.Read(fingerprints.First().ValueSpan)); + } + + private Statistics GetStatistics(IDb db, string name, Hash? oldFingerprint) + { + var timestampAttr = MnemonicDB.Abstractions.BuiltInEntities.Transaction.Timestamp; + + var timestamp = (DateTimeOffset)db.Get(PartitionId.Transactions.MakeEntityId(1)).Resolved(db.Connection).First(t => t.A == timestampAttr).ObjectValue; + + return new Statistics + { + Name = name, + OldFingerprint = oldFingerprint?.ToString() ?? "None", + NewFingerprint = RecordedFingerprint(db)?.ToString() ?? "None", + Loadouts = Loadout.All(db).Count, + LoadoutItemGroups = LoadoutItemGroup.All(db).Count, + Files = LoadoutItemWithTargetPath.All(db).Count, + Collections = CollectionGroup.All(db).Count, + Created = timestamp.ToString("yyyy-MM-dd HH:mm:ss") + }; + } + + /// + /// Statistics about the data in a database + /// + record Statistics + { + public string Name { get; init; } + + public string OldFingerprint { get; init; } + + public string NewFingerprint { get; init; } + + public int Loadouts { get; init; } + public int LoadoutItemGroups { get; init; } + public int Files { get; init; } + public int Collections { get; init; } + public string Created { get; init; } + } + + + public static IEnumerable DatabaseNames() + { + var databaseFolder = DatabaseFolder(); + foreach (var file in databaseFolder.EnumerateFiles("*.zip").Order()) + { + yield return [file.Name]; + } + } + + private static AbsolutePath DatabaseFolder() + { + var basePath = FileSystem.Shared.GetKnownPath(KnownPath.EntryDirectory).Parent.Parent.Parent; + var databaseFolder = basePath.Combine("Resources/Databases"); + return databaseFolder; + } +} diff --git a/tests/NexusMods.DataModel.SchemaVersions.Tests/NexusMods.DataModel.SchemaVersions.Tests.csproj b/tests/NexusMods.DataModel.SchemaVersions.Tests/NexusMods.DataModel.SchemaVersions.Tests.csproj new file mode 100644 index 000000000..58ac6da60 --- /dev/null +++ b/tests/NexusMods.DataModel.SchemaVersions.Tests/NexusMods.DataModel.SchemaVersions.Tests.csproj @@ -0,0 +1,15 @@ + + + NexusMods.DataModel.SchemaVersions.Tests + + + + + + + + + + + + diff --git a/tests/NexusMods.DataModel.SchemaVersions.Tests/Resources/Databases/Descriptions.md b/tests/NexusMods.DataModel.SchemaVersions.Tests/Resources/Databases/Descriptions.md new file mode 100644 index 000000000..3416017f2 --- /dev/null +++ b/tests/NexusMods.DataModel.SchemaVersions.Tests/Resources/Databases/Descriptions.md @@ -0,0 +1,19 @@ +## Database Descriptions +This is a document that provides information about each of the database snapshots +in this test project. Please keep this stable sorted by date. + +## Adding new test data +To add new data, set up the app in the state you want to capture. Next log out of +the app. This is critical as it will clear out your personal data from the database (via excise). + +Next zip up the `MnemonicDB.rocksdb` folder from the app folder, and place it in the `Resources/Databases` folder with a name +that includes the date of the snapshot. Note, the tests expect there to be a `MnemonicDB.rocksdb` folder in the zip file at the root. +So don't zip up the contents of the folder, zip up the folder itself. + +Finally, update the `Descriptions.md` file with the new data. + +## Descriptions + +| Date | Database Name | Description | +|------------------|----------------------------|----------------------------------------------------------------------------------------------| +| 2024-11-04 | `SDV.4_11_2024.rocksdb.zip` | A snapshot of SDV managed and with the "Stardew Valley VERY Expanded" collection installed. | diff --git a/tests/NexusMods.DataModel.SchemaVersions.Tests/Resources/Databases/SDV.4_11_2024.rocksdb.zip b/tests/NexusMods.DataModel.SchemaVersions.Tests/Resources/Databases/SDV.4_11_2024.rocksdb.zip new file mode 100644 index 000000000..e8f28d746 --- /dev/null +++ b/tests/NexusMods.DataModel.SchemaVersions.Tests/Resources/Databases/SDV.4_11_2024.rocksdb.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8dd7635d4d31d8c923ade0d94cb9f4baeca01512f4bf3a03b33231b466b6b8be +size 8235851 diff --git a/tests/NexusMods.DataModel.SchemaVersions.Tests/Schema.verified.md b/tests/NexusMods.DataModel.SchemaVersions.Tests/Schema.verified.md new file mode 100644 index 000000000..79c23c9e7 --- /dev/null +++ b/tests/NexusMods.DataModel.SchemaVersions.Tests/Schema.verified.md @@ -0,0 +1,140 @@ +## NexusMods app schema +This schema is written to a markdown file for both documentation and validation reasons. DO NOT EDIT THIS FILE MANUALLY. Instead change the +models in the app, then validate the tests to update this file. + +## Statistics + - Fingerprint: 0x1DDE8ED4581368B7 + - Total attributes: 128 + - Total namespaces: 51 + +## Attributes +| AttributeId | Type | Indexed | Many | NoHistory | +| ------------------------------------------------------------------------------------------------ | ----------------------- | ------- | ----- | --------- | +| NexusMods.Abstractions.Collections.NexusCollectionLoadoutItem/LibraryFile | Reference | False | False | False | +| NexusMods.Abstractions.Collections/LogicalFileName | Utf8 | False | False | False | +| NexusMods.Abstractions.Collections/Md5 | UInt128 | True | False | False | +| NexusMods.Abstractions.Loadouts.CollectionGroup/IsReadOnly | UInt8 | True | False | False | +| NexusMods.Abstractions.Loadouts.GameMetadata/GameId | UInt32 | False | False | False | +| NexusMods.Abstractions.Loadouts.GameMetadata/InitialDiskStateTransaction | Reference | False | False | False | +| NexusMods.Abstractions.Loadouts.GameMetadata/LastScannedDiskStateTransaction | Reference | False | False | False | +| NexusMods.Abstractions.Loadouts.GameMetadata/LastSyncedLoadout | Reference | False | False | False | +| NexusMods.Abstractions.Loadouts.GameMetadata/LastSyncedLoadoutTransaction | Reference | False | False | False | +| NexusMods.Abstractions.Loadouts.GameMetadata/Name | Utf8 | False | False | False | +| NexusMods.Abstractions.Loadouts.GameMetadata/Path | Utf8 | True | False | False | +| NexusMods.Abstractions.Loadouts.GameMetadata/Store | Ascii | False | False | False | +| NexusMods.Abstractions.Loadouts.Loadout/Installation | Reference | False | False | False | +| NexusMods.Abstractions.Loadouts.Loadout/LastAppliedDateTime | Int64 | False | False | False | +| NexusMods.Abstractions.Loadouts.Loadout/LoadoutKind | UInt8 | False | False | False | +| NexusMods.Abstractions.Loadouts.Loadout/Name | Utf8 | True | False | False | +| NexusMods.Abstractions.Loadouts.Loadout/Revision | UInt64 | False | False | False | +| NexusMods.Abstractions.Loadouts.Loadout/ShortName | Utf8 | True | False | False | +| NexusMods.Abstractions.NexusModsLibrary.CollectionTag/Name | Utf8 | True | False | False | +| NexusMods.Abstractions.NexusModsLibrary.CollectionTag/NexusId | UInt64 | True | False | False | +| NexusMods.Abstractions.NexusModsLibrary.Models.NexusModsCollectionLibraryFile/CollectionRevision | Reference | False | False | False | +| NexusMods.Abstractions.NexusModsLibrary.User/Avatar | Utf8 | False | False | False | +| NexusMods.Abstractions.NexusModsLibrary.User/AvatarImage | HashedBlob | False | False | False | +| NexusMods.Abstractions.NexusModsLibrary.User/Name | Utf8 | True | False | False | +| NexusMods.Abstractions.NexusModsLibrary.User/NexusId | UInt64 | True | False | False | +| NexusMods.App.UI.Windows.WindowData/Data | Utf8 | False | False | False | +| NexusMods.Collections.NexusCollectionBundledLoadoutGroup/CollectionLibraryFile | Reference | False | False | False | +| NexusMods.DataModel.ArchiveContents.ArchivedFileContainer/Path | Utf8Insensitive | False | False | False | +| NexusMods.DataModel.ArchivedFile/Container | Reference | False | False | False | +| NexusMods.DataModel.ArchivedFile/Hash | UInt64 | True | False | False | +| NexusMods.DataModel.ArchivedFile/NxFileEntry | Blob | False | False | False | +| NexusMods.DataModel.DiskStateEntry/Game | Reference | False | False | False | +| NexusMods.DataModel.DiskStateEntry/Hash | UInt64 | False | False | False | +| NexusMods.DataModel.DiskStateEntry/LastModified | Int64 | False | False | False | +| NexusMods.DataModel.DiskStateEntry/Path | Tuple3_Ref_UShort_Utf8I | False | False | False | +| NexusMods.DataModel.DiskStateEntry/Size | UInt64 | False | False | False | +| NexusMods.DataModel.SchemaVersioning.SchemaVersionModel/Fingerprint | UInt64 | False | False | False | +| NexusMods.DataModel.Settings/Name | Utf8 | True | False | False | +| NexusMods.DataModel.Settings/Value | Utf8 | False | False | False | +| NexusMods.Games.FOMOD.EmptyDirectory/EmptyDirectory | UInt8 | False | False | False | +| NexusMods.Games.RedEngine.Cyberpunk2077.RedModInfoFile/Name | Utf8 | False | False | False | +| NexusMods.Games.RedEngine.Cyberpunk2077.RedModInfoFile/Version | Utf8 | False | False | False | +| NexusMods.Games.RedEngine.Cyberpunk2077.RedModLoadoutGroup/RedModInfoFile | Reference | False | False | False | +| NexusMods.Library.CollectionRevisionModFile/CollectionRevision | Reference | False | False | False | +| NexusMods.Library.CollectionRevisionModFile/FileId | UInt64 | True | False | False | +| NexusMods.Library.CollectionRevisionModFile/IsOptional | UInt8 | False | False | False | +| NexusMods.Library.CollectionRevisionModFile/NexusModFile | Reference | False | False | False | +| NexusMods.Library.DownloadedFile/DownloadPageUri | Utf8 | False | False | False | +| NexusMods.Library.LibraryArchive/Archive | Null | False | False | False | +| NexusMods.Library.LibraryArchiveFileEntry/Parent | Reference | False | False | False | +| NexusMods.Library.LibraryArchiveFileEntry/Path | Utf8Insensitive | False | False | False | +| NexusMods.Library.LibraryFile/FileName | Utf8Insensitive | False | False | False | +| NexusMods.Library.LibraryFile/Hash | UInt64 | True | False | False | +| NexusMods.Library.LibraryFile/Size | UInt64 | False | False | False | +| NexusMods.Library.LibraryItem/Name | Utf8 | False | False | False | +| NexusMods.Library.LocalFile/OriginalPath | Utf8 | False | False | False | +| NexusMods.Library.NexusModsCollectionMetadata/Author | Reference | False | False | False | +| NexusMods.Library.NexusModsCollectionMetadata/BackgroundImageResource | Reference | False | False | False | +| NexusMods.Library.NexusModsCollectionMetadata/BackgroundImageUri | Utf8 | False | False | False | +| NexusMods.Library.NexusModsCollectionMetadata/Endorsements | UInt64 | False | False | False | +| NexusMods.Library.NexusModsCollectionMetadata/Name | Utf8 | False | False | False | +| NexusMods.Library.NexusModsCollectionMetadata/Slug | Ascii | True | False | False | +| NexusMods.Library.NexusModsCollectionMetadata/Summary | Utf8 | False | False | False | +| NexusMods.Library.NexusModsCollectionMetadata/Tags | Reference | False | True | False | +| NexusMods.Library.NexusModsCollectionMetadata/TileImageResource | Reference | False | False | False | +| NexusMods.Library.NexusModsCollectionMetadata/TileImageUri | Utf8 | False | False | False | +| NexusMods.Library.NexusModsCollectionRevision/Collection | Reference | False | False | False | +| NexusMods.Library.NexusModsCollectionRevision/Downloads | UInt64 | False | False | False | +| NexusMods.Library.NexusModsCollectionRevision/OverallRating | Float32 | False | False | False | +| NexusMods.Library.NexusModsCollectionRevision/RevisionId | UInt64 | True | False | False | +| NexusMods.Library.NexusModsCollectionRevision/RevisionNumber | UInt64 | True | False | False | +| NexusMods.Library.NexusModsCollectionRevision/TotalRatings | UInt64 | False | False | False | +| NexusMods.Library.NexusModsCollectionRevision/TotalSize | UInt64 | False | False | False | +| NexusMods.Library.NexusModsFileMetadata/ModPage | Reference | False | False | False | +| NexusMods.Library.NexusModsFileMetadata/Name | Utf8 | False | False | False | +| NexusMods.Library.NexusModsFileMetadata/Size | UInt64 | False | False | False | +| NexusMods.Library.NexusModsFileMetadata/Uid | UInt64 | True | False | False | +| NexusMods.Library.NexusModsFileMetadata/UploadedAt | Int64 | False | False | False | +| NexusMods.Library.NexusModsFileMetadata/Version | Utf8 | False | False | False | +| NexusMods.Library.NexusModsLibraryItem/FileMetadata | Reference | False | False | False | +| NexusMods.Library.NexusModsLibraryItem/ModPageMetadata | Reference | False | False | False | +| NexusMods.Library.NexusModsModPageMetadata/FullSizedPictureUri | Utf8 | False | False | False | +| NexusMods.Library.NexusModsModPageMetadata/GameDomain | Ascii | True | False | False | +| NexusMods.Library.NexusModsModPageMetadata/Name | Utf8 | False | False | False | +| NexusMods.Library.NexusModsModPageMetadata/ThumbnailResource | Reference | False | False | False | +| NexusMods.Library.NexusModsModPageMetadata/ThumbnailUri | Utf8 | False | False | False | +| NexusMods.Library.NexusModsModPageMetadata/Uid | UInt64 | True | False | False | +| NexusMods.Library.NexusModsModPageMetadata/UpdatedAt | Int64 | False | False | False | +| NexusMods.Loadouts.DeletedFile/Reason | Utf8 | False | False | False | +| NexusMods.Loadouts.LibraryLinkedLoadoutItem/LibraryItem | Reference | True | False | False | +| NexusMods.Loadouts.LoadoutFile/Hash | UInt64 | True | False | False | +| NexusMods.Loadouts.LoadoutFile/Size | UInt64 | False | False | False | +| NexusMods.Loadouts.LoadoutGameFilesGroup/GameMetadata | Reference | False | False | False | +| NexusMods.Loadouts.LoadoutItem/Disabled | Null | False | False | False | +| NexusMods.Loadouts.LoadoutItem/Loadout | Reference | True | False | False | +| NexusMods.Loadouts.LoadoutItem/Name | Utf8 | False | False | False | +| NexusMods.Loadouts.LoadoutItem/Parent | Reference | True | False | False | +| NexusMods.Loadouts.LoadoutItemGroup/Group | Null | True | False | False | +| NexusMods.Loadouts.LoadoutItemWithTargetPath/TargetPath | Tuple3_Ref_UShort_Utf8I | True | False | False | +| NexusMods.Loadouts.LoadoutOverridesGroup/OverridesFor | Reference | False | False | False | +| NexusMods.MnemonicDB.DatomStore/Cardinality | UInt8 | False | False | False | +| NexusMods.MnemonicDB.DatomStore/Documentation | Utf8 | False | False | False | +| NexusMods.MnemonicDB.DatomStore/Indexed | Null | False | False | False | +| NexusMods.MnemonicDB.DatomStore/NoHistory | Null | False | False | False | +| NexusMods.MnemonicDB.DatomStore/Optional | Null | False | False | False | +| NexusMods.MnemonicDB.DatomStore/UniqueId | Ascii | True | False | False | +| NexusMods.MnemonicDB.DatomStore/ValueType | UInt8 | False | False | False | +| NexusMods.MnemonicDB.Transaction/ExcisedDatoms | UInt64 | False | False | False | +| NexusMods.MnemonicDB.Transaction/Timestamp | Int64 | False | False | False | +| NexusMods.MountAndBlade2Bannerlord.ModLoadoutItem/ModuleInfo | Reference | False | False | False | +| NexusMods.MountAndBlade2Bannerlord.ModuleInfoLoadoutFile/ModuleInfoFile | Null | False | False | False | +| NexusMods.Networking.NexusWebApi.Auth.ApiKey/Key | Utf8 | False | False | False | +| NexusMods.Networking.NexusWebApi.Auth.JWTToken/AccessToken | Utf8 | False | False | False | +| NexusMods.Networking.NexusWebApi.Auth.JWTToken/ExpiresAt | Int64 | False | False | False | +| NexusMods.Networking.NexusWebApi.Auth.JWTToken/RefreshToken | Utf8 | False | False | False | +| NexusMods.NexusWebApi.GameDomainToGameIdMapping/Domain | Ascii | True | False | False | +| NexusMods.NexusWebApi.GameDomainToGameIdMapping/GameId | UInt32 | True | False | False | +| NexusMods.Resources.PersistedResource/Data | Blob | False | False | False | +| NexusMods.Resources.PersistedResource/ExpiresAt | Int64 | False | False | False | +| NexusMods.Resources.PersistedResource/ResourceIdentifierHash | UInt64 | False | False | False | +| NexusMods.StandardGameLocators.ManuallyAddedGame/GameId | UInt32 | True | False | False | +| NexusMods.StandardGameLocators.ManuallyAddedGame/Path | Utf8 | True | False | False | +| NexusMods.StandardGameLocators.ManuallyAddedGame/Version | Utf8 | False | False | False | +| NexusMods.StardewValley.SMAPILoadoutItem/ModDatabase | Reference | False | False | False | +| NexusMods.StardewValley.SMAPILoadoutItem/Version | Utf8 | False | False | False | +| NexusMods.StardewValley.SMAPIManifestLoadoutFile/ManifestFile | Null | False | False | False | +| NexusMods.StardewValley.SMAPIModDatabaseLoadoutFile/ModDatabaseFile | Null | False | False | False | +| NexusMods.StardewValley.SMAPIModLoadoutItem/Manifest | Reference | False | False | False | diff --git a/tests/NexusMods.DataModel.SchemaVersions.Tests/SchemaFailsafes.cs b/tests/NexusMods.DataModel.SchemaVersions.Tests/SchemaFailsafes.cs new file mode 100644 index 000000000..465f96060 --- /dev/null +++ b/tests/NexusMods.DataModel.SchemaVersions.Tests/SchemaFailsafes.cs @@ -0,0 +1,87 @@ +using System.Text; +using FluentAssertions; +using NexusMods.DataModel.SchemaVersions; +using NexusMods.Hashing.xxHash3; +using NexusMods.MnemonicDB.Abstractions; + +namespace NexusMods.DataModel.Migrations.Tests; + +public class SchemaFailsafes(IConnection connection) +{ + + [Fact] + public async Task SchemaFingerprintHasntChanged() + { + var db = connection.Db; + var records = db.AttributeCache.AllAttributeIds + .OrderBy(id => id.Id, StringComparer.Ordinal) + .Select(id => + { + var aid = db.AttributeCache.GetAttributeId(id); + return new string[] + { + id.ToString(), + db.AttributeCache.GetValueTag(aid).ToString(), + db.AttributeCache.IsIndexed(aid).ToString(), + db.AttributeCache.IsCardinalityMany(aid).ToString(), + db.AttributeCache.IsNoHistory(aid).ToString(), + }; + } + ).ToArray(); + + var prefix = $""" + ## NexusMods app schema + This schema is written to a markdown file for both documentation and validation reasons. DO NOT EDIT THIS FILE MANUALLY. Instead change the + models in the app, then validate the tests to update this file. + + ## Statistics + - Fingerprint: {SchemaFingerprint.GenerateFingerprint(db)} + - Total attributes: {records.Length} + - Total namespaces: {db.AttributeCache.AllAttributeIds.Select(id => id.Namespace).Distinct().Count()} + + ## Attributes + """; + + await VerifyTable(prefix, records, "AttributeId", "Type", + "Indexed", "Many", "NoHistory" + ).UseFileName("Schema"); + } + + private SettingsTask VerifyTable(string prefix, string[][] rows, params string[] columnNames) + { + var rowsMaxSize = columnNames + .Select((name, index) => Math.Max(rows.Max(row => row[index].Length), name.Length)) + .ToArray(); + + var sb = new StringBuilder(); + sb.AppendLine(prefix); + sb.Append("| "); + for (var i = 0; i < columnNames.Length; i++) + { + sb.Append(columnNames[i].PadRight(rowsMaxSize[i])); + sb.Append(" | "); + } + sb.AppendLine(); + sb.Append("| "); + for (var i = 0; i < columnNames.Length; i++) + { + sb.Append(new string('-', rowsMaxSize[i])); + sb.Append(" | "); + } + sb.AppendLine(); + + foreach (var row in rows) + { + sb.Append("| "); + for (var i = 0; i < row.Length; i++) + { + sb.Append(row[i].PadRight(rowsMaxSize[i])); + sb.Append(" | "); + } + sb.AppendLine(); + } + + return Verify(sb.ToString(), extension: "md"); + } + +} diff --git a/tests/NexusMods.DataModel.SchemaVersions.Tests/Startup.cs b/tests/NexusMods.DataModel.SchemaVersions.Tests/Startup.cs new file mode 100644 index 000000000..e6f8251b5 --- /dev/null +++ b/tests/NexusMods.DataModel.SchemaVersions.Tests/Startup.cs @@ -0,0 +1,38 @@ +using Microsoft.Extensions.DependencyInjection; +using NexusMods.Abstractions.Settings; +using NexusMods.App; +using NexusMods.Paths; + +namespace NexusMods.DataModel.Migrations.Tests; + +public class Startup +{ + /// + /// Why are Cyberpunk tests in a generic DataModel project, well it's, so we can test something that's close to real-world data. + /// + /// + public void ConfigureServices(IServiceCollection container) + { + var mode = new StartupMode { RunAsMain = true }; + + const KnownPath baseKnownPath = KnownPath.EntryDirectory; + var baseDirectory = $"NexusMods.UI.Tests.Tests-{Guid.NewGuid()}"; + + var path = FileSystem.Shared.GetKnownPath(KnownPath.EntryDirectory).Combine("temp").Combine(Guid.NewGuid().ToString()); + path.CreateDirectory(); + + container + .AddApp(startupMode: mode) + .OverrideSettingsForTests(settings => settings with + { + UseInMemoryDataModel = true, + MnemonicDBPath = new ConfigurablePath(baseKnownPath, $"{baseDirectory}/MnemonicDB.rocksdb"), + ArchiveLocations = + [ + new ConfigurablePath(baseKnownPath, $"{baseDirectory}/Archives"), + ], + } + ); + } +} +