-
Notifications
You must be signed in to change notification settings - Fork 47
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* Tests and start of the migration failsafes * Add db snapshot via lfs * Added the ability to load old rocksdb snapshots * add rocksdb.zip files to LFS * Provide some documentation and tests * Update src/NexusMods.DataModel.SchemaVersions/SchemaFingerprint.cs Co-authored-by: erri120 <[email protected]> * Update src/NexusMods.DataModel.SchemaVersions/NexusMods.DataModel.SchemaVersions.csproj Co-authored-by: erri120 <[email protected]> * Update src/NexusMods.DataModel.SchemaVersions/Migrations/UpsertFingerprint.cs Co-authored-by: erri120 <[email protected]> * Update tests/NexusMods.DataModel.SchemaVersions.Tests/NexusMods.DataModel.SchemaVersions.Tests.csproj Co-authored-by: erri120 <[email protected]> * Update src/NexusMods.DataModel.SchemaVersions/Migrations/UpsertFingerprint.cs Co-authored-by: erri120 <[email protected]> * Update tests/NexusMods.DataModel.SchemaVersions.Tests/NexusMods.DataModel.SchemaVersions.Tests.csproj Co-authored-by: erri120 <[email protected]> * Fix tests * Fix extra dependency I didn't intend to commit * Try and fix the tests on Mac * Missed two files somehow * Platform independent newlines * Switch to standardized newlines and ascii for fingerprints. They're already ASCII this just removes another layer of complexity * Update to build version that uses lfs * Fix how we store the `Created` date so it works with MacOS --------- Co-authored-by: erri120 <[email protected]>
- Loading branch information
Showing
23 changed files
with
662 additions
and
16 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,65 +1,53 @@ | ||
| # Auto detect text files and perform LF normalization | ||
| * text=auto | ||
|
|
||
| # Documents | ||
| *.md text diff=markdown | ||
|
|
||
| # Graphics | ||
| *.png binary | ||
| *.jpg binary | ||
| *.jpeg binary | ||
| *.gif binary | ||
| *.ico binary | ||
| *.svg binary | ||
|
|
||
| # Scripts (Unix) | ||
| *.bash text eol=lf | ||
| *.sh text eol=lf | ||
| *.zsh text eol=lf | ||
|
|
||
| # Scripts (Windows) | ||
| *.bat text eol=crlf | ||
| *.cmd text eol=crlf | ||
| *.ps1 text eol=crlf | ||
|
|
||
| # Archives | ||
| *.7z binary | ||
| *.gz binary | ||
| *.tar binary | ||
| *.tgz binary | ||
| *.zip binary | ||
|
|
||
| # Code files | ||
| *.cs text diff=csharp | ||
|
|
||
| # Project files | ||
| *.sln text eol=crlf | ||
| *.csproj text eol=crlf | ||
|
|
||
| *.targets text eol=crlf | ||
| *.filters text eol=crlf | ||
| *.filters text eol=crlf | ||
| *.vcxitems text eol=crlf | ||
|
|
||
| # Dynamic libraries | ||
| *.so binary | ||
| *.dylib binary | ||
| *.dll binary | ||
|
|
||
| # Executables | ||
| *.exe binary | ||
| *.out binary | ||
| *.app binary | ||
|
|
||
| # Text files where line endings should be preserved | ||
| *.patch -text | ||
|
|
||
| # Exclude files from exporting | ||
| .gitattributes export-ignore | ||
| .gitignore export-ignore | ||
| .gitkeep export-ignore | ||
|
|
||
| # Verify | ||
| *.verified.txt text eol=lf working-tree-encoding=UTF-8 | ||
| *.verified.xml text eol=lf working-tree-encoding=UTF-8 | ||
| *.verified.json text eol=lf working-tree-encoding=UTF-8 | ||
| *.rocksdb.zip filter=lfs diff=lfs merge=lfs -text |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,36 @@ | ||
| using NexusMods.MnemonicDB.Abstractions; | ||
|
|
||
| namespace NexusMods.DataModel.Migrations; | ||
|
|
||
| /// <summary> | ||
| /// A definition of a single data migration | ||
| /// </summary> | ||
| public interface IMigration | ||
| { | ||
| /// <summary> | ||
| /// The name of the migration | ||
| /// </summary> | ||
| public string Name { get; } | ||
|
|
||
| /// <summary> | ||
| /// A long description of the migration | ||
| /// </summary> | ||
| public string Description { get; } | ||
|
|
||
| /// <summary> | ||
| /// A date for the migration's creation. Not used for anything other than sorting. Migrations | ||
| /// will be run in order of this date. | ||
| /// </summary> | ||
| public DateTimeOffset CreatedAt { get; } | ||
|
|
||
| /// <summary> | ||
| /// Returns true if the migration should run. This function should do any sort of querying and processing to make sure | ||
| /// data is in the format expected by the migration. | ||
| /// </summary> | ||
| public bool ShouldRun(IDb db); | ||
|
|
||
| /// <summary> | ||
| /// Runs the migration | ||
| /// </summary> | ||
| public void Migrate(IDb basis, ITransaction tx); | ||
| } |
44 changes: 44 additions & 0 deletions
44
src/NexusMods.DataModel.SchemaVersions/MigrationService.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,44 @@ | ||
| using Microsoft.Extensions.Logging; | ||
| using NexusMods.MnemonicDB.Abstractions; | ||
|
|
||
| namespace NexusMods.DataModel.Migrations; | ||
|
|
||
| /// <summary> | ||
| /// Updates the state of a database and provides hooks for migrating schemas | ||
| /// and transforming data between versions. | ||
| /// </summary> | ||
| public class MigrationService | ||
| { | ||
| private readonly ILogger<MigrationService> _logger; | ||
| private readonly IConnection _connection; | ||
| private readonly List<IMigration> _migrations; | ||
|
|
||
| public MigrationService(ILogger<MigrationService> logger, IConnection connection, IEnumerable<IMigration> migrations) | ||
| { | ||
| _logger = logger; | ||
| _connection = connection; | ||
| _migrations = migrations.OrderBy(m => m.CreatedAt).ToList(); | ||
| } | ||
|
|
||
| public async Task Run() | ||
| { | ||
| // Run all migrations, for now this interface works by handing a transaction to each migration, in the future we'll need | ||
| // to add support for changing history of the datoms and not just the most recent state. But until we need such a migration | ||
| // we'll go with this approach as it's simpler. | ||
| foreach (var migration in _migrations) | ||
| { | ||
| var db = _connection.Db; | ||
| if (!migration.ShouldRun(db)) | ||
| { | ||
| _logger.LogInformation("Migration {Name} skipped", migration.Name); | ||
| continue; | ||
| } | ||
|
|
||
| _logger.LogInformation("Running migration {Name}", migration.Name); | ||
| using var tx = _connection.BeginTransaction(); | ||
| migration.Migrate(db, tx); | ||
| await tx.Commit(); | ||
| _logger.LogInformation("Migration {Name} completed", migration.Name); | ||
| } | ||
| } | ||
| } |
42 changes: 42 additions & 0 deletions
42
src/NexusMods.DataModel.SchemaVersions/Migrations/UpsertFingerprint.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| using NexusMods.DataModel.SchemaVersions; | ||
| using NexusMods.Hashing.xxHash3; | ||
| using NexusMods.MnemonicDB.Abstractions; | ||
| using NexusMods.MnemonicDB.Abstractions.ElementComparers; | ||
| using NexusMods.MnemonicDB.Abstractions.ValueSerializers; | ||
|
|
||
| namespace NexusMods.DataModel.Migrations.Migrations; | ||
|
|
||
| public class UpsertFingerprint : IMigration | ||
| { | ||
| public string Name => "Upsert Fingerprint"; | ||
| public string Description => "Upserts the fingerprint of the database, creating it if it does not exist."; | ||
|
|
||
| /// <summary> | ||
| /// Max value so it always runs last | ||
| /// </summary> | ||
| public DateTimeOffset CreatedAt => DateTimeOffset.MaxValue; | ||
|
|
||
| public bool ShouldRun(IDb db) | ||
| { | ||
| if (!db.AttributeCache.Has(SchemaVersion.Fingerprint.Id)) | ||
| return true; | ||
|
|
||
| var fingerprints = db.Datoms(SchemaVersion.Fingerprint); | ||
| // No fingerprint, we need to create it | ||
| if (fingerprints.Count == 0) | ||
| return true; | ||
|
|
||
| var currentFingerprint = SchemaFingerprint.GenerateFingerprint(db); | ||
| var dbFingerprint = Hash.From(UInt64Serializer.Read(fingerprints[0].ValueSpan)); | ||
| // Is the fingerprint up to date? | ||
| return currentFingerprint != dbFingerprint; | ||
| } | ||
|
|
||
| public void Migrate(IDb basis, ITransaction tx) | ||
| { | ||
| var eid = basis.Datoms(SchemaVersion.Fingerprint).Select(d => d.E) | ||
| .FirstOrDefault(tx.TempId()); | ||
|
|
||
| tx.Add(eid, SchemaVersion.Fingerprint, SchemaFingerprint.GenerateFingerprint(basis)); | ||
| } | ||
| } |
16 changes: 16 additions & 0 deletions
16
src/NexusMods.DataModel.SchemaVersions/NexusMods.DataModel.SchemaVersions.csproj
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| <Project Sdk="Microsoft.NET.Sdk"> | ||
| <!-- NuGet Package Shared Details --> | ||
| <Import Project="$([MSBuild]::GetPathOfFileAbove('NuGet.Build.props', '$(MSBuildThisFileDirectory)../'))" /> | ||
|
|
||
| <ItemGroup> | ||
| <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" /> | ||
| <PackageReference Include="NexusMods.Hashing.xxHash3" /> | ||
| <PackageReference Include="NexusMods.MnemonicDB.Abstractions" /> | ||
| <PackageReference Include="NexusMods.MnemonicDB.SourceGenerator" PrivateAssets="all" OutputItemType="Analyzer" ReferenceOutputAssembly="false" /> | ||
| </ItemGroup> | ||
|
|
||
| <ItemGroup> | ||
| <ProjectReference Include="..\Abstractions\NexusMods.Abstractions.MnemonicDB.Attributes\NexusMods.Abstractions.MnemonicDB.Attributes.csproj" /> | ||
| </ItemGroup> | ||
|
|
||
| </Project> |
40 changes: 40 additions & 0 deletions
40
src/NexusMods.DataModel.SchemaVersions/SchemaFingerprint.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| using System.Text; | ||
| using NexusMods.Hashing.xxHash3; | ||
| using NexusMods.MnemonicDB.Abstractions; | ||
|
|
||
| namespace NexusMods.DataModel.SchemaVersions; | ||
|
|
||
| /// <summary> | ||
| /// Tools for generating a hash of all the attributes of a schema so that we can detect changes. | ||
| /// </summary> | ||
| public class SchemaFingerprint | ||
| { | ||
| public static Hash GenerateFingerprint(IDb db) | ||
| { | ||
| StringBuilder sb = new(); | ||
| var cache = db.AttributeCache; | ||
|
|
||
|
|
||
| void AppendLine(string s) | ||
| { | ||
| // We want platform independent newlines. | ||
| sb.Append(s); | ||
| sb.Append("\n"); | ||
| } | ||
|
|
||
| foreach (var id in cache.AllAttributeIds.OrderBy(id => id.Id, StringComparer.Ordinal)) | ||
| { | ||
| var aid = cache.GetAttributeId(id); | ||
| AppendLine(id.ToString()); | ||
| AppendLine(cache.GetValueTag(aid).ToString()); | ||
| AppendLine(cache.IsIndexed(aid).ToString()); | ||
| AppendLine(cache.IsCardinalityMany(aid).ToString()); | ||
| AppendLine(cache.IsNoHistory(aid).ToString()); | ||
| AppendLine("--"); | ||
| } | ||
| // Use ascii as the attribute names must be ascii and this makes data comparisons simpler. | ||
| var bytes = Encoding.ASCII.GetBytes(sb.ToString()); | ||
| return bytes.xxHash3(); | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| using NexusMods.Abstractions.MnemonicDB.Attributes; | ||
| using NexusMods.MnemonicDB.Abstractions.Models; | ||
|
|
||
| namespace NexusMods.DataModel.Migrations; | ||
|
|
||
| public partial class SchemaVersion : IModelDefinition | ||
| { | ||
| public const string Namespace = "NexusMods.DataModel.SchemaVersioning.SchemaVersionModel"; | ||
|
|
||
| /// <summary> | ||
| /// The current fingerprint of the database. This is used to detect when schema updates do not need to be performend, | ||
| /// and the app can start without the rather expensive upgrade process. | ||
| /// </summary> | ||
| public static readonly HashAttribute Fingerprint = new(Namespace, "Fingerprint"); | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| using Microsoft.Extensions.DependencyInjection; | ||
| using NexusMods.DataModel.Migrations; | ||
| using NexusMods.DataModel.Migrations.Migrations; | ||
|
|
||
| namespace NexusMods.DataModel.SchemaVersions; | ||
|
|
||
| public static class Services | ||
| { | ||
| public static IServiceCollection AddMigrations(this IServiceCollection services) | ||
| { | ||
| services.AddSchemaVersionModel(); | ||
| services.AddSingleton<IMigration, UpsertFingerprint>(); | ||
| services.AddSingleton<MigrationService>(); | ||
| return services; | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.