From fd50b83343f226cc4d0e2aaf13ad8261755a979d Mon Sep 17 00:00:00 2001 From: James Gunn Date: Thu, 25 Jul 2024 11:42:47 +0100 Subject: [PATCH] Sync changes from TRS DB into DQT reporting database --- .github/workflows/pr.yml | 13 +- README.md | 18 +- ...ommands.DropDqtReportingReplicationSlot.cs | 42 + .../src/TeachingRecordSystem.Cli/Program.cs | 1 + ...gSyncPublicationQualifications.Designer.cs | 2381 +++++++++++++++++ ...tReportingSyncPublicationQualifications.cs | 23 + .../DistributedLockKeys.cs | 1 + .../BackfillDqtReportingQualifications.cs | 80 + .../Jobs/HostApplicationBuilderExtensions.cs | 5 + .../DqtReporting/DqtReportingOptions.cs | 4 + .../DqtReporting/DqtReportingService.cs | 208 +- .../Migrations/0026_TrsQualifications.sql | 23 + .../TeachingRecordSystem.Core.csproj | 2 + .../CrmClientFixture.cs | 11 +- .../DqtReporting/DqtReportingFixture.cs | 17 +- .../DqtReporting/DqtReportingServiceTests.cs | 158 +- .../DbFixture.cs | 9 + 17 files changed, 2957 insertions(+), 39 deletions(-) create mode 100644 TeachingRecordSystem/src/TeachingRecordSystem.Cli/Commands.DropDqtReportingReplicationSlot.cs create mode 100644 TeachingRecordSystem/src/TeachingRecordSystem.Core/DataStore/Postgres/Migrations/20240822100611_DqtReportingSyncPublicationQualifications.Designer.cs create mode 100644 TeachingRecordSystem/src/TeachingRecordSystem.Core/DataStore/Postgres/Migrations/20240822100611_DqtReportingSyncPublicationQualifications.cs create mode 100644 TeachingRecordSystem/src/TeachingRecordSystem.Core/Jobs/BackfillDqtReportingQualifications.cs create mode 100644 TeachingRecordSystem/src/TeachingRecordSystem.Core/Services/DqtReporting/Migrations/0026_TrsQualifications.sql diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 00b2aa655..23478fc0e 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -20,7 +20,7 @@ jobs: services: postgres: - image: postgres + image: postgres:15 env: POSTGRES_PASSWORD: trs POSTGRES_DB: trs @@ -192,11 +192,12 @@ jobs: services: postgres: - image: postgres + image: postgres:15 env: POSTGRES_PASSWORD: trs POSTGRES_DB: trs options: >- + --name postgres --health-cmd pg_isready --health-interval 10s --health-timeout 5s @@ -218,6 +219,14 @@ jobs: --health-retries 3 steps: + - name: Set postgres wal_level to logical + run: | + docker exec -i postgres bash << EOF + echo "wal_level = logical" >> /var/lib/postgresql/data/postgresql.conf + EOF + + docker restart --time 0 postgres + - uses: actions/checkout@v4 - uses: extractions/setup-just@v2 diff --git a/README.md b/README.md index 8e1dec2bc..e45353bf9 100644 --- a/README.md +++ b/README.md @@ -103,18 +103,32 @@ The databases will be created automatically when running the tests. #### DQT Reporting database setup -This solution contains a service that synchronises changes from CRM into a SQL Server database used for reporting (this replaces the now-deprecated Data Export Service). By default this is disabled for local development. For the tests to pass, you will need a test database and a connection string defined in user secrets e.g. +This solution contains a service that synchronises changes from CRM into a SQL Server database used for reporting (this replaces the now-deprecated Data Export Service). +It also synronises selected tables from TRS. +By default this is disabled for local development. For the tests to pass, you will need a test database and a connection string defined in user secrets e.g. ```shell just set-tests-secret DqtReporting:ReportingDbConnectionString "Data Source=(local);Initial Catalog=DqtReportingTests;Integrated Security=Yes;TrustServerCertificate=True" ``` -To run the service locally, override the configuration option to run the service and ensure a connection string is provided e.g. +Your postgres server's `wal_level` must be set to `logical`: +``` +ALTER SYSTEM SET wal_level = logical; +``` +You will have to restart the server after amending this configuration. + +To run the service locally override the configuration option to run the service and ensure a connection string is provided e.g. ```shell just set-secret DqtReporting:RunService true just set-secret DqtReporting:ReportingDbConnectionString "Data Source=(local);Initial Catalog=DqtReporting;Integrated Security=Yes;TrustServerCertificate=True" ``` The service will now run as a background service of the `Worker` project. +It is a good idea to remove the replication slot when you're not working on this service to avoid a backlog on unprocessed changes accumulating in postgres. +```shell +just set-secret DqtReporting:RunService false +just cli drop-dqt-reporting-replication-slot +``` + ### Admin user setup diff --git a/TeachingRecordSystem/src/TeachingRecordSystem.Cli/Commands.DropDqtReportingReplicationSlot.cs b/TeachingRecordSystem/src/TeachingRecordSystem.Cli/Commands.DropDqtReportingReplicationSlot.cs new file mode 100644 index 000000000..35de7287d --- /dev/null +++ b/TeachingRecordSystem/src/TeachingRecordSystem.Cli/Commands.DropDqtReportingReplicationSlot.cs @@ -0,0 +1,42 @@ +using Npgsql; +using TeachingRecordSystem.Core.DataStore.Postgres; +using TeachingRecordSystem.Core.Services.DqtReporting; + +namespace TeachingRecordSystem.Cli; + +public partial class Commands +{ + public static Command CreateDropDqtReportingReplicationSlotCommand(IConfiguration configuration) + { + var connectionStringOption = new Option("--connection-string") { IsRequired = true }; + + var configuredConnectionString = configuration.GetConnectionString("DefaultConnection"); + if (configuredConnectionString is not null) + { + connectionStringOption.SetDefaultValue(configuredConnectionString); + } + + var command = new Command("drop-dqt-reporting-replication-slot", "Drops the logical replication slot for the DQT Reporting Service.") + { + connectionStringOption + }; + + command.SetHandler( + async (string connectionString) => + { + using var dbContext = TrsDbContext.Create(connectionString, commandTimeout: (int)TimeSpan.FromMinutes(10).TotalSeconds); + + // Ensure the user has the replication permission + var user = new NpgsqlConnectionStringBuilder(connectionString).Username; +#pragma warning disable EF1002 // Risk of vulnerability to SQL injection. + await dbContext.Database.ExecuteSqlRawAsync($"alter user {user} with replication"); +#pragma warning restore EF1002 // Risk of vulnerability to SQL injection. + + await dbContext.Database.ExecuteSqlRawAsync( + $"select pg_drop_replication_slot('{DqtReportingOptions.DefaultTrsDbReplicationSlotName}');"); + }, + connectionStringOption); + + return command; + } +} diff --git a/TeachingRecordSystem/src/TeachingRecordSystem.Cli/Program.cs b/TeachingRecordSystem/src/TeachingRecordSystem.Cli/Program.cs index 247cc4e83..64423bdc6 100644 --- a/TeachingRecordSystem/src/TeachingRecordSystem.Cli/Program.cs +++ b/TeachingRecordSystem/src/TeachingRecordSystem.Cli/Program.cs @@ -15,6 +15,7 @@ Commands.CreateCreateAdminCommand(configuration), Commands.CreateSyncPersonCommand(configuration), Commands.CreateGenerateKeyCommand(configuration), + Commands.CreateDropDqtReportingReplicationSlotCommand(configuration), }; return await rootCommand.InvokeAsync(args); diff --git a/TeachingRecordSystem/src/TeachingRecordSystem.Core/DataStore/Postgres/Migrations/20240822100611_DqtReportingSyncPublicationQualifications.Designer.cs b/TeachingRecordSystem/src/TeachingRecordSystem.Core/DataStore/Postgres/Migrations/20240822100611_DqtReportingSyncPublicationQualifications.Designer.cs new file mode 100644 index 000000000..5752b6f80 --- /dev/null +++ b/TeachingRecordSystem/src/TeachingRecordSystem.Core/DataStore/Postgres/Migrations/20240822100611_DqtReportingSyncPublicationQualifications.Designer.cs @@ -0,0 +1,2381 @@ +// +using System; +using System.Collections.Generic; +using System.Text.Json; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using TeachingRecordSystem.Core.DataStore.Postgres; + +#nullable disable + +namespace TeachingRecordSystem.Core.DataStore.Postgres.Migrations +{ + [DbContext(typeof(TrsDbContext))] + [Migration("20240822100611_DqtReportingSyncPublicationQualifications")] + partial class DqtReportingSyncPublicationQualifications + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "8.0.7") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreApplication", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("ApplicationType") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("application_type"); + + b.Property("ClientId") + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("client_id"); + + b.Property("ClientSecret") + .HasColumnType("text") + .HasColumnName("client_secret"); + + b.Property("ClientType") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("client_type"); + + b.Property("ConcurrencyToken") + .IsConcurrencyToken() + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("concurrency_token"); + + b.Property("ConsentType") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("consent_type"); + + b.Property("DisplayName") + .HasColumnType("text") + .HasColumnName("display_name"); + + b.Property("DisplayNames") + .HasColumnType("text") + .HasColumnName("display_names"); + + b.Property("JsonWebKeySet") + .HasColumnType("text") + .HasColumnName("json_web_key_set"); + + b.Property("Permissions") + .HasColumnType("text") + .HasColumnName("permissions"); + + b.Property("PostLogoutRedirectUris") + .HasColumnType("text") + .HasColumnName("post_logout_redirect_uris"); + + b.Property("Properties") + .HasColumnType("text") + .HasColumnName("properties"); + + b.Property("RedirectUris") + .HasColumnType("text") + .HasColumnName("redirect_uris"); + + b.Property("Requirements") + .HasColumnType("text") + .HasColumnName("requirements"); + + b.Property("Settings") + .HasColumnType("text") + .HasColumnName("settings"); + + b.HasKey("Id") + .HasName("pk_oidc_applications"); + + b.HasIndex("ClientId") + .IsUnique() + .HasDatabaseName("ix_oidc_applications_client_id"); + + b.ToTable("oidc_applications", (string)null); + }); + + modelBuilder.Entity("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreAuthorization", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("ApplicationId") + .HasColumnType("uuid") + .HasColumnName("application_id"); + + b.Property("ConcurrencyToken") + .IsConcurrencyToken() + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("concurrency_token"); + + b.Property("CreationDate") + .HasColumnType("timestamp with time zone") + .HasColumnName("creation_date"); + + b.Property("Properties") + .HasColumnType("text") + .HasColumnName("properties"); + + b.Property("Scopes") + .HasColumnType("text") + .HasColumnName("scopes"); + + b.Property("Status") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("status"); + + b.Property("Subject") + .HasMaxLength(400) + .HasColumnType("character varying(400)") + .HasColumnName("subject"); + + b.Property("Type") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("type"); + + b.HasKey("Id") + .HasName("pk_oidc_authorizations"); + + b.HasIndex("ApplicationId", "Status", "Subject", "Type") + .HasDatabaseName("ix_oidc_authorizations_application_id_status_subject_type"); + + b.ToTable("oidc_authorizations", (string)null); + }); + + modelBuilder.Entity("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreScope", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("ConcurrencyToken") + .IsConcurrencyToken() + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("concurrency_token"); + + b.Property("Description") + .HasColumnType("text") + .HasColumnName("description"); + + b.Property("Descriptions") + .HasColumnType("text") + .HasColumnName("descriptions"); + + b.Property("DisplayName") + .HasColumnType("text") + .HasColumnName("display_name"); + + b.Property("DisplayNames") + .HasColumnType("text") + .HasColumnName("display_names"); + + b.Property("Name") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("name"); + + b.Property("Properties") + .HasColumnType("text") + .HasColumnName("properties"); + + b.Property("Resources") + .HasColumnType("text") + .HasColumnName("resources"); + + b.HasKey("Id") + .HasName("pk_oidc_scopes"); + + b.HasIndex("Name") + .IsUnique() + .HasDatabaseName("ix_oidc_scopes_name"); + + b.ToTable("oidc_scopes", (string)null); + }); + + modelBuilder.Entity("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreToken", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("ApplicationId") + .HasColumnType("uuid") + .HasColumnName("application_id"); + + b.Property("AuthorizationId") + .HasColumnType("uuid") + .HasColumnName("authorization_id"); + + b.Property("ConcurrencyToken") + .IsConcurrencyToken() + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("concurrency_token"); + + b.Property("CreationDate") + .HasColumnType("timestamp with time zone") + .HasColumnName("creation_date"); + + b.Property("ExpirationDate") + .HasColumnType("timestamp with time zone") + .HasColumnName("expiration_date"); + + b.Property("Payload") + .HasColumnType("text") + .HasColumnName("payload"); + + b.Property("Properties") + .HasColumnType("text") + .HasColumnName("properties"); + + b.Property("RedemptionDate") + .HasColumnType("timestamp with time zone") + .HasColumnName("redemption_date"); + + b.Property("ReferenceId") + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("reference_id"); + + b.Property("Status") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("status"); + + b.Property("Subject") + .HasMaxLength(400) + .HasColumnType("character varying(400)") + .HasColumnName("subject"); + + b.Property("Type") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("type"); + + b.HasKey("Id") + .HasName("pk_oidc_tokens"); + + b.HasIndex("ReferenceId") + .IsUnique() + .HasDatabaseName("ix_oidc_tokens_reference_id"); + + b.HasIndex("ApplicationId", "Status", "Subject", "Type") + .HasDatabaseName("ix_oidc_tokens_application_id_status_subject_type"); + + b.ToTable("oidc_tokens", (string)null); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.Alert", b => + { + b.Property("AlertId") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("alert_id"); + + b.Property("AlertTypeId") + .HasColumnType("uuid") + .HasColumnName("alert_type_id"); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("DeletedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("deleted_on"); + + b.Property("Details") + .HasColumnType("text") + .HasColumnName("details"); + + b.Property("EndDate") + .HasColumnType("date") + .HasColumnName("end_date"); + + b.Property("ExternalLink") + .HasColumnType("text") + .HasColumnName("external_link"); + + b.Property("PersonId") + .HasColumnType("uuid") + .HasColumnName("person_id"); + + b.Property("StartDate") + .HasColumnType("date") + .HasColumnName("start_date"); + + b.Property("UpdatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_on"); + + b.HasKey("AlertId") + .HasName("pk_alerts"); + + b.HasIndex("AlertTypeId") + .HasDatabaseName("ix_alerts_alert_type_id"); + + b.HasIndex("PersonId") + .HasDatabaseName("ix_alerts_person_id"); + + b.ToTable("alerts", (string)null); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.AlertCategory", b => + { + b.Property("AlertCategoryId") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("alert_category_id"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("name") + .UseCollation("case_insensitive"); + + b.HasKey("AlertCategoryId") + .HasName("pk_alert_categories"); + + b.ToTable("alert_categories", (string)null); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.AlertType", b => + { + b.Property("AlertTypeId") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("alert_type_id"); + + b.Property("AlertCategoryId") + .HasColumnType("uuid") + .HasColumnName("alert_category_id"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("name") + .UseCollation("case_insensitive"); + + b.HasKey("AlertTypeId") + .HasName("pk_alert_types"); + + b.HasIndex("AlertCategoryId") + .HasDatabaseName("ix_alert_types_alert_category_id"); + + b.ToTable("alert_types", (string)null); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.ApiKey", b => + { + b.Property("ApiKeyId") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("api_key_id"); + + b.Property("ApplicationUserId") + .HasColumnType("uuid") + .HasColumnName("application_user_id"); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("Expires") + .HasColumnType("timestamp with time zone") + .HasColumnName("expires"); + + b.Property("Key") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("key"); + + b.Property("UpdatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_on"); + + b.HasKey("ApiKeyId") + .HasName("pk_api_keys"); + + b.HasIndex("ApplicationUserId") + .HasDatabaseName("ix_api_keys_application_user_id"); + + b.HasIndex("Key") + .IsUnique() + .HasDatabaseName("ix_api_keys_key"); + + b.ToTable("api_keys", (string)null); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.EntityChangesJournal", b => + { + b.Property("Key") + .HasColumnType("text") + .HasColumnName("key"); + + b.Property("EntityLogicalName") + .HasColumnType("text") + .HasColumnName("entity_logical_name"); + + b.Property("DataToken") + .HasColumnType("text") + .HasColumnName("data_token"); + + b.Property("LastUpdated") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_updated"); + + b.Property("LastUpdatedBy") + .HasColumnType("text") + .HasColumnName("last_updated_by"); + + b.Property("NextQueryPageNumber") + .HasColumnType("integer") + .HasColumnName("next_query_page_number"); + + b.Property("NextQueryPageSize") + .HasColumnType("integer") + .HasColumnName("next_query_page_size"); + + b.Property("NextQueryPagingCookie") + .HasColumnType("text") + .HasColumnName("next_query_paging_cookie"); + + b.HasKey("Key", "EntityLogicalName") + .HasName("pk_entity_changes_journals"); + + b.ToTable("entity_changes_journals", (string)null); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.Establishment", b => + { + b.Property("EstablishmentId") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("establishment_id"); + + b.Property("Address3") + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("address3") + .UseCollation("case_insensitive"); + + b.Property("County") + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("county") + .UseCollation("case_insensitive"); + + b.Property("EstablishmentName") + .IsRequired() + .HasMaxLength(120) + .HasColumnType("character varying(120)") + .HasColumnName("establishment_name") + .UseCollation("case_insensitive"); + + b.Property("EstablishmentNumber") + .HasMaxLength(4) + .HasColumnType("character(4)") + .HasColumnName("establishment_number") + .IsFixedLength(); + + b.Property("EstablishmentSourceId") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasDefaultValue(1) + .HasColumnName("establishment_source_id"); + + b.Property("EstablishmentStatusCode") + .HasColumnType("integer") + .HasColumnName("establishment_status_code"); + + b.Property("EstablishmentStatusName") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("establishment_status_name"); + + b.Property("EstablishmentTypeCode") + .HasMaxLength(3) + .HasColumnType("character varying(3)") + .HasColumnName("establishment_type_code"); + + b.Property("EstablishmentTypeGroupCode") + .HasColumnType("integer") + .HasColumnName("establishment_type_group_code"); + + b.Property("EstablishmentTypeGroupName") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("establishment_type_group_name"); + + b.Property("EstablishmentTypeName") + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("establishment_type_name") + .UseCollation("case_insensitive"); + + b.Property("LaCode") + .IsRequired() + .HasMaxLength(3) + .HasColumnType("character(3)") + .HasColumnName("la_code") + .IsFixedLength(); + + b.Property("LaName") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("la_name") + .UseCollation("case_insensitive"); + + b.Property("Locality") + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("locality") + .UseCollation("case_insensitive"); + + b.Property("Postcode") + .HasMaxLength(10) + .HasColumnType("character varying(10)") + .HasColumnName("postcode") + .UseCollation("case_insensitive"); + + b.Property("Street") + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("street") + .UseCollation("case_insensitive"); + + b.Property("Town") + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("town") + .UseCollation("case_insensitive"); + + b.Property("Urn") + .HasMaxLength(6) + .HasColumnType("integer") + .HasColumnName("urn") + .IsFixedLength(); + + b.HasKey("EstablishmentId") + .HasName("pk_establishments"); + + b.HasIndex("EstablishmentSourceId") + .HasDatabaseName("ix_establishment_establishment_source_id"); + + b.HasIndex("Urn") + .HasDatabaseName("ix_establishment_urn"); + + b.HasIndex("LaCode", "EstablishmentNumber") + .HasDatabaseName("ix_establishment_la_code_establishment_number"); + + b.ToTable("establishments", (string)null); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.EstablishmentSource", b => + { + b.Property("EstablishmentSourceId") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("establishment_source_id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("EstablishmentSourceId")); + + b.Property("Name") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("name") + .UseCollation("case_insensitive"); + + b.HasKey("EstablishmentSourceId") + .HasName("pk_establishment_sources"); + + b.ToTable("establishment_sources", (string)null); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.Event", b => + { + b.Property("EventId") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("event_id"); + + b.Property("Created") + .HasColumnType("timestamp with time zone") + .HasColumnName("created"); + + b.Property("EventName") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("event_name"); + + b.Property("Inserted") + .HasColumnType("timestamp with time zone") + .HasColumnName("inserted"); + + b.Property("Key") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("key"); + + b.Property("Payload") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("payload"); + + b.Property("PersonId") + .HasColumnType("uuid") + .HasColumnName("person_id"); + + b.Property("Published") + .HasColumnType("boolean") + .HasColumnName("published"); + + b.HasKey("EventId") + .HasName("pk_events"); + + b.HasIndex("Key") + .IsUnique() + .HasDatabaseName("ix_events_key") + .HasFilter("key is not null"); + + b.HasIndex("Payload") + .HasDatabaseName("ix_events_payload"); + + NpgsqlIndexBuilderExtensions.HasMethod(b.HasIndex("Payload"), "gin"); + + b.HasIndex("PersonId", "EventName") + .HasDatabaseName("ix_events_person_id_event_name") + .HasFilter("person_id is not null"); + + NpgsqlIndexBuilderExtensions.IncludeProperties(b.HasIndex("PersonId", "EventName"), new[] { "Payload" }); + + b.ToTable("events", (string)null); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.EytsAwardedEmailsJob", b => + { + b.Property("EytsAwardedEmailsJobId") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("eyts_awarded_emails_job_id"); + + b.Property("AwardedToUtc") + .HasColumnType("timestamp with time zone") + .HasColumnName("awarded_to_utc"); + + b.Property("ExecutedUtc") + .HasColumnType("timestamp with time zone") + .HasColumnName("executed_utc"); + + b.HasKey("EytsAwardedEmailsJobId") + .HasName("pk_eyts_awarded_emails_jobs"); + + b.HasIndex("ExecutedUtc") + .HasDatabaseName("ix_eyts_awarded_emails_jobs_executed_utc"); + + b.ToTable("eyts_awarded_emails_jobs", (string)null); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.EytsAwardedEmailsJobItem", b => + { + b.Property("EytsAwardedEmailsJobId") + .HasColumnType("uuid") + .HasColumnName("eyts_awarded_emails_job_id"); + + b.Property("PersonId") + .HasColumnType("uuid") + .HasColumnName("person_id"); + + b.Property("EmailAddress") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("email_address"); + + b.Property("EmailSent") + .HasColumnType("boolean") + .HasColumnName("email_sent"); + + b.Property("Personalization") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("personalization"); + + b.Property("Trn") + .IsRequired() + .HasMaxLength(7) + .HasColumnType("character(7)") + .HasColumnName("trn") + .IsFixedLength(); + + b.HasKey("EytsAwardedEmailsJobId", "PersonId") + .HasName("pk_eyts_awarded_emails_job_items"); + + b.HasIndex("Personalization") + .HasDatabaseName("ix_eyts_awarded_emails_job_items_personalization"); + + NpgsqlIndexBuilderExtensions.HasMethod(b.HasIndex("Personalization"), "gin"); + + b.ToTable("eyts_awarded_emails_job_items", (string)null); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.InductionCompletedEmailsJob", b => + { + b.Property("InductionCompletedEmailsJobId") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("induction_completed_emails_job_id"); + + b.Property("AwardedToUtc") + .HasColumnType("timestamp with time zone") + .HasColumnName("awarded_to_utc"); + + b.Property("ExecutedUtc") + .HasColumnType("timestamp with time zone") + .HasColumnName("executed_utc"); + + b.HasKey("InductionCompletedEmailsJobId") + .HasName("pk_induction_completed_emails_jobs"); + + b.HasIndex("ExecutedUtc") + .HasDatabaseName("ix_induction_completed_emails_jobs_executed_utc"); + + b.ToTable("induction_completed_emails_jobs", (string)null); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.InductionCompletedEmailsJobItem", b => + { + b.Property("InductionCompletedEmailsJobId") + .HasColumnType("uuid") + .HasColumnName("induction_completed_emails_job_id"); + + b.Property("PersonId") + .HasColumnType("uuid") + .HasColumnName("person_id"); + + b.Property("EmailAddress") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("email_address"); + + b.Property("EmailSent") + .HasColumnType("boolean") + .HasColumnName("email_sent"); + + b.Property("Personalization") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("personalization"); + + b.Property("Trn") + .IsRequired() + .HasMaxLength(7) + .HasColumnType("character(7)") + .HasColumnName("trn") + .IsFixedLength(); + + b.HasKey("InductionCompletedEmailsJobId", "PersonId") + .HasName("pk_induction_completed_emails_job_items"); + + b.HasIndex("Personalization") + .HasDatabaseName("ix_induction_completed_emails_job_items_personalization"); + + NpgsqlIndexBuilderExtensions.HasMethod(b.HasIndex("Personalization"), "gin"); + + b.ToTable("induction_completed_emails_job_items", (string)null); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.InternationalQtsAwardedEmailsJob", b => + { + b.Property("InternationalQtsAwardedEmailsJobId") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("international_qts_awarded_emails_job_id"); + + b.Property("AwardedToUtc") + .HasColumnType("timestamp with time zone") + .HasColumnName("awarded_to_utc"); + + b.Property("ExecutedUtc") + .HasColumnType("timestamp with time zone") + .HasColumnName("executed_utc"); + + b.HasKey("InternationalQtsAwardedEmailsJobId") + .HasName("pk_international_qts_awarded_emails_jobs"); + + b.HasIndex("ExecutedUtc") + .HasDatabaseName("ix_international_qts_awarded_emails_jobs_executed_utc"); + + b.ToTable("international_qts_awarded_emails_jobs", (string)null); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.InternationalQtsAwardedEmailsJobItem", b => + { + b.Property("InternationalQtsAwardedEmailsJobId") + .HasColumnType("uuid") + .HasColumnName("international_qts_awarded_emails_job_id"); + + b.Property("PersonId") + .HasColumnType("uuid") + .HasColumnName("person_id"); + + b.Property("EmailAddress") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("email_address"); + + b.Property("EmailSent") + .HasColumnType("boolean") + .HasColumnName("email_sent"); + + b.Property("Personalization") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("personalization"); + + b.Property("Trn") + .IsRequired() + .HasMaxLength(7) + .HasColumnType("character(7)") + .HasColumnName("trn") + .IsFixedLength(); + + b.HasKey("InternationalQtsAwardedEmailsJobId", "PersonId") + .HasName("pk_international_qts_awarded_emails_job_items"); + + b.HasIndex("Personalization") + .HasDatabaseName("ix_international_qts_awarded_emails_job_items_personalization"); + + NpgsqlIndexBuilderExtensions.HasMethod(b.HasIndex("Personalization"), "gin"); + + b.ToTable("international_qts_awarded_emails_job_items", (string)null); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.JourneyState", b => + { + b.Property("InstanceId") + .HasMaxLength(300) + .HasColumnType("character varying(300)") + .HasColumnName("instance_id"); + + b.Property("Completed") + .HasColumnType("timestamp with time zone") + .HasColumnName("completed"); + + b.Property("Created") + .HasColumnType("timestamp with time zone") + .HasColumnName("created"); + + b.Property("State") + .IsRequired() + .HasColumnType("text") + .HasColumnName("state"); + + b.Property("Updated") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated"); + + b.Property("UserId") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("user_id"); + + b.HasKey("InstanceId") + .HasName("pk_journey_states"); + + b.ToTable("journey_states", (string)null); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.MandatoryQualificationProvider", b => + { + b.Property("MandatoryQualificationProviderId") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("mandatory_qualification_provider_id"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("name"); + + b.HasKey("MandatoryQualificationProviderId") + .HasName("pk_mandatory_qualification_providers"); + + b.ToTable("mandatory_qualification_providers", (string)null); + + b.HasData( + new + { + MandatoryQualificationProviderId = new Guid("e28ea41d-408d-4c89-90cc-8b9b04ac68f5"), + Name = "University of Birmingham" + }, + new + { + MandatoryQualificationProviderId = new Guid("89f9a1aa-3d68-4985-a4ce-403b6044c18c"), + Name = "University of Leeds" + }, + new + { + MandatoryQualificationProviderId = new Guid("aa5c300e-3b7c-456c-8183-3520b3d55dca"), + Name = "University of Manchester" + }, + new + { + MandatoryQualificationProviderId = new Guid("f417e73e-e2ad-40eb-85e3-55865be7f6be"), + Name = "Mary Hare School / University of Hertfordshire" + }, + new + { + MandatoryQualificationProviderId = new Guid("fbf22e04-b274-4c80-aba8-79fb6a7a32ce"), + Name = "University of Edinburgh" + }, + new + { + MandatoryQualificationProviderId = new Guid("26204149-349c-4ad6-9466-bb9b83723eae"), + Name = "Liverpool John Moores University" + }, + new + { + MandatoryQualificationProviderId = new Guid("0c30f666-647c-4ea8-8883-0fc6010b56be"), + Name = "University of Oxford/Oxford Polytechnic" + }, + new + { + MandatoryQualificationProviderId = new Guid("d0e6d54c-5e90-438a-945d-f97388c2b352"), + Name = "University of Cambridge" + }, + new + { + MandatoryQualificationProviderId = new Guid("aec32252-ef25-452e-a358-34a04e03369c"), + Name = "University of Newcastle-upon-Tyne" + }, + new + { + MandatoryQualificationProviderId = new Guid("d9ee7054-7fde-4cfd-9a5e-4b99511d1b3d"), + Name = "University of Plymouth" + }, + new + { + MandatoryQualificationProviderId = new Guid("707d58ca-1953-413b-9a46-41e9b0be885e"), + Name = "University of Hertfordshire" + }, + new + { + MandatoryQualificationProviderId = new Guid("3fc648a7-18e4-49e7-8a4b-1612616b72d5"), + Name = "University of London" + }, + new + { + MandatoryQualificationProviderId = new Guid("374dceb8-8224-45b8-b7dc-a6b0282b1065"), + Name = "Bristol Polytechnic" + }, + new + { + MandatoryQualificationProviderId = new Guid("d4fc958b-21de-47ec-9f03-36ae237a1b11"), + Name = "University College, Swansea" + }); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.NameSynonyms", b => + { + b.Property("NameSynonymsId") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("name_synonyms_id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("NameSynonymsId")); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("name") + .UseCollation("case_insensitive"); + + b.Property("Synonyms") + .IsRequired() + .HasColumnType("text[]") + .HasColumnName("synonyms") + .UseCollation("case_insensitive"); + + b.HasKey("NameSynonymsId") + .HasName("pk_name_synonyms"); + + b.HasIndex("Name") + .IsUnique() + .HasDatabaseName("ix_name_synonyms_name"); + + b.ToTable("name_synonyms", (string)null); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.OneLoginUser", b => + { + b.Property("Subject") + .HasMaxLength(255) + .HasColumnType("character varying(255)") + .HasColumnName("subject"); + + b.Property("Email") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("email"); + + b.Property("FirstOneLoginSignIn") + .HasColumnType("timestamp with time zone") + .HasColumnName("first_one_login_sign_in"); + + b.Property("FirstSignIn") + .HasColumnType("timestamp with time zone") + .HasColumnName("first_sign_in"); + + b.Property("LastCoreIdentityVc") + .HasColumnType("jsonb") + .HasColumnName("last_core_identity_vc"); + + b.Property("LastOneLoginSignIn") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_one_login_sign_in"); + + b.Property("LastSignIn") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_sign_in"); + + b.Property("MatchRoute") + .HasColumnType("integer") + .HasColumnName("match_route"); + + b.Property("MatchedAttributes") + .HasColumnType("jsonb") + .HasColumnName("matched_attributes"); + + b.Property("PersonId") + .HasColumnType("uuid") + .HasColumnName("person_id"); + + b.Property("VerificationRoute") + .HasColumnType("integer") + .HasColumnName("verification_route"); + + b.Property("VerifiedDatesOfBirth") + .HasColumnType("jsonb") + .HasColumnName("verified_dates_of_birth"); + + b.Property("VerifiedNames") + .HasColumnType("jsonb") + .HasColumnName("verified_names"); + + b.Property("VerifiedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("verified_on"); + + b.HasKey("Subject") + .HasName("pk_one_login_users"); + + b.ToTable("one_login_users", (string)null); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.Person", b => + { + b.Property("PersonId") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("person_id"); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("DateOfBirth") + .HasColumnType("date") + .HasColumnName("date_of_birth"); + + b.Property("DeletedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("deleted_on"); + + b.Property("DqtContactId") + .HasColumnType("uuid") + .HasColumnName("dqt_contact_id"); + + b.Property("DqtCreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("dqt_created_on"); + + b.Property("DqtFirstName") + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("dqt_first_name") + .UseCollation("case_insensitive"); + + b.Property("DqtFirstSync") + .HasColumnType("timestamp with time zone") + .HasColumnName("dqt_first_sync"); + + b.Property("DqtLastName") + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("dqt_last_name") + .UseCollation("case_insensitive"); + + b.Property("DqtLastSync") + .HasColumnType("timestamp with time zone") + .HasColumnName("dqt_last_sync"); + + b.Property("DqtMiddleName") + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("dqt_middle_name") + .UseCollation("case_insensitive"); + + b.Property("DqtModifiedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("dqt_modified_on"); + + b.Property("DqtState") + .HasColumnType("integer") + .HasColumnName("dqt_state"); + + b.Property("EmailAddress") + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("email_address") + .UseCollation("case_insensitive"); + + b.Property("FirstName") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("first_name") + .UseCollation("case_insensitive"); + + b.Property("LastName") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("last_name") + .UseCollation("case_insensitive"); + + b.Property("MiddleName") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("middle_name") + .UseCollation("case_insensitive"); + + b.Property("NationalInsuranceNumber") + .HasMaxLength(9) + .HasColumnType("character(9)") + .HasColumnName("national_insurance_number") + .IsFixedLength(); + + b.Property("Trn") + .HasMaxLength(7) + .HasColumnType("character(7)") + .HasColumnName("trn") + .IsFixedLength(); + + b.Property("UpdatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_on"); + + b.HasKey("PersonId") + .HasName("pk_persons"); + + b.HasIndex("DqtContactId") + .IsUnique() + .HasDatabaseName("ix_persons_dqt_contact_id") + .HasFilter("dqt_contact_id is not null"); + + b.HasIndex("Trn") + .IsUnique() + .HasDatabaseName("ix_persons_trn") + .HasFilter("trn is not null"); + + b.ToTable("persons", (string)null); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.PersonSearchAttribute", b => + { + b.Property("PersonSearchAttributeId") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("person_search_attribute_id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("PersonSearchAttributeId")); + + b.Property("AttributeKey") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("attribute_key") + .UseCollation("case_insensitive"); + + b.Property("AttributeType") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("attribute_type") + .UseCollation("case_insensitive"); + + b.Property("AttributeValue") + .IsRequired() + .HasMaxLength(1000) + .HasColumnType("character varying(1000)") + .HasColumnName("attribute_value") + .UseCollation("case_insensitive"); + + b.Property("PersonId") + .HasColumnType("uuid") + .HasColumnName("person_id"); + + b.Property("Tags") + .IsRequired() + .HasColumnType("text[]") + .HasColumnName("tags"); + + b.HasKey("PersonSearchAttributeId") + .HasName("pk_person_search_attributes"); + + b.HasIndex("PersonId") + .HasDatabaseName("ix_person_search_attributes_person_id"); + + b.HasIndex("AttributeType", "AttributeValue") + .HasDatabaseName("ix_person_search_attributes_attribute_type_and_value"); + + b.ToTable("person_search_attributes", (string)null); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.QtsAwardedEmailsJob", b => + { + b.Property("QtsAwardedEmailsJobId") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("qts_awarded_emails_job_id"); + + b.Property("AwardedToUtc") + .HasColumnType("timestamp with time zone") + .HasColumnName("awarded_to_utc"); + + b.Property("ExecutedUtc") + .HasColumnType("timestamp with time zone") + .HasColumnName("executed_utc"); + + b.HasKey("QtsAwardedEmailsJobId") + .HasName("pk_qts_awarded_emails_jobs"); + + b.HasIndex("ExecutedUtc") + .HasDatabaseName("ix_qts_awarded_emails_jobs_executed_utc"); + + b.ToTable("qts_awarded_emails_jobs", (string)null); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.QtsAwardedEmailsJobItem", b => + { + b.Property("QtsAwardedEmailsJobId") + .HasColumnType("uuid") + .HasColumnName("qts_awarded_emails_job_id"); + + b.Property("PersonId") + .HasColumnType("uuid") + .HasColumnName("person_id"); + + b.Property("EmailAddress") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("email_address"); + + b.Property("EmailSent") + .HasColumnType("boolean") + .HasColumnName("email_sent"); + + b.Property("Personalization") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("personalization"); + + b.Property("Trn") + .IsRequired() + .HasMaxLength(7) + .HasColumnType("character(7)") + .HasColumnName("trn") + .IsFixedLength(); + + b.HasKey("QtsAwardedEmailsJobId", "PersonId") + .HasName("pk_qts_awarded_emails_job_items"); + + b.HasIndex("Personalization") + .HasDatabaseName("ix_qts_awarded_emails_job_items_personalization"); + + NpgsqlIndexBuilderExtensions.HasMethod(b.HasIndex("Personalization"), "gin"); + + b.ToTable("qts_awarded_emails_job_items", (string)null); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.Qualification", b => + { + b.Property("QualificationId") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("qualification_id"); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("DeletedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("deleted_on"); + + b.Property("DqtCreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("dqt_created_on"); + + b.Property("DqtFirstSync") + .HasColumnType("timestamp with time zone") + .HasColumnName("dqt_first_sync"); + + b.Property("DqtLastSync") + .HasColumnType("timestamp with time zone") + .HasColumnName("dqt_last_sync"); + + b.Property("DqtModifiedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("dqt_modified_on"); + + b.Property("DqtQualificationId") + .HasColumnType("uuid") + .HasColumnName("dqt_qualification_id"); + + b.Property("DqtState") + .HasColumnType("integer") + .HasColumnName("dqt_state"); + + b.Property("PersonId") + .HasColumnType("uuid") + .HasColumnName("person_id"); + + b.Property("QualificationType") + .HasColumnType("integer") + .HasColumnName("qualification_type"); + + b.Property("UpdatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_on"); + + b.HasKey("QualificationId") + .HasName("pk_qualifications"); + + b.HasIndex("DqtQualificationId") + .IsUnique() + .HasDatabaseName("ix_qualifications_dqt_qualification_id") + .HasFilter("dqt_qualification_id is not null"); + + b.HasIndex("PersonId") + .HasDatabaseName("ix_qualifications_person_id"); + + b.ToTable("qualifications", (string)null); + + b.HasDiscriminator("QualificationType"); + + b.UseTphMappingStrategy(); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.SupportTask", b => + { + b.Property("SupportTaskReference") + .HasMaxLength(16) + .HasColumnType("character varying(16)") + .HasColumnName("support_task_reference"); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("OneLoginUserSubject") + .HasColumnType("character varying(255)") + .HasColumnName("one_login_user_subject"); + + b.Property("PersonId") + .HasColumnType("uuid") + .HasColumnName("person_id"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.Property("SupportTaskType") + .HasColumnType("integer") + .HasColumnName("support_task_type"); + + b.Property("UpdatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_on"); + + b.Property("_data") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("data"); + + b.HasKey("SupportTaskReference") + .HasName("pk_support_tasks"); + + b.HasIndex("OneLoginUserSubject") + .HasDatabaseName("ix_support_tasks_one_login_user_subject"); + + b.HasIndex("PersonId") + .HasDatabaseName("ix_support_tasks_person_id"); + + b.ToTable("support_tasks", (string)null); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.TpsCsvExtract", b => + { + b.Property("TpsCsvExtractId") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("tps_csv_extract_id"); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("Filename") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("filename"); + + b.HasKey("TpsCsvExtractId") + .HasName("pk_tps_csv_extracts"); + + b.ToTable("tps_csv_extracts", (string)null); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.TpsCsvExtractItem", b => + { + b.Property("TpsCsvExtractItemId") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("tps_csv_extract_item_id"); + + b.Property("Created") + .HasColumnType("timestamp with time zone") + .HasColumnName("created"); + + b.Property("DateOfBirth") + .HasColumnType("date") + .HasColumnName("date_of_birth"); + + b.Property("DateOfDeath") + .HasColumnType("date") + .HasColumnName("date_of_death"); + + b.Property("EmploymentEndDate") + .HasColumnType("date") + .HasColumnName("employment_end_date"); + + b.Property("EmploymentStartDate") + .HasColumnType("date") + .HasColumnName("employment_start_date"); + + b.Property("EmploymentType") + .HasColumnType("integer") + .HasColumnName("employment_type"); + + b.Property("EstablishmentEmailAddress") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("establishment_email_address"); + + b.Property("EstablishmentNumber") + .HasMaxLength(4) + .HasColumnType("character(4)") + .HasColumnName("establishment_number") + .IsFixedLength(); + + b.Property("EstablishmentPostcode") + .HasMaxLength(10) + .HasColumnType("character varying(10)") + .HasColumnName("establishment_postcode"); + + b.Property("ExtractDate") + .HasColumnType("date") + .HasColumnName("extract_date"); + + b.Property("Gender") + .IsRequired() + .HasMaxLength(10) + .HasColumnType("character varying(10)") + .HasColumnName("gender"); + + b.Property("Key") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("key"); + + b.Property("LocalAuthorityCode") + .IsRequired() + .HasMaxLength(3) + .HasColumnType("character(3)") + .HasColumnName("local_authority_code") + .IsFixedLength(); + + b.Property("MemberEmailAddress") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("member_email_address"); + + b.Property("MemberId") + .HasColumnType("integer") + .HasColumnName("member_id"); + + b.Property("MemberPostcode") + .HasMaxLength(10) + .HasColumnType("character varying(10)") + .HasColumnName("member_postcode"); + + b.Property("NationalInsuranceNumber") + .IsRequired() + .HasMaxLength(9) + .HasColumnType("character(9)") + .HasColumnName("national_insurance_number") + .IsFixedLength(); + + b.Property("Result") + .HasColumnType("integer") + .HasColumnName("result"); + + b.Property("TpsCsvExtractId") + .HasColumnType("uuid") + .HasColumnName("tps_csv_extract_id"); + + b.Property("TpsCsvExtractLoadItemId") + .HasColumnType("uuid") + .HasColumnName("tps_csv_extract_load_item_id"); + + b.Property("Trn") + .IsRequired() + .HasMaxLength(7) + .HasColumnType("character(7)") + .HasColumnName("trn") + .IsFixedLength(); + + b.Property("WithdrawalIndicator") + .HasMaxLength(1) + .HasColumnType("character(1)") + .HasColumnName("withdrawal_indicator") + .IsFixedLength(); + + b.HasKey("TpsCsvExtractItemId") + .HasName("pk_tps_csv_extract_items"); + + b.HasIndex("Key") + .HasDatabaseName("ix_tps_csv_extract_items_key"); + + b.HasIndex("TpsCsvExtractId") + .HasDatabaseName("ix_tps_csv_extract_items_tps_csv_extract_id"); + + b.HasIndex("TpsCsvExtractLoadItemId") + .HasDatabaseName("ix_tps_csv_extract_items_tps_csv_extract_load_item_id"); + + b.HasIndex("Trn") + .HasDatabaseName("ix_tps_csv_extract_items_trn"); + + b.HasIndex("LocalAuthorityCode", "EstablishmentNumber") + .HasDatabaseName("ix_tps_csv_extract_items_la_code_establishment_number"); + + b.ToTable("tps_csv_extract_items", (string)null); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.TpsCsvExtractLoadItem", b => + { + b.Property("TpsCsvExtractLoadItemId") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("tps_csv_extract_load_item_id"); + + b.Property("Created") + .HasColumnType("timestamp with time zone") + .HasColumnName("created"); + + b.Property("DateOfBirth") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("date_of_birth"); + + b.Property("DateOfDeath") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("date_of_death"); + + b.Property("EmploymentEndDate") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("employment_end_date"); + + b.Property("EmploymentStartDate") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("employment_start_date"); + + b.Property("Errors") + .HasColumnType("integer") + .HasColumnName("errors"); + + b.Property("EstablishmentEmailAddress") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("establishment_email_address"); + + b.Property("EstablishmentNumber") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("establishment_number"); + + b.Property("EstablishmentPostcode") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("establishment_postcode"); + + b.Property("ExtractDate") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("extract_date"); + + b.Property("FullOrPartTimeIndicator") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("full_or_part_time_indicator"); + + b.Property("Gender") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("gender"); + + b.Property("LocalAuthorityCode") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("local_authority_code"); + + b.Property("MemberEmailAddress") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("member_email_address"); + + b.Property("MemberId") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("member_id"); + + b.Property("MemberPostcode") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("member_postcode"); + + b.Property("NationalInsuranceNumber") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("national_insurance_number"); + + b.Property("TpsCsvExtractId") + .HasColumnType("uuid") + .HasColumnName("tps_csv_extract_id"); + + b.Property("Trn") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("trn"); + + b.Property("WithdrawalIndicator") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("withdrawal_indicator"); + + b.HasKey("TpsCsvExtractLoadItemId") + .HasName("pk_tps_csv_extract_load_items"); + + b.ToTable("tps_csv_extract_load_items", (string)null); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.TpsEmployment", b => + { + b.Property("TpsEmploymentId") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("tps_employment_id"); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("EmploymentType") + .HasColumnType("integer") + .HasColumnName("employment_type"); + + b.Property("EndDate") + .HasColumnType("date") + .HasColumnName("end_date"); + + b.Property("EstablishmentId") + .HasColumnType("uuid") + .HasColumnName("establishment_id"); + + b.Property("Key") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("key"); + + b.Property("LastExtractDate") + .HasColumnType("date") + .HasColumnName("last_extract_date"); + + b.Property("LastKnownTpsEmployedDate") + .HasColumnType("date") + .HasColumnName("last_known_tps_employed_date"); + + b.Property("NationalInsuranceNumber") + .HasMaxLength(9) + .HasColumnType("character(9)") + .HasColumnName("national_insurance_number") + .IsFixedLength(); + + b.Property("PersonId") + .HasColumnType("uuid") + .HasColumnName("person_id"); + + b.Property("PersonPostcode") + .HasMaxLength(10) + .HasColumnType("character varying(10)") + .HasColumnName("person_postcode"); + + b.Property("StartDate") + .HasColumnType("date") + .HasColumnName("start_date"); + + b.Property("UpdatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_on"); + + b.Property("WithdrawalConfirmed") + .HasColumnType("boolean") + .HasColumnName("withdrawal_confirmed"); + + b.HasKey("TpsEmploymentId") + .HasName("pk_tps_employments"); + + b.HasIndex("EstablishmentId") + .HasDatabaseName("ix_tps_employments_establishment_id"); + + b.HasIndex("Key") + .HasDatabaseName("ix_tps_employments_key"); + + b.HasIndex("PersonId") + .HasDatabaseName("ix_tps_employments_person_id"); + + b.ToTable("tps_employments", (string)null); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.TpsEstablishment", b => + { + b.Property("TpsEstablishmentId") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("tps_establishment_id"); + + b.Property("EmployersName") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("employers_name"); + + b.Property("EstablishmentCode") + .IsRequired() + .HasMaxLength(4) + .HasColumnType("character(4)") + .HasColumnName("establishment_code") + .IsFixedLength(); + + b.Property("LaCode") + .IsRequired() + .HasMaxLength(3) + .HasColumnType("character(3)") + .HasColumnName("la_code") + .IsFixedLength(); + + b.Property("SchoolClosedDate") + .HasColumnType("date") + .HasColumnName("school_closed_date"); + + b.Property("SchoolGiasName") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("school_gias_name"); + + b.HasKey("TpsEstablishmentId") + .HasName("pk_tps_establishments"); + + b.HasIndex("LaCode", "EstablishmentCode") + .HasDatabaseName("ix_tps_establishments_la_code_establishment_number"); + + b.ToTable("tps_establishments", (string)null); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.TpsEstablishmentType", b => + { + b.Property("TpsEstablishmentTypeId") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("tps_establishment_type_id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("TpsEstablishmentTypeId")); + + b.Property("Description") + .IsRequired() + .HasMaxLength(300) + .HasColumnType("character varying(300)") + .HasColumnName("description"); + + b.Property("EstablishmentRangeFrom") + .IsRequired() + .HasMaxLength(4) + .HasColumnType("character(4)") + .HasColumnName("establishment_range_from") + .IsFixedLength(); + + b.Property("EstablishmentRangeTo") + .IsRequired() + .HasMaxLength(4) + .HasColumnType("character(4)") + .HasColumnName("establishment_range_to") + .IsFixedLength(); + + b.Property("ShortDescription") + .IsRequired() + .HasMaxLength(120) + .HasColumnType("character varying(120)") + .HasColumnName("short_description"); + + b.HasKey("TpsEstablishmentTypeId") + .HasName("pk_tps_establishment_types"); + + b.ToTable("tps_establishment_types", (string)null); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.TrnRequest", b => + { + b.Property("TrnRequestId") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("trn_request_id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("TrnRequestId")); + + b.Property("ClientId") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("client_id"); + + b.Property("IdentityUserId") + .HasColumnType("uuid") + .HasColumnName("identity_user_id"); + + b.Property("LinkedToIdentity") + .HasColumnType("boolean") + .HasColumnName("linked_to_identity"); + + b.Property("RequestId") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("request_id"); + + b.Property("TeacherId") + .HasColumnType("uuid") + .HasColumnName("teacher_id"); + + b.Property("TrnToken") + .HasMaxLength(128) + .HasColumnType("character varying(128)") + .HasColumnName("trn_token"); + + b.HasKey("TrnRequestId") + .HasName("pk_trn_requests"); + + b.HasIndex("ClientId", "RequestId") + .IsUnique() + .HasDatabaseName("ix_trn_requests_client_id_request_id"); + + b.ToTable("trn_requests", (string)null); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.UserBase", b => + { + b.Property("UserId") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("user_id"); + + b.Property("Active") + .HasColumnType("boolean") + .HasColumnName("active"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("name"); + + b.Property("UserType") + .HasColumnType("integer") + .HasColumnName("user_type"); + + b.HasKey("UserId") + .HasName("pk_users"); + + b.ToTable("users", (string)null); + + b.HasDiscriminator("UserType"); + + b.UseTphMappingStrategy(); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.MandatoryQualification", b => + { + b.HasBaseType("TeachingRecordSystem.Core.DataStore.Postgres.Models.Qualification"); + + b.Property("DqtMqEstablishmentId") + .HasColumnType("uuid") + .HasColumnName("dqt_mq_establishment_id"); + + b.Property("DqtSpecialismId") + .HasColumnType("uuid") + .HasColumnName("dqt_specialism_id"); + + b.Property("EndDate") + .HasColumnType("date") + .HasColumnName("end_date"); + + b.Property("ProviderId") + .HasColumnType("uuid") + .HasColumnName("mq_provider_id"); + + b.Property("Specialism") + .HasColumnType("integer") + .HasColumnName("mq_specialism"); + + b.Property("StartDate") + .HasColumnType("date") + .HasColumnName("start_date"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("mq_status"); + + b.HasDiscriminator().HasValue(0); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.ApplicationUser", b => + { + b.HasBaseType("TeachingRecordSystem.Core.DataStore.Postgres.Models.UserBase"); + + b.Property("ApiRoles") + .HasColumnType("varchar[]") + .HasColumnName("api_roles"); + + b.Property("ClientId") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("client_id"); + + b.Property("ClientSecret") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("client_secret"); + + b.Property("IsOidcClient") + .HasColumnType("boolean") + .HasColumnName("is_oidc_client"); + + b.Property("OneLoginAuthenticationSchemeName") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("one_login_authentication_scheme_name"); + + b.Property("OneLoginClientId") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("one_login_client_id"); + + b.Property("OneLoginPostLogoutRedirectUriPath") + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("one_login_post_logout_redirect_uri_path"); + + b.Property("OneLoginPrivateKeyPem") + .HasMaxLength(2000) + .HasColumnType("character varying(2000)") + .HasColumnName("one_login_private_key_pem"); + + b.Property("OneLoginRedirectUriPath") + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("one_login_redirect_uri_path"); + + b.Property>("PostLogoutRedirectUris") + .HasColumnType("varchar[]") + .HasColumnName("post_logout_redirect_uris"); + + b.Property>("RedirectUris") + .HasColumnType("varchar[]") + .HasColumnName("redirect_uris"); + + b.HasIndex("ClientId") + .IsUnique() + .HasDatabaseName("ix_users_client_id") + .HasFilter("client_id is not null"); + + b.HasIndex("OneLoginAuthenticationSchemeName") + .IsUnique() + .HasDatabaseName("ix_users_one_login_authentication_scheme_name") + .HasFilter("one_login_authentication_scheme_name is not null"); + + b.HasDiscriminator().HasValue(2); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.SystemUser", b => + { + b.HasBaseType("TeachingRecordSystem.Core.DataStore.Postgres.Models.UserBase"); + + b.ToTable("users", (string)null); + + b.HasDiscriminator().HasValue(3); + + b.HasData( + new + { + UserId = new Guid("a81394d1-a498-46d8-af3e-e077596ab303"), + Active = true, + Name = "System", + UserType = 0 + }); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.User", b => + { + b.HasBaseType("TeachingRecordSystem.Core.DataStore.Postgres.Models.UserBase"); + + b.Property("AzureAdUserId") + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("azure_ad_user_id"); + + b.Property("DqtUserId") + .HasColumnType("uuid") + .HasColumnName("dqt_user_id"); + + b.Property("Email") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("email") + .UseCollation("case_insensitive"); + + b.Property("Roles") + .IsRequired() + .HasColumnType("varchar[]") + .HasColumnName("roles"); + + b.HasIndex("AzureAdUserId") + .IsUnique() + .HasDatabaseName("ix_users_azure_ad_user_id"); + + b.HasDiscriminator().HasValue(1); + }); + + modelBuilder.Entity("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreAuthorization", b => + { + b.HasOne("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreApplication", "Application") + .WithMany("Authorizations") + .HasForeignKey("ApplicationId") + .HasConstraintName("fk_oidc_authorizations_oidc_applications_application_id"); + + b.Navigation("Application"); + }); + + modelBuilder.Entity("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreToken", b => + { + b.HasOne("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreApplication", "Application") + .WithMany("Tokens") + .HasForeignKey("ApplicationId") + .HasConstraintName("fk_oidc_tokens_oidc_applications_application_id"); + + b.HasOne("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreAuthorization", "Authorization") + .WithMany("Tokens") + .HasForeignKey("AuthorizationId") + .HasConstraintName("fk_oidc_tokens_oidc_authorizations_authorization_id"); + + b.Navigation("Application"); + + b.Navigation("Authorization"); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.Alert", b => + { + b.HasOne("TeachingRecordSystem.Core.DataStore.Postgres.Models.AlertType", null) + .WithMany() + .HasForeignKey("AlertTypeId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_alerts_alert_type"); + + b.HasOne("TeachingRecordSystem.Core.DataStore.Postgres.Models.Person", null) + .WithMany() + .HasForeignKey("PersonId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_alerts_person"); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.AlertType", b => + { + b.HasOne("TeachingRecordSystem.Core.DataStore.Postgres.Models.AlertCategory", null) + .WithMany() + .HasForeignKey("AlertCategoryId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_alert_types_alert_category"); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.ApiKey", b => + { + b.HasOne("TeachingRecordSystem.Core.DataStore.Postgres.Models.ApplicationUser", "ApplicationUser") + .WithMany("ApiKeys") + .HasForeignKey("ApplicationUserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_api_key_application_user"); + + b.Navigation("ApplicationUser"); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.Establishment", b => + { + b.HasOne("TeachingRecordSystem.Core.DataStore.Postgres.Models.EstablishmentSource", null) + .WithMany() + .HasForeignKey("EstablishmentSourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_establishments_establishment_source_id"); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.EytsAwardedEmailsJobItem", b => + { + b.HasOne("TeachingRecordSystem.Core.DataStore.Postgres.Models.EytsAwardedEmailsJob", "EytsAwardedEmailsJob") + .WithMany("JobItems") + .HasForeignKey("EytsAwardedEmailsJobId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_eyts_awarded_emails_job_items_eyts_awarded_emails_jobs_eyts"); + + b.Navigation("EytsAwardedEmailsJob"); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.InductionCompletedEmailsJobItem", b => + { + b.HasOne("TeachingRecordSystem.Core.DataStore.Postgres.Models.InductionCompletedEmailsJob", "InductionCompletedEmailsJob") + .WithMany("JobItems") + .HasForeignKey("InductionCompletedEmailsJobId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_induction_completed_emails_job_items_induction_completed_em"); + + b.Navigation("InductionCompletedEmailsJob"); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.InternationalQtsAwardedEmailsJobItem", b => + { + b.HasOne("TeachingRecordSystem.Core.DataStore.Postgres.Models.InternationalQtsAwardedEmailsJob", "InternationalQtsAwardedEmailsJob") + .WithMany("JobItems") + .HasForeignKey("InternationalQtsAwardedEmailsJobId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_international_qts_awarded_emails_job_items_international_qt"); + + b.Navigation("InternationalQtsAwardedEmailsJob"); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.OneLoginUser", b => + { + b.HasOne("TeachingRecordSystem.Core.DataStore.Postgres.Models.Person", "Person") + .WithOne() + .HasForeignKey("TeachingRecordSystem.Core.DataStore.Postgres.Models.OneLoginUser", "PersonId") + .HasConstraintName("fk_one_login_users_persons_person_id"); + + b.Navigation("Person"); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.QtsAwardedEmailsJobItem", b => + { + b.HasOne("TeachingRecordSystem.Core.DataStore.Postgres.Models.QtsAwardedEmailsJob", "QtsAwardedEmailsJob") + .WithMany("JobItems") + .HasForeignKey("QtsAwardedEmailsJobId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_qts_awarded_emails_job_items_qts_awarded_emails_jobs_qts_aw"); + + b.Navigation("QtsAwardedEmailsJob"); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.Qualification", b => + { + b.HasOne("TeachingRecordSystem.Core.DataStore.Postgres.Models.Person", null) + .WithMany() + .HasForeignKey("PersonId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_qualifications_person"); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.SupportTask", b => + { + b.HasOne("TeachingRecordSystem.Core.DataStore.Postgres.Models.OneLoginUser", null) + .WithMany() + .HasForeignKey("OneLoginUserSubject") + .HasConstraintName("fk_support_tasks_one_login_user"); + + b.HasOne("TeachingRecordSystem.Core.DataStore.Postgres.Models.Person", null) + .WithMany() + .HasForeignKey("PersonId") + .HasConstraintName("fk_support_tasks_person"); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.TpsCsvExtractItem", b => + { + b.HasOne("TeachingRecordSystem.Core.DataStore.Postgres.Models.TpsCsvExtract", null) + .WithMany() + .HasForeignKey("TpsCsvExtractId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_tps_csv_extract_items_tps_csv_extract_id"); + + b.HasOne("TeachingRecordSystem.Core.DataStore.Postgres.Models.TpsCsvExtractLoadItem", null) + .WithMany() + .HasForeignKey("TpsCsvExtractLoadItemId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_tps_csv_extract_items_tps_csv_extract_load_item_id"); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.TpsCsvExtractLoadItem", b => + { + b.HasOne("TeachingRecordSystem.Core.DataStore.Postgres.Models.TpsCsvExtract", null) + .WithMany() + .HasForeignKey("TpsCsvExtractId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_tps_csv_extract_load_items_tps_csv_extract_id"); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.TpsEmployment", b => + { + b.HasOne("TeachingRecordSystem.Core.DataStore.Postgres.Models.Establishment", null) + .WithMany() + .HasForeignKey("EstablishmentId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_tps_employments_establishment_id"); + + b.HasOne("TeachingRecordSystem.Core.DataStore.Postgres.Models.Person", null) + .WithMany() + .HasForeignKey("PersonId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_tps_employments_person_id"); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.MandatoryQualification", b => + { + b.HasOne("TeachingRecordSystem.Core.DataStore.Postgres.Models.MandatoryQualificationProvider", "Provider") + .WithMany() + .HasForeignKey("ProviderId") + .HasConstraintName("fk_qualifications_mandatory_qualification_provider"); + + b.Navigation("Provider"); + }); + + modelBuilder.Entity("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreApplication", b => + { + b.Navigation("Authorizations"); + + b.Navigation("Tokens"); + }); + + modelBuilder.Entity("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreAuthorization", b => + { + b.Navigation("Tokens"); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.EytsAwardedEmailsJob", b => + { + b.Navigation("JobItems"); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.InductionCompletedEmailsJob", b => + { + b.Navigation("JobItems"); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.InternationalQtsAwardedEmailsJob", b => + { + b.Navigation("JobItems"); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.QtsAwardedEmailsJob", b => + { + b.Navigation("JobItems"); + }); + + modelBuilder.Entity("TeachingRecordSystem.Core.DataStore.Postgres.Models.ApplicationUser", b => + { + b.Navigation("ApiKeys"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/TeachingRecordSystem/src/TeachingRecordSystem.Core/DataStore/Postgres/Migrations/20240822100611_DqtReportingSyncPublicationQualifications.cs b/TeachingRecordSystem/src/TeachingRecordSystem.Core/DataStore/Postgres/Migrations/20240822100611_DqtReportingSyncPublicationQualifications.cs new file mode 100644 index 000000000..3e79e9657 --- /dev/null +++ b/TeachingRecordSystem/src/TeachingRecordSystem.Core/DataStore/Postgres/Migrations/20240822100611_DqtReportingSyncPublicationQualifications.cs @@ -0,0 +1,23 @@ +using Microsoft.EntityFrameworkCore.Migrations; +using TeachingRecordSystem.Core.Services.DqtReporting; + +#nullable disable + +namespace TeachingRecordSystem.Core.DataStore.Postgres.Migrations +{ + /// + public partial class DqtReportingSyncPublicationQualifications : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.Sql($"CREATE PUBLICATION {DqtReportingService.TrsDbPublicationName} FOR TABLE qualifications;"); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.Sql($"DROP PUBLICATION {DqtReportingService.TrsDbPublicationName};"); + } + } +} diff --git a/TeachingRecordSystem/src/TeachingRecordSystem.Core/DistributedLockKeys.cs b/TeachingRecordSystem/src/TeachingRecordSystem.Core/DistributedLockKeys.cs index 750c47e03..0b7418384 100644 --- a/TeachingRecordSystem/src/TeachingRecordSystem.Core/DistributedLockKeys.cs +++ b/TeachingRecordSystem/src/TeachingRecordSystem.Core/DistributedLockKeys.cs @@ -6,4 +6,5 @@ public static class DistributedLockKeys public static string Husid(string husid) => $"husid:{husid}"; public static string Trn(string trn) => $"trn:{trn}"; public static string TrnRequestId(string clientId, string requestId) => $"trn-request:{clientId}/{requestId}"; + public static string DqtReportingReplicationSlot() => nameof(DqtReportingReplicationSlot); } diff --git a/TeachingRecordSystem/src/TeachingRecordSystem.Core/Jobs/BackfillDqtReportingQualifications.cs b/TeachingRecordSystem/src/TeachingRecordSystem.Core/Jobs/BackfillDqtReportingQualifications.cs new file mode 100644 index 000000000..298b8d14a --- /dev/null +++ b/TeachingRecordSystem/src/TeachingRecordSystem.Core/Jobs/BackfillDqtReportingQualifications.cs @@ -0,0 +1,80 @@ +using System.Data; +using Microsoft.Data.SqlClient; +using Microsoft.Extensions.Options; +using TeachingRecordSystem.Core.DataStore.Postgres; +using TeachingRecordSystem.Core.Services.DqtReporting; + +namespace TeachingRecordSystem.Core.Jobs; + +public class BackfillDqtReportingQualifications(IOptions dqtReportingOptionsAccessor, TrsDbContext dbContext, IClock clock) +{ + public async Task ExecuteAsync(CancellationToken cancellationToken) + { + var dataTable = new DataTable(); + dataTable.Columns.Add("qualification_id", typeof(Guid)); + dataTable.Columns.Add("created_on", typeof(DateTime)); + dataTable.Columns.Add("updated_on", typeof(DateTime)); + dataTable.Columns.Add("deleted_on", typeof(DateTime)); + dataTable.Columns.Add("qualification_type", typeof(int)); + dataTable.Columns.Add("person_id", typeof(Guid)); + dataTable.Columns.Add("dqt_qualification_id", typeof(Guid)); + dataTable.Columns.Add("dqt_first_sync", typeof(DateTime)); + dataTable.Columns.Add("dqt_last_sync", typeof(DateTime)); + dataTable.Columns.Add("dqt_state", typeof(int)); + dataTable.Columns.Add("dqt_created_on", typeof(DateTime)); + dataTable.Columns.Add("dqt_modified_on", typeof(DateTime)); + dataTable.Columns.Add("mq_specialism", typeof(int)); + dataTable.Columns.Add("mq_status", typeof(int)); + dataTable.Columns.Add("start_date", typeof(DateOnly)); + dataTable.Columns.Add("end_date", typeof(DateOnly)); + dataTable.Columns.Add("dqt_mq_establishment_id", typeof(Guid)); + dataTable.Columns.Add("dqt_specialism_id", typeof(Guid)); + dataTable.Columns.Add("mq_provider_id", typeof(Guid)); + dataTable.Columns.Add("__Inserted", typeof(DateTime)); + dataTable.Columns.Add("__Updated", typeof(DateTime)); + + var mqs = await dbContext.MandatoryQualifications.ToListAsync(); + + foreach (var mq in mqs) + { + dataTable.Rows.Add( + mq.QualificationId, + mq.CreatedOn, + mq.UpdatedOn, + mq.DeletedOn, + mq.QualificationType, + mq.PersonId, + mq.DqtQualificationId, + mq.DqtFirstSync, + mq.DqtLastSync, + mq.DqtState, + mq.DqtCreatedOn, + mq.DqtModifiedOn, + mq.Specialism, + mq.Status, + mq.StartDate, + mq.EndDate, + mq.DqtMqEstablishmentId, + mq.DqtSpecialismId, + mq.ProviderId, + clock.UtcNow, + clock.UtcNow); + } + + using var conn = new SqlConnection(dqtReportingOptionsAccessor.Value.ReportingDbConnectionString); + conn.Open(); + + using (var sqlBulkCopy = new SqlBulkCopy(conn)) + { + foreach (DataColumn column in dataTable.Columns) + { + sqlBulkCopy.ColumnMappings.Add(new SqlBulkCopyColumnMapping(column.ColumnName, column.ColumnName)); + } + + sqlBulkCopy.BulkCopyTimeout = 0; + sqlBulkCopy.DestinationTableName = "trs_qualifications"; + + await sqlBulkCopy.WriteToServerAsync(dataTable, cancellationToken); + } + } +} diff --git a/TeachingRecordSystem/src/TeachingRecordSystem.Core/Jobs/HostApplicationBuilderExtensions.cs b/TeachingRecordSystem/src/TeachingRecordSystem.Core/Jobs/HostApplicationBuilderExtensions.cs index 8650e34c9..2466f6e07 100644 --- a/TeachingRecordSystem/src/TeachingRecordSystem.Core/Jobs/HostApplicationBuilderExtensions.cs +++ b/TeachingRecordSystem/src/TeachingRecordSystem.Core/Jobs/HostApplicationBuilderExtensions.cs @@ -133,6 +133,11 @@ public static IHostApplicationBuilder AddBackgroundJobs(this IHostApplicationBui job => job.Execute(CancellationToken.None), DeleteOldAttachmentsJob.JobSchedule); + recurringJobManager.AddOrUpdate( + nameof(BackfillDqtReportingQualifications), + job => job.ExecuteAsync(CancellationToken.None), + Cron.Never); + return Task.CompletedTask; }); } diff --git a/TeachingRecordSystem/src/TeachingRecordSystem.Core/Services/DqtReporting/DqtReportingOptions.cs b/TeachingRecordSystem/src/TeachingRecordSystem.Core/Services/DqtReporting/DqtReportingOptions.cs index ec8f5a73b..022afc79c 100644 --- a/TeachingRecordSystem/src/TeachingRecordSystem.Core/Services/DqtReporting/DqtReportingOptions.cs +++ b/TeachingRecordSystem/src/TeachingRecordSystem.Core/Services/DqtReporting/DqtReportingOptions.cs @@ -4,6 +4,8 @@ namespace TeachingRecordSystem.Core.Services.DqtReporting; public class DqtReportingOptions { + public const string DefaultTrsDbReplicationSlotName = "dqt_rep_sync_slot"; + [Required] public required int PollIntervalSeconds { get; set; } @@ -21,4 +23,6 @@ public class DqtReportingOptions [Required] public required bool RunService { get; set; } + + public required string TrsDbReplicationSlotName { get; set; } = DefaultTrsDbReplicationSlotName; } diff --git a/TeachingRecordSystem/src/TeachingRecordSystem.Core/Services/DqtReporting/DqtReportingService.cs b/TeachingRecordSystem/src/TeachingRecordSystem.Core/Services/DqtReporting/DqtReportingService.cs index 710698c4a..2ed198a2b 100644 --- a/TeachingRecordSystem/src/TeachingRecordSystem.Core/Services/DqtReporting/DqtReportingService.cs +++ b/TeachingRecordSystem/src/TeachingRecordSystem.Core/Services/DqtReporting/DqtReportingService.cs @@ -1,8 +1,10 @@ using System.Data; using System.Text; +using Medallion.Threading; using Microsoft.ApplicationInsights; using Microsoft.ApplicationInsights.DataContracts; using Microsoft.Data.SqlClient; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; @@ -10,6 +12,11 @@ using Microsoft.Xrm.Sdk; using Microsoft.Xrm.Sdk.Metadata; using Microsoft.Xrm.Sdk.Query; +using Npgsql; +using Npgsql.Replication; +using Npgsql.Replication.PgOutput; +using Npgsql.Replication.PgOutput.Messages; +using NpgsqlTypes; using Polly; using TeachingRecordSystem.Core.Dqt; using TeachingRecordSystem.Core.Dqt.Queries; @@ -22,6 +29,7 @@ public partial class DqtReportingService : BackgroundService public const string ChangesKey = "DqtReporting"; public const string CrmClientName = "DqtReporting"; public const string ProcessChangesOperationName = "DqtReporting: process changes"; + public const string TrsDbPublicationName = "dqt_rep_sync"; private const int MaxParameters = 1024; private const int PageSize = 500; @@ -40,8 +48,10 @@ public partial class DqtReportingService : BackgroundService private readonly DqtReportingOptions _options; private readonly ICrmEntityChangesService _crmEntityChangesService; private readonly ICrmQueryDispatcher _crmQueryDispatcher; + private readonly IDistributedLockProvider _distributedLockProvider; private readonly IClock _clock; private readonly TelemetryClient _telemetryClient; + private readonly IConfiguration _configuration; private readonly ILogger _logger; private readonly Dictionary _entityMetadata = new(); @@ -49,52 +59,70 @@ public DqtReportingService( IOptions optionsAccessor, [FromKeyedServices(CrmClientName)] ICrmEntityChangesService crmEntityChangesService, [FromKeyedServices(CrmClientName)] ICrmQueryDispatcher crmQueryDispatcher, + IDistributedLockProvider distributedLockProvider, IClock clock, TelemetryClient telemetryClient, + IConfiguration configuration, ILogger logger) { _options = optionsAccessor.Value; _crmEntityChangesService = crmEntityChangesService; _crmQueryDispatcher = crmQueryDispatcher; + _distributedLockProvider = distributedLockProvider; _clock = clock; _telemetryClient = telemetryClient; + _configuration = configuration; _logger = logger; } - protected override async Task ExecuteAsync(CancellationToken stoppingToken) + protected override Task ExecuteAsync(CancellationToken stoppingToken) { - await LoadEntityMetadata(); + return Task.WhenAll(ProcessCrmChangesWrapper(), ProcessTrsChangesWrapper()); - using var timer = new PeriodicTimer(TimeSpan.FromSeconds(_options.PollIntervalSeconds)); + async Task ProcessCrmChangesWrapper() + { + await LoadEntityMetadata(); + + using var timer = new PeriodicTimer(TimeSpan.FromSeconds(_options.PollIntervalSeconds)); - do + do + { + try + { + await _resiliencePipeline.ExecuteAsync(async ct => await ProcessCrmChanges(ct), stoppingToken); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + } + catch (ProcessCrmChangesException ex) + { + _logger.LogError(ex.InnerException, ex.Message); + return; + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed processing entity changes."); + return; + } + } + while (await timer.WaitForNextTickAsync(stoppingToken)); + } + + async Task ProcessTrsChangesWrapper() { try { - await _resiliencePipeline.ExecuteAsync(async ct => await ProcessChanges(ct), stoppingToken); + await _resiliencePipeline.ExecuteAsync(async ct => await ProcessTrsChanges(observer: null, ct)); } catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { } - catch (ProcessChangesException ex) - when (ex.InnerException is SqlException sqlException && - (sqlException.IsTransient || sqlException.Message.StartsWith("Execution Timeout Expired."))) - { - _logger.LogWarning(ex, "Transient SQL exception thrown."); - continue; - } - catch (ProcessChangesException ex) - { - _logger.LogError(ex.InnerException, ex.Message); - return; - } catch (Exception ex) { - _logger.LogError(ex, "Failed processing entity changes."); + _logger.LogError(ex, "Failed processing TRS changes."); return; } } - while (await timer.WaitForNextTickAsync(stoppingToken)); } internal async Task LoadEntityMetadata() @@ -139,7 +167,7 @@ internal async Task LoadEntityMetadata() } } - internal async Task ProcessChanges(CancellationToken cancellationToken) + internal async Task ProcessCrmChanges(CancellationToken cancellationToken) { using var operation = _telemetryClient.StartOperation(ProcessChangesOperationName); @@ -154,7 +182,7 @@ await Parallel.ForEachAsync( { try { - await ProcessChangesForEntityType(entityType, ct); + await ProcessCrmChangesForEntityType(entityType, ct); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { @@ -162,12 +190,12 @@ await Parallel.ForEachAsync( } catch (Exception ex) { - throw new ProcessChangesException(entityType, ex); + throw new ProcessCrmChangesException(entityType, ex); } }); } - internal async Task ProcessChangesForEntityType(string entityLogicalName, CancellationToken cancellationToken) + internal async Task ProcessCrmChangesForEntityType(string entityLogicalName, CancellationToken cancellationToken) { var totalProcessed = 0; @@ -428,11 +456,141 @@ private async Task HandleRemovedOrDeletedItems( await command.ExecuteNonQueryAsync(cancellationToken); } } + + internal async Task ProcessTrsChanges( + IObserver? observer, + CancellationToken cancellationToken) + { + await using var @lock = await _distributedLockProvider.TryAcquireLockAsync(DistributedLockKeys.DqtReportingReplicationSlot()); + if (@lock is null) + { + return; + } + + await using var replicationConn = new LogicalReplicationConnection(_configuration.GetPostgresConnectionString()); + await replicationConn.Open(); + + var slot = await GetReplicationSlot(replicationConn, cancellationToken); + var replicationOptions = new PgOutputReplicationOptions(TrsDbPublicationName, protocolVersion: 1, binary: true); + + await foreach (var message in replicationConn.StartReplication(slot, replicationOptions, cancellationToken)) + { + if (message is InsertMessage or UpdateMessage) + { + var tuple = (message as InsertMessage)?.NewRow ?? (message as UpdateMessage)!.NewRow; + var relation = (message as InsertMessage)?.Relation ?? (message as UpdateMessage)!.Relation; + + var targetTableName = $"trs_{relation.RelationName}"; + var values = await GetTupleValues(tuple); + var columns = relation.Columns.ToArray(); + var idColumn = columns.Single(c => c.Flags == RelationMessage.Column.ColumnFlags.PartOfKey); + var columnValues = columns.Zip(values, (c, v) => (Column: c, Value: v)).ToDictionary(t => t.Column.ColumnName, t => t.Value); + var id = values[Array.IndexOf(columns, idColumn)]!; + + await UpsertRowFromTrs(targetTableName, idColumn.ColumnName, id, columnValues); + + observer?.OnNext(message); + } + else if (message is DeleteMessage or TruncateMessage) + { + throw new NotSupportedException(); + } + + replicationConn.SetReplicationStatus(message.WalEnd); + } + + static ValueTask GetTupleValues(ReplicationTuple tuple) + { + return Core().ToArrayAsync(); + + async IAsyncEnumerable Core() + { + await foreach (var value in tuple) + { + yield return value.IsDBNull ? null : await value.Get(); + } + } + } + } + + private async Task GetReplicationSlot( + LogicalReplicationConnection replicationConn, + CancellationToken cancellationToken) + { + await using var dataSource = NpgsqlDataSource.Create(_configuration.GetPostgresConnectionString()); + + var slotName = _options.TrsDbReplicationSlotName; + var startLsn = NpgsqlLogSequenceNumber.Invalid; + + await using (var cmd = dataSource.CreateCommand("SELECT confirmed_flush_lsn FROM pg_replication_slots WHERE slot_name = $1")) + { + cmd.Parameters.AddWithValue(slotName); + + await using (var reader = await cmd.ExecuteReaderAsync(cancellationToken)) + { + if (await reader.ReadAsync()) + { + startLsn = reader.GetFieldValue(0); + } + } + } + + if (startLsn == NpgsqlLogSequenceNumber.Invalid) + { + return await replicationConn.CreatePgOutputReplicationSlot(slotName); + } + else + { + return new PgOutputReplicationSlot(new ReplicationSlotOptions(slotName, startLsn)); + } + } + + private async Task UpsertRowFromTrs(string destinationTableName, string idColumnName, object id, IReadOnlyDictionary columnValues) + { + var parameters = new List(); + var columnNames = new List(); + + foreach (var (columnName, columnValue) in columnValues) + { + var parameterName = $"@p{parameters.Count + 1}"; + parameters.Add(new SqlParameter(parameterName, columnValue ?? DBNull.Value)); + columnNames.Add(columnName); + } + + var parametersAndColumns = parameters.Zip(columnNames, (p, c) => (ParameterName: p.ParameterName, ColumnName: c)).ToArray(); + + var nowParameterName = "@UtcNow"; + var idParameterName = "@id"; + + var sql = $"""" + merge {destinationTableName} as target + using ( + select {(string.Join(",\n\t", parametersAndColumns.Select(p => $"{p.ParameterName} as {p.ColumnName}")))} + ) as source + on target.{idColumnName} = source.{idColumnName} + when not matched then + insert ({(string.Join(", ", parametersAndColumns.Select(p => p.ColumnName)))}, __Inserted, __Updated) + values ({(string.Join(", ", parametersAndColumns.Select(p => $"source.{p.ColumnName}")))}, {nowParameterName}, {nowParameterName}) + when matched then + update set {(string.Join(", ", parametersAndColumns.Where(p => p.ColumnName != idColumnName).Select(p => $"{p.ColumnName} = source.{p.ColumnName}")))}, __Updated = {nowParameterName} + ; + """"; + + parameters.Add(new SqlParameter(nowParameterName, _clock.UtcNow)); + parameters.Add(new SqlParameter(idParameterName, id)); + + using var conn = new SqlConnection(_options.ReportingDbConnectionString); + await conn.OpenAsync(); + + using var cmd = new SqlCommand(sql, conn); + cmd.Parameters.AddRange(parameters.ToArray()); + await cmd.ExecuteNonQueryAsync(); + } } -file class ProcessChangesException : Exception +file class ProcessCrmChangesException : Exception { - public ProcessChangesException(string entityType, Exception innerException) + public ProcessCrmChangesException(string entityType, Exception innerException) : base(GetMessage(entityType), innerException) { EntityType = entityType; diff --git a/TeachingRecordSystem/src/TeachingRecordSystem.Core/Services/DqtReporting/Migrations/0026_TrsQualifications.sql b/TeachingRecordSystem/src/TeachingRecordSystem.Core/Services/DqtReporting/Migrations/0026_TrsQualifications.sql new file mode 100644 index 000000000..d06b57c18 --- /dev/null +++ b/TeachingRecordSystem/src/TeachingRecordSystem.Core/Services/DqtReporting/Migrations/0026_TrsQualifications.sql @@ -0,0 +1,23 @@ +create table trs_qualifications ( + qualification_id uniqueidentifier primary key, + created_on datetime, + updated_on datetime, + deleted_on datetime, + qualification_type integer, + person_id uniqueidentifier, + dqt_qualification_id uniqueidentifier, + dqt_first_sync datetime, + dqt_last_sync datetime, + dqt_state integer, + dqt_created_on datetime, + dqt_modified_on datetime, + mq_specialism integer, + mq_status integer, + start_date date, + end_date date, + dqt_mq_establishment_id uniqueidentifier, + dqt_specialism_id uniqueidentifier, + mq_provider_id uniqueidentifier, + [__Inserted] datetime, + [__Updated] datetime +) diff --git a/TeachingRecordSystem/src/TeachingRecordSystem.Core/TeachingRecordSystem.Core.csproj b/TeachingRecordSystem/src/TeachingRecordSystem.Core/TeachingRecordSystem.Core.csproj index 553f0518b..219579c73 100644 --- a/TeachingRecordSystem/src/TeachingRecordSystem.Core/TeachingRecordSystem.Core.csproj +++ b/TeachingRecordSystem/src/TeachingRecordSystem.Core/TeachingRecordSystem.Core.csproj @@ -58,6 +58,7 @@ + @@ -114,6 +115,7 @@ + diff --git a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Dqt.CrmIntegrationTests/CrmClientFixture.cs b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Dqt.CrmIntegrationTests/CrmClientFixture.cs index 883d2e566..2b1d0f98f 100644 --- a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Dqt.CrmIntegrationTests/CrmClientFixture.cs +++ b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Dqt.CrmIntegrationTests/CrmClientFixture.cs @@ -15,7 +15,6 @@ namespace TeachingRecordSystem.Core.Dqt.CrmIntegrationTests; public sealed class CrmClientFixture : IDisposable { private readonly ServiceClient _baseServiceClient; - private readonly DbFixture _dbFixture; private readonly CancellationTokenSource _completedCts; private readonly EnvironmentLockManager _lockManager; private readonly IMemoryCache _memoryCache; @@ -27,7 +26,7 @@ public CrmClientFixture(ServiceClient serviceClient, DbFixture dbFixture, IConfi Clock = new Clock(); Configuration = configuration; _baseServiceClient = serviceClient; - _dbFixture = dbFixture; + DbFixture = dbFixture; _completedCts = new CancellationTokenSource(); _lockManager = new EnvironmentLockManager(Configuration); _lockManager.AcquireLock(_completedCts.Token); @@ -42,6 +41,8 @@ public CrmClientFixture(ServiceClient serviceClient, DbFixture dbFixture, IConfi public IConfiguration Configuration { get; } + public DbFixture DbFixture { get; } + public CrmQueryDispatcher CreateQueryDispatcher() => new CrmQueryDispatcher(CreateQueryServiceProvider(_baseServiceClient, _referenceDataCache), serviceClientName: null); @@ -49,17 +50,17 @@ public CrmQueryDispatcher CreateQueryDispatcher() => /// Creates a scope that owns an implementation of that tracks the entities created through it. /// When is called the created entities will be deleted from CRM. /// - public TestDataScope CreateTestDataScope() => new( + public TestDataScope CreateTestDataScope(bool withSync = false) => new( _baseServiceClient, orgService => new DataverseAdapter(orgService, Clock, _memoryCache, _trnGenerationApiClient), orgService => new CrmQueryDispatcher(CreateQueryServiceProvider(orgService, _referenceDataCache), serviceClientName: null), orgService => TestData.CreateWithCustomTrnGeneration( - _dbFixture.GetDbContextFactory(), + DbFixture.GetDbContextFactory(), orgService, _referenceDataCache, Clock, () => _trnGenerationApiClient.GenerateTrn(), - TestDataSyncConfiguration.NoSync()), + withSync ? TestDataSyncConfiguration.Sync(new(DbFixture.GetDataSource(), orgService, _referenceDataCache, Clock)) : TestDataSyncConfiguration.NoSync()), _memoryCache); public void Dispose() diff --git a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Dqt.CrmIntegrationTests/Services/DqtReporting/DqtReportingFixture.cs b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Dqt.CrmIntegrationTests/Services/DqtReporting/DqtReportingFixture.cs index 6aa9b8dd6..d8f9d1732 100644 --- a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Dqt.CrmIntegrationTests/Services/DqtReporting/DqtReportingFixture.cs +++ b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Dqt.CrmIntegrationTests/Services/DqtReporting/DqtReportingFixture.cs @@ -1,3 +1,4 @@ +using Medallion.Threading.FileSystem; using Microsoft.ApplicationInsights; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; @@ -23,15 +24,21 @@ public DqtReportingFixture(CrmClientFixture crmClientFixture) public IClock Clock => _crmClientFixture.Clock; + public DbFixture DbFixture => _crmClientFixture.DbFixture; + public string ReportingDbConnectionString { get; } + public string TrsDbReplicationSlotName { get; } = "dqt_rep_sync_slot_test"; + + public CrmClientFixture.TestDataScope CreateTestDataScope(bool withSync) => _crmClientFixture.CreateTestDataScope(withSync); + public Task PublishChangedItemsAndConsume(params IChangedItem[] changedItems) => WithService(async (service, changesObserver) => { await service.LoadEntityMetadata(); changesObserver.OnNext(changedItems); - var processTask = service.ProcessChangesForEntityType(Contact.EntityLogicalName, CancellationToken.None); + var processTask = service.ProcessCrmChangesForEntityType(Contact.EntityLogicalName, CancellationToken.None); changesObserver.OnCompleted(); await processTask; }); @@ -45,12 +52,16 @@ public async Task WithService(Func(); @@ -59,8 +70,10 @@ public async Task WithService(Func { private static readonly TimeSpan _dateTimeComparisonTolerance = TimeSpan.FromMilliseconds(500); @@ -150,6 +155,153 @@ public async Task ProcessChangesForEntityType_SameRecordMultipleTimesInBatch_Wri Assert.Equal(fixture.Clock.UtcNow, (DateTime)row["__Updated"]!, _dateTimeComparisonTolerance); } + [Fact(Skip = "Flaky on CI")] + public Task ProcessTrsChanges_NewRow_IsInsertedIntoDbWithCorrectValues() => ProcessTrsChangesSingle(async singleMessageConsumed => + { + // Arrange + await using var testDataScope = fixture.CreateTestDataScope(withSync: true); + + var mqProviderId = MandatoryQualificationProvider.All.First().MandatoryQualificationProviderId; + var mqSpecialism = MandatoryQualificationSpecialism.MultiSensory; + var mqStatus = MandatoryQualificationStatus.Passed; + var mqStartDate = new DateOnly(2022, 5, 1); + var mqEndDate = new DateOnly(2023, 4, 1); + + // Act + var createPersonResult = await testDataScope.TestData.CreatePerson(p => p + .WithMandatoryQualification(m => m + .WithProvider(mqProviderId) + .WithSpecialism(mqSpecialism) + .WithStartDate(mqStartDate) + .WithStatus(mqStatus, mqEndDate))); + var mq = createPersonResult.MandatoryQualifications.Single(); + + await singleMessageConsumed; + + // Assert + var row = await GetRowById("trs_qualifications", mq.QualificationId, idColumnName: "qualification_id"); + Assert.NotNull(row); + Assert.Equal(createPersonResult.PersonId, row["person_id"]); + Assert.Equal((int)QualificationType.MandatoryQualification, row["qualification_type"]); + Assert.Equal(mqProviderId, row["mq_provider_id"]); + Assert.Equal((int)mqSpecialism, row["mq_specialism"]); + Assert.Equal((int)mqStatus, row["mq_status"]); + Assert.Equal(mqStartDate, ConvertDateTimeColumn(row["start_date"])); + Assert.Equal(mqEndDate, ConvertDateTimeColumn(row["end_date"])); + Assert.Equal(fixture.Clock.UtcNow, (DateTime)row["__Inserted"]!, _dateTimeComparisonTolerance); + Assert.Equal(fixture.Clock.UtcNow, (DateTime)row["__Updated"]!, _dateTimeComparisonTolerance); + + static DateOnly? ConvertDateTimeColumn(object? value) => value is null ? null : DateOnly.FromDateTime((DateTime)value); + }); + + [Fact(Skip = "Flaky on CI")] + public async Task ProcessTrsChanges_ExistingRow_IsUpdatedInDbWithCorrectValues() + { + // Arrange + await using var testDataScope = fixture.CreateTestDataScope(withSync: true); + + var insertedTime = fixture.Clock.UtcNow.AddDays(-10); + + var mqProviderId = MandatoryQualificationProvider.All.First().MandatoryQualificationProviderId; + var mqSpecialism = MandatoryQualificationSpecialism.MultiSensory; + var mqStatus = MandatoryQualificationStatus.Passed; + var mqStartDate = new DateOnly(2022, 5, 1); + var mqEndDate = new DateOnly(2023, 4, 1); + + var createPersonResult = await testDataScope.TestData.CreatePerson(p => p.WithMandatoryQualification()); + var mq = createPersonResult.MandatoryQualifications.Single(); + + await InsertRow("trs_qualifications", new Dictionary() + { + { "qualification_id", mq.QualificationId }, + { "person_id", createPersonResult.PersonId }, + { "qualification_type", (int)QualificationType.MandatoryQualification }, + { "mq_provider_id", mq.ProviderId }, + { "mq_specialism", mq.Specialism }, + { "mq_status", mq.Status }, + { "start_date", mq.StartDate }, + { "end_date", mq.EndDate }, + { "__Inserted", insertedTime } + }); + + await ProcessTrsChangesSingle(async singleMessageConsumed => + { + // Act + await fixture.DbFixture.WithDbContext(async dbContext => + { + var qualification = await dbContext.MandatoryQualifications.SingleAsync(q => q.QualificationId == mq.QualificationId); + qualification.ProviderId = mqProviderId; + qualification.Specialism = mqSpecialism; + qualification.Status = mqStatus; + qualification.StartDate = mqStartDate; + qualification.EndDate = mqEndDate; + await dbContext.SaveChangesAsync(); + }); + + await singleMessageConsumed; + + // Assert + var row = await GetRowById("trs_qualifications", mq.QualificationId, idColumnName: "qualification_id"); + Assert.NotNull(row); + Assert.Equal(createPersonResult.PersonId, row["person_id"]); + Assert.Equal((int)QualificationType.MandatoryQualification, row["qualification_type"]); + Assert.Equal(mqProviderId, row["mq_provider_id"]); + Assert.Equal((int)mqSpecialism, row["mq_specialism"]); + Assert.Equal((int)mqStatus, row["mq_status"]); + Assert.Equal(mqStartDate, ConvertDateTimeColumn(row["start_date"])); + Assert.Equal(mqEndDate, ConvertDateTimeColumn(row["end_date"])); + Assert.Equal(fixture.Clock.UtcNow, (DateTime)row["__Updated"]!, _dateTimeComparisonTolerance); + }); + + static DateOnly? ConvertDateTimeColumn(object? value) => value is null ? null : DateOnly.FromDateTime((DateTime)value); + } + + private Task ProcessTrsChangesSingle(Func action) => fixture.WithService(async (service, _) => + { + try + { + // ProcessTrsChanges will call OnNext on any provided IObserver to signal when it's processed a message; + // this Subject provides that IObserver. + var replicationMessageSubject = new System.Reactive.Subjects.ReplaySubject(); + + using var cts = new CancellationTokenSource(); + + var processChangesTask = Task.Run(() => service.ProcessTrsChanges(observer: replicationMessageSubject, cancellationToken: cts.Token)); + + // We need to pass a Task to the `action` delegate that it can await so that it knows when we've consumed a single message from the replication stream. + // We also need to ensure any exceptions from ProcessTrsChanges are surfaced. + // The CancellationToken ensures that we're not waiting forever. + async Task ConsumeSingleMessage() + { + try + { + await replicationMessageSubject.FirstAsync().ToTask(cts.Token); + } + catch (TaskCanceledException) + { + await processChangesTask; + throw; + } + } + + cts.CancelAfter(15000); + + try + { + await action(ConsumeSingleMessage()); + } + finally + { + await processChangesTask.ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing); + cts.Cancel(); + } + } + finally + { + await fixture.DbFixture.DropReplicationSlot(fixture.TrsDbReplicationSlotName); + } + }); + private async Task AssertInDeleteLog(string entityLogicalName, Guid entityId, DateTime expectedDeleted) { using var sqlConnection = new SqlConnection(fixture.ReportingDbConnectionString); @@ -171,12 +323,12 @@ private async Task AssertInDeleteLog(string entityLogicalName, Guid entityId, Da Assert.Equal(expectedDeleted, deleted, _dateTimeComparisonTolerance); } - private async Task?> GetRowById(string tableName, Guid id) + private async Task?> GetRowById(string tableName, Guid id, string idColumnName = "id") { using var sqlConnection = new SqlConnection(fixture.ReportingDbConnectionString); await sqlConnection.OpenAsync(); - var cmd = new SqlCommand($"select * from {tableName} where id = @id"); + var cmd = new SqlCommand($"select * from {tableName} where {idColumnName} = @id"); cmd.Connection = sqlConnection; cmd.Parameters.Add(new SqlParameter("@id", id)); @@ -207,7 +359,7 @@ private async Task InsertRow(string tableName, IReadOnlyDictionary Services.GetRequiredService(); + public Task CreateReplicationSlot(string slot) => WithDbContext(dbContext => + dbContext.Database.ExecuteSqlAsync($"select * from pg_create_logical_replication_slot({slot}, 'pgoutput');")); + + public Task DropReplicationSlot(string slot) => WithDbContext(dbContext => + dbContext.Database.ExecuteSqlAsync($"select pg_drop_replication_slot({slot});")); + + public Task AdvanceReplicationSlotToCurrentWalLsn(string slot) => WithDbContext(dbContext => + dbContext.Database.ExecuteSqlAsync($"select * from pg_replication_slot_advance({slot}, pg_current_wal_lsn());")); + public TrsDbContext GetDbContext() => Services.GetRequiredService(); public IDbContextFactory GetDbContextFactory() => Services.GetRequiredService>();