Sync changes from TRS DB into DQT reporting database (#1451)

.github/workflows/pr.yml

@@ -20,7 +20,7 @@ jobs:
 
     services:
       postgres:
-        image: postgres
+        image: postgres:15
         env:
           POSTGRES_PASSWORD: trs
           POSTGRES_DB: trs
@@ -192,11 +192,12 @@ jobs:
 
     services:
       postgres:
-        image: postgres
+        image: postgres:15
         env:
           POSTGRES_PASSWORD: trs
           POSTGRES_DB: trs
         options: >-
+          --name postgres
           --health-cmd pg_isready
           --health-interval 10s
           --health-timeout 5s
@@ -218,6 +219,14 @@ jobs:
           --health-retries 3
 
     steps:
+    - name: Set postgres wal_level to logical
+      run: |
+        docker exec -i postgres bash << EOF
+        echo "wal_level = logical" >> /var/lib/postgresql/data/postgresql.conf
+        EOF
+
+        docker restart --time 0 postgres
+
     - uses: actions/checkout@v4
 
     - uses: extractions/setup-just@v2

README.md

@@ -103,18 +103,32 @@ The databases will be created automatically when running the tests.
 
 #### DQT Reporting database setup
 
-This solution contains a service that synchronises changes from CRM into a SQL Server database used for reporting (this replaces the now-deprecated Data Export Service). By default this is disabled for local development. For the tests to pass, you will need a test database and a connection string defined in user secrets e.g.
+This solution contains a service that synchronises changes from CRM into a SQL Server database used for reporting (this replaces the now-deprecated Data Export Service).
+It also synronises selected tables from TRS.
+By default this is disabled for local development. For the tests to pass, you will need a test database and a connection string defined in user secrets e.g.
 ```shell
 just set-tests-secret DqtReporting:ReportingDbConnectionString "Data Source=(local);Initial Catalog=DqtReportingTests;Integrated Security=Yes;TrustServerCertificate=True"
 ```
 
-To run the service locally, override the configuration option to run the service and ensure a connection string is provided e.g.
+Your postgres server's `wal_level` must be set to `logical`:
+```
+ALTER SYSTEM SET wal_level = logical;
+```
+You will have to restart the server after amending this configuration.
+
+To run the service locally override the configuration option to run the service and ensure a connection string is provided e.g.
 ```shell
 just set-secret DqtReporting:RunService true
 just set-secret DqtReporting:ReportingDbConnectionString "Data Source=(local);Initial Catalog=DqtReporting;Integrated Security=Yes;TrustServerCertificate=True"
 ```
 The service will now run as a background service of the `Worker` project.
 
+It is a good idea to remove the replication slot when you're not working on this service to avoid a backlog on unprocessed changes accumulating in postgres.
+```shell
+just set-secret DqtReporting:RunService false
+just cli drop-dqt-reporting-replication-slot
+```
+
 
 ### Admin user setup
 

TeachingRecordSystem/src/TeachingRecordSystem.Cli/Commands.DropDqtReportingReplicationSlot.cs

@@ -0,0 +1,42 @@
+using Npgsql;
+using TeachingRecordSystem.Core.DataStore.Postgres;
+using TeachingRecordSystem.Core.Services.DqtReporting;
+
+namespace TeachingRecordSystem.Cli;
+
+public partial class Commands
+{
+    public static Command CreateDropDqtReportingReplicationSlotCommand(IConfiguration configuration)
+    {
+        var connectionStringOption = new Option<string>("--connection-string") { IsRequired = true };
+
+        var configuredConnectionString = configuration.GetConnectionString("DefaultConnection");
+        if (configuredConnectionString is not null)
+        {
+            connectionStringOption.SetDefaultValue(configuredConnectionString);
+        }
+
+        var command = new Command("drop-dqt-reporting-replication-slot", "Drops the logical replication slot for the DQT Reporting Service.")
+        {
+            connectionStringOption
+        };
+
+        command.SetHandler(
+            async (string connectionString) =>
+            {
+                using var dbContext = TrsDbContext.Create(connectionString, commandTimeout: (int)TimeSpan.FromMinutes(10).TotalSeconds);
+
+                // Ensure the user has the replication permission
+                var user = new NpgsqlConnectionStringBuilder(connectionString).Username;
+#pragma warning disable EF1002 // Risk of vulnerability to SQL injection.
+                await dbContext.Database.ExecuteSqlRawAsync($"alter user {user} with replication");
+#pragma warning restore EF1002 // Risk of vulnerability to SQL injection.
+
+                await dbContext.Database.ExecuteSqlRawAsync(
+                    $"select pg_drop_replication_slot('{DqtReportingOptions.DefaultTrsDbReplicationSlotName}');");
+            },
+            connectionStringOption);
+
+        return command;
+    }
+}

TeachingRecordSystem/src/TeachingRecordSystem.Cli/Program.cs

@@ -15,6 +15,7 @@
     Commands.CreateCreateAdminCommand(configuration),
     Commands.CreateSyncPersonCommand(configuration),
     Commands.CreateGenerateKeyCommand(configuration),
+    Commands.CreateDropDqtReportingReplicationSlotCommand(configuration),
 };
 
 return await rootCommand.InvokeAsync(args);