From 5187ed29cc23a3cb4d77260f5ca784c1cbb897bf Mon Sep 17 00:00:00 2001
From: James Gunn <james@gunn.io>
Date: Thu, 2 Jan 2025 11:11:00 +0000
Subject: [PATCH] Add jobs for syncing ITT and QTS audit records to blob
 storage (#1781)

Given how long the audit sync for the `contact` entity took, I'm getting
these in now well before we need the data.
---
 .../Jobs/SyncAllDqtIttAuditsJob.cs            | 77 +++++++++++++++++++
 .../Jobs/SyncAllDqtQtsAuditsJob.cs            | 77 +++++++++++++++++++
 2 files changed, 154 insertions(+)
 create mode 100644 TeachingRecordSystem/src/TeachingRecordSystem.Core/Jobs/SyncAllDqtIttAuditsJob.cs
 create mode 100644 TeachingRecordSystem/src/TeachingRecordSystem.Core/Jobs/SyncAllDqtQtsAuditsJob.cs

diff --git a/TeachingRecordSystem/src/TeachingRecordSystem.Core/Jobs/SyncAllDqtIttAuditsJob.cs b/TeachingRecordSystem/src/TeachingRecordSystem.Core/Jobs/SyncAllDqtIttAuditsJob.cs
new file mode 100644
index 000000000..9a82778b0
--- /dev/null
+++ b/TeachingRecordSystem/src/TeachingRecordSystem.Core/Jobs/SyncAllDqtIttAuditsJob.cs
@@ -0,0 +1,77 @@
+using System.ServiceModel;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using Microsoft.PowerPlatform.Dataverse.Client;
+using Microsoft.Xrm.Sdk;
+using Microsoft.Xrm.Sdk.Query;
+using TeachingRecordSystem.Core.Dqt;
+using TeachingRecordSystem.Core.Services.TrsDataSync;
+
+namespace TeachingRecordSystem.Core.Jobs;
+
+public class SyncAllDqtIttAuditsJob(
+    [FromKeyedServices(TrsDataSyncService.CrmClientName)] IOrganizationServiceAsync2 organizationService,
+    TrsDataSyncHelper trsDataSyncHelper,
+    ILogger<SyncAllDqtIttAuditsJob> logger)
+{
+    public async Task ExecuteAsync(CancellationToken cancellationToken)
+    {
+        const int pageSize = 1000;
+
+        var query = new QueryExpression(dfeta_initialteachertraining.EntityLogicalName)
+        {
+            ColumnSet = new ColumnSet(),
+            Orders =
+            {
+                new OrderExpression("createdon", OrderType.Ascending),
+                new OrderExpression(dfeta_initialteachertraining.PrimaryIdAttribute, OrderType.Ascending)
+            },
+            PageInfo = new PagingInfo()
+            {
+                Count = pageSize,
+                PageNumber = 1
+            }
+        };
+
+        var fetched = 0;
+
+        while (true)
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+
+            EntityCollection result;
+            try
+            {
+                result = await organizationService.RetrieveMultipleAsync(query);
+            }
+            catch (FaultException<OrganizationServiceFault> fex) when (fex.IsCrmRateLimitException(out var retryAfter))
+            {
+                await Task.Delay(retryAfter, cancellationToken);
+                continue;
+            }
+
+            fetched += result.Entities.Count;
+
+            await trsDataSyncHelper.SyncAuditAsync(
+                dfeta_initialteachertraining.EntityLogicalName,
+                result.Entities.Select(e => e.Id),
+                skipIfExists: true,
+                cancellationToken);
+
+            if (fetched > 0 && fetched % 50000 == 0)
+            {
+                logger.LogWarning("Synced {Count} dfeta_initialteachertraining audit records.", fetched);
+            }
+
+            if (result.MoreRecords)
+            {
+                query.PageInfo.PageNumber++;
+                query.PageInfo.PagingCookie = result.PagingCookie;
+            }
+            else
+            {
+                break;
+            }
+        }
+    }
+}
diff --git a/TeachingRecordSystem/src/TeachingRecordSystem.Core/Jobs/SyncAllDqtQtsAuditsJob.cs b/TeachingRecordSystem/src/TeachingRecordSystem.Core/Jobs/SyncAllDqtQtsAuditsJob.cs
new file mode 100644
index 000000000..60052f1a5
--- /dev/null
+++ b/TeachingRecordSystem/src/TeachingRecordSystem.Core/Jobs/SyncAllDqtQtsAuditsJob.cs
@@ -0,0 +1,77 @@
+using System.ServiceModel;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using Microsoft.PowerPlatform.Dataverse.Client;
+using Microsoft.Xrm.Sdk;
+using Microsoft.Xrm.Sdk.Query;
+using TeachingRecordSystem.Core.Dqt;
+using TeachingRecordSystem.Core.Services.TrsDataSync;
+
+namespace TeachingRecordSystem.Core.Jobs;
+
+public class SyncAllDqtQtsAuditsJob(
+    [FromKeyedServices(TrsDataSyncService.CrmClientName)] IOrganizationServiceAsync2 organizationService,
+    TrsDataSyncHelper trsDataSyncHelper,
+    ILogger<SyncAllDqtQtsAuditsJob> logger)
+{
+    public async Task ExecuteAsync(CancellationToken cancellationToken)
+    {
+        const int pageSize = 1000;
+
+        var query = new QueryExpression(dfeta_qtsregistration.EntityLogicalName)
+        {
+            ColumnSet = new ColumnSet(),
+            Orders =
+            {
+                new OrderExpression("createdon", OrderType.Ascending),
+                new OrderExpression(dfeta_qtsregistration.PrimaryIdAttribute, OrderType.Ascending)
+            },
+            PageInfo = new PagingInfo()
+            {
+                Count = pageSize,
+                PageNumber = 1
+            }
+        };
+
+        var fetched = 0;
+
+        while (true)
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+
+            EntityCollection result;
+            try
+            {
+                result = await organizationService.RetrieveMultipleAsync(query);
+            }
+            catch (FaultException<OrganizationServiceFault> fex) when (fex.IsCrmRateLimitException(out var retryAfter))
+            {
+                await Task.Delay(retryAfter, cancellationToken);
+                continue;
+            }
+
+            fetched += result.Entities.Count;
+
+            await trsDataSyncHelper.SyncAuditAsync(
+                dfeta_qtsregistration.EntityLogicalName,
+                result.Entities.Select(e => e.Id),
+                skipIfExists: true,
+                cancellationToken);
+
+            if (fetched > 0 && fetched % 50000 == 0)
+            {
+                logger.LogWarning("Synced {Count} dfeta_qtsregistration audit records.", fetched);
+            }
+
+            if (result.MoreRecords)
+            {
+                query.PageInfo.PageNumber++;
+                query.PageInfo.PagingCookie = result.PagingCookie;
+            }
+            else
+            {
+                break;
+            }
+        }
+    }
+}