Amend queries to return paged results

DFE-Digital · Aug 21, 2024 · aeec6db · aeec6db
1 parent 4744c0c
commit aeec6db
Show file tree

Hide file tree

Showing 10 changed files with 127 additions and 15 deletions.
diff --git a/TeachingRecordSystem/src/TeachingRecordSystem.Core/Dqt/CrmQueryDispatcher.cs b/TeachingRecordSystem/src/TeachingRecordSystem.Core/Dqt/CrmQueryDispatcher.cs
@@ -1,3 +1,4 @@
+using System.Runtime.CompilerServices;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.PowerPlatform.Dataverse.Client;
 
@@ -20,6 +21,24 @@ public async Task<TResult> ExecuteQuery<TResult>(ICrmQuery<TResult> query)
         return await wrappedHandler.Execute(query, organizationService);
     }
 
+    public async IAsyncEnumerable<TResult> ExecuteQuery<TResult>(IEnumerableCrmQuery<TResult> query, [EnumeratorCancellation] CancellationToken cancellationToken)
+    {
+        using var scope = serviceProvider.CreateScope();
+
+        var organizationService = scope.ServiceProvider.GetRequiredKeyedService<IOrganizationServiceAsync>(serviceClientName);
+
+        var handlerType = typeof(IEnumerableCrmQueryHandler<,>).MakeGenericType(query.GetType(), typeof(TResult));
+        var handler = scope.ServiceProvider.GetRequiredService(handlerType);
+
+        var wrapperHandlerType = typeof(EnumerableQueryHandler<,>).MakeGenericType(query.GetType(), typeof(TResult));
+        var wrappedHandler = (EnumerableQueryHandler<TResult>)Activator.CreateInstance(wrapperHandlerType, handler)!;
+
+        await foreach (var result in wrappedHandler.Execute(query, organizationService, cancellationToken))
+        {
+            yield return result;
+        }
+    }
+
     private abstract class QueryHandler<T>
     {
         public abstract Task<T> Execute(ICrmQuery<T> query, IOrganizationServiceAsync organizationService);
@@ -35,4 +54,21 @@ public override Task<TResult> Execute(
             return innerHandler.Execute((TQuery)query, organizationService);
         }
     }
+
+    private abstract class EnumerableQueryHandler<T>
+    {
+        public abstract IAsyncEnumerable<T> Execute(IEnumerableCrmQuery<T> query, IOrganizationServiceAsync organizationService, CancellationToken cancellationToken);
+    }
+
+    private class EnumerableQueryHandler<TQuery, TResult>(IEnumerableCrmQueryHandler<TQuery, TResult> innerHandler) : EnumerableQueryHandler<TResult>
+        where TQuery : IEnumerableCrmQuery<TResult>
+    {
+        public override IAsyncEnumerable<TResult> Execute(
+            IEnumerableCrmQuery<TResult> query,
+            IOrganizationServiceAsync organizationService,
+            CancellationToken cancellationToken)
+        {
+            return innerHandler.Execute((TQuery)query, organizationService, cancellationToken);
+        }
+    }
 }
diff --git a/TeachingRecordSystem/src/TeachingRecordSystem.Core/Dqt/ICrmQueryDispatcher.cs b/TeachingRecordSystem/src/TeachingRecordSystem.Core/Dqt/ICrmQueryDispatcher.cs
@@ -3,4 +3,5 @@ namespace TeachingRecordSystem.Core.Dqt;
 public interface ICrmQueryDispatcher
 {
     Task<TResult> ExecuteQuery<TResult>(ICrmQuery<TResult> query);
+    IAsyncEnumerable<TResult> ExecuteQuery<TResult>(IEnumerableCrmQuery<TResult> query, CancellationToken cancellationToken = default);
 }
diff --git a/TeachingRecordSystem/src/TeachingRecordSystem.Core/Dqt/IEnumerableCrmQuery.cs b/TeachingRecordSystem/src/TeachingRecordSystem.Core/Dqt/IEnumerableCrmQuery.cs
@@ -0,0 +1,5 @@
+namespace TeachingRecordSystem.Core.Dqt;
+
+public interface IEnumerableCrmQuery<TResult>
+{
+}
diff --git a/TeachingRecordSystem/src/TeachingRecordSystem.Core/Dqt/IEnumerableCrmQueryHandler.cs b/TeachingRecordSystem/src/TeachingRecordSystem.Core/Dqt/IEnumerableCrmQueryHandler.cs
@@ -0,0 +1,9 @@
+using Microsoft.PowerPlatform.Dataverse.Client;
+
+namespace TeachingRecordSystem.Core.Dqt;
+
+public interface IEnumerableCrmQueryHandler<TQuery, TResult>
+    where TQuery : IEnumerableCrmQuery<TResult>
+{
+    IAsyncEnumerable<TResult> Execute(TQuery query, IOrganizationServiceAsync organizationService, CancellationToken cancellationToken);
+}
diff --git a/...gRecordSystem/src/TeachingRecordSystem.Core/Dqt/Queries/GetNonOpenTaskAnnotationsQuery.cs b/...gRecordSystem/src/TeachingRecordSystem.Core/Dqt/Queries/GetNonOpenTaskAnnotationsQuery.cs
@@ -2,4 +2,4 @@
 
 namespace TeachingRecordSystem.Core.Dqt.Queries;
 
-public record GetNonOpenTaskAnnotationsQuery(string[] Subjects, DateTime ModifiedBefore, ColumnSet ColumnSet) : ICrmQuery<Annotation[]>;
+public record GetNonOpenTaskAnnotationsQuery(string[] Subjects, DateTime ModifiedBefore, ColumnSet ColumnSet) : IEnumerableCrmQuery<Annotation[]>;
diff --git a/...rdSystem/src/TeachingRecordSystem.Core/Dqt/Queries/GetResolvedIncidentAnnotationsQuery.cs b/...rdSystem/src/TeachingRecordSystem.Core/Dqt/Queries/GetResolvedIncidentAnnotationsQuery.cs
@@ -2,4 +2,4 @@
 
 namespace TeachingRecordSystem.Core.Dqt.Queries;
 
-public record GetResolvedIncidentAnnotationsQuery(Guid[] SubjectIds, DateTime ModifiedBefore, ColumnSet ColumnSet) : ICrmQuery<Annotation[]>;
+public record GetResolvedIncidentAnnotationsQuery(Guid[] SubjectIds, DateTime ModifiedBefore, ColumnSet ColumnSet) : IEnumerableCrmQuery<Annotation[]>;
diff --git a/...ystem/src/TeachingRecordSystem.Core/Dqt/QueryHandlers/GetNonOpenTaskAnnotationsHandler.cs b/...ystem/src/TeachingRecordSystem.Core/Dqt/QueryHandlers/GetNonOpenTaskAnnotationsHandler.cs
@@ -1,13 +1,18 @@
+using System.Runtime.CompilerServices;
 using Microsoft.PowerPlatform.Dataverse.Client;
+using Microsoft.Xrm.Sdk;
 using Microsoft.Xrm.Sdk.Messages;
 using Microsoft.Xrm.Sdk.Query;
 using TeachingRecordSystem.Core.Dqt.Queries;
 
 namespace TeachingRecordSystem.Core.Dqt.QueryHandlers;
 
-public class GetNonOpenTaskAnnotationsHandler : ICrmQueryHandler<GetNonOpenTaskAnnotationsQuery, Annotation[]>
+public class GetNonOpenTaskAnnotationsHandler : IEnumerableCrmQueryHandler<GetNonOpenTaskAnnotationsQuery, Annotation[]>
 {
-    public async Task<Annotation[]> Execute(GetNonOpenTaskAnnotationsQuery query, IOrganizationServiceAsync organizationService)
+    public async IAsyncEnumerable<Annotation[]> Execute(
+        GetNonOpenTaskAnnotationsQuery query,
+        IOrganizationServiceAsync organizationService,
+        [EnumeratorCancellation] CancellationToken cancellationToken)
     {
         var queryExpression = new QueryExpression()
         {
@@ -26,13 +31,33 @@ public async Task<Annotation[]> Execute(GetNonOpenTaskAnnotationsQuery query, IO
         taskLink.LinkCriteria.AddCondition(CrmTask.Fields.Subject, ConditionOperator.In, query.Subjects.Cast<object>().ToArray());
         queryExpression.LinkEntities.Add(taskLink);
 
+        queryExpression.PageInfo = new()
+        {
+            Count = 50,
+            PageNumber = 1
+        };
+
         var request = new RetrieveMultipleRequest()
         {
             Query = queryExpression
         };
 
-        var response = await organizationService.RetrieveMultipleAsync(queryExpression);
+        EntityCollection response;
+        do
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+
+            response = await organizationService.RetrieveMultipleAsync(queryExpression);
+
+            var annotations = response.Entities.Select(e => e.ToEntity<Annotation>()).ToArray();
+            if (annotations.Length > 0)
+            {
+                yield return annotations;
+            }
 
-        return response.Entities.Select(e => e.ToEntity<Annotation>()).ToArray();
+            queryExpression.PageInfo.PageNumber++;
+            queryExpression.PageInfo.PagingCookie = response.PagingCookie;
+        }
+        while (response.MoreRecords);
     }
 }
diff --git a/.../src/TeachingRecordSystem.Core/Dqt/QueryHandlers/GetResolvedIncidentAnnotationsHandler.cs b/.../src/TeachingRecordSystem.Core/Dqt/QueryHandlers/GetResolvedIncidentAnnotationsHandler.cs
@@ -1,13 +1,18 @@
+using System.Runtime.CompilerServices;
 using Microsoft.PowerPlatform.Dataverse.Client;
+using Microsoft.Xrm.Sdk;
 using Microsoft.Xrm.Sdk.Messages;
 using Microsoft.Xrm.Sdk.Query;
 using TeachingRecordSystem.Core.Dqt.Queries;
 
 namespace TeachingRecordSystem.Core.Dqt.QueryHandlers;
 
-public class GetResolvedIncidentAnnotationsHandler : ICrmQueryHandler<GetResolvedIncidentAnnotationsQuery, Annotation[]>
+public class GetResolvedIncidentAnnotationsHandler : IEnumerableCrmQueryHandler<GetResolvedIncidentAnnotationsQuery, Annotation[]>
 {
-    public async Task<Annotation[]> Execute(GetResolvedIncidentAnnotationsQuery query, IOrganizationServiceAsync organizationService)
+    public async IAsyncEnumerable<Annotation[]> Execute(
+        GetResolvedIncidentAnnotationsQuery query,
+        IOrganizationServiceAsync organizationService,
+        [EnumeratorCancellation] CancellationToken cancellationToken)
     {
         var queryExpression = new QueryExpression()
         {
@@ -34,13 +39,33 @@ public async Task<Annotation[]> Execute(GetResolvedIncidentAnnotationsQuery quer
         incidentLink.LinkCriteria.AddCondition(Incident.Fields.SubjectId, ConditionOperator.In, query.SubjectIds.Cast<object>().ToArray());
         documentLink.LinkEntities.Add(incidentLink);
 
+        queryExpression.PageInfo = new()
+        {
+            Count = 50,
+            PageNumber = 1
+        };
+
         var request = new RetrieveMultipleRequest()
         {
             Query = queryExpression
         };
 
-        var response = await organizationService.RetrieveMultipleAsync(queryExpression);
+        EntityCollection response;
+        do
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+
+            response = await organizationService.RetrieveMultipleAsync(queryExpression);
+
+            var annotations = response.Entities.Select(e => e.ToEntity<Annotation>()).ToArray();
+            if (annotations.Length > 0)
+            {
+                yield return annotations;
+            }
 
-        return response.Entities.Select(e => e.ToEntity<Annotation>()).ToArray();
+            queryExpression.PageInfo.PageNumber++;
+            queryExpression.PageInfo.PagingCookie = response.PagingCookie;
+        }
+        while (response.MoreRecords);
     }
 }
diff --git a/TeachingRecordSystem/src/TeachingRecordSystem.Core/Jobs/DeleteOldIncidentAttachmentsJob.cs b/TeachingRecordSystem/src/TeachingRecordSystem.Core/Jobs/DeleteOldIncidentAttachmentsJob.cs
@@ -16,13 +16,19 @@ public async Task Execute(CancellationToken cancellationToken)
 
         var modifiedBefore = clock.UtcNow.Subtract(_modifiedBeforeWindow);
 
-        var incidentAnnotations = await crmQueryDispatcher.ExecuteQuery(
-            new GetResolvedIncidentAnnotationsQuery(SubjectIds: [changeDateOfBirthSubject.Id, changeNameSubject.Id], modifiedBefore, ColumnSet: new()));
+        var annotationIds = new List<Guid>();
 
-        var taskAnnotations = await crmQueryDispatcher.ExecuteQuery(
-            new GetNonOpenTaskAnnotationsQuery(Subjects: [CreateTrnRequestTaskQuery.TaskSubject], modifiedBefore, ColumnSet: new()));
+        await foreach (var annotations in crmQueryDispatcher.ExecuteQuery(
+            new GetResolvedIncidentAnnotationsQuery(SubjectIds: [changeDateOfBirthSubject.Id, changeNameSubject.Id], modifiedBefore, ColumnSet: new()), cancellationToken))
+        {
+            annotationIds.AddRange(annotations.Select(i => i.AnnotationId!.Value));
+        }
 
-        var annotationIds = incidentAnnotations.Select(i => i.AnnotationId!.Value).Concat(taskAnnotations.Select(i => i.AnnotationId!.Value));
+        await foreach (var annotations in crmQueryDispatcher.ExecuteQuery(
+            new GetNonOpenTaskAnnotationsQuery(Subjects: [CreateTrnRequestTaskQuery.TaskSubject], modifiedBefore, ColumnSet: new()), cancellationToken))
+        {
+            annotationIds.AddRange(annotations.Select(i => i.AnnotationId!.Value));
+        }
 
         foreach (var annotationId in annotationIds)
         {

diff --git a/TeachingRecordSystem/tests/TeachingRecordSystem.TestCommon/CrmQueryDispatcherSpy.cs b/TeachingRecordSystem/tests/TeachingRecordSystem.TestCommon/CrmQueryDispatcherSpy.cs
@@ -47,4 +47,9 @@ public async Task<TResult> ExecuteQuery<TResult>(ICrmQuery<TResult> query)
         spy.RegisterQuery(query, result);
         return result;
     }
+
+    public IAsyncEnumerable<TResult> ExecuteQuery<TResult>(IEnumerableCrmQuery<TResult> query, CancellationToken cancellationToken = default)
+    {
+        return innerDispatcher.ExecuteQuery(query, cancellationToken);
+    }
 }
Original file line number	Diff line number	Diff line change
Expand Up		@@ -2,4 +2,4 @@

		namespace TeachingRecordSystem.Core.Dqt.Queries;

		public record GetNonOpenTaskAnnotationsQuery(string[] Subjects, DateTime ModifiedBefore, ColumnSet ColumnSet) : ICrmQuery<Annotation[]>;
		public record GetNonOpenTaskAnnotationsQuery(string[] Subjects, DateTime ModifiedBefore, ColumnSet ColumnSet) : IEnumerableCrmQuery<Annotation[]>;