Add retries to TrsDataSyncService and DqtReportingService (#1117)

DFE-Digital · Jan 29, 2024 · ff53e06 · ff53e06
1 parent 16ec56b
commit ff53e06
Show file tree

Hide file tree

Showing 4 changed files with 25 additions and 3 deletions.
diff --git a/TeachingRecordSystem/Directory.Packages.props b/TeachingRecordSystem/Directory.Packages.props
@@ -63,6 +63,7 @@
     <PackageVersion Include="NSwag.Examples" Version="1.0.11" />
     <PackageVersion Include="Optional" Version="4.0.0" />
     <PackageVersion Include="PdfSharpCore" Version="1.3.62" />
+    <PackageVersion Include="Polly.Core" Version="8.2.1" />
     <PackageVersion Include="RedisRateLimiting.AspNetCore" Version="1.1.0" />
     <PackageVersion Include="Respawn" Version="6.1.0" />
     <PackageVersion Include="Scrutor" Version="4.2.2" />
@@ -86,4 +87,4 @@
     <PackageVersion Include="Xunit.DependencyInjection" Version="8.9.0" />
     <PackageVersion Include="xunit.runner.visualstudio" Version="2.5.5" />
   </ItemGroup>
-</Project>
+</Project>
diff --git a/...ngRecordSystem/src/TeachingRecordSystem.Core/Services/DqtReporting/DqtReportingService.cs b/...ngRecordSystem/src/TeachingRecordSystem.Core/Services/DqtReporting/DqtReportingService.cs
@@ -10,6 +10,7 @@
 using Microsoft.Xrm.Sdk;
 using Microsoft.Xrm.Sdk.Metadata;
 using Microsoft.Xrm.Sdk.Query;
+using Polly;
 using TeachingRecordSystem.Core.Dqt;
 using TeachingRecordSystem.Core.Dqt.Queries;
 using TeachingRecordSystem.Core.Services.CrmEntityChanges;
@@ -27,6 +28,15 @@ public partial class DqtReportingService : BackgroundService
     private const int MaxUpsertBatchSize = 100;
     private const int MaxEntityTypesToProcessConcurrently = 10;
 
+    private static readonly ResiliencePipeline _resiliencePipeline = new ResiliencePipelineBuilder()
+        .AddRetry(new Polly.Retry.RetryStrategyOptions()
+        {
+            BackoffType = DelayBackoffType.Exponential,
+            Delay = TimeSpan.FromSeconds(30),
+            MaxRetryAttempts = 3
+        })
+        .Build();
+
     private readonly DqtReportingOptions _options;
     private readonly ICrmEntityChangesService _crmEntityChangesService;
     private readonly ICrmQueryDispatcher _crmQueryDispatcher;
@@ -61,7 +71,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
         {
             try
             {
-                await ProcessChanges(stoppingToken);
+                await _resiliencePipeline.ExecuteAsync(async ct => await ProcessChanges(ct), stoppingToken);
             }
             catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
             {

diff --git a/...hingRecordSystem/src/TeachingRecordSystem.Core/Services/TrsDataSync/TrsDataSyncService.cs b/...hingRecordSystem/src/TeachingRecordSystem.Core/Services/TrsDataSync/TrsDataSyncService.cs
@@ -4,6 +4,7 @@
 using Microsoft.Extensions.Options;
 using Microsoft.Xrm.Sdk;
 using Microsoft.Xrm.Sdk.Query;
+using Polly;
 using TeachingRecordSystem.Core.Services.CrmEntityChanges;
 
 namespace TeachingRecordSystem.Core.Services.TrsDataSync;
@@ -17,6 +18,15 @@ public class TrsDataSyncService(
     public const string CrmClientName = "TrsDataSync";
     private const string ChangesKeyPrefix = "TrsDataSync";
 
+    private static readonly ResiliencePipeline _resiliencePipeline = new ResiliencePipelineBuilder()
+        .AddRetry(new Polly.Retry.RetryStrategyOptions()
+        {
+            BackoffType = DelayBackoffType.Exponential,
+            Delay = TimeSpan.FromSeconds(30),
+            MaxRetryAttempts = 3
+        })
+        .Build();
+
     private const int PageSize = 1000;
 
     protected override async Task ExecuteAsync(CancellationToken stoppingToken)
@@ -27,7 +37,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
         {
             try
             {
-                await ProcessChanges(stoppingToken);
+                await _resiliencePipeline.ExecuteAsync(async ct => await ProcessChanges(ct), stoppingToken);
             }
             catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
             {

diff --git a/TeachingRecordSystem/src/TeachingRecordSystem.Core/TeachingRecordSystem.Core.csproj b/TeachingRecordSystem/src/TeachingRecordSystem.Core/TeachingRecordSystem.Core.csproj
@@ -74,6 +74,7 @@
     <PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" />
     <PackageReference Include="Optional" />
     <PackageReference Include="PdfSharpCore" />
+    <PackageReference Include="Polly.Core" />
     <PackageReference Include="Scrutor" />
     <PackageReference Include="System.Net.Http.Json" />
     <PackageReference Include="System.Reactive" />