SkillsFundingAgency · David-B-Read · Jun 6, 2024 · Jun 6, 2024 · Jun 6, 2024 · Jun 6, 2024
diff --git a/.gitignore b/.gitignore
@@ -9,6 +9,7 @@
 *.user
 *.userosscache
 *.sln.docstates
+local.settings.json
 
 # User-specific files (MonoDevelop/Xamarin Studio)
 *.userprefs

diff --git a/README.md b/README.md
@@ -26,9 +26,30 @@ Setup instructions: https://skillsfundingagency.atlassian.net/wiki/spaces/NDL/pa
 
 ### Config
 
-As detailed in: https://skillsfundingagency.atlassian.net/wiki/spaces/NDL/pages/644972941/Developer+Configuration+Settings
-
-Select the configuration for the Earning Events application
+To run locally, create a file `local.settings.json` in the `SFA.DAS.Payments.ScheduledJobs` project, containing the following:
+
+```
+{
+  "IsEncrypted": false,
+  "Values": {
+    "AzureWebJobsStorage": "UseDevelopmentStorage=true",
+    "FUNCTIONS_WORKER_RUNTIME": "dotnet-isolated",
+    "DataLockAuditDataCleanUpQueue": "{use queue name from your Payments V2 service bus namespace}",
+    "EarningAuditDataCleanUpQueue": "{use queue name from your Payments V2 service bus namespace}",
+    "FundingSourceAuditDataCleanUpQueue": "{use queue name from your Payments V2 service bus namespace}",
+    "LevyAccountSchedule": "0 6 * * *",
+    "RequiredPaymentAuditDataCleanUpQueue": "{use queue name from your Payments V2 service bus namespace}",
+    "ApprenticeshipValidationSchedule": "0 6 * * *",
+    "AuditDataCleanUpSchedule": "0 6 * * *",
+    "LevyAccountValidationSchedule": "0 6 * * *"
+  },
+  "ConnectionStrings": {
+    "ServiceBusConnectionString": "{use connection string from your Payments V2 service bus namespace}"
+  }
+}
+```
+
+The CRON settings in the above configuration are just examples, to change the frequency of the jobs to a time other than at 6am daily, refer to the website https://crontab.guru/ for the relevant CRON syntax.
 
 ## 🔗 External Dependencies
 

diff --git a/...s.ScheduledJobs.UnitTests/Monitoring/LevyAccountData/LevyAccountValidationServiceTests.cs b/...s.ScheduledJobs.UnitTests/Monitoring/LevyAccountData/LevyAccountValidationServiceTests.cs
@@ -60,15 +60,20 @@ public void Cleanup()
         }
 
         [Test]
-        public void Validate_Should_RaiseInvalidDataValidationEventWhenDasLevyAccountApiWrapperReturnsNullOrEmptyList()
+        public async Task Validate_Should_RaiseInvalidDataValidationEventWhenDasLevyAccountApiWrapperReturnsNullOrEmptyList()
         {
             accountApiWrapper.Setup(x => x.GetDasLevyAccountDetails())
                              .ReturnsAsync((List<LevyAccountModel>)null);
 
-            Func<Task> act = async () => await sut.Validate();
-
-            act.Should().NotThrow();
-
+            try
+            {
+                await sut.Validate();
+            }
+            catch (Exception ex)
+            {
+                Assert.Fail($"Exception thrown: {ex.Message}");
+            }
+
             telemetry.Verify(x => x.TrackEvent(It.Is<string>(s => s == "EmployerAccountReferenceData.Comparison.InvalidData"), It.IsAny<Dictionary<string, string>>(), null), Times.Once);
             telemetry.Verify(x => x.TrackEvent(It.IsAny<string>(), It.IsAny<Dictionary<string, double>>()), Times.Never);
         }
@@ -82,9 +87,14 @@ public async Task Validate_Should_RaiseInvalidDataValidationEventWhenGetPayments
             accountApiWrapper.Setup(x => x.GetDasLevyAccountDetails())
                              .ReturnsAsync(levyAccountBuilder.Build(1).ToList());
 
-            Func<Task> act = async () => await sut.Validate();
-
-            act.Should().NotThrow();
+            try
+            {
+                await sut.Validate();
+            }
+            catch (Exception ex)
+            {
+                Assert.Fail($"Exception thrown: {ex.Message}");
+            }
 
             telemetry.Verify(x => x.TrackEvent(It.Is<string>(s => s == "EmployerAccountReferenceData.Comparison.InvalidData"), It.IsAny<Dictionary<string, string>>(), null), Times.Once);
             telemetry.Verify(x => x.TrackEvent(It.IsAny<string>(), It.IsAny<Dictionary<string, double>>()), Times.Never);

diff --git a/src/SFA.DAS.Payments.ScheduledJobs.UnitTests/SFA.DAS.Payments.ScheduledJobs.UnitTests.csproj b/src/SFA.DAS.Payments.ScheduledJobs.UnitTests/SFA.DAS.Payments.ScheduledJobs.UnitTests.csproj
@@ -1,25 +1,24 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-    <PropertyGroup>
-        <TargetFramework>netcoreapp2.1</TargetFramework>
-
-        <IsPackable>false</IsPackable>
-    </PropertyGroup>
-
-    <ItemGroup>
-        <PackageReference Include="Autofac.Extras.Moq" Version="4.3.0" />
-        <PackageReference Include="AutoMoqCore" Version="1.0.0" />
-        <PackageReference Include="FluentAssertions" Version="5.9.0" />
-        <PackageReference Include="Microsoft.EntityFrameworkCore.InMemory" Version="2.2.1" />
-        <PackageReference Include="Moq" Version="4.10.0" />
-        <PackageReference Include="NUnit" Version="3.12.0" />
-        <PackageReference Include="NUnit3TestAdapter" Version="3.16.1" />
-        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.5.0" />
-        <PackageReference Include="SFA.DAS.Account.Api.Client" Version="1.6.1469" />
-    </ItemGroup>
-
-    <ItemGroup>
-      <ProjectReference Include="..\SFA.DAS.Payments.ScheduledJobs\SFA.DAS.Payments.ScheduledJobs.csproj" />
-    </ItemGroup>
-
-</Project>
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFramework>net6.0</TargetFramework>
+    <IsPackable>false</IsPackable>
+  </PropertyGroup>
+  <ItemGroup>
+	<PackageReference Include="Autofac" Version="7.0.1" />
+	<PackageReference Include="Autofac.Extras.Moq" Version="6.1.1" />
+    <PackageReference Include="AutoMoqCore" Version="1.0.0" />
+    <PackageReference Include="FluentAssertions" Version="6.12.0" />
+    <PackageReference Include="Moq" Version="4.18.4" />
+    <PackageReference Include="NUnit" Version="4.1.0" />
+    <PackageReference Include="NUnit3TestAdapter" Version="4.5.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.9.0" />
+    <PackageReference Include="SFA.DAS.Account.Api.Client" Version="1.6.1469" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore" Version="6.0.31" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore.InMemory" Version="6.0.31" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+    <PackageReference Include="SFA.DAS.Payments.Application" Version="2.0.0-prerelease-18" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\SFA.DAS.Payments.ScheduledJobs\SFA.DAS.Payments.ScheduledJobs.csproj" />
+  </ItemGroup>
+</Project>
diff --git a/src/SFA.DAS.Payments.ScheduledJobs.sln b/src/SFA.DAS.Payments.ScheduledJobs.sln
@@ -3,8 +3,6 @@ Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio Version 17
 VisualStudioVersion = 17.8.34330.188
 MinimumVisualStudioVersion = 10.0.40219.1
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{E176BAAB-30C6-4E7E-B534-EEE8E15DA8CE}"
-EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SFA.DAS.Payments.ScheduledJobs.UnitTests", "SFA.DAS.Payments.ScheduledJobs.UnitTests\SFA.DAS.Payments.ScheduledJobs.UnitTests.csproj", "{DA5ABC45-22D6-4661-BBED-53A322360612}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SFA.DAS.Payments.ScheduledJobs", "SFA.DAS.Payments.ScheduledJobs\SFA.DAS.Payments.ScheduledJobs.csproj", "{C80E98C8-C964-41DA-BF80-F8080C18BF89}"
@@ -27,10 +25,6 @@ Global
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
 	EndGlobalSection
-	GlobalSection(NestedProjects) = preSolution
-		{DA5ABC45-22D6-4661-BBED-53A322360612} = {E176BAAB-30C6-4E7E-B534-EEE8E15DA8CE}
-		{C80E98C8-C964-41DA-BF80-F8080C18BF89} = {E176BAAB-30C6-4E7E-B534-EEE8E15DA8CE}
-	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {04DA2764-13CE-4481-9053-E12803A0B9E0}
 	EndGlobalSection

diff --git a/src/SFA.DAS.Payments.ScheduledJobs/AuditDataCleanUp/AuditDataCleanUpService.cs b/src/SFA.DAS.Payments.ScheduledJobs/AuditDataCleanUp/AuditDataCleanUpService.cs
@@ -124,7 +124,7 @@ public async Task DataLockEventAuditDataCleanUp(SubmissionJobsToBeDeletedBatch b
 
         private async Task DeleteEarningEventData(IList<SqlParameter> sqlParameters, string sqlParamName, string paramValues)
         {
-            var earningEventPeriodCount = await dataContext.Database.ExecuteSqlCommandAsync(
+            var earningEventPeriodCount = await dataContext.Database.ExecuteSqlRawAsync(
                     $@"DELETE Payments2.EarningEventPeriod 
                        FROM Payments2.EarningEventPeriod AS EEP 
                            INNER JOIN Payments2.EarningEvent AS EE ON EEP.EarningEventId = EE.EventId 
@@ -133,7 +133,7 @@ WHERE EE.JobId IN ({sqlParamName})",
 
             paymentLogger.LogInfo($"DELETED {earningEventPeriodCount} earningEventPeriods for JobIds {paramValues}");
 
-            var earningEventPriceEpisodeCount = await dataContext.Database.ExecuteSqlCommandAsync(
+            var earningEventPriceEpisodeCount = await dataContext.Database.ExecuteSqlRawAsync(
                     $@"DELETE Payments2.EarningEventPriceEpisode 
                        FROM Payments2.EarningEventPriceEpisode AS EEPE 
                           INNER JOIN Payments2.EarningEvent AS EE ON EEPE.EarningEventId = EE.EventId 
@@ -142,7 +142,7 @@ WHERE EE.JobId IN ({sqlParamName})",
 
             paymentLogger.LogInfo($"DELETED {earningEventPriceEpisodeCount} earningEventPriceEpisodes for JobIds {paramValues}");
 
-            var earningEventCount = await dataContext.Database.ExecuteSqlCommandAsync(
+            var earningEventCount = await dataContext.Database.ExecuteSqlRawAsync(
                     $"DELETE Payments2.EarningEvent WHERE JobId IN ({sqlParamName})",
                     sqlParameters);
 
@@ -151,7 +151,7 @@ WHERE EE.JobId IN ({sqlParamName})",
 
         private async Task DeleteFundingSourceEvent(IList<SqlParameter> sqlParameters, string sqlParamName, string paramValues)
         {
-            var fundingSourceEventCount = await dataContext.Database.ExecuteSqlCommandAsync(
+            var fundingSourceEventCount = await dataContext.Database.ExecuteSqlRawAsync(
                     $"DELETE Payments2.FundingSourceEvent WHERE JobId IN ({sqlParamName})",
                     sqlParameters);
 
@@ -160,7 +160,7 @@ private async Task DeleteFundingSourceEvent(IList<SqlParameter> sqlParameters, s
 
         private async Task DeleteRequiredPaymentEvent(IList<SqlParameter> sqlParameters, string sqlParamName, string paramValues)
         {
-            var requiredPaymentEventCount = await dataContext.Database.ExecuteSqlCommandAsync(
+            var requiredPaymentEventCount = await dataContext.Database.ExecuteSqlRawAsync(
                     $"DELETE Payments2.RequiredPaymentEvent WHERE JobId IN ({sqlParamName})",
                     sqlParameters);
 
@@ -169,7 +169,7 @@ private async Task DeleteRequiredPaymentEvent(IList<SqlParameter> sqlParameters,
 
         private async Task DeleteDataLockEvent(IList<SqlParameter> sqlParameters, string sqlParamName, string paramValues)
         {
-            var dataLockEventNonPayablePeriodFailuresCount = await dataContext.Database.ExecuteSqlCommandAsync(
+            var dataLockEventNonPayablePeriodFailuresCount = await dataContext.Database.ExecuteSqlRawAsync(
                     $@"DELETE Payments2.DataLockEventNonPayablePeriodFailures 
                        FROM Payments2.DataLockEventNonPayablePeriodFailures AS DLENPPF 
                            INNER JOIN Payments2.DataLockEventNonPayablePeriod AS DLENPP ON DLENPPF.DataLockEventNonPayablePeriodId = DLENPP.DataLockEventNonPayablePeriodId 
@@ -179,7 +179,7 @@ WHERE DL.JobId IN ({sqlParamName})",
 
             paymentLogger.LogInfo($"DELETED {dataLockEventNonPayablePeriodFailuresCount} DataLockEventNonPayablePeriodFailures for JobIds {paramValues}");
 
-            var dataLockEventNonPayablePeriodCount = await dataContext.Database.ExecuteSqlCommandAsync(
+            var dataLockEventNonPayablePeriodCount = await dataContext.Database.ExecuteSqlRawAsync(
                     $@"DELETE Payments2.DataLockEventNonPayablePeriod 
                        FROM Payments2.DataLockEventNonPayablePeriod AS DLENPP
                            INNER JOIN Payments2.DataLockEvent AS DL ON DLENPP.DataLockEventId = DL.EventId
@@ -188,7 +188,7 @@ WHERE DL.JobId IN ({sqlParamName})",
 
             paymentLogger.LogInfo($"DELETED {dataLockEventNonPayablePeriodCount} DataLockEventNonPayablePeriods for JobIds {paramValues}");
 
-            var dataLockEventPayablePeriodCount = await dataContext.Database.ExecuteSqlCommandAsync(
+            var dataLockEventPayablePeriodCount = await dataContext.Database.ExecuteSqlRawAsync(
                     $@"DELETE Payments2.DataLockEventPayablePeriod 
                        FROM Payments2.DataLockEventPayablePeriod AS DLEPP
                            INNER JOIN Payments2.DataLockEvent AS DL ON DLEPP.DataLockEventId = DL.EventId
@@ -197,7 +197,7 @@ WHERE DL.JobId IN ({sqlParamName})",
 
             paymentLogger.LogInfo($"DELETED {dataLockEventPayablePeriodCount} DataLockEventPayablePeriods for JobIds {paramValues}");
 
-            var dataLockEventPriceEpisodeCount = await dataContext.Database.ExecuteSqlCommandAsync(
+            var dataLockEventPriceEpisodeCount = await dataContext.Database.ExecuteSqlRawAsync(
                     $@"DELETE Payments2.DataLockEventPriceEpisode 
                        FROM Payments2.DataLockEventPriceEpisode AS DLEPP
                           INNER JOIN Payments2.DataLockEvent AS DL ON DLEPP.DataLockEventId = DL.EventId
@@ -206,7 +206,7 @@ WHERE DL.JobId IN ({sqlParamName})",
 
             paymentLogger.LogInfo($"DELETED {dataLockEventPriceEpisodeCount} DataLockEventPriceEpisodes for JobIds {paramValues}");
 
-            var dataLockEventCount = await dataContext.Database.ExecuteSqlCommandAsync(
+            var dataLockEventCount = await dataContext.Database.ExecuteSqlRawAsync(
                     $"DELETE Payments2.DataLockEvent WHERE JobId IN ({sqlParamName})",
                     sqlParameters);
 
@@ -243,7 +243,7 @@ SELECT JobId FROM Payments2.DataLockEvent
                 SELECT JobId AS DcJobId FROM #JobDataToBeDeleted";
 
             return (await dataContext.SubmissionJobsToBeDeleted
-                .FromSql(selectJobsToBeDeleted,
+                .FromSqlRaw(selectJobsToBeDeleted,
                     new SqlParameter("collectionPeriod", collectionPeriod),
                     new SqlParameter("academicYear", academicYear))
                 .ToListAsync())

diff --git a/src/SFA.DAS.Payments.ScheduledJobs/AuditDataCleanUp/AuditDataCleanUpTrigger.cs b/src/SFA.DAS.Payments.ScheduledJobs/AuditDataCleanUp/AuditDataCleanUpTrigger.cs
@@ -1,31 +1,39 @@
 using System.Threading.Tasks;
 using AzureFunctions.Autofac;
 using Microsoft.AspNetCore.Http;
-using Microsoft.Azure.WebJobs;
-using Microsoft.Azure.WebJobs.Extensions.Http;
 using SFA.DAS.Payments.ScheduledJobs.Infrastructure.IoC;
+using Microsoft.Extensions.Logging;
+using Microsoft.Azure.Functions.Worker;
+using Microsoft.Azure.WebJobs;
 
 // ReSharper disable UnusedMember.Global
 
 namespace SFA.DAS.Payments.ScheduledJobs.AuditDataCleanUp
 {
     [DependencyInjectionConfig(typeof(DependencyInjectionConfig))]
-    public static class AuditDataCleanUpTrigger
+    public class AuditDataCleanUpTrigger
     {
-        [FunctionName("TimerTriggerAuditDataCleanup")]
-        public static async Task TimerTriggerAuditDataCleanup(
-            [TimerTrigger("%AuditDataCleanUpSchedule%", RunOnStartup = false)] TimerInfo timerInfo,
-            [Inject] IAuditDataCleanUpService auditDataCleanUpService)
+        private readonly ILogger _logger;
+        private readonly IAuditDataCleanUpService _auditDataCleanUpService;
+
+        public AuditDataCleanUpTrigger(ILogger<AuditDataCleanUpTrigger> logger, IAuditDataCleanUpService auditDataCleanUpService)
+        {
+            _logger = logger;
+            _auditDataCleanUpService = auditDataCleanUpService;
+        }
+
+        [Function("TimerTriggerAuditDataCleanup")]
+        public async Task TimerTriggerAuditDataCleanup(
+            [TimerTrigger("%AuditDataCleanUpSchedule%", RunOnStartup = false)] TimerInfo timerInfo)
         {
-            await auditDataCleanUpService.TriggerAuditDataCleanup();
+            await _auditDataCleanUpService.TriggerAuditDataCleanup();
         }
 
-        [FunctionName("HttpTriggerAuditDataCleanup")]
-        public static async Task HttpTriggerAuditDataCleanup(
-            [HttpTrigger(AuthorizationLevel.Function, "get", Route = null)] HttpRequest httpRequest,
-            [Inject] IAuditDataCleanUpService auditDataCleanUpService)
+        [Function("HttpTriggerAuditDataCleanup")]
+        public async Task HttpTriggerAuditDataCleanup(
+            [HttpTrigger(AuthorizationLevel.Function, "get", Route = null)] HttpRequest httpRequest)
         {
-            await auditDataCleanUpService.TriggerAuditDataCleanup();
+            await _auditDataCleanUpService.TriggerAuditDataCleanup();
         }
     }
 }
diff --git a/src/SFA.DAS.Payments.ScheduledJobs/AuditDataCleanUp/DataLockAuditDataCleanUp.cs b/src/SFA.DAS.Payments.ScheduledJobs/AuditDataCleanUp/DataLockAuditDataCleanUp.cs
@@ -1,22 +1,33 @@
 using System.Threading.Tasks;
 using AzureFunctions.Autofac;
-using Microsoft.Azure.WebJobs;
 using Newtonsoft.Json;
 using SFA.DAS.Payments.ScheduledJobs.Infrastructure.IoC;
+using Microsoft.Extensions.Logging;
+using Microsoft.Azure.Functions.Worker;
+using Microsoft.Azure.WebJobs;
+
 // ReSharper disable UnusedMember.Global
 
 namespace SFA.DAS.Payments.ScheduledJobs.AuditDataCleanUp
 {
     [DependencyInjectionConfig(typeof(DependencyInjectionConfig))]
-    public static class DataLockAuditDataCleanUp
+    public class DataLockAuditDataCleanUp
     {
-        [FunctionName("DataLockEventAuditDataCleanUp")]
-        public static async Task DataLockEventAuditDataCleanUp([ServiceBusTrigger("%DataLockAuditDataCleanUpQueue%", Connection = "ServiceBusConnectionString")] string message,
-                                                               [Inject] IAuditDataCleanUpService auditDataCleanUpService)
+        private readonly ILogger _logger;
+        private readonly IAuditDataCleanUpService _auditDataCleanUpService;
+
+        public DataLockAuditDataCleanUp(ILogger<DataLockAuditDataCleanUp> logger, IAuditDataCleanUpService auditDataCleanUpService)
+        {
+            _logger = logger;
+            _auditDataCleanUpService = auditDataCleanUpService;
+        }
+
+        [Function("DataLockEventAuditDataCleanUp")]
+        public async Task DataLockEventAuditDataCleanUp([ServiceBusTrigger("%DataLockAuditDataCleanUpQueue%", Connection = "ServiceBusConnectionString")] string message)
         {
             var batch = JsonConvert.DeserializeObject<SubmissionJobsToBeDeletedBatch>(message);
 
-            await auditDataCleanUpService.DataLockEventAuditDataCleanUp(batch);
+            await _auditDataCleanUpService.DataLockEventAuditDataCleanUp(batch);
         }
     }
 }
diff --git a/src/SFA.DAS.Payments.ScheduledJobs/AuditDataCleanUp/EarningAuditDataCleanUp.cs b/src/SFA.DAS.Payments.ScheduledJobs/AuditDataCleanUp/EarningAuditDataCleanUp.cs
@@ -1,22 +1,33 @@
 using System.Threading.Tasks;
 using AzureFunctions.Autofac;
-using Microsoft.Azure.WebJobs;
 using Newtonsoft.Json;
 using SFA.DAS.Payments.ScheduledJobs.Infrastructure.IoC;
+using Microsoft.Extensions.Logging;
+using Microsoft.Azure.Functions.Worker;
+using Microsoft.Azure.WebJobs;
+
 // ReSharper disable UnusedMember.Global
 
 namespace SFA.DAS.Payments.ScheduledJobs.AuditDataCleanUp
 {
     [DependencyInjectionConfig(typeof(DependencyInjectionConfig))]
-    public static class EarningAuditDataCleanUp
+    public class EarningAuditDataCleanUp
     {
-        [FunctionName("EarningEventAuditDataCleanUp")]
-        public static async Task EarningEventAuditDataCleanUp([ServiceBusTrigger("%EarningAuditDataCleanUpQueue%", Connection = "ServiceBusConnectionString")] string message,
-                                                              [Inject] IAuditDataCleanUpService auditDataCleanUpService)
+        private readonly ILogger _logger;
+        private readonly IAuditDataCleanUpService _auditDataCleanUpService;
+
+        public EarningAuditDataCleanUp(ILogger<EarningAuditDataCleanUp> logger, IAuditDataCleanUpService auditDataCleanUpService)
+        {
+            _logger = logger;
+            _auditDataCleanUpService = auditDataCleanUpService;
+        }
+
+        [Function("EarningEventAuditDataCleanUp")]
+        public async Task EarningEventAuditDataCleanUp([ServiceBusTrigger("%EarningAuditDataCleanUpQueue%", Connection = "ServiceBusConnectionString")] string message)
         {
             var batch = JsonConvert.DeserializeObject<SubmissionJobsToBeDeletedBatch>(message);
-            
-            await auditDataCleanUpService.EarningEventAuditDataCleanUp(batch);
+
+            await _auditDataCleanUpService.EarningEventAuditDataCleanUp(batch);
         }
     }
 }