diff --git a/.gitignore b/.gitignore index dfcfd56..0f58556 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ *.user *.userosscache *.sln.docstates +local.settings.json # User-specific files (MonoDevelop/Xamarin Studio) *.userprefs diff --git a/README.md b/README.md index eb7204b..e706ace 100644 --- a/README.md +++ b/README.md @@ -26,9 +26,30 @@ Setup instructions: https://skillsfundingagency.atlassian.net/wiki/spaces/NDL/pa ### Config -As detailed in: https://skillsfundingagency.atlassian.net/wiki/spaces/NDL/pages/644972941/Developer+Configuration+Settings - -Select the configuration for the Earning Events application +To run locally, create a file `local.settings.json` in the `SFA.DAS.Payments.ScheduledJobs` project, containing the following: + +``` +{ + "IsEncrypted": false, + "Values": { + "AzureWebJobsStorage": "UseDevelopmentStorage=true", + "FUNCTIONS_WORKER_RUNTIME": "dotnet-isolated", + "DataLockAuditDataCleanUpQueue": "{use queue name from your Payments V2 service bus namespace}", + "EarningAuditDataCleanUpQueue": "{use queue name from your Payments V2 service bus namespace}", + "FundingSourceAuditDataCleanUpQueue": "{use queue name from your Payments V2 service bus namespace}", + "LevyAccountSchedule": "0 6 * * *", + "RequiredPaymentAuditDataCleanUpQueue": "{use queue name from your Payments V2 service bus namespace}", + "ApprenticeshipValidationSchedule": "0 6 * * *", + "AuditDataCleanUpSchedule": "0 6 * * *", + "LevyAccountValidationSchedule": "0 6 * * *" + }, + "ConnectionStrings": { + "ServiceBusConnectionString": "{use connection string from your Payments V2 service bus namespace}" + } +} +``` + +The CRON settings in the above configuration are just examples, to change the frequency of the jobs to a time other than at 6am daily, refer to the website https://crontab.guru/ for the relevant CRON syntax. ## 🔗 External Dependencies diff --git a/src/SFA.DAS.Payments.ScheduledJobs.UnitTests/Monitoring/LevyAccountData/LevyAccountValidationServiceTests.cs b/src/SFA.DAS.Payments.ScheduledJobs.UnitTests/Monitoring/LevyAccountData/LevyAccountValidationServiceTests.cs index 5130f51..b63859c 100644 --- a/src/SFA.DAS.Payments.ScheduledJobs.UnitTests/Monitoring/LevyAccountData/LevyAccountValidationServiceTests.cs +++ b/src/SFA.DAS.Payments.ScheduledJobs.UnitTests/Monitoring/LevyAccountData/LevyAccountValidationServiceTests.cs @@ -60,15 +60,20 @@ public void Cleanup() } [Test] - public void Validate_Should_RaiseInvalidDataValidationEventWhenDasLevyAccountApiWrapperReturnsNullOrEmptyList() + public async Task Validate_Should_RaiseInvalidDataValidationEventWhenDasLevyAccountApiWrapperReturnsNullOrEmptyList() { accountApiWrapper.Setup(x => x.GetDasLevyAccountDetails()) .ReturnsAsync((List)null); - Func act = async () => await sut.Validate(); - - act.Should().NotThrow(); - + try + { + await sut.Validate(); + } + catch (Exception ex) + { + Assert.Fail($"Exception thrown: {ex.Message}"); + } + telemetry.Verify(x => x.TrackEvent(It.Is(s => s == "EmployerAccountReferenceData.Comparison.InvalidData"), It.IsAny>(), null), Times.Once); telemetry.Verify(x => x.TrackEvent(It.IsAny(), It.IsAny>()), Times.Never); } @@ -82,9 +87,14 @@ public async Task Validate_Should_RaiseInvalidDataValidationEventWhenGetPayments accountApiWrapper.Setup(x => x.GetDasLevyAccountDetails()) .ReturnsAsync(levyAccountBuilder.Build(1).ToList()); - Func act = async () => await sut.Validate(); - - act.Should().NotThrow(); + try + { + await sut.Validate(); + } + catch (Exception ex) + { + Assert.Fail($"Exception thrown: {ex.Message}"); + } telemetry.Verify(x => x.TrackEvent(It.Is(s => s == "EmployerAccountReferenceData.Comparison.InvalidData"), It.IsAny>(), null), Times.Once); telemetry.Verify(x => x.TrackEvent(It.IsAny(), It.IsAny>()), Times.Never); diff --git a/src/SFA.DAS.Payments.ScheduledJobs.UnitTests/SFA.DAS.Payments.ScheduledJobs.UnitTests.csproj b/src/SFA.DAS.Payments.ScheduledJobs.UnitTests/SFA.DAS.Payments.ScheduledJobs.UnitTests.csproj index 362d00f..f248947 100644 --- a/src/SFA.DAS.Payments.ScheduledJobs.UnitTests/SFA.DAS.Payments.ScheduledJobs.UnitTests.csproj +++ b/src/SFA.DAS.Payments.ScheduledJobs.UnitTests/SFA.DAS.Payments.ScheduledJobs.UnitTests.csproj @@ -1,25 +1,29 @@ - - - - netcoreapp2.1 - - false - - - - - - - - - - - - - - - - - - - + + + net6.0 + false + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/SFA.DAS.Payments.ScheduledJobs.sln b/src/SFA.DAS.Payments.ScheduledJobs.sln index 79d8012..97d18f3 100644 --- a/src/SFA.DAS.Payments.ScheduledJobs.sln +++ b/src/SFA.DAS.Payments.ScheduledJobs.sln @@ -3,8 +3,6 @@ Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 17 VisualStudioVersion = 17.8.34330.188 MinimumVisualStudioVersion = 10.0.40219.1 -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{E176BAAB-30C6-4E7E-B534-EEE8E15DA8CE}" -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SFA.DAS.Payments.ScheduledJobs.UnitTests", "SFA.DAS.Payments.ScheduledJobs.UnitTests\SFA.DAS.Payments.ScheduledJobs.UnitTests.csproj", "{DA5ABC45-22D6-4661-BBED-53A322360612}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SFA.DAS.Payments.ScheduledJobs", "SFA.DAS.Payments.ScheduledJobs\SFA.DAS.Payments.ScheduledJobs.csproj", "{C80E98C8-C964-41DA-BF80-F8080C18BF89}" @@ -27,10 +25,6 @@ Global GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection - GlobalSection(NestedProjects) = preSolution - {DA5ABC45-22D6-4661-BBED-53A322360612} = {E176BAAB-30C6-4E7E-B534-EEE8E15DA8CE} - {C80E98C8-C964-41DA-BF80-F8080C18BF89} = {E176BAAB-30C6-4E7E-B534-EEE8E15DA8CE} - EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {04DA2764-13CE-4481-9053-E12803A0B9E0} EndGlobalSection diff --git a/src/SFA.DAS.Payments.ScheduledJobs/AuditDataCleanUp/AuditDataCleanUpService.cs b/src/SFA.DAS.Payments.ScheduledJobs/AuditDataCleanUp/AuditDataCleanUpService.cs index 0c62b0f..fce9a3d 100644 --- a/src/SFA.DAS.Payments.ScheduledJobs/AuditDataCleanUp/AuditDataCleanUpService.cs +++ b/src/SFA.DAS.Payments.ScheduledJobs/AuditDataCleanUp/AuditDataCleanUpService.cs @@ -1,10 +1,10 @@ using System; using System.Collections.Generic; -using System.Data.SqlClient; using System.Linq; using System.Threading.Tasks; -using NServiceBus; +using Microsoft.Data.SqlClient; using Microsoft.EntityFrameworkCore; +using NServiceBus; using SFA.DAS.Payments.Application.Infrastructure.Logging; using SFA.DAS.Payments.Application.Messaging; using SFA.DAS.Payments.Application.Repositories; @@ -20,7 +20,8 @@ public class AuditDataCleanUpService : IAuditDataCleanUpService private readonly IScheduledJobsConfiguration config; private readonly IPaymentLogger paymentLogger; - public AuditDataCleanUpService(IScheduledJobsConfiguration config, IPaymentsDataContext dataContext, IEndpointInstanceFactory endpointInstanceFactory, IPaymentLogger paymentLogger) + public AuditDataCleanUpService(IScheduledJobsConfiguration config, IPaymentsDataContext dataContext, + IEndpointInstanceFactory endpointInstanceFactory, IPaymentLogger paymentLogger) { this.dataContext = dataContext ?? throw new ArgumentNullException(nameof(dataContext)); this.endpointInstanceFactory = endpointInstanceFactory ?? throw new ArgumentNullException(nameof(endpointInstanceFactory)); @@ -42,6 +43,7 @@ public async Task TriggerAuditDataCleanup() var submissionJobsToBeDeletedBatches = previousSubmissionJobsToBeDeletedBatches.Union(currentSubmissionJobsToBeDeletedBatches); var submissionJobsToBeDeletedBatchesList = submissionJobsToBeDeletedBatches.ToList(); + var endpointInstance = await endpointInstanceFactory.GetEndpointInstance().ConfigureAwait(false); paymentLogger.LogInfo($"Triggering Audit Data Cleanup for {submissionJobsToBeDeletedBatchesList.Count} submission job batches. " + @@ -124,91 +126,91 @@ public async Task DataLockEventAuditDataCleanUp(SubmissionJobsToBeDeletedBatch b private async Task DeleteEarningEventData(IList sqlParameters, string sqlParamName, string paramValues) { - var earningEventPeriodCount = await dataContext.Database.ExecuteSqlCommandAsync( - $@"DELETE Payments2.EarningEventPeriod + var earningEventPeriodCount = await dataContext.Database.ExecuteSqlRawAsync( + $@"DELETE Payments2.EarningEventPeriod FROM Payments2.EarningEventPeriod AS EEP INNER JOIN Payments2.EarningEvent AS EE ON EEP.EarningEventId = EE.EventId WHERE EE.JobId IN ({sqlParamName})", - sqlParameters); + sqlParameters); paymentLogger.LogInfo($"DELETED {earningEventPeriodCount} earningEventPeriods for JobIds {paramValues}"); - var earningEventPriceEpisodeCount = await dataContext.Database.ExecuteSqlCommandAsync( - $@"DELETE Payments2.EarningEventPriceEpisode + var earningEventPriceEpisodeCount = await dataContext.Database.ExecuteSqlRawAsync( + $@"DELETE Payments2.EarningEventPriceEpisode FROM Payments2.EarningEventPriceEpisode AS EEPE INNER JOIN Payments2.EarningEvent AS EE ON EEPE.EarningEventId = EE.EventId WHERE EE.JobId IN ({sqlParamName})", - sqlParameters); + sqlParameters); paymentLogger.LogInfo($"DELETED {earningEventPriceEpisodeCount} earningEventPriceEpisodes for JobIds {paramValues}"); - var earningEventCount = await dataContext.Database.ExecuteSqlCommandAsync( - $"DELETE Payments2.EarningEvent WHERE JobId IN ({sqlParamName})", - sqlParameters); + var earningEventCount = await dataContext.Database.ExecuteSqlRawAsync( + $"DELETE Payments2.EarningEvent WHERE JobId IN ({sqlParamName})", + sqlParameters); paymentLogger.LogInfo($"DELETED {earningEventCount} EarningEvents for JobIds {paramValues}"); } private async Task DeleteFundingSourceEvent(IList sqlParameters, string sqlParamName, string paramValues) { - var fundingSourceEventCount = await dataContext.Database.ExecuteSqlCommandAsync( - $"DELETE Payments2.FundingSourceEvent WHERE JobId IN ({sqlParamName})", - sqlParameters); + var fundingSourceEventCount = await dataContext.Database.ExecuteSqlRawAsync( + $"DELETE Payments2.FundingSourceEvent WHERE JobId IN ({sqlParamName})", + sqlParameters); paymentLogger.LogInfo($"DELETED {fundingSourceEventCount} FundingSourceEvents for JobIds {paramValues}"); } private async Task DeleteRequiredPaymentEvent(IList sqlParameters, string sqlParamName, string paramValues) { - var requiredPaymentEventCount = await dataContext.Database.ExecuteSqlCommandAsync( - $"DELETE Payments2.RequiredPaymentEvent WHERE JobId IN ({sqlParamName})", - sqlParameters); + var requiredPaymentEventCount = await dataContext.Database.ExecuteSqlRawAsync( + $"DELETE Payments2.RequiredPaymentEvent WHERE JobId IN ({sqlParamName})", + sqlParameters); paymentLogger.LogInfo($"DELETED {requiredPaymentEventCount} RequiredPaymentEvents for JobIds {paramValues}"); } private async Task DeleteDataLockEvent(IList sqlParameters, string sqlParamName, string paramValues) { - var dataLockEventNonPayablePeriodFailuresCount = await dataContext.Database.ExecuteSqlCommandAsync( - $@"DELETE Payments2.DataLockEventNonPayablePeriodFailures + var dataLockEventNonPayablePeriodFailuresCount = await dataContext.Database.ExecuteSqlRawAsync( + $@"DELETE Payments2.DataLockEventNonPayablePeriodFailures FROM Payments2.DataLockEventNonPayablePeriodFailures AS DLENPPF INNER JOIN Payments2.DataLockEventNonPayablePeriod AS DLENPP ON DLENPPF.DataLockEventNonPayablePeriodId = DLENPP.DataLockEventNonPayablePeriodId INNER JOIN Payments2.DataLockEvent AS DL ON DLENPP.DataLockEventId = DL.EventId WHERE DL.JobId IN ({sqlParamName})", - sqlParameters); + sqlParameters); paymentLogger.LogInfo($"DELETED {dataLockEventNonPayablePeriodFailuresCount} DataLockEventNonPayablePeriodFailures for JobIds {paramValues}"); - var dataLockEventNonPayablePeriodCount = await dataContext.Database.ExecuteSqlCommandAsync( - $@"DELETE Payments2.DataLockEventNonPayablePeriod + var dataLockEventNonPayablePeriodCount = await dataContext.Database.ExecuteSqlRawAsync( + $@"DELETE Payments2.DataLockEventNonPayablePeriod FROM Payments2.DataLockEventNonPayablePeriod AS DLENPP INNER JOIN Payments2.DataLockEvent AS DL ON DLENPP.DataLockEventId = DL.EventId WHERE DL.JobId IN ({sqlParamName})", - sqlParameters); + sqlParameters); paymentLogger.LogInfo($"DELETED {dataLockEventNonPayablePeriodCount} DataLockEventNonPayablePeriods for JobIds {paramValues}"); - var dataLockEventPayablePeriodCount = await dataContext.Database.ExecuteSqlCommandAsync( - $@"DELETE Payments2.DataLockEventPayablePeriod + var dataLockEventPayablePeriodCount = await dataContext.Database.ExecuteSqlRawAsync( + $@"DELETE Payments2.DataLockEventPayablePeriod FROM Payments2.DataLockEventPayablePeriod AS DLEPP INNER JOIN Payments2.DataLockEvent AS DL ON DLEPP.DataLockEventId = DL.EventId WHERE DL.JobId IN ({sqlParamName})", - sqlParameters); + sqlParameters); paymentLogger.LogInfo($"DELETED {dataLockEventPayablePeriodCount} DataLockEventPayablePeriods for JobIds {paramValues}"); - var dataLockEventPriceEpisodeCount = await dataContext.Database.ExecuteSqlCommandAsync( - $@"DELETE Payments2.DataLockEventPriceEpisode + var dataLockEventPriceEpisodeCount = await dataContext.Database.ExecuteSqlRawAsync( + $@"DELETE Payments2.DataLockEventPriceEpisode FROM Payments2.DataLockEventPriceEpisode AS DLEPP INNER JOIN Payments2.DataLockEvent AS DL ON DLEPP.DataLockEventId = DL.EventId WHERE DL.JobId IN ({sqlParamName})", - sqlParameters); + sqlParameters); paymentLogger.LogInfo($"DELETED {dataLockEventPriceEpisodeCount} DataLockEventPriceEpisodes for JobIds {paramValues}"); - var dataLockEventCount = await dataContext.Database.ExecuteSqlCommandAsync( - $"DELETE Payments2.DataLockEvent WHERE JobId IN ({sqlParamName})", - sqlParameters); + var dataLockEventCount = await dataContext.Database.ExecuteSqlRawAsync( + $"DELETE Payments2.DataLockEvent WHERE JobId IN ({sqlParamName})", + sqlParameters); paymentLogger.LogInfo($"DELETED {dataLockEventCount} DataLockEvents for JobIds {paramValues}"); } @@ -243,10 +245,10 @@ SELECT JobId FROM Payments2.DataLockEvent SELECT JobId AS DcJobId FROM #JobDataToBeDeleted"; return (await dataContext.SubmissionJobsToBeDeleted - .FromSql(selectJobsToBeDeleted, - new SqlParameter("collectionPeriod", collectionPeriod), - new SqlParameter("academicYear", academicYear)) - .ToListAsync()) + .FromSqlRaw(selectJobsToBeDeleted, + new SqlParameter("collectionPeriod", collectionPeriod), + new SqlParameter("academicYear", academicYear)) + .ToListAsync()) .ToBatch(100); } } diff --git a/src/SFA.DAS.Payments.ScheduledJobs/AuditDataCleanUp/AuditDataCleanUpTrigger.cs b/src/SFA.DAS.Payments.ScheduledJobs/AuditDataCleanUp/AuditDataCleanUpTrigger.cs index 2efdad8..64cf4f1 100644 --- a/src/SFA.DAS.Payments.ScheduledJobs/AuditDataCleanUp/AuditDataCleanUpTrigger.cs +++ b/src/SFA.DAS.Payments.ScheduledJobs/AuditDataCleanUp/AuditDataCleanUpTrigger.cs @@ -1,31 +1,37 @@ using System.Threading.Tasks; -using AzureFunctions.Autofac; using Microsoft.AspNetCore.Http; -using Microsoft.Azure.WebJobs; -using Microsoft.Azure.WebJobs.Extensions.Http; using SFA.DAS.Payments.ScheduledJobs.Infrastructure.IoC; +using Microsoft.Extensions.Logging; +using Microsoft.Azure.Functions.Worker; +using Microsoft.Azure.WebJobs; // ReSharper disable UnusedMember.Global namespace SFA.DAS.Payments.ScheduledJobs.AuditDataCleanUp { - [DependencyInjectionConfig(typeof(DependencyInjectionConfig))] - public static class AuditDataCleanUpTrigger + public class AuditDataCleanUpTrigger { - [FunctionName("TimerTriggerAuditDataCleanup")] - public static async Task TimerTriggerAuditDataCleanup( - [TimerTrigger("%AuditDataCleanUpSchedule%", RunOnStartup = false)] TimerInfo timerInfo, - [Inject] IAuditDataCleanUpService auditDataCleanUpService) + private readonly ILogger _logger; + private readonly IAuditDataCleanUpService _auditDataCleanUpService; + + public AuditDataCleanUpTrigger(ILogger logger, IAuditDataCleanUpService auditDataCleanUpService) + { + _logger = logger; + _auditDataCleanUpService = auditDataCleanUpService; + } + + [Function("TimerTriggerAuditDataCleanup")] + public async Task TimerTriggerAuditDataCleanup( + [TimerTrigger("%AuditDataCleanUpSchedule%", RunOnStartup = false)] TimerInfo timerInfo) { - await auditDataCleanUpService.TriggerAuditDataCleanup(); + await _auditDataCleanUpService.TriggerAuditDataCleanup(); } - [FunctionName("HttpTriggerAuditDataCleanup")] - public static async Task HttpTriggerAuditDataCleanup( - [HttpTrigger(AuthorizationLevel.Function, "get", Route = null)] HttpRequest httpRequest, - [Inject] IAuditDataCleanUpService auditDataCleanUpService) + [Function("HttpTriggerAuditDataCleanup")] + public async Task HttpTriggerAuditDataCleanup( + [HttpTrigger(AuthorizationLevel.Function, "get", Route = null)] HttpRequest httpRequest) { - await auditDataCleanUpService.TriggerAuditDataCleanup(); + await _auditDataCleanUpService.TriggerAuditDataCleanup(); } } } diff --git a/src/SFA.DAS.Payments.ScheduledJobs/AuditDataCleanUp/DataLockAuditDataCleanUp.cs b/src/SFA.DAS.Payments.ScheduledJobs/AuditDataCleanUp/DataLockAuditDataCleanUp.cs index 8330248..c8563e1 100644 --- a/src/SFA.DAS.Payments.ScheduledJobs/AuditDataCleanUp/DataLockAuditDataCleanUp.cs +++ b/src/SFA.DAS.Payments.ScheduledJobs/AuditDataCleanUp/DataLockAuditDataCleanUp.cs @@ -1,22 +1,31 @@ using System.Threading.Tasks; -using AzureFunctions.Autofac; -using Microsoft.Azure.WebJobs; using Newtonsoft.Json; using SFA.DAS.Payments.ScheduledJobs.Infrastructure.IoC; +using Microsoft.Extensions.Logging; +using Microsoft.Azure.Functions.Worker; +using Microsoft.Azure.WebJobs; + // ReSharper disable UnusedMember.Global namespace SFA.DAS.Payments.ScheduledJobs.AuditDataCleanUp { - [DependencyInjectionConfig(typeof(DependencyInjectionConfig))] - public static class DataLockAuditDataCleanUp + public class DataLockAuditDataCleanUp { - [FunctionName("DataLockEventAuditDataCleanUp")] - public static async Task DataLockEventAuditDataCleanUp([ServiceBusTrigger("%DataLockAuditDataCleanUpQueue%", Connection = "ServiceBusConnectionString")] string message, - [Inject] IAuditDataCleanUpService auditDataCleanUpService) + private readonly ILogger _logger; + private readonly IAuditDataCleanUpService _auditDataCleanUpService; + + public DataLockAuditDataCleanUp(ILogger logger, IAuditDataCleanUpService auditDataCleanUpService) + { + _logger = logger; + _auditDataCleanUpService = auditDataCleanUpService; + } + + [Function("DataLockEventAuditDataCleanUp")] + public async Task DataLockEventAuditDataCleanUp([ServiceBusTrigger("%DataLockAuditDataCleanUpQueue%", Connection = "ServiceBusConnectionString")] string message) { var batch = JsonConvert.DeserializeObject(message); - await auditDataCleanUpService.DataLockEventAuditDataCleanUp(batch); + await _auditDataCleanUpService.DataLockEventAuditDataCleanUp(batch); } } } \ No newline at end of file diff --git a/src/SFA.DAS.Payments.ScheduledJobs/AuditDataCleanUp/EarningAuditDataCleanUp.cs b/src/SFA.DAS.Payments.ScheduledJobs/AuditDataCleanUp/EarningAuditDataCleanUp.cs index a770cb2..74fe2b0 100644 --- a/src/SFA.DAS.Payments.ScheduledJobs/AuditDataCleanUp/EarningAuditDataCleanUp.cs +++ b/src/SFA.DAS.Payments.ScheduledJobs/AuditDataCleanUp/EarningAuditDataCleanUp.cs @@ -1,22 +1,31 @@ using System.Threading.Tasks; -using AzureFunctions.Autofac; -using Microsoft.Azure.WebJobs; using Newtonsoft.Json; using SFA.DAS.Payments.ScheduledJobs.Infrastructure.IoC; +using Microsoft.Extensions.Logging; +using Microsoft.Azure.Functions.Worker; +using Microsoft.Azure.WebJobs; + // ReSharper disable UnusedMember.Global namespace SFA.DAS.Payments.ScheduledJobs.AuditDataCleanUp { - [DependencyInjectionConfig(typeof(DependencyInjectionConfig))] - public static class EarningAuditDataCleanUp + public class EarningAuditDataCleanUp { - [FunctionName("EarningEventAuditDataCleanUp")] - public static async Task EarningEventAuditDataCleanUp([ServiceBusTrigger("%EarningAuditDataCleanUpQueue%", Connection = "ServiceBusConnectionString")] string message, - [Inject] IAuditDataCleanUpService auditDataCleanUpService) + private readonly ILogger _logger; + private readonly IAuditDataCleanUpService _auditDataCleanUpService; + + public EarningAuditDataCleanUp(ILogger logger, IAuditDataCleanUpService auditDataCleanUpService) + { + _logger = logger; + _auditDataCleanUpService = auditDataCleanUpService; + } + + [Function("EarningEventAuditDataCleanUp")] + public async Task EarningEventAuditDataCleanUp([ServiceBusTrigger("%EarningAuditDataCleanUpQueue%", Connection = "ServiceBusConnectionString")] string message) { var batch = JsonConvert.DeserializeObject(message); - - await auditDataCleanUpService.EarningEventAuditDataCleanUp(batch); + + await _auditDataCleanUpService.EarningEventAuditDataCleanUp(batch); } } } \ No newline at end of file diff --git a/src/SFA.DAS.Payments.ScheduledJobs/AuditDataCleanUp/Extensions.cs b/src/SFA.DAS.Payments.ScheduledJobs/AuditDataCleanUp/Extensions.cs index 70119ed..7d929d0 100644 --- a/src/SFA.DAS.Payments.ScheduledJobs/AuditDataCleanUp/Extensions.cs +++ b/src/SFA.DAS.Payments.ScheduledJobs/AuditDataCleanUp/Extensions.cs @@ -1,6 +1,6 @@ using System.Collections.Generic; -using System.Data.SqlClient; using System.Linq; +using Microsoft.Data.SqlClient; using SFA.DAS.Payments.Model.Core.Audit; namespace SFA.DAS.Payments.ScheduledJobs.AuditDataCleanUp diff --git a/src/SFA.DAS.Payments.ScheduledJobs/AuditDataCleanUp/FundingSourceAuditDataCleanUp.cs b/src/SFA.DAS.Payments.ScheduledJobs/AuditDataCleanUp/FundingSourceAuditDataCleanUp.cs index 979710a..8ba70d3 100644 --- a/src/SFA.DAS.Payments.ScheduledJobs/AuditDataCleanUp/FundingSourceAuditDataCleanUp.cs +++ b/src/SFA.DAS.Payments.ScheduledJobs/AuditDataCleanUp/FundingSourceAuditDataCleanUp.cs @@ -1,22 +1,30 @@ using System.Threading.Tasks; -using AzureFunctions.Autofac; -using Microsoft.Azure.WebJobs; using Newtonsoft.Json; using SFA.DAS.Payments.ScheduledJobs.Infrastructure.IoC; +using Microsoft.Extensions.Logging; +using Microsoft.Azure.Functions.Worker; + // ReSharper disable UnusedMember.Global namespace SFA.DAS.Payments.ScheduledJobs.AuditDataCleanUp { - [DependencyInjectionConfig(typeof(DependencyInjectionConfig))] - public static class FundingSourceAuditDataCleanUp + public class FundingSourceAuditDataCleanUp { - [FunctionName("FundingSourceEventAuditDataCleanUp")] - public static async Task FundingSourceEventAuditDataCleanUp([ServiceBusTrigger("%FundingSourceAuditDataCleanUpQueue%", Connection = "ServiceBusConnectionString")] string message, - [Inject] IAuditDataCleanUpService auditDataCleanUpService) + private readonly ILogger _logger; + private readonly IAuditDataCleanUpService _auditDataCleanUpService; + + public FundingSourceAuditDataCleanUp(ILogger logger, IAuditDataCleanUpService auditDataCleanUpService) + { + _logger = logger; + _auditDataCleanUpService = auditDataCleanUpService; + } + + [Function("FundingSourceEventAuditDataCleanUp")] + public async Task FundingSourceEventAuditDataCleanUp([ServiceBusTrigger("%FundingSourceAuditDataCleanUpQueue%", Connection = "ServiceBusConnectionString")] string message) { var batch = JsonConvert.DeserializeObject(message); - await auditDataCleanUpService.FundingSourceEventAuditDataCleanUp(batch); + await _auditDataCleanUpService.FundingSourceEventAuditDataCleanUp(batch); } } } \ No newline at end of file diff --git a/src/SFA.DAS.Payments.ScheduledJobs/AuditDataCleanUp/RequiredPaymentAuditDataCleanUp.cs b/src/SFA.DAS.Payments.ScheduledJobs/AuditDataCleanUp/RequiredPaymentAuditDataCleanUp.cs index c001824..6f72a6d 100644 --- a/src/SFA.DAS.Payments.ScheduledJobs/AuditDataCleanUp/RequiredPaymentAuditDataCleanUp.cs +++ b/src/SFA.DAS.Payments.ScheduledJobs/AuditDataCleanUp/RequiredPaymentAuditDataCleanUp.cs @@ -1,22 +1,31 @@ using System.Threading.Tasks; -using AzureFunctions.Autofac; -using Microsoft.Azure.WebJobs; using Newtonsoft.Json; using SFA.DAS.Payments.ScheduledJobs.Infrastructure.IoC; +using Microsoft.Extensions.Logging; +using Microsoft.Azure.Functions.Worker; +using Microsoft.Azure.WebJobs; + // ReSharper disable UnusedMember.Global namespace SFA.DAS.Payments.ScheduledJobs.AuditDataCleanUp { - [DependencyInjectionConfig(typeof(DependencyInjectionConfig))] - public static class RequiredPaymentAuditDataCleanUp + public class RequiredPaymentAuditDataCleanUp { - [FunctionName("RequiredPaymentEventAuditDataCleanUp")] - public static async Task RequiredPaymentEventAuditDataCleanUp([ServiceBusTrigger("%RequiredPaymentAuditDataCleanUpQueue%", Connection = "ServiceBusConnectionString")] string message, - [Inject] IAuditDataCleanUpService auditDataCleanUpService) + private readonly ILogger _logger; + private readonly IAuditDataCleanUpService _auditDataCleanUpService; + + public RequiredPaymentAuditDataCleanUp(ILogger logger, IAuditDataCleanUpService auditDataCleanUpService) + { + _logger = logger; + _auditDataCleanUpService = auditDataCleanUpService; + } + + [Function("RequiredPaymentEventAuditDataCleanUp")] + public async Task RequiredPaymentEventAuditDataCleanUp([ServiceBusTrigger("%RequiredPaymentAuditDataCleanUpQueue%", Connection = "ServiceBusConnectionString")] string message) { var batch = JsonConvert.DeserializeObject(message); - await auditDataCleanUpService.RequiredPaymentEventAuditDataCleanUp(batch); + await _auditDataCleanUpService.RequiredPaymentEventAuditDataCleanUp(batch); } } } \ No newline at end of file diff --git a/src/SFA.DAS.Payments.ScheduledJobs/Infrastructure/IoC/DependencyInjectionConfig.cs b/src/SFA.DAS.Payments.ScheduledJobs/Infrastructure/IoC/DependencyInjectionConfig.cs deleted file mode 100644 index d69f72b..0000000 --- a/src/SFA.DAS.Payments.ScheduledJobs/Infrastructure/IoC/DependencyInjectionConfig.cs +++ /dev/null @@ -1,28 +0,0 @@ -using Autofac; -using AzureFunctions.Autofac.Configuration; -using SFA.DAS.Payments.Application.Infrastructure.Ioc.Modules; -using SFA.DAS.Payments.ScheduledJobs.Infrastructure.IoC.Modules; - -namespace SFA.DAS.Payments.ScheduledJobs.Infrastructure.IoC -{ - public class DependencyInjectionConfig - { - public DependencyInjectionConfig(string functionName) - { - DependencyInjection.Initialize(RegisterModules, functionName); - } - - private static void RegisterModules(ContainerBuilder builder) - { - builder.RegisterModule(); - builder.RegisterModule(); - builder.RegisterModule(); - builder.RegisterModule(); - - builder.RegisterModule(); - builder.RegisterModule(); - builder.RegisterModule(); - builder.RegisterModule(); - } - } -} diff --git a/src/SFA.DAS.Payments.ScheduledJobs/Infrastructure/IoC/Modules/MessagingModule.cs b/src/SFA.DAS.Payments.ScheduledJobs/Infrastructure/IoC/Modules/MessagingModule.cs index fc3872f..95a7005 100644 --- a/src/SFA.DAS.Payments.ScheduledJobs/Infrastructure/IoC/Modules/MessagingModule.cs +++ b/src/SFA.DAS.Payments.ScheduledJobs/Infrastructure/IoC/Modules/MessagingModule.cs @@ -1,12 +1,10 @@ -using System.Linq; using System.Net; using System.Threading.Tasks; using Autofac; using NServiceBus; -using NServiceBus.Features; +using SFA.DAS.Payments.Application.Infrastructure.Logging; using SFA.DAS.Payments.Application.Messaging; using SFA.DAS.Payments.ScheduledJobs.Infrastructure.Configuration; -using SFA.DAS.Payments.Application.Infrastructure.Logging; namespace SFA.DAS.Payments.ScheduledJobs.Infrastructure.IoC.Modules { @@ -23,17 +21,17 @@ protected override void Load(ContainerBuilder builder) var endpointConfiguration = new EndpointConfiguration(config.EndpointName); var logger = c.Resolve(); - - endpointConfiguration.CustomDiagnosticsWriter(diagnostics => - { - logger.Info(diagnostics); - return Task.CompletedTask; - }); + + endpointConfiguration.CustomDiagnosticsWriter( + (diagnostics, ct) => + { + logger.Info(diagnostics); + return Task.CompletedTask; + }); var conventions = endpointConfiguration.Conventions(); conventions.DefiningCommandsAs(type => ( type.Namespace?.StartsWith("SFA.DAS.Payments") ?? false ) && (bool) type.Namespace?.Contains(".Messages.Commands")); - endpointConfiguration.DisableFeature(); if (!string.IsNullOrEmpty(config.DasNServiceBusLicenseKey)) { var license = WebUtility.HtmlDecode(config.DasNServiceBusLicenseKey); @@ -41,10 +39,10 @@ protected override void Load(ContainerBuilder builder) } var transport = endpointConfiguration.UseTransport(); - transport.ConnectionString(config.ServiceBusConnectionString).Transactions(TransportTransactionMode.ReceiveOnly).RuleNameShortener(ruleName => ruleName.Split('.').LastOrDefault() ?? ruleName); + transport.ConnectionString(config.ServiceBusConnectionString); transport.PrefetchCount(20); builder.RegisterInstance(transport).As>().SingleInstance(); - endpointConfiguration.UseSerialization(); + endpointConfiguration.UseSerialization(); endpointConfiguration.EnableInstallers(); return endpointConfiguration; diff --git a/src/SFA.DAS.Payments.ScheduledJobs/LevyAccountImport.cs b/src/SFA.DAS.Payments.ScheduledJobs/LevyAccountImport.cs index 1c4ecfa..84928a7 100644 --- a/src/SFA.DAS.Payments.ScheduledJobs/LevyAccountImport.cs +++ b/src/SFA.DAS.Payments.ScheduledJobs/LevyAccountImport.cs @@ -1,34 +1,45 @@ using System; using System.Threading.Tasks; -using AzureFunctions.Autofac; using Microsoft.AspNetCore.Http; -using Microsoft.Azure.WebJobs; -using Microsoft.Azure.WebJobs.Extensions.Http; using NServiceBus; using SFA.DAS.Payments.Application.Infrastructure.Logging; using SFA.DAS.Payments.Application.Messaging; using SFA.DAS.Payments.FundingSource.Messages.Commands; using SFA.DAS.Payments.ScheduledJobs.Infrastructure.Configuration; -using SFA.DAS.Payments.ScheduledJobs.Infrastructure.IoC; +using Microsoft.Extensions.Logging; +using Microsoft.Azure.Functions.Worker; + // ReSharper disable UnusedMember.Global namespace SFA.DAS.Payments.ScheduledJobs { - [DependencyInjectionConfig(typeof(DependencyInjectionConfig))] - public static class LevyAccountImport + public class LevyAccountImport { - [FunctionName("LevyAccountImport")] - public static async Task Run([TimerTrigger("%LevyAccountSchedule%", RunOnStartup=false)]TimerInfo myTimer, [Inject]IEndpointInstanceFactory endpointInstanceFactory, [Inject]IScheduledJobsConfiguration config, [Inject]IPaymentLogger log) + private readonly ILogger _logger; + private readonly IEndpointInstanceFactory _endpointInstanceFactory; + private readonly IScheduledJobsConfiguration _config; + private readonly IPaymentLogger _log; + + public LevyAccountImport(ILogger logger, IEndpointInstanceFactory endpointInstanceFactory, IScheduledJobsConfiguration config, IPaymentLogger log) + { + _logger = logger; + _endpointInstanceFactory = endpointInstanceFactory; + _config = config; + _log = log; + } + + [Function("LevyAccountImport")] + public async Task Run([TimerTrigger("%LevyAccountSchedule%", RunOnStartup = false)] TimerInfo myTimer) { - await RunLevyAccountImport(endpointInstanceFactory, config, log); + await RunLevyAccountImport(_endpointInstanceFactory, _config, _log); } - [FunctionName("HttpLevyAccountImport")] - public static async Task HttpLevyAccountImport([HttpTrigger(AuthorizationLevel.Function, "get", Route = null)] HttpRequest req, [Inject]IEndpointInstanceFactory endpointInstanceFactory, [Inject]IScheduledJobsConfiguration config, [Inject]IPaymentLogger log) + [Function("HttpLevyAccountImport")] + public async Task HttpLevyAccountImport([HttpTrigger(AuthorizationLevel.Function, "get", Route = null)] HttpRequest req) { - await RunLevyAccountImport(endpointInstanceFactory, config, log); + await RunLevyAccountImport(_endpointInstanceFactory, _config, _log); } - + private static async Task RunLevyAccountImport(IEndpointInstanceFactory endpointInstanceFactory, IScheduledJobsConfiguration config, IPaymentLogger log) { try diff --git a/src/SFA.DAS.Payments.ScheduledJobs/Monitoring/ApprenticeshipData/ApprenticeshipDataTrigger.cs b/src/SFA.DAS.Payments.ScheduledJobs/Monitoring/ApprenticeshipData/ApprenticeshipDataTrigger.cs index 07b6aa2..0ab49f5 100644 --- a/src/SFA.DAS.Payments.ScheduledJobs/Monitoring/ApprenticeshipData/ApprenticeshipDataTrigger.cs +++ b/src/SFA.DAS.Payments.ScheduledJobs/Monitoring/ApprenticeshipData/ApprenticeshipDataTrigger.cs @@ -1,29 +1,36 @@ using System; using System.Threading.Tasks; -using AzureFunctions.Autofac; using Microsoft.AspNetCore.Http; -using Microsoft.Azure.WebJobs; -using Microsoft.Azure.WebJobs.Extensions.Http; using SFA.DAS.Payments.Application.Infrastructure.Logging; using SFA.DAS.Payments.ScheduledJobs.Infrastructure.IoC; +using Microsoft.Extensions.Logging; +using Microsoft.Azure.Functions.Worker; namespace SFA.DAS.Payments.ScheduledJobs.Monitoring.ApprenticeshipData { - [DependencyInjectionConfig(typeof(DependencyInjectionConfig))] - public static class ApprenticeshipDataTrigger + public class ApprenticeshipDataTrigger { - [FunctionName("TimerTriggerApprenticeshipsReferenceDataComparison")] - public static async Task TimerTriggerApprenticeshipsReferenceDataComparison([TimerTrigger("%ApprenticeshipValidationSchedule%", RunOnStartup = false)]TimerInfo myTimer, [Inject]IApprenticeshipsDataService service, [Inject] IPaymentLogger log) + private readonly ILogger _logger; + private readonly IApprenticeshipsDataService _service; + private readonly IPaymentLogger _log; + + public ApprenticeshipDataTrigger(ILogger logger, IApprenticeshipsDataService service, IPaymentLogger log) + { + _logger = logger; + _service = service; + _log = log; + } + + [Function("TimerTriggerApprenticeshipsReferenceDataComparison")] + public async Task TimerTriggerApprenticeshipsReferenceDataComparison([TimerTrigger("%ApprenticeshipValidationSchedule%", RunOnStartup = false)] TimerInfo myTimer) { - await RunApprenticeshipsReferenceDataComparison(service, log); + await RunApprenticeshipsReferenceDataComparison(_service, _log); } - [FunctionName("HttpTriggerApprenticeshipsReferenceDataComparison")] - public static async Task HttpTriggerApprenticeshipsReferenceDataComparison( - [HttpTrigger(AuthorizationLevel.Function, "get", Route = null)] HttpRequest httpRequest, - [Inject]IApprenticeshipsDataService service, [Inject] IPaymentLogger log) + [Function("HttpTriggerApprenticeshipsReferenceDataComparison")] + public async Task HttpTriggerApprenticeshipsReferenceDataComparison([HttpTrigger(AuthorizationLevel.Function, "get", Route = null)] HttpRequest httpRequest) { - await RunApprenticeshipsReferenceDataComparison(service, log); + await RunApprenticeshipsReferenceDataComparison(_service, _log); } private static async Task RunApprenticeshipsReferenceDataComparison(IApprenticeshipsDataService service, IPaymentLogger log) diff --git a/src/SFA.DAS.Payments.ScheduledJobs/Monitoring/LevyAccountData/LevyAccountValidationFunction.cs b/src/SFA.DAS.Payments.ScheduledJobs/Monitoring/LevyAccountData/LevyAccountValidationFunction.cs index c41086d..4590c0d 100644 --- a/src/SFA.DAS.Payments.ScheduledJobs/Monitoring/LevyAccountData/LevyAccountValidationFunction.cs +++ b/src/SFA.DAS.Payments.ScheduledJobs/Monitoring/LevyAccountData/LevyAccountValidationFunction.cs @@ -1,29 +1,30 @@ using System.Threading.Tasks; -using AzureFunctions.Autofac; using Microsoft.AspNetCore.Http; -using Microsoft.Azure.WebJobs; -using Microsoft.Azure.WebJobs.Extensions.Http; using SFA.DAS.Payments.ScheduledJobs.Infrastructure.IoC; +using Microsoft.Azure.Functions.Worker; +using Microsoft.Azure.WebJobs; namespace SFA.DAS.Payments.ScheduledJobs.Monitoring.LevyAccountData { - [DependencyInjectionConfig(typeof(DependencyInjectionConfig))] public class LevyAccountValidationFunction { - [FunctionName("TimerTriggerLevyAccountValidation")] - public static async Task TimerTriggerLevyAccountValidation( - [TimerTrigger("%LevyAccountValidationSchedule%", RunOnStartup = false)] TimerInfo timerInfo, - [Inject] ILevyAccountValidationService levyAccountValidationService) + private readonly ILevyAccountValidationService _levyAccountValidationService; + + public LevyAccountValidationFunction(ILevyAccountValidationService levyAccountValidationService) + { + _levyAccountValidationService = levyAccountValidationService; + } + + [Function("TimerTriggerLevyAccountValidation")] + public async Task TimerTriggerLevyAccountValidation([TimerTrigger("%LevyAccountValidationSchedule%", RunOnStartup = false)] TimerInfo timerInfo) { - await levyAccountValidationService.Validate(); + await _levyAccountValidationService.Validate(); } - [FunctionName("HttpTriggerLevyAccountValidation")] - public static async Task HttpTriggerLevyAccountValidation( - [HttpTrigger(AuthorizationLevel.Function, "get", Route = null)] HttpRequest httpRequest, - [Inject] ILevyAccountValidationService levyAccountValidationService) + [Function("HttpTriggerLevyAccountValidation")] + public async Task HttpTriggerLevyAccountValidation([HttpTrigger(AuthorizationLevel.Function, "get", Route = null)] HttpRequest httpRequest) { - await levyAccountValidationService.Validate(); + await _levyAccountValidationService.Validate(); } } } diff --git a/src/SFA.DAS.Payments.ScheduledJobs/Program.cs b/src/SFA.DAS.Payments.ScheduledJobs/Program.cs new file mode 100644 index 0000000..e5c3fb9 --- /dev/null +++ b/src/SFA.DAS.Payments.ScheduledJobs/Program.cs @@ -0,0 +1,31 @@ +using Autofac; +using Autofac.Extensions.DependencyInjection; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Hosting; +using SFA.DAS.Payments.Application.Infrastructure.Ioc.Modules; +using SFA.DAS.Payments.ScheduledJobs.Infrastructure.IoC.Modules; +using Modules = SFA.DAS.Payments.ScheduledJobs.Infrastructure.IoC.Modules; + + +var host = new HostBuilder() + .ConfigureAppConfiguration(config => + { +#if DEBUG + config.AddJsonFile("local.settings.json", optional: true); +#endif + })/*.UseServiceProviderFactory(new AutofacServiceProviderFactory()) + .ConfigureContainer(builder => + { + builder.RegisterModule(new TelemetryModule()); + builder.RegisterModule(new LoggingModule()); + builder.RegisterModule(new FunctionsModule()); + builder.RegisterModule(new LevyAccountValidationModule()); + + builder.RegisterModule(new Modules.PaymentDataContextModule()); + builder.RegisterModule(new CommitmentsDataContextModule()); + builder.RegisterModule(new Modules.ConfigurationModule()); + builder.RegisterModule(new Modules.MessagingModule()); + })*/ + .Build(); + +host.Run(); diff --git a/src/SFA.DAS.Payments.ScheduledJobs/SFA.DAS.Payments.ScheduledJobs.csproj b/src/SFA.DAS.Payments.ScheduledJobs/SFA.DAS.Payments.ScheduledJobs.csproj index ee4e4ed..be7d611 100644 --- a/src/SFA.DAS.Payments.ScheduledJobs/SFA.DAS.Payments.ScheduledJobs.csproj +++ b/src/SFA.DAS.Payments.ScheduledJobs/SFA.DAS.Payments.ScheduledJobs.csproj @@ -1,24 +1,37 @@  - netcoreapp2.1 - v2 + net6.0 + v4 + Exe + enabled - - + + + + + - - - - - - - + + + + + + + + - - + + + + + + + + + - + PreserveNewest @@ -31,4 +44,7 @@ Never + + + \ No newline at end of file diff --git a/src/SFA.DAS.Payments.ScheduledJobs/host.json b/src/SFA.DAS.Payments.ScheduledJobs/host.json index ab0bef0..2dcb00a 100644 --- a/src/SFA.DAS.Payments.ScheduledJobs/host.json +++ b/src/SFA.DAS.Payments.ScheduledJobs/host.json @@ -1,13 +1,21 @@ { - "version": "2.0", - "functionTimeout": "00:10:00", - "extensions": { - "serviceBus": { - "prefetchCount": 1, - "messageHandlerOptions": { - "maxConcurrentCalls": 1, - "maxAutoRenewDuration": "00:10:00" - } - } + "version": "2.0", + "functionTimeout": "00:10:00", + "extensions": { + "serviceBus": { + "prefetchCount": 1, + "messageHandlerOptions": { + "maxConcurrentCalls": 1, + "maxAutoRenewDuration": "00:10:00" + } } + }, + "logging": { + "applicationInsights": { + "samplingSettings": { + "isEnabled": true, + "excludedTypes": "Request" + } + } + } } \ No newline at end of file