Merge pull request #921 from SkillsFundingAgency/CON-954-house-keepin…

…g-communications

CON-954 : v2 housekeeping tasks: communications

src/Communication/Communication.Types/ICommunicationRepository.cs

@@ -12,5 +12,6 @@ public interface ICommunicationRepository
         Task UpdateAsync(CommunicationMessage commMsg);
         Task<IEnumerable<CommunicationMessage>> GetScheduledMessagesSinceAsync(string requestType, DeliveryFrequency frequency, DateTime from, DateTime to);
         Task UpdateScheduledMessagesAsSentAsync(IEnumerable<Guid> msgIds, Guid aggregatedMessageId);
+        Task HardDelete(DateTime deleteCommunicationMessagesStaleDateTime);
     }
 }

src/Data/CosmosDb/Console.RecruitSeedDataWriter/Data/RecruitWebJobsSystem.json

@@ -3,5 +3,6 @@
     "disabledJobs" : ["VacancyAnalyticsSummaryGeneratorJob"],
     "queryStoreDocumentsStaleByDays" : 90,
     "draftVacanciesStaleByDays" : 180,
-    "referredVacanciesStaleByDays" : 90
+    "referredVacanciesStaleByDays" : 90,
+	  "hardDeleteCommunicationMessagesStaleByDays" : 180
 }

src/Jobs/Recruit.Vacancies.Jobs.UnitTests/Triggers/QueueTriggers/CommunicationsHouseKeepingQueueTriggerTests.cs

@@ -0,0 +1,40 @@
+using System;
+using System.Threading.Tasks;
+using Communication.Types;
+using Esfa.Recruit.Vacancies.Client.Application.Providers;
+using Esfa.Recruit.Vacancies.Client.Application.Queues.Messages;
+using Esfa.Recruit.Vacancies.Jobs.Configuration;
+using Esfa.Recruit.Vacancies.Jobs.Triggers.QueueTriggers;
+using Microsoft.Extensions.Logging;
+using Moq;
+using Newtonsoft.Json;
+using Xunit;
+
+namespace Recruit.Vacancies.Jobs.UnitTests.Triggers.QueueTriggers
+{
+    public class CommunicationsHouseKeepingQueueTriggerTests
+    {
+        private const int Days = 180;
+        private readonly Mock<ILogger<CommunicationsHouseKeepingQueueTrigger>> _loggerMock = new Mock<ILogger<CommunicationsHouseKeepingQueueTrigger>>();
+        private readonly RecruitWebJobsSystemConfiguration _jobsConfig = new RecruitWebJobsSystemConfiguration() { HardDeleteCommunicationMessagesStaleByDays = Days };
+        private readonly Mock<ITimeProvider> _timeProviderMock = new Mock<ITimeProvider>();
+        private readonly Mock<ICommunicationRepository> _communicationRepositoryMock = new Mock<ICommunicationRepository>();
+
+        [Fact]
+        public async Task Delete_Communication_Messages_older_than_180_days()
+        {
+            //Arrange            
+            var datetime = new DateTime(2020, 08, 30).AddDays(-180);
+            _communicationRepositoryMock.Setup(x => x.HardDelete(datetime)).Returns(Task.CompletedTask);
+            var sut = new CommunicationsHouseKeepingQueueTrigger(_loggerMock.Object, _jobsConfig, _timeProviderMock.Object, _communicationRepositoryMock.Object);
+            var message = new CommunicationsHouseKeepingQueueMessage() { CreatedByScheduleDate = new DateTime(2020, 08, 30) };
+
+            //Act
+            await sut.CommunicationsHouseKeepingAsync(JsonConvert.SerializeObject(message), null);
+
+            //Assert
+            _communicationRepositoryMock.Verify(c => c.HardDelete(datetime), Times.AtLeastOnce);
+
+        }
+    }
+}

src/Jobs/Recruit.Vacancies.Jobs/Configuration/RecruitWebJobsSystemConfiguration.cs

@@ -9,5 +9,6 @@ public class RecruitWebJobsSystemConfiguration
         public int? QueryStoreDocumentsStaleByDays { get; set; }
         public int? DraftVacanciesStaleByDays { get; set; }
         public int? ReferredVacanciesStaleByDays { get; set; }
+        public int? HardDeleteCommunicationMessagesStaleByDays { get; set; }
     }
 }

src/Jobs/Recruit.Vacancies.Jobs/Schedules.cs

@@ -11,5 +11,6 @@ internal static class Schedules
         internal const string Hourly = "0 0 */1 * * *";
         internal const string WeeklyFourAmSunday = "0 4 * * SUN";
         internal const string WeeklySevenAmSunday = "0 7 * * SUN";
+        internal const string WeeklyTenAmSunday = "0 10 * * SUN";
     }
 }

src/Jobs/Recruit.Vacancies.Jobs/Triggers/QueueTriggers/CommunicationsHouseKeepingQueueTrigger.cs

@@ -0,0 +1,63 @@
+using System;
+using System.IO;
+using System.Threading.Tasks;
+using Communication.Types;
+using Esfa.Recruit.Vacancies.Client.Application.Providers;
+using Esfa.Recruit.Vacancies.Client.Application.Queues.Messages;
+using Esfa.Recruit.Vacancies.Client.Infrastructure.StorageQueue;
+using Esfa.Recruit.Vacancies.Jobs.Configuration;
+using Microsoft.Azure.WebJobs;
+using Microsoft.Extensions.Logging;
+using Newtonsoft.Json;
+
+namespace Esfa.Recruit.Vacancies.Jobs.Triggers.QueueTriggers
+{
+    public class CommunicationsHouseKeepingQueueTrigger
+    {
+        private readonly ILogger<CommunicationsHouseKeepingQueueTrigger> _logger;
+        private readonly RecruitWebJobsSystemConfiguration _jobsConfig;
+        private readonly ITimeProvider _timeProvider;
+        private readonly ICommunicationRepository _communicationRepository;
+        private const int DefaultStaleByDays = 180;
+        private string JobName => GetType().Name;
+
+        public CommunicationsHouseKeepingQueueTrigger(ILogger<CommunicationsHouseKeepingQueueTrigger> logger,
+          RecruitWebJobsSystemConfiguration jobsConfig,
+          ITimeProvider timeProvider,
+          ICommunicationRepository communicationRepository)
+        {
+            _logger = logger;
+            _jobsConfig = jobsConfig;
+            _timeProvider = timeProvider;
+            _communicationRepository = communicationRepository;
+        }
+
+        public async Task CommunicationsHouseKeepingAsync([QueueTrigger(QueueNames.CommunicationsHouseKeepingQueueName, Connection = "QueueStorage")] string message, TextWriter log)
+        {
+            try
+            {
+                if (_jobsConfig.DisabledJobs.Contains(JobName))
+                {
+                    _logger.LogDebug($"{JobName} is disabled, skipping ...");
+                    return;
+                }
+
+                var payload = JsonConvert.DeserializeObject<CommunicationsHouseKeepingQueueMessage>(message);
+
+                var targetDate = payload?.CreatedByScheduleDate ?? _timeProvider.Today;
+
+                var deleteCommunicationsMessagesBefore180Days = targetDate.AddDays((_jobsConfig.HardDeleteCommunicationMessagesStaleByDays ?? DefaultStaleByDays) * -1);                
+
+                await _communicationRepository.HardDelete(deleteCommunicationsMessagesBefore180Days);
+
+                _logger.LogInformation($"Deleted Communication Mesages created before 180 days {deleteCommunicationsMessagesBefore180Days}");
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, $"Failed to run {JobName}");
+                throw;
+            }
+        }
+
+    }
+}

src/Jobs/Recruit.Vacancies.Jobs/Triggers/TimerTriggers/CommunicationsHouseKeepingTimerTrigger.cs

@@ -0,0 +1,37 @@
+using System;
+using System.IO;
+using System.Threading.Tasks;
+using Esfa.Recruit.Vacancies.Client.Application.Providers;
+using Esfa.Recruit.Vacancies.Client.Application.Queues;
+using Esfa.Recruit.Vacancies.Client.Application.Queues.Messages;
+using Microsoft.Azure.WebJobs;
+using Microsoft.Extensions.Logging;
+
+namespace Esfa.Recruit.Vacancies.Jobs.Triggers.TimerTriggers
+{
+    public class CommunicationsHouseKeepingTimerTrigger
+    {
+        private readonly ILogger<CommunicationsHouseKeepingTimerTrigger> _logger;
+        private readonly IRecruitQueueService _queue;
+        private readonly ITimeProvider _timeProvider;
+
+        public CommunicationsHouseKeepingTimerTrigger(ILogger<CommunicationsHouseKeepingTimerTrigger> logger, IRecruitQueueService queue, ITimeProvider timeProvider)
+        {
+            _logger = logger;
+            _queue = queue;
+            _timeProvider = timeProvider;
+        }
+
+        public Task CommunicationsHouseKeepingAsync([TimerTrigger(Schedules.WeeklyTenAmSunday)] TimerInfo timerInfo, TextWriter log)        
+        {
+            _logger.LogInformation($"Timer trigger {this.GetType().Name} fired");
+
+            var message = new CommunicationsHouseKeepingQueueMessage
+            {
+                CreatedByScheduleDate = _timeProvider.Now
+            };
+
+            return _queue.AddMessageAsync(message);
+        }
+    }
+}

src/Shared/Recruit.Vacancies.Client/Application/Queues/Messages/CommunicationsHouseKeepingQueueMessage.cs

@@ -0,0 +1,11 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Esfa.Recruit.Vacancies.Client.Application.Queues.Messages
+{
+    public class CommunicationsHouseKeepingQueueMessage
+    {
+        public DateTime? CreatedByScheduleDate { get; set; }
+    }
+}

src/Shared/Recruit.Vacancies.Client/Infrastructure/Repositories/MongoDbCommunicationRepository.cs

@@ -86,5 +86,18 @@ public Task UpdateScheduledMessagesAsSentAsync(IEnumerable<Guid> msgIds, Guid ag
                 collection.UpdateManyAsync(filter, updateDef),
                 new Context(nameof(UpdateScheduledMessagesAsSentAsync)));
         }
+
+        public Task HardDelete(DateTime dispatchDateTime)
+        {
+            var builder = Builders<CommunicationMessage>.Filter;
+            var filter = (builder.Eq(cm => cm.Status, CommunicationMessageStatus.Sent) |
+                        builder.Eq(cm => cm.Status, CommunicationMessageStatus.NotSent)) &
+                        builder.Lte(cm => cm.DispatchDateTime, dispatchDateTime);
+            var collection = GetCollection<CommunicationMessage>();
+
+            return RetryPolicy.ExecuteAsync(_ =>
+                collection.DeleteManyAsync(filter),
+                new Context(nameof(HardDelete)));
+        }
     }
 }

src/Shared/Recruit.Vacancies.Client/Infrastructure/StorageQueue/QueueNames.cs

@@ -28,5 +28,6 @@ public static class QueueNames
         public const string DeleteStaleVacanciesQueueName = "delete-stale-vacancies-queue";
         public const string BeginMigrationQueueName = "begin-migration";
         public const string DataMigrationQueueName = "data-migration";
+        public const string CommunicationsHouseKeepingQueueName = "communications-house-keeping-queue";
     }
 }

src/Shared/Recruit.Vacancies.Client/Infrastructure/StorageQueue/RecruitStorageQueueService.cs

@@ -25,7 +25,8 @@ internal class RecruitStorageQueueService : StorageQueueServiceBase, IRecruitQue
             { typeof(VacancyStatusQueueMessage), QueueNames.VacancyStatusQueueName },
             { typeof(UpdateEmployerUserAccountQueueMessage), QueueNames.UpdateEmployerUserAccountQueueName },
             { typeof(DeleteStaleQueryStoreDocumentsQueueMessage), QueueNames.DeleteStaleQueryStoreDocumentsQueueName },
-            { typeof(DataMigrationQueueMessage), QueueNames.DataMigrationQueueName }
+            { typeof(DataMigrationQueueMessage), QueueNames.DataMigrationQueueName },
+            { typeof(CommunicationsHouseKeepingQueueMessage), QueueNames.CommunicationsHouseKeepingQueueName}
         };
 
         protected override string ConnectionString { get; }