From f03f4f44dd4211bf7b215f3d365590f4ad3c3140 Mon Sep 17 00:00:00 2001 From: LarryOG <43647901+LarryOG@users.noreply.github.com> Date: Fri, 8 Sep 2023 10:54:30 +0100 Subject: [PATCH] removed remaining cosmos references from code --- .../Sql/Queries/UpsertProvidersFromCosmos.cs | 57 ---- .../Sql/QueryHandlers/CourseMappingHelper.cs | 5 +- .../UpsertProvidersFromCosmosHandler.cs | 268 ------------------ .../RemoveZombieCoursesFromSql.cs | 44 --- .../local.settings.json | 1 - src/Dfc.CourseDirectory.WebV2/README.md | 2 +- 6 files changed, 2 insertions(+), 375 deletions(-) delete mode 100644 src/Dfc.CourseDirectory.Core/DataStore/Sql/Queries/UpsertProvidersFromCosmos.cs delete mode 100644 src/Dfc.CourseDirectory.Core/DataStore/Sql/QueryHandlers/UpsertProvidersFromCosmosHandler.cs delete mode 100644 src/Dfc.CourseDirectory.Functions/RemoveZombieCoursesFromSql.cs diff --git a/src/Dfc.CourseDirectory.Core/DataStore/Sql/Queries/UpsertProvidersFromCosmos.cs b/src/Dfc.CourseDirectory.Core/DataStore/Sql/Queries/UpsertProvidersFromCosmos.cs deleted file mode 100644 index 731fd61e84..0000000000 --- a/src/Dfc.CourseDirectory.Core/DataStore/Sql/Queries/UpsertProvidersFromCosmos.cs +++ /dev/null @@ -1,57 +0,0 @@ -using System; -using System.Collections.Generic; -using Dfc.CourseDirectory.Core.Models; -using OneOf.Types; - -namespace Dfc.CourseDirectory.Core.DataStore.Sql.Queries -{ - public class UpsertProvidersFromCosmos : ISqlQuery - { - public IEnumerable Records { get; set; } - public DateTime LastSyncedFromCosmos { get; set; } - } - - public class UpsertProvidersRecord - { - public Guid ProviderId { get; set; } - public int Ukprn { get; set; } - public ProviderStatus ProviderStatus { get; set; } - public ProviderType ProviderType { get; set; } - public string ProviderName { get; set; } - public string UkrlpProviderStatusDescription { get; set; } - public string MarketingInformation { get; set; } - public string CourseDirectoryName { get; set; } - public string TradingName { get; set; } - public string Alias { get; set; } - public IEnumerable Contacts { get; set; } - public DateTime? UpdatedOn { get; set; } - public string UpdatedBy { get; set; } - public int? TribalProviderId { get; set; } - public bool? BulkUploadInProgress { get; set; } - public bool? BulkUploadPublishInProgress { get; set; } - public DateTime? BulkUploadStartedDateTime { get; set; } - public int? BulkUploadTotalRowCount { get; set; } - } - - public class UpsertProvidersRecordContact - { - public string ContactType { get; set; } - public string ContactRole { get; set; } - public string AddressSaonDescription { get; set; } - public string AddressPaonDescription { get; set; } - public string AddressStreetDescription { get; set; } - public string AddressLocality { get; set; } - public string AddressItems { get; set; } - public string AddressPostTown { get; set; } - public string AddressCounty { get; set; } - public string AddressPostcode { get; set; } - public string PersonalDetailsPersonNameTitle { get; set; } - public string PersonalDetailsPersonNameGivenName { get; set; } - public string PersonalDetailsPersonNameFamilyName { get; set; } - public string Telephone1 { get; set; } - public string Telephone2 { get; set; } - public string Fax { get; set; } - public string WebsiteAddress { get; set; } - public string Email { get; set; } - } -} diff --git a/src/Dfc.CourseDirectory.Core/DataStore/Sql/QueryHandlers/CourseMappingHelper.cs b/src/Dfc.CourseDirectory.Core/DataStore/Sql/QueryHandlers/CourseMappingHelper.cs index 3df3154a7a..5aef4436db 100644 --- a/src/Dfc.CourseDirectory.Core/DataStore/Sql/QueryHandlers/CourseMappingHelper.cs +++ b/src/Dfc.CourseDirectory.Core/DataStore/Sql/QueryHandlers/CourseMappingHelper.cs @@ -23,10 +23,7 @@ public static async Task> MapCourses(SqlMapper.GridR .GroupBy(r => r.CourseRunId) .ToDictionary(g => g.Key, g => g.Select(r => r.RegionId).AsEnumerable()); - // N.B. We need to normalize HTML-encoded data here. The legacy projects HTML-encoded everything before - // persisting it in Cosmos. We want to move the HTML encoding to the edge, where it should be done. - // The existing data synced from Cosmos has a DataIsHtmlEncoded column set to true; read that here and - // decode the relevant fields if it's set. + return courses.Select(MapCourse).ToArray(); diff --git a/src/Dfc.CourseDirectory.Core/DataStore/Sql/QueryHandlers/UpsertProvidersFromCosmosHandler.cs b/src/Dfc.CourseDirectory.Core/DataStore/Sql/QueryHandlers/UpsertProvidersFromCosmosHandler.cs deleted file mode 100644 index 2ba0a21900..0000000000 --- a/src/Dfc.CourseDirectory.Core/DataStore/Sql/QueryHandlers/UpsertProvidersFromCosmosHandler.cs +++ /dev/null @@ -1,268 +0,0 @@ -using System.Data.SqlClient; -using System.Linq; -using System.Threading.Tasks; -using Dapper; -using Dfc.CourseDirectory.Core.DataStore.Sql.Queries; -using OneOf.Types; - -namespace Dfc.CourseDirectory.Core.DataStore.Sql.QueryHandlers -{ - public class UpsertProvidersFromCosmosHandler : ISqlQueryHandler - { - public async Task Execute(SqlTransaction transaction, UpsertProvidersFromCosmos query) - { - await UpsertProvider(); - await UpsertProviderContacts(); - - async Task UpsertProvider() - { - var createTableSql = @" -CREATE TABLE #Providers ( - ProviderId UNIQUEIDENTIFIER, - Ukprn INT, - ProviderStatus TINYINT, - ProviderType TINYINT, - ProviderName NVARCHAR(MAX), - UkrlpProviderStatusDescription NVARCHAR(MAX), - MarketingInformation NVARCHAR(MAX), - CourseDirectoryName NVARCHAR(MAX), - TradingName NVARCHAR(MAX), - Alias NVARCHAR(MAX), - UpdatedOn DATETIME, - UpdatedBy NVARCHAR(MAX), - TribalProviderId INT, - BulkUploadInProgress BIT NULL, - BulkUploadPublishInProgress BIT NULL, - BulkUploadStartedDateTime DATETIME2 (7) NULL, - BulkUploadTotalRowCount INT NULL -)"; - - await transaction.Connection.ExecuteAsync(createTableSql, transaction: transaction); - - await BulkCopyHelper.WriteRecords( - query.Records.Select(r => new - { - r.ProviderId, - r.Ukprn, - ProviderStatus = (byte)r.ProviderStatus, - ProviderType = (byte)r.ProviderType, - r.ProviderName, - r.UkrlpProviderStatusDescription, - r.MarketingInformation, - r.CourseDirectoryName, - r.TradingName, - r.Alias, - r.UpdatedOn, - r.UpdatedBy, - r.TribalProviderId, - r.BulkUploadInProgress, - r.BulkUploadPublishInProgress, - r.BulkUploadStartedDateTime, - r.BulkUploadTotalRowCount - }), - tableName: "#Providers", - transaction); - - var sql = @" -MERGE Pttcd.Providers AS target -USING (SELECT * FROM #Providers) AS source -ON target.ProviderId = source.ProviderId -WHEN NOT MATCHED THEN - INSERT ( - ProviderId, - LastSyncedFromCosmos, - Ukprn, - ProviderStatus, - ProviderType, - ProviderName, - UkrlpProviderStatusDescription, - MarketingInformation, - CourseDirectoryName, - TradingName, - Alias, - UpdatedOn, - UpdatedBy, - TribalProviderId, - BulkUploadInProgress, - BulkUploadPublishInProgress, - BulkUploadStartedDateTime, - BulkUploadTotalRowCount - ) VALUES ( - source.ProviderId, - @LastSyncedFromCosmos, - source.Ukprn, - source.ProviderStatus, - source.ProviderType, - source.ProviderName, - source.UkrlpProviderStatusDescription, - source.MarketingInformation, - source.CourseDirectoryName, - source.TradingName, - source.Alias, - source.UpdatedOn, - source.UpdatedBy, - source.TribalProviderId, - source.BulkUploadInProgress, - source.BulkUploadPublishInProgress, - source.BulkUploadStartedDateTime, - source.BulkUploadTotalRowCount - ) -WHEN MATCHED THEN - UPDATE SET - Ukprn = source.Ukprn, - LastSyncedFromCosmos = @LastSyncedFromCosmos, - ProviderStatus = source.ProviderStatus, - ProviderType = source.ProviderType, - ProviderName = source.ProviderName, - UkrlpProviderStatusDescription = source.UkrlpProviderStatusDescription, - MarketingInformation = source.MarketingInformation, - CourseDirectoryName = source.CourseDirectoryName, - TradingName = source.TradingName, - Alias = source.Alias, - UpdatedOn = source.UpdatedOn, - UpdatedBy = source.UpdatedBy, - TribalProviderId = source.TribalProviderId, - BulkUploadInProgress = source.BulkUploadInProgress, - BulkUploadPublishInProgress = source.BulkUploadPublishInProgress, - BulkUploadStartedDateTime = source.BulkUploadStartedDateTime, - BulkUploadTotalRowCount = source.BulkUploadTotalRowCount;"; - - await transaction.Connection.ExecuteAsync( - sql, - param: new { query.LastSyncedFromCosmos }, - transaction: transaction); - } - - async Task UpsertProviderContacts() - { - var createTableSql = @" -CREATE TABLE #ProviderContacts ( - ProviderId UNIQUEIDENTIFIER, - ProviderContactIndex INT, - ContactType CHAR, - ContactRole NVARCHAR(MAX), - AddressSaonDescription NVARCHAR(MAX), - AddressPaonDescription NVARCHAR(MAX), - AddressStreetDescription NVARCHAR(MAX), - AddressLocality NVARCHAR(MAX), - AddressItems NVARCHAR(MAX), - AddressPostTown NVARCHAR(MAX), - AddressCounty NVARCHAR(MAX), - AddressPostcode NVARCHAR(MAX), - PersonalDetailsPersonNameTitle NVARCHAR(MAX), - PersonalDetailsPersonNameGivenName NVARCHAR(MAX), - PersonalDetailsPersonNameFamilyName NVARCHAR(MAX), - Telephone1 NVARCHAR(MAX), - Telephone2 NVARCHAR(MAX), - Fax NVARCHAR(MAX), - WebsiteAddress NVARCHAR(MAX), - Email NVARCHAR(MAX) -)"; - - await transaction.Connection.ExecuteAsync(createTableSql, transaction: transaction); - - await BulkCopyHelper.WriteRecords( - query.Records - .SelectMany(p => p.Contacts.Select((c, i) => new - { - p.ProviderId, - ProviderContactIndex = i, - c.ContactType, - c.ContactRole, - c.AddressSaonDescription, - c.AddressPaonDescription, - c.AddressStreetDescription, - c.AddressLocality, - c.AddressItems, - c.AddressPostTown, - c.AddressCounty, - c.AddressPostcode, - c.PersonalDetailsPersonNameTitle, - c.PersonalDetailsPersonNameGivenName, - c.PersonalDetailsPersonNameFamilyName, - c.Telephone1, - c.Telephone2, - c.Fax, - c.WebsiteAddress, - c.Email - })), - tableName: "#ProviderContacts", - transaction); - - var mergeSql = @" -MERGE Pttcd.ProviderContacts AS target -USING (SELECT * FROM #ProviderContacts) AS source -ON target.ProviderId = source.ProviderId AND target.ProviderContactIndex = source.ProviderContactIndex -WHEN NOT MATCHED THEN - INSERT ( - ProviderId, - ProviderContactIndex, - ContactType, - ContactRole, - AddressSaonDescription, - AddressPaonDescription, - AddressStreetDescription, - AddressLocality, - AddressItems, - AddressPostTown, - AddressCounty, - AddressPostcode, - PersonalDetailsPersonNameTitle, - PersonalDetailsPersonNameGivenName, - PersonalDetailsPersonNameFamilyName, - Telephone1, - Telephone2, - Fax, - WebsiteAddress, - Email - ) VALUES ( - source.ProviderId, - source.ProviderContactIndex, - source.ContactType, - source.ContactRole, - source.AddressSaonDescription, - source.AddressPaonDescription, - source.AddressStreetDescription, - source.AddressLocality, - source.AddressItems, - source.AddressPostTown, - source.AddressCounty, - source.AddressPostcode, - source.PersonalDetailsPersonNameTitle, - source.PersonalDetailsPersonNameGivenName, - source.PersonalDetailsPersonNameFamilyName, - source.Telephone1, - source.Telephone2, - source.Fax, - source.WebsiteAddress, - source.Email - ) -WHEN MATCHED THEN UPDATE SET - ProviderContactIndex = source.ProviderContactIndex, - ContactType = source.ContactType, - ContactRole = source.ContactRole, - AddressSaonDescription = source.AddressSaonDescription, - AddressPaonDescription = source.AddressPaonDescription, - AddressStreetDescription = source.AddressStreetDescription, - AddressLocality = source.AddressLocality, - AddressItems = source.AddressItems, - AddressPostTown = source.AddressPostTown, - AddressCounty = source.AddressCounty, - AddressPostcode = source.AddressPostcode, - PersonalDetailsPersonNameTitle = source.PersonalDetailsPersonNameTitle, - PersonalDetailsPersonNameGivenName = source.PersonalDetailsPersonNameGivenName, - PersonalDetailsPersonNameFamilyName = source.PersonalDetailsPersonNameFamilyName, - Telephone1 = source.Telephone1, - Telephone2 = source.Telephone2, - Fax = source.Fax, - WebsiteAddress = source.WebsiteAddress, - Email = source.Email -WHEN NOT MATCHED BY SOURCE AND target.ProviderId IN (SELECT ProviderId FROM #Providers) THEN DELETE;"; - - await transaction.Connection.ExecuteAsync(mergeSql, transaction: transaction); - } - - return new None(); - } - } -} diff --git a/src/Dfc.CourseDirectory.Functions/RemoveZombieCoursesFromSql.cs b/src/Dfc.CourseDirectory.Functions/RemoveZombieCoursesFromSql.cs deleted file mode 100644 index baca130765..0000000000 --- a/src/Dfc.CourseDirectory.Functions/RemoveZombieCoursesFromSql.cs +++ /dev/null @@ -1,44 +0,0 @@ -using System.Threading.Tasks; -using Dapper; -using Dfc.CourseDirectory.Core.DataStore.Sql; -using Microsoft.Azure.WebJobs; - -namespace Dfc.CourseDirectory.Functions -{ - public class RemoveZombieCoursesFromSql - { - private readonly ISqlQueryDispatcher _sqlQueryDispatcher; - - public RemoveZombieCoursesFromSql(ISqlQueryDispatcher sqlQueryDispatcher) - { - _sqlQueryDispatcher = sqlQueryDispatcher; - } - - [FunctionName(nameof(RemoveZombieCoursesFromSql))] - [NoAutomaticTrigger] - public Task Run(string input) - { - var sql = @" -select CourseId into #CourseIds from Pttcd.Courses where LastSyncedFromCosmos is null - -delete Pttcd.CourseRunRegions -from Pttcd.CourseRunRegions crr -join Pttcd.CourseRuns cr on crr.CourseRunId = cr.CourseRunId -join #CourseIds x on x.CourseId = cr.CourseId - -delete Pttcd.CourseRuns -from Pttcd.CourseRuns cr -join #CourseIds x on x.CourseId = cr.CourseId - -delete Pttcd.Courses -from Pttcd.Courses c -join #CourseIds x on x.CourseId = c.CourseId - -drop table #CourseIds"; - - return _sqlQueryDispatcher.Transaction.Connection.ExecuteAsync( - sql, - transaction: _sqlQueryDispatcher.Transaction); - } - } -} diff --git a/src/Dfc.CourseDirectory.Functions/local.settings.json b/src/Dfc.CourseDirectory.Functions/local.settings.json index 4bbc5894ad..3c4670f80d 100644 --- a/src/Dfc.CourseDirectory.Functions/local.settings.json +++ b/src/Dfc.CourseDirectory.Functions/local.settings.json @@ -2,7 +2,6 @@ "IsEncrypted": false, "Values": { "AzureWebJobs.SyncUkrlpChanges.Disabled": "true", - "AzureWebJobs.RealTimeSyncCosmosUkrlpToSql.Disabled": "true", "AzureWebJobs.ImportLarsData.Disabled": "true", "AzureWebJobs.RefreshFindACourseIndex.Disabled": "true", "AzureWebJobsStorage": "UseDevelopmentStorage=true", diff --git a/src/Dfc.CourseDirectory.WebV2/README.md b/src/Dfc.CourseDirectory.WebV2/README.md index 6d6f433e5a..89e967fdb3 100644 --- a/src/Dfc.CourseDirectory.WebV2/README.md +++ b/src/Dfc.CourseDirectory.WebV2/README.md @@ -30,7 +30,7 @@ For an intro to MediatR in controllers see: There are no controller tests because the controller actions are very thin mappings. -The tests are mostly integration tests that test an endpoint + behaviour, with things like CosmosDB calls mocked out. +The tests are mostly integration tests that test an endpoint + behaviour. ### Multi-page transactions - FormFlow