From c37d2a4650892358245fef51d393bd530fbb4b40 Mon Sep 17 00:00:00 2001 From: Andrew Horth Date: Thu, 14 Mar 2024 10:56:12 +0000 Subject: [PATCH] Tweaked queries to only match on LA Code and Postcode for HEI if there isn't already a match on LA Code and Establishment Number --- .../WorkforceData/TpsCsvExtractProcessor.cs | 98 +++++++++++++++---- .../TpsCsvExtractProcessorTests.cs | 36 +++++++ 2 files changed, 114 insertions(+), 20 deletions(-) diff --git a/TeachingRecordSystem/src/TeachingRecordSystem.Core/Services/WorkforceData/TpsCsvExtractProcessor.cs b/TeachingRecordSystem/src/TeachingRecordSystem.Core/Services/WorkforceData/TpsCsvExtractProcessor.cs index b37b472ac8..872194d964 100644 --- a/TeachingRecordSystem/src/TeachingRecordSystem.Core/Services/WorkforceData/TpsCsvExtractProcessor.cs +++ b/TeachingRecordSystem/src/TeachingRecordSystem.Core/Services/WorkforceData/TpsCsvExtractProcessor.cs @@ -30,22 +30,64 @@ public async Task ProcessNonMatchingTrns(Guid tpsCsvExtractId, CancellationToken public async Task ProcessNonMatchingEstablishments(Guid tpsCsvExtractId, CancellationToken cancellationToken) { - int i = 0; using var dbContext = dbContextFactory.CreateDbContext(); - foreach (var item in await dbContext.TpsCsvExtractItems.Where(x => x.TpsCsvExtractId == tpsCsvExtractId && dbContext.Persons.Any(p => p.Trn == x.Trn) && !dbContext.Establishments.Any(e => e.LaCode == x.LocalAuthorityCode && e.EstablishmentNumber == x.EstablishmentNumber)).ToListAsync()) - { - item.Result = TpsCsvExtractItemResult.InvalidEstablishment; - i++; - if (i % 1000 == 0) - { - await dbContext.SaveChangesAsync(cancellationToken); - } - } + dbContext.Database.SetCommandTimeout(300); - if (dbContext.ChangeTracker.HasChanges()) - { - await dbContext.SaveChangesAsync(cancellationToken); - } + FormattableString updateSql = + $""" + WITH unique_establishments AS ( + SELECT + establishment_id, + la_code, + establishment_number, + establishment_name, + establishment_type_code, + postcode + FROM + (SELECT + establishment_id, + la_code, + establishment_number, + establishment_name, + establishment_type_code, + postcode, + ROW_NUMBER() OVER (PARTITION BY la_code, establishment_number ORDER BY translate(establishment_status_code::text, '1234', '1324')) as row_number + FROM + establishments) e + WHERE + e.row_number = 1 + ) + UPDATE + tps_csv_extract_items x + SET + result = {TpsCsvExtractItemResult.InvalidEstablishment} + WHERE + x.tps_csv_extract_id = {tpsCsvExtractId} + AND EXISTS (SELECT + 1 + FROM + persons p + WHERE + p.trn = x.trn) + AND NOT EXISTS (SELECT + 1 + FROM + unique_establishments e + WHERE + x.local_authority_code = e.la_code + AND (e.establishment_number = x.establishment_number + OR (e.establishment_type_code = '29' + AND x.establishment_postcode = e.postcode + AND NOT EXISTS (SELECT + 1 + FROM + unique_establishments e2 + WHERE + e2.la_code = x.local_authority_code + AND e2.establishment_number = x.establishment_number)))) + """; + + await dbContext.Database.ExecuteSqlAsync(updateSql, cancellationToken); } public async Task ProcessNewEmploymentHistory(Guid tpsCsvExtractId, CancellationToken cancellationToken) @@ -91,16 +133,24 @@ tps_csv_extract_items x persons p ON x.trn = p.trn JOIN unique_establishments e ON x.local_authority_code = e.la_code - AND (x.establishment_number = e.establishment_number OR - (e.establishment_type_code = '29' AND x.establishment_postcode = e.postcode)) + AND (e.establishment_number = x.establishment_number + OR (e.establishment_type_code = '29' + AND x.establishment_postcode = e.postcode + AND NOT EXISTS (SELECT + 1 + FROM + unique_establishments e2 + WHERE + e2.la_code = x.local_authority_code + AND e2.establishment_number = x.establishment_number))) WHERE x.tps_csv_extract_id = {tpsCsvExtractId} AND x.result IS NULL AND NOT EXISTS (SELECT 1 - FROM + FROM person_employments pe - WHERE + WHERE pe.person_id = p.person_id AND pe.establishment_id = e.establishment_id AND pe.start_date = x.employment_start_date) @@ -212,8 +262,16 @@ tps_csv_extract_items x persons p ON x.trn = p.trn JOIN unique_establishments e ON x.local_authority_code = e.la_code - AND (x.establishment_number = e.establishment_number OR - (e.establishment_type_code = '29' AND x.establishment_postcode = e.postcode)) + AND (e.establishment_number = x.establishment_number + OR (e.establishment_type_code = '29' + AND x.establishment_postcode = e.postcode + AND NOT EXISTS (SELECT + 1 + FROM + unique_establishments e2 + WHERE + e2.la_code = x.local_authority_code + AND e2.establishment_number = x.establishment_number))) JOIN person_employments pe ON pe.person_id = p.person_id AND pe.establishment_id = e.establishment_id diff --git a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/WorkforceData/TpsCsvExtractProcessorTests.cs b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/WorkforceData/TpsCsvExtractProcessorTests.cs index 81094dfd25..99c88fa135 100644 --- a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/WorkforceData/TpsCsvExtractProcessorTests.cs +++ b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/WorkforceData/TpsCsvExtractProcessorTests.cs @@ -134,6 +134,42 @@ public async Task ProcessNewEmploymentHistory_ForLaCodeAndEstablishmentNumberWit Assert.Equal(openEstablishment.EstablishmentId, personEmployment.EstablishmentId); } + [Fact] + public async Task ProcessNewEmploymentHistory_WithValidData_OnlyMatchesToLaCodeAndPostCodeForHigherEducationIfNoMatchOnLaCodeAndEstablishment() + { + // Arrange + var person = await TestData.CreatePerson(); + var tpsCsvExtractId = Guid.NewGuid(); + var laCode1 = "322"; + var establishmentNumber1 = "4322"; + var postcode1 = Faker.Address.UkPostCode(); + var laCode2 = "323"; + var establishmentNumber2 = "4323"; + var postcode2 = Faker.Address.UkPostCode(); + var nonHigherEducationEstablishment1 = await TestData.CreateEstablishment(laCode1, establishmentNumber: establishmentNumber1, postcode: postcode1); + var higherEductionEstablishment1 = await TestData.CreateEstablishment(laCode1, establishmentNumber: establishmentNumber1, postcode: postcode1, isHigherEducationInstitution: true); + var higherEductionEstablishment2 = await TestData.CreateEstablishment(laCode2, postcode: postcode2, isHigherEducationInstitution: true); + await TestData.CreateTpsCsvExtract( + b => b.WithTpsCsvExtractId(tpsCsvExtractId) + .WithItem(person!.Trn!, laCode1, establishmentNumber1, postcode1, new DateOnly(2023, 02, 03)) + .WithItem(person!.Trn!, laCode2, establishmentNumber2, postcode2, new DateOnly(2023, 02, 03))); + + // Act + var processor = new TpsCsvExtractProcessor( + TestData.DbContextFactory, + TestData.Clock); + await processor.ProcessNewEmploymentHistory(tpsCsvExtractId, CancellationToken.None); + + // Assert + using var dbContext = TestData.DbContextFactory.CreateDbContext(); + var items = await dbContext.TpsCsvExtractItems.Where(i => i.TpsCsvExtractId == tpsCsvExtractId).ToListAsync(); + Assert.All(items, i => Assert.Equal(TpsCsvExtractItemResult.ValidDataAdded, i.Result)); + var employmentHistory = await dbContext.PersonEmployments.Where(e => e.PersonId == person.PersonId).ToListAsync(); + Assert.Equal(2, employmentHistory.Count); + Assert.Contains(nonHigherEducationEstablishment1.EstablishmentId, employmentHistory.Select(pe => pe.EstablishmentId)); + Assert.Contains(higherEductionEstablishment2.EstablishmentId, employmentHistory.Select(pe => pe.EstablishmentId)); + } + [Fact] public async Task ProcessUpdatedEmploymentHistory_WhenCalledWithUpdatedEmploymentHistory_UpdatesPersonEmploymentRecord() {