Skip to content

Commit

Permalink
Extended live sync to sync induction details
Browse files Browse the repository at this point in the history
Tweak to ensure required CRM entity attributes are retrieved for sync

Amended to clear down data between tests which was causing indeterminate test results
  • Loading branch information
hortha committed Dec 10, 2024
1 parent 735ece1 commit 375426b
Show file tree
Hide file tree
Showing 6 changed files with 167 additions and 57 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@ public static Command CreateSyncPersonCommand(IConfiguration configuration)
}

await syncHelper.SyncPersonAsync(contact, ignoreInvalid: false, dryRun: false);

await syncHelper.SyncInductionsAsync([contact], ignoreInvalid: false, createMigratedEvent: false, dryRun: false);
//return 0;
},
connectionStringOption,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ public class Person
public required DateOnly? DateOfBirth { get; set; } // A few DQT records in prod have a null DOB
public string? EmailAddress { get; set; }
public string? NationalInsuranceNumber { get; set; }
public InductionStatus InductionStatus { get; private set; }
public InductionStatus InductionStatus { get; set; }
public InductionExemptionReasons InductionExemptionReasons { get; private set; }
public DateOnly? InductionStartDate { get; private set; }
public DateOnly? InductionCompletedDate { get; private set; }
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -319,7 +319,7 @@ public async Task<int> SyncInductionsAsync(
bool ignoreInvalid,
bool createMigratedEvent,
bool dryRun,
CancellationToken cancellationToken)
CancellationToken cancellationToken = default)
{
var inductionAttributeNames = new[]
{
Expand All @@ -328,6 +328,8 @@ public async Task<int> SyncInductionsAsync(
dfeta_induction.Fields.dfeta_InductionExemptionReason,
dfeta_induction.Fields.dfeta_StartDate,
dfeta_induction.Fields.dfeta_InductionStatus,
dfeta_induction.Fields.CreatedOn,
dfeta_induction.Fields.CreatedBy,
dfeta_induction.Fields.ModifiedOn
};

Expand Down Expand Up @@ -774,7 +776,7 @@ private async Task<IReadOnlyDictionary<Guid, AuditDetailCollection>> GetAuditRec
IEnumerable<Guid> ids,
CancellationToken cancellationToken)
{
return (await Task.WhenAll(ids
return (IsFakeXrm ? new Dictionary<Guid, AuditDetailCollection>() : (await Task.WhenAll(ids
.Chunk(MaxAuditRequestsPerBatch)
.Select(async chunk =>
{
Expand All @@ -788,6 +790,7 @@ private async Task<IReadOnlyDictionary<Guid, AuditDetailCollection>> GetAuditRec
}
};

// The following is not supported by FakeXrmEasy hence the check above to allow more test coverage
request.Requests.AddRange(chunk.Select(e => new RetrieveRecordChangeHistoryRequest() { Target = e.ToEntityReference(entityLogicalName) }));

ExecuteMultipleResponse response;
Expand Down Expand Up @@ -829,7 +832,7 @@ private async Task<IReadOnlyDictionary<Guid, AuditDetailCollection>> GetAuditRec
(r, e) => (Id: e, ((RetrieveRecordChangeHistoryResponse)r.Response).AuditDetailCollection));
})))
.SelectMany(b => b)
.ToDictionary(t => t.Id, t => t.AuditDetailCollection);
.ToDictionary(t => t.Id, t => t.AuditDetailCollection));
}

private async Task<TEntity[]> GetEntitiesAsync<TEntity>(
Expand Down Expand Up @@ -876,7 +879,8 @@ private static ModelTypeSyncInfo GetModelTypeSyncInfoForPerson()
"dqt_modified_on",
"dqt_first_name",
"dqt_middle_name",
"dqt_last_name"
"dqt_last_name",
"induction_status",
};

var columnsToUpdate = columnNames.Except(new[] { "person_id", "dqt_contact_id" }).ToArray();
Expand Down Expand Up @@ -905,6 +909,7 @@ WHERE t.dqt_modified_on < EXCLUDED.dqt_modified_on
Contact.Fields.ContactId,
Contact.Fields.StateCode,
Contact.Fields.CreatedOn,
Contact.Fields.CreatedBy,
Contact.Fields.ModifiedOn,
Contact.Fields.dfeta_TRN,
Contact.Fields.FirstName,
Expand All @@ -915,7 +920,8 @@ WHERE t.dqt_modified_on < EXCLUDED.dqt_modified_on
Contact.Fields.dfeta_StatedLastName,
Contact.Fields.BirthDate,
Contact.Fields.dfeta_NINumber,
Contact.Fields.EMailAddress1
Contact.Fields.EMailAddress1,
Contact.Fields.dfeta_InductionStatus
};

Action<NpgsqlBinaryImporter, Person> writeRecord = (writer, person) =>
Expand All @@ -937,6 +943,7 @@ WHERE t.dqt_modified_on < EXCLUDED.dqt_modified_on
writer.WriteValueOrNull(person.DqtFirstName, NpgsqlDbType.Varchar);
writer.WriteValueOrNull(person.DqtMiddleName, NpgsqlDbType.Varchar);
writer.WriteValueOrNull(person.DqtLastName, NpgsqlDbType.Varchar);
writer.WriteValueOrNull((int?)person.InductionStatus, NpgsqlDbType.Integer);
};

return new ModelTypeSyncInfo<Person>()
Expand Down Expand Up @@ -1171,38 +1178,11 @@ private static List<Person> MapPersons(IEnumerable<Contact> contacts) => contact
DqtModifiedOn = c.ModifiedOn!.Value,
DqtFirstName = c.FirstName ?? string.Empty,
DqtMiddleName = c.MiddleName ?? string.Empty,
DqtLastName = c.LastName ?? string.Empty
DqtLastName = c.LastName ?? string.Empty,
InductionStatus = c.dfeta_InductionStatus.ToInductionStatus()
})
.ToList();

private static List<InductionInfo> MapInductions(IReadOnlyCollection<Contact> contacts, IEnumerable<dfeta_induction> inductions, bool ignoreInvalid)
{
var inductionLookup = inductions
.GroupBy(i => i.dfeta_PersonId.Id)
.ToDictionary(g => g.Key, g => g.ToArray());

return contacts
.Select(contact =>
{
dfeta_induction? induction = null;
if (inductionLookup.TryGetValue(contact.ContactId!.Value, out var personInductions))
{
// We shouldn't have multiple induction records for the same person in prod at all but we might in other environments
// so we'll just take the most recently modified one.
induction = personInductions.OrderByDescending(i => i.ModifiedOn).First();
if (personInductions.Length > 1 && !ignoreInvalid)
{
throw new InvalidOperationException($"Contact '{contact.ContactId!.Value}' has multiple induction records.");
}
}

return MapInductionInfoFromDqtInduction(induction, contact, ignoreInvalid);
})
.Where(i => i is not null)
.Cast<InductionInfo>()
.ToList();
}

private (List<InductionInfo> Inductions, List<EventBase> Events) MapInductionsAndAudits(
IReadOnlyCollection<Contact> contacts,
IEnumerable<dfeta_induction> inductionEntities,
Expand Down Expand Up @@ -1247,39 +1227,46 @@ private static List<InductionInfo> MapInductions(IReadOnlyCollection<Contact> co
dfeta_induction.Fields.dfeta_InductionStatus,
dfeta_induction.Fields.ModifiedOn
};
var inductionAudits = auditDetails[induction!.Id].AuditDetails;
var inductionVersions = GetEntityVersions(induction, inductionAudits, inductionAttributeNames);

events.Add(inductionAudits.Any(a => a.AuditRecord.ToEntity<Audit>().Action == Audit_Action.Create) ?
MapCreatedEvent(inductionVersions.First()) :
MapImportedEvent(inductionVersions.First()));

foreach (var (thisVersion, previousVersion) in inductionVersions.Skip(1).Zip(inductionVersions, (thisVersion, previousVersion) => (thisVersion, previousVersion)))
if (auditDetails.TryGetValue(induction!.Id, out var inductionAudits))
{
var mappedEvent = MapUpdatedEvent(thisVersion, previousVersion);
var inductionAuditDetails = inductionAudits.AuditDetails;
var inductionVersions = GetEntityVersions(induction, inductionAuditDetails, inductionAttributeNames);

if (mappedEvent is not null)
events.Add(inductionAuditDetails.Any(a => a.AuditRecord.ToEntity<Audit>().Action == Audit_Action.Create) ?
MapCreatedEvent(inductionVersions.First()) :
MapImportedEvent(inductionVersions.First()));

foreach (var (thisVersion, previousVersion) in inductionVersions.Skip(1).Zip(inductionVersions, (thisVersion, previousVersion) => (thisVersion, previousVersion)))
{
events.Add(mappedEvent);
var mappedEvent = MapUpdatedEvent(thisVersion, previousVersion);

if (mappedEvent is not null)
{
events.Add(mappedEvent);
}
}
}

if (createMigratedEvent)
{
events.Add(MapMigratedEvent(inductionVersions.Last(), mapped));
if (createMigratedEvent)
{
events.Add(MapMigratedEvent(inductionVersions.Last(), mapped));
}
}
}

var contactAudits = auditDetails[contact.ContactId!.Value].AuditDetails;
var contactVersions = GetEntityVersions(contact, contactAudits, GetModelTypeSyncInfo(ModelTypes.Person).AttributeNames);

foreach (var (thisVersion, previousVersion) in contactVersions.Skip(1).Zip(contactVersions, (thisVersion, previousVersion) => (thisVersion, previousVersion)))
if (auditDetails.TryGetValue(contact.ContactId!.Value, out var contactAudits))
{
var mappedEvent = MapContactInductionStatusChangedEvent(thisVersion, previousVersion);
var contactAuditDetails = contactAudits.AuditDetails;
var contactVersions = GetEntityVersions(contact, contactAuditDetails, GetModelTypeSyncInfo(ModelTypes.Person).AttributeNames);

if (mappedEvent is not null)
foreach (var (thisVersion, previousVersion) in contactVersions.Skip(1).Zip(contactVersions, (thisVersion, previousVersion) => (thisVersion, previousVersion)))
{
events.Add(mappedEvent);
var mappedEvent = MapContactInductionStatusChangedEvent(thisVersion, previousVersion);

if (mappedEvent is not null)
{
events.Add(mappedEvent);
}
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ internal async Task ProcessChangesAsync(CancellationToken cancellationToken)
var modelTypesToSync = optionsAccessor.Value.ModelTypes;

// Order is important here; the dependees should come before dependents
await SyncIfEnabledAsync(TrsDataSyncHelper.ModelTypes.Induction);
await SyncIfEnabledAsync(TrsDataSyncHelper.ModelTypes.Person);
await SyncIfEnabledAsync(TrsDataSyncHelper.ModelTypes.Event);
await SyncIfEnabledAsync(TrsDataSyncHelper.ModelTypes.Alert);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
using Microsoft.Xrm.Sdk;
using Microsoft.Xrm.Sdk.Messages;
using TeachingRecordSystem.Core.Dqt;
using TeachingRecordSystem.Core.Dqt.Models;
using TeachingRecordSystem.Core.Services.TrsDataSync;

namespace TeachingRecordSystem.Core.Tests.Services.TrsDataSync;

public partial class TrsDataSyncServiceTests
{
[Theory]
[InlineData(true)]
[InlineData(false)]
public async Task Induction_NewRecord_WritesUpdatedPersonRecordToDatabase(bool personAlreadySynced)
{
// Arrange
var createPersonResult = await TestData.CreatePersonAsync(p => p.WithSyncOverride(personAlreadySynced));
var contactId = createPersonResult.ContactId;

var inductionStartDate = Clock.Today.AddYears(-1);
var inductionEndDate = Clock.Today.AddDays(-10);
var induction = new dfeta_induction()
{
Id = Guid.NewGuid(),
dfeta_PersonId = new EntityReference(Contact.EntityLogicalName, contactId),
dfeta_InductionStatus = dfeta_InductionStatus.Pass,
dfeta_StartDate = inductionStartDate.ToDateTimeWithDqtBstFix(isLocalTime: true),
dfeta_CompletionDate = inductionEndDate.ToDateTimeWithDqtBstFix(isLocalTime: true),
CreatedOn = Clock.UtcNow,
ModifiedOn = Clock.UtcNow
};

// Keep the contact induction status in sync with dfeta_induction otherwise the sync will fail
await TestData.OrganizationService.ExecuteAsync(new UpdateRequest()
{
Target = new Contact()
{
Id = contactId,
dfeta_InductionStatus = dfeta_InductionStatus.Pass
}
});

var newItem = new NewOrUpdatedItem(ChangeType.NewOrUpdated, induction);

// Act
await fixture.PublishChangedItemAndConsumeAsync(TrsDataSyncHelper.ModelTypes.Induction, newItem);

// Assert
await fixture.DbFixture.WithDbContextAsync(async dbContext =>
{
var person = await dbContext.Persons.SingleOrDefaultAsync(p => p.DqtContactId == contactId);
Assert.NotNull(person);
Assert.Equal(InductionStatus.Passed, person.InductionStatus);
Assert.Equal(inductionStartDate, person.InductionStartDate);
Assert.Equal(inductionEndDate, person.InductionCompletedDate);
});
}

[Fact]
public async Task Induction_UpdatedRecord_WritesUpdatedPersonRecordToDatabase()
{
// Arrange
var originalInductionStatus = dfeta_InductionStatus.InProgress;
var originalInductionStartDate = Clock.Today.AddYears(-1);
var originalInductionEndDate = Clock.Today.AddDays(-10);

var createPersonResult = await TestData.CreatePersonAsync(
p => p.WithSyncOverride(true)
.WithDqtInduction(originalInductionStatus, null, originalInductionStartDate, null));
var contactId = createPersonResult.ContactId;
var existingInduction = createPersonResult.DqtInductions.Single();

var updatedInductionStatus = dfeta_InductionStatus.Pass;
var updatedInductionStartDate = Clock.Today.AddYears(-2);
var updatedInductionEndDate = Clock.Today.AddDays(-20);
var createdOn = Clock.UtcNow;
var modifiedOn = Clock.Advance();
var updatedInduction = new dfeta_induction()
{
Id = existingInduction.InductionId,
dfeta_PersonId = new EntityReference(Contact.EntityLogicalName, contactId),
dfeta_InductionStatus = updatedInductionStatus,
dfeta_StartDate = updatedInductionStartDate.ToDateTimeWithDqtBstFix(isLocalTime: true),
dfeta_CompletionDate = updatedInductionEndDate.ToDateTimeWithDqtBstFix(isLocalTime: true),
CreatedOn = Clock.UtcNow,
ModifiedOn = modifiedOn
};

// Keep the contact induction status in sync with dfeta_induction otherwise the sync will fail
await TestData.OrganizationService.ExecuteAsync(new UpdateRequest()
{
Target = new Contact()
{
Id = contactId,
dfeta_InductionStatus = dfeta_InductionStatus.Pass
}
});

var updatedItem = new NewOrUpdatedItem(ChangeType.NewOrUpdated, updatedInduction);

// Act
await fixture.PublishChangedItemAndConsumeAsync(TrsDataSyncHelper.ModelTypes.Induction, updatedItem);

// Assert
await fixture.DbFixture.WithDbContextAsync(async dbContext =>
{
var person = await dbContext.Persons.SingleOrDefaultAsync(p => p.DqtContactId == contactId);
Assert.NotNull(person);
Assert.Equal(InductionStatus.Passed, person.InductionStatus);
Assert.Equal(updatedInductionStartDate, person.InductionStartDate);
Assert.Equal(updatedInductionEndDate, person.InductionCompletedDate);
Assert.Equal(Clock.UtcNow, person.DqtInductionModifiedOn);
});
}
}
Original file line number Diff line number Diff line change
@@ -1,9 +1,14 @@

namespace TeachingRecordSystem.Core.Tests.Services.TrsDataSync;

[Collection(nameof(TrsDataSyncTestCollection))]
public partial class TrsDataSyncServiceTests(TrsDataSyncServiceFixture fixture) : IClassFixture<TrsDataSyncServiceFixture>
public partial class TrsDataSyncServiceTests(TrsDataSyncServiceFixture fixture) : IClassFixture<TrsDataSyncServiceFixture>, IAsyncLifetime
{
private TestableClock Clock => fixture.Clock;

private TestData TestData => fixture.TestData;

Task IAsyncLifetime.DisposeAsync() => Task.CompletedTask;

Task IAsyncLifetime.InitializeAsync() => fixture.DbFixture.DbHelper.ClearDataAsync();
}

0 comments on commit 375426b

Please sign in to comment.