Implemented clearing down removed links from notifications

Btms.Business/Pipelines/PreProcessing/MovementPreProcessor.cs

@@ -12,7 +12,6 @@ public class MovementPreProcessor(IMongoDbContext dbContext, ILogger<MovementPre
 {
     public async Task<PreProcessingResult<Movement>> Process(PreProcessingContext<AlvsClearanceRequest> preProcessingContext)
     {
-
         var internalClearanceRequest = AlvsClearanceRequestMapper.Map(preProcessingContext.Message);
         var movement = BuildMovement(internalClearanceRequest);
         var existingMovement = await dbContext.Movements.Find(movement.Id!);
@@ -35,7 +34,6 @@ public async Task<PreProcessingResult<Movement>> Process(PreProcessingContext<Al
         {
             var changeSet = movement.ClearanceRequests[^1].GenerateChangeSet(existingMovement.ClearanceRequests[0]);
 
-
             var auditEntry = AuditEntry.CreateUpdated(changeSet,
                 preProcessingContext.MessageId,
                 movement.ClearanceRequests[0].Header!.EntryVersionNumber.GetValueOrDefault(),
@@ -61,7 +59,6 @@ public async Task<PreProcessingResult<Movement>> Process(PreProcessingContext<Al
 
         logger.MessageSkipped(preProcessingContext.MessageId, preProcessingContext.Message.Header?.EntryReference!);
         return PreProcessResult.Skipped(existingMovement);
-
     }
 
     public static Movement BuildMovement(Model.Cds.CdsClearanceRequest request)

Btms.Business/Services/Linking/LinkingService.cs

@@ -54,6 +54,8 @@ public async Task<LinkResult> Link(LinkContext linkContext, CancellationToken ca
                             logger.LinkNotAttempted(linkContext.GetType().Name, linkContext.GetIdentifiers());
                             return new LinkResult(LinkOutcome.NotLinked);
                         }
+
+                        await CheckMovementForRemovedLinks(movementLinkContext, cancellationToken);
 
                         result = await FindMovementLinks(movementLinkContext.PersistedMovement, cancellationToken);
                         break;
@@ -70,7 +72,6 @@ public async Task<LinkResult> Link(LinkContext linkContext, CancellationToken ca
                     default: throw new ArgumentException("context type not supported");
                 }
 
-
                 if (result.Outcome == LinkOutcome.NotLinked)
                 {
                     logger.LinkNotFound(linkContext.GetType().Name, linkContext.GetIdentifiers());
@@ -83,9 +84,9 @@ public async Task<LinkResult> Link(LinkContext linkContext, CancellationToken ca
                 metrics.Linked<ImportNotification>(result.Notifications.Count);
 
                 using var transaction = await dbContext.StartTransaction(cancellationToken);
-                foreach (var notification in result.Notifications)
+                foreach (var movement in result.Movements)
                 {
-                    foreach (var movement in result.Movements)
+                    foreach (var notification in result.Notifications)
                     {
                         notification.AddRelationship(new TdmRelationshipObject
                         {
@@ -129,11 +130,46 @@ await dbContext.Notifications.Update(notification, notification._Etag, transacti
             }
         }
 
-
-
         return result;
     }
 
+    private async Task CheckMovementForRemovedLinks(MovementLinkContext linkContext,
+        CancellationToken cancellationToken = default)
+    {
+        var chedRefs = linkContext.ChangeSet?.GetPreviousValue<List<string>>($"$.{nameof(Movement._MatchReferences)}");
+
+        if (chedRefs?.Count > 0)
+        {
+            var removedRefs = chedRefs.Except(linkContext.PersistedMovement._MatchReferences).ToList();
+            if (removedRefs.Any())
+            {
+                foreach (var chedRef in chedRefs)
+                {
+                    await RemoveMovementLinkFromNotification(linkContext.PersistedMovement.Id, chedRef,
+                        cancellationToken);
+                }
+            }
+        }
+    }
+
+    private async Task RemoveMovementLinkFromNotification(string? movementId, string chedRef,
+        CancellationToken cancellationToken = default)
+    {
+        var notification = dbContext.Notifications.SingleOrDefault(x => x._MatchReference == chedRef);
+
+        if (notification != null)
+        {
+            var relationshipLink = notification.Relationships.Movements.Data?
+                .SingleOrDefault(x => x.Id == movementId && x.Type == "movements");
+
+            if (relationshipLink != null)
+            {
+                notification.RemoveRelationship(relationshipLink);
+                await dbContext.Notifications.Update(notification, notification._Etag);
+            }
+        }
+    }
+
     private static bool ShouldLinkMovement(ChangeSet? changeSet)
     {
         return changeSet is null || changeSet.HasDocumentsChanged();
@@ -194,4 +230,20 @@ public static bool HasDocumentsChanged(this ChangeSet changeSet)
             .Any(x => ItemsRegex().IsMatch(x.Path.ToString())
                       || DocumentsRegex().IsMatch(x.Path.ToString()));
     }
+
+    public static bool LinksToRemove(this ChangeSet changeSet)
+    {
+        return changeSet.JsonPatch.Operations
+            .Any(x => ItemsRegex().IsMatch(x.Path.ToString())
+                      || DocumentsRegex().IsMatch(x.Path.ToString()));
+    }
+
+    private static void TidyRemovedLinks(this MovementLinkContext linkContext)
+    {
+        var changeSet = linkContext.ChangeSet;
+        changeSet!.JsonPatch.Operations
+            .Any(x => ItemsRegex().IsMatch(x.Path.ToString())
+                      || DocumentsRegex().IsMatch(x.Path.ToString()));
+        bool derp = changeSet is null || changeSet.HasDocumentsChanged();
+    }
 }

Btms.Model/Auditing/AuditEntry.cs

@@ -120,6 +120,19 @@ public static AuditEntry CreateLinked(string id, int version)
             Status = "Linked"
         };
     }
+
+    public static AuditEntry CreateUnlinked(string id, int version, DateTime? lastUpdated)
+    {
+        return new AuditEntry
+        {
+            Id = id,
+            Version = version,
+            CreatedSource = lastUpdated,
+            CreatedBy = CreatedBySystem,
+            CreatedLocal = DateTime.UtcNow,
+            Status = "Unlinked"
+        };
+    }
 
     public static AuditEntry CreateMatch(string id, int version)
     {

Btms.Model/Btms.Model.csproj

@@ -10,6 +10,7 @@
   <ItemGroup>
     <PackageReference Include="JsonApiDotNetCore" Version="5.6.0" />
     <PackageReference Include="JsonApiDotNetCore.MongoDb" Version="5.6.0" />
+    <PackageReference Include="JsonPath.Net" Version="2.0.0" />
     <PackageReference Include="Macross.Json.Extensions" Version="3.0.0" />
     <PackageReference Include="MongoDB.Bson" Version="2.30.0" />
     <PackageReference Include="JsonPatch.Net" Version="3.1.1" />

Btms.Model/ChangeLog/ChangeSet.cs

@@ -3,12 +3,13 @@
 using System.Text.Json.Serialization;
 using Btms.Model.Extensions;
 using Json.Patch;
+using Json.Path;
 
 namespace Btms.Model.ChangeLog;
 
-public class ChangeSet(JsonPatch jsonPatch)
+public class ChangeSet(JsonPatch jsonPatch, JsonNode jsonNodePrevious)
 {
-    private static JsonSerializerOptions _jsonOptions = new()
+    private static readonly JsonSerializerOptions _jsonOptions = new()
     {
         TypeInfoResolver = new ChangeSetTypeInfoResolver(),
         PropertyNameCaseInsensitive = true,
@@ -17,15 +18,30 @@ public class ChangeSet(JsonPatch jsonPatch)
 
     public JsonPatch JsonPatch { get; } = jsonPatch;
 
+    public JsonNode Previous { get; } = jsonNodePrevious;
+
     public static ChangeSet CreateChangeSet<T>(T current, T previous)
     {
         var previousNode = JsonNode.Parse(previous.ToJsonString(_jsonOptions));
         var currentNode = JsonNode.Parse(current.ToJsonString(_jsonOptions));
         var diff = previousNode.CreatePatch(currentNode);
-
+        
         //exclude fields from patch, like _ts, audit entries etc
         var operations = diff.Operations.Where(x => !x.Path.ToString().Contains("_ts"));
 
-        return new ChangeSet(new JsonPatch(operations));
+        return new ChangeSet(new JsonPatch(operations), previousNode!);
+    }
+
+    public T? GetPreviousValue<T>(string path)
+    {
+        var jp = JsonPath.Parse($"$.{nameof(Movement._MatchReferences)}");
+        var pathResult = jp.Evaluate(Previous);
+
+        if (pathResult.Matches.Any())
+        {
+            return pathResult.Matches.First().Value.Deserialize<T>();
+        }
+
+        return default;
     }
 }

Btms.Model/Ipaffs/ImportNotification.cs

@@ -147,6 +147,24 @@ public void AddRelationship(TdmRelationshipObject relationship)
             AuditEntries.Add(AuditEntry.CreateLinked(string.Empty, Version.GetValueOrDefault()));
         }
     }
+
+    public void RemoveRelationship(RelationshipDataItem relationship)
+    {
+        var unlinked = false;
+
+        if (Relationships.Movements.Data.Contains(relationship))
+        {
+            Relationships.Movements.Data.Remove(relationship);
+            unlinked = true;
+        }
+
+        Relationships.Movements.Matched = Relationships.Movements.Data.TrueForAll(x => x.Matched.GetValueOrDefault());
+
+        if (unlinked)
+        {
+            AuditEntries.Add(AuditEntry.CreateUnlinked(string.Empty, Version.GetValueOrDefault(), UpdatedSource));
+        }
+    }
 
     public void Changed(AuditEntry auditEntry)
     {

Btms.Model/Movement.cs

@@ -68,7 +68,6 @@ public class Movement : IMongoIdentifiable, IDataEntity, IAuditable
 
 
     [BsonElement("_matchReferences")]
-    [ChangeSetIgnore]
     public List<string> _MatchReferences
     {
         get