Save all datawallet modification payloads of a push in a single blob (#…

…315) * feat: add property to DatawalletModification * feat: add migration * chore: code cleanup * chore: add words to dictionary * feat: update FinalizeSyncRun handler * feat: update PushDatawalletModifications handler * feat: update GetModifications handler * refactor: no hyphens in guids * test: fix compiler error * fix: make new column fix length and unicode * chore: formatting of migration * test: fix compiler error * ci: ignore audit issue * test: fix tests * chore: remove log message * chore: fix PR comments * test: remove logger parameter from tests * chore: formatting
nmshd · Oct 4, 2023 · 5e68ea1 · 5e68ea1
1 parent e7f73fe
commit 5e68ea1
Show file tree

Hide file tree

Showing 20 changed files with 1,442 additions and 761 deletions.
diff --git a/.ci/runAdminUiChecks.sh b/.ci/runAdminUiChecks.sh
@@ -7,5 +7,5 @@ npm ci
 npx eslint --ext ts ./src
 npx prettier --check .
 npx license-check --ignorePackages [email protected]
-npx better-npm-audit audit
+npx better-npm-audit audit --exclude 1094239
 cd $INITIAL_DIR
diff --git a/Backbone.sln.DotSettings b/Backbone.sln.DotSettings
@@ -27,6 +27,9 @@
 	<s:String x:Key="/Default/Environment/UnitTesting/ExcludedCategoriesList/@EntryValue">ignore("skipping_due_to_required_backbone_changes"),ignore("skipping_due_to_required_adminAPI_changes")</s:String>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=Apns/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=appsettings/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/UserDictionary/Words/=Datawallet/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/UserDictionary/Words/=Datawallets/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/UserDictionary/Words/=Dtos/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=enrichers/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=firebase/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=inexistent/@EntryIndexedValue">True</s:Boolean>

diff --git a/...c/Synchronization.Application/Datawallets/Commands/PushDatawalletModifications/Handler.cs b/...c/Synchronization.Application/Datawallets/Commands/PushDatawalletModifications/Handler.cs
@@ -1,4 +1,5 @@
-using AutoMapper;
+using System.Text.Json;
+using AutoMapper;
 using Backbone.Modules.Synchronization.Application.Datawallets.DTOs;
 using Backbone.Modules.Synchronization.Application.Infrastructure;
 using Backbone.Modules.Synchronization.Application.IntegrationEvents.Outgoing;
@@ -89,18 +90,20 @@ private void EnsureSufficientSupportedDatawalletVersion()
 
     private async Task CreateModifications()
     {
-        var newModifications = _request.Modifications.Select(CreateModification);
+        var blobName = Guid.NewGuid().ToString("N");
+
+        var newModifications = _request.Modifications.Select(m => CreateModification(m, blobName));
 
         _dbContext.Set<Datawallet>().Update(_datawallet);
 
         var modificationsArray = newModifications.ToArray();
 
-        await Save(modificationsArray);
+        await Save(modificationsArray, blobName);
 
         _modifications = modificationsArray;
     }
 
-    private DatawalletModification CreateModification(PushDatawalletModificationItem modificationDto)
+    private DatawalletModification CreateModification(PushDatawalletModificationItem modificationDto, string blobReference)
     {
         return _datawallet.AddModification(
             _mapper.Map<DatawalletModificationType>(modificationDto.Type),
@@ -109,7 +112,8 @@ private DatawalletModification CreateModification(PushDatawalletModificationItem
             modificationDto.ObjectIdentifier,
             modificationDto.PayloadCategory,
             modificationDto.EncryptedPayload,
-            _activeDevice
+            _activeDevice,
+            blobReference
         );
     }
 
@@ -125,14 +129,17 @@ private void BuildResponse()
         _response = new PushDatawalletModificationsResponse { Modifications = responseItems, NewIndex = responseItems.Max(i => i.Index) };
     }
 
-    private async Task Save(DatawalletModification[] modifications)
+    private async Task Save(DatawalletModification[] modifications, string blobName)
     {
         await _dbContext.Set<DatawalletModification>().AddRangeAsync(modifications, _cancellationToken);
-        foreach (var newModification in modifications)
-        {
-            if (newModification.EncryptedPayload != null)
-                _blobStorage.Add(_blobOptions.RootFolder, newModification.Id, newModification.EncryptedPayload);
-        }
+
+        var payloads = modifications
+            .Where(newModification => newModification.EncryptedPayload != null)
+            .ToDictionary(m => m.Index, m => m.EncryptedPayload);
+
+        var blobContent = JsonSerializer.SerializeToUtf8Bytes(payloads);
+
+        _blobStorage.Add(_blobOptions.RootFolder, blobName, blobContent);
 
         await _blobStorage.SaveAsync();
 

diff --git a/...onization/src/Synchronization.Application/Datawallets/Queries/GetModifications/Handler.cs b/...onization/src/Synchronization.Application/Datawallets/Queries/GetModifications/Handler.cs
@@ -1,11 +1,13 @@
-using AutoMapper;
+using System.Text.Json;
+using AutoMapper;
 using Backbone.Modules.Synchronization.Application.Datawallets.DTOs;
 using Backbone.Modules.Synchronization.Application.Infrastructure;
 using Backbone.Modules.Synchronization.Domain.Entities;
 using Enmeshed.BuildingBlocks.Application.Abstractions.Exceptions;
 using Enmeshed.BuildingBlocks.Application.Abstractions.Infrastructure.Persistence.BlobStorage;
 using Enmeshed.BuildingBlocks.Application.Abstractions.Infrastructure.UserContext;
 using Enmeshed.DevelopmentKit.Identity.ValueObjects;
+using Enmeshed.Tooling.Extensions;
 using MediatR;
 using Microsoft.Extensions.Options;
 
@@ -39,27 +41,59 @@ public async Task<GetModificationsResponse> Handle(GetModificationsQuery request
 
         var dbPaginationResult = await _dbContext.GetDatawalletModifications(_activeIdentity, request.LocalIndex, request.PaginationFilter, cancellationToken);
 
-        var dtos = _mapper.Map<IEnumerable<DatawalletModificationDTO>>(dbPaginationResult.ItemsOnPage).ToArray();
-
-        await FillEncryptedPayloads(dtos);
+        var dtos = await MapToDtos(dbPaginationResult.ItemsOnPage);
 
         return new GetModificationsResponse(dtos, request.PaginationFilter, dbPaginationResult.TotalNumberOfItems);
     }
 
-    private async Task FillEncryptedPayloads(IEnumerable<DatawalletModificationDTO> modifications)
+    private async Task<List<DatawalletModificationDTO>> MapToDtos(IEnumerable<DatawalletModification> modifications)
     {
-        await Task.WhenAll(modifications.Select(FillEncryptedPayload));
+        var datawalletModifications = modifications as DatawalletModification[] ?? modifications.ToArray();
+
+        var blobReferences = datawalletModifications.Where(m => !m.BlobReference.IsNullOrEmpty()).Select(m => m.BlobReference).Distinct();
+        var blobs = await Task.WhenAll(blobReferences.Select(r =>
+        {
+            try
+            {
+                return _blobStorage.FindAsync(_blobOptions.RootFolder, r);
+            }
+            catch (NotFoundException)
+            {
+                throw new Exception($"Blob with reference '{r}' not found.");
+            }
+        }));
+
+        var payloads = blobs
+            .Select(b => JsonSerializer.Deserialize<Dictionary<long, byte[]>>(b))
+            .SelectMany(b => b)
+            .ToDictionary(b => b.Key, b => b.Value);
+
+        var mappingTasks = datawalletModifications.Select(m => MapToDto(m, payloads));
+
+        return (await Task.WhenAll(mappingTasks)).ToList();
     }
 
-    private async Task FillEncryptedPayload(DatawalletModificationDTO datawalletModification)
+    private async Task<DatawalletModificationDTO> MapToDto(DatawalletModification modification, Dictionary<long, byte[]> payloads)
     {
-        try
+        var dto = _mapper.Map<DatawalletModificationDTO>(modification);
+
+        if (modification.BlobReference.IsNullOrEmpty())
         {
-            datawalletModification.EncryptedPayload = await _blobStorage.FindAsync(_blobOptions.RootFolder, datawalletModification.Id);
+            try
+            {
+                dto.EncryptedPayload = await _blobStorage.FindAsync(_blobOptions.RootFolder, modification.Id);
+            }
+            catch (NotFoundException)
+            {
+                // blob not found means that there is no payload for this modification
+            }
         }
-        catch (NotFoundException)
+        else
         {
-            // if the payload was not found, it means that the modification had no payload
+            payloads.TryGetValue(modification.Index, out var payload);
+            dto.EncryptedPayload = payload;
         }
+
+        return dto;
     }
 }
diff --git a/...chronization/src/Synchronization.Application/SyncRuns/Commands/FinalizeSyncRun/Handler.cs b/...chronization/src/Synchronization.Application/SyncRuns/Commands/FinalizeSyncRun/Handler.cs
@@ -1,4 +1,5 @@
-using AutoMapper;
+using System.Text.Json;
+using AutoMapper;
 using Backbone.Modules.Synchronization.Application.Datawallets.DTOs;
 using Backbone.Modules.Synchronization.Application.Infrastructure;
 using Backbone.Modules.Synchronization.Application.IntegrationEvents.Outgoing;
@@ -15,7 +16,8 @@
 
 namespace Backbone.Modules.Synchronization.Application.SyncRuns.Commands.FinalizeSyncRun;
 
-public class Handler : IRequestHandler<FinalizeExternalEventSyncSyncRunCommand, FinalizeExternalEventSyncSyncRunResponse>, IRequestHandler<FinalizeDatawalletVersionUpgradeSyncRunCommand, FinalizeDatawalletVersionUpgradeSyncRunResponse>
+public class Handler : IRequestHandler<FinalizeExternalEventSyncSyncRunCommand, FinalizeExternalEventSyncSyncRunResponse>,
+    IRequestHandler<FinalizeDatawalletVersionUpgradeSyncRunCommand, FinalizeDatawalletVersionUpgradeSyncRunResponse>
 {
     private readonly DeviceId _activeDevice;
     private readonly IdentityAddress _activeIdentity;
@@ -136,7 +138,10 @@ private List<DatawalletModification> AddModificationsToDatawallet(List<PushDataw
         if (!modifications.Any())
             return new List<DatawalletModification>();
 
+        var blobName = Guid.NewGuid().ToString("N");
+
         var newModifications = new List<DatawalletModification>();
+        var payloads = new Dictionary<long, byte[]>();
         foreach (var modificationDto in modifications)
         {
             var newModification = _datawallet.AddModification(
@@ -146,14 +151,17 @@ private List<DatawalletModification> AddModificationsToDatawallet(List<PushDataw
                 modificationDto.ObjectIdentifier,
                 modificationDto.PayloadCategory,
                 modificationDto.EncryptedPayload,
-                _activeDevice);
-
-            if (newModification.EncryptedPayload != null)
-                _blobStorage.Add(_blobOptions.RootFolder, newModification.Id, newModification.EncryptedPayload);
+                _activeDevice,
+                blobName);
 
             newModifications.Add(newModification);
+
+            if (newModification.EncryptedPayload != null)
+                payloads.Add(newModification.Index, modificationDto.EncryptedPayload);
         }
 
+        _blobStorage.Add(_blobOptions.RootFolder, blobName, JsonSerializer.SerializeToUtf8Bytes(payloads));
+
         return newModifications;
     }
 

diff --git a/Modules/Synchronization/src/Synchronization.Domain/Entities/Datawallet.cs b/Modules/Synchronization/src/Synchronization.Domain/Entities/Datawallet.cs
@@ -31,23 +31,18 @@ public void Upgrade(DatawalletVersion targetVersion)
         Version = targetVersion;
     }
 
-    public DatawalletModification AddModification(DatawalletModificationType type, DatawalletVersion datawalletVersionOfModification, string collection, string objectIdentifier, string payloadCategory, byte[] encryptedPayload, DeviceId createdByDevice)
+    public DatawalletModification AddModification(DatawalletModificationType type, DatawalletVersion datawalletVersionOfModification, string collection, string objectIdentifier, string payloadCategory, byte[] encryptedPayload, DeviceId createdByDevice, string blobReference)
     {
         if (datawalletVersionOfModification > Version)
             throw new DomainException(DomainErrors.Datawallet.DatawalletVersionOfModificationTooHigh(Version, datawalletVersionOfModification));
 
         var indexOfNewModification = Modifications.Count > 0 ? Modifications.Max(m => m.Index) + 1 : 0;
-        var newModification = new DatawalletModification(this, datawalletVersionOfModification, indexOfNewModification, type, collection, objectIdentifier, payloadCategory, encryptedPayload, createdByDevice);
+
+        var newModification = new DatawalletModification(this, datawalletVersionOfModification, indexOfNewModification, type, collection, objectIdentifier, payloadCategory, encryptedPayload, createdByDevice, blobReference);
         Modifications.Add(newModification);
         return newModification;
     }
 
-    public DatawalletModification AddModification(DatawalletModification modification)
-    {
-        Modifications.Add(modification);
-        return modification;
-    }
-
     public class DatawalletVersion : SimpleValueObject<ushort>
     {
         public DatawalletVersion(ushort value) : base(value) { }

diff --git a/Modules/Synchronization/src/Synchronization.Domain/Entities/DatawalletModification.cs b/Modules/Synchronization/src/Synchronization.Domain/Entities/DatawalletModification.cs
@@ -6,10 +6,12 @@ namespace Backbone.Modules.Synchronization.Domain.Entities;
 public class DatawalletModification
 {
 #pragma warning disable CS8618
-    private DatawalletModification() { }
+    private DatawalletModification()
+    {
+    }
 #pragma warning restore CS8618
 
-    public DatawalletModification(Datawallet datawallet, Datawallet.DatawalletVersion datawalletVersion, long index, DatawalletModificationType type, string collection, string objectIdentifier, string payloadCategory, byte[] encryptedPayload, DeviceId createdByDevice)
+    public DatawalletModification(Datawallet datawallet, Datawallet.DatawalletVersion datawalletVersion, long index, DatawalletModificationType type, string collection, string objectIdentifier, string payloadCategory, byte[] encryptedPayload, DeviceId createdByDevice, string blobReference)
     {
         Id = DatawalletModificationId.New();
 
@@ -26,6 +28,7 @@ public DatawalletModification(Datawallet datawallet, Datawallet.DatawalletVersio
 
         CreatedAt = SystemTime.UtcNow;
         CreatedByDevice = createdByDevice;
+        BlobReference = blobReference;
     }
 
     public DatawalletModificationId Id { get; }
@@ -40,6 +43,7 @@ public DatawalletModification(Datawallet datawallet, Datawallet.DatawalletVersio
     public string Collection { get; }
     public DatawalletModificationType Type { get; }
     public byte[]? EncryptedPayload { get; }
+    public string BlobReference { get; }
 }
 
 public enum DatawalletModificationType