test: added validation to stress test and mocks for entici and gpas

.config/dotnet-tools.json

@@ -3,11 +3,11 @@
   "isRoot": true,
   "tools": {
     "csharpier": {
-      "version": "0.25.0",
+      "version": "0.26.3",
       "commands": ["dotnet-csharpier"]
     },
     "dotnet-outdated-tool": {
-      "version": "4.5.3",
+      "version": "4.6.0",
       "commands": ["dotnet-outdated"]
     }
   }

.github/workflows/ci.yaml

@@ -132,25 +132,17 @@ jobs:
       - name: List images in cluster
         run: docker exec kind-control-plane crictl images
 
-      - name: Install the latest version of vfps as a pseudonymization service
-        run: |
-          helm repo add chgl https://chgl.github.io/charts
-          helm install \
-            --wait \
-            --timeout=10m \
-            vfps chgl/vfps
-
       - name: Install "fhir-pseudonymizer"
         env:
           IMAGE_TAG: ${{ needs.build.outputs.image-version }}
         run: |
-          helm repo add miracum https://miracum.github.io/charts
           helm install \
             --set="image.tag=${IMAGE_TAG}" \
             -f tests/iter8/values.yaml \
             --wait \
             --timeout=10m \
-            fhir-pseudonymizer miracum/fhir-pseudonymizer
+            fhir-pseudonymizer \
+            oci://ghcr.io/miracum/charts/fhir-pseudonymizer
 
       - name: Launch iter8 experiment
         run: kubectl apply -f tests/iter8/experiment.yaml

.gitignore

@@ -370,3 +370,5 @@ tests/chaos/argo
 *-iid.txt
 
 *.received.*
+
+.task/

compose.dev.yaml

@@ -76,6 +76,30 @@ services:
       # Http2-only for plaintext gRPC
       - "127.0.0.1:8081:8081"
 
+  gpas-entici-mock:
+    image: docker.io/mockserver/mockserver:5.15.0@sha256:0f9ef78c94894ac3e70135d156193b25e23872575d58e2228344964273b4af6b
+    ipc: none
+    security_opt:
+      - "no-new-privileges:true"
+    cap_drop:
+      - ALL
+    privileged: false
+    deploy:
+      resources:
+        limits:
+          memory: 512m
+          cpus: "1"
+        reservations:
+          memory: 512m
+          cpus: "1"
+    environment:
+      MOCKSERVER_INITIALIZATION_JSON_PATH: /config/initializer.json
+      MOCKSERVER_WATCH_INITIALIZATION_JSON: "true"
+    ports:
+      - 127.0.0.1:1080:1080
+    volumes:
+      - ./hack/mocks:/config:ro
+
   keycloak:
     image: quay.io/keycloak/keycloak:22.0.5@sha256:bfa8852e52c279f0857fe8da239c0ad6bbd2cc07793a28a6770f7e24c1e25444
     restart: unless-stopped

hack/mocks/README.md

@@ -0,0 +1,12 @@
+# Generating MockServer's initialization config
+
+Because it's easier to read, the initializers are managed as YAML and converted to JSON
+for MockServer.
+
+Run:
+
+```sh
+yq -o json hack/mocks/initializer.yaml > hack/mocks/initializer.json
+```
+
+to convert.

hack/mocks/initializer.json

@@ -0,0 +1,24 @@
+[
+  {
+    "id": "gpas-pseudonymize",
+    "httpRequest": {
+      "method": "POST",
+      "path": "/ttp-fhir/fhir/gpas/$pseudonymizeAllowCreate"
+    },
+    "httpResponseTemplate": {
+      "templateType": "VELOCITY",
+      "template": "{\n  \"body\": {\n    \"resourceType\": \"Parameters\",\n    \"parameter\": [\n      {\n        \"name\": \"pseudonym\",\n        \"part\": [\n          {\n            \"name\": \"original\",\n            \"valueIdentifier\": {\n              \"system\": \"https://ths-greifswald.de/gpas\",\n              \"value\": \"test\"\n            }\n          },\n          {\n            \"name\": \"target\",\n            \"valueIdentifier\": {\n              \"system\": \"https://ths-greifswald.de/gpas\",\n              \"value\": \"benchmark\"\n            }\n          },\n          {\n            \"name\": \"pseudonym\",\n            \"valueIdentifier\": {\n              \"system\": \"https://ths-greifswald.de/gpas\",\n              #set($jsonBody = $json.parse($!request.body))\n              #set($originalValue = \"\")\n              #foreach($parameter in $jsonBody.parameter)\n                  #if($parameter.name == 'original')\n                      #set($originalValue = $parameter.valueString)\n                  #end\n              #end\n              \"value\": \"pseuded-$originalValue\"\n            }\n          }\n        ]\n      }\n    ]\n  }\n}\n"
+    }
+  },
+  {
+    "id": "entici-pseudonymize",
+    "httpRequest": {
+      "method": "POST",
+      "path": "/entici/$pseudonymize"
+    },
+    "httpResponseTemplate": {
+      "templateType": "VELOCITY",
+      "template": "{\n  \"body\": {\n    \"resourceType\": \"Parameters\",\n    \"parameter\": [\n      {\n        \"name\": \"pseudonym\",\n        \"valueIdentifier\": {\n          \"use\": \"secondary\",\n          \"system\": \"urn:fdc:difuture.de:trustcenter.plain\",\n          #set($jsonBody = $json.parse($!request.body))\n            #set($originalValue = \"\")\n            #foreach($parameter in $jsonBody.parameter)\n              #if($parameter.name == 'identifier')\n                #set($originalValue = $parameter.valueIdentifier.value)\n              #end\n            #end\n          \"value\": \"pseuded-$originalValue\"\n        }\n      }\n    ]\n  }\n}\n"
+    }
+  }
+]

hack/mocks/initializer.yaml

@@ -0,0 +1,76 @@
+- id: gpas-pseudonymize
+  httpRequest:
+    method: POST
+    path: /ttp-fhir/fhir/gpas/$pseudonymizeAllowCreate
+  httpResponseTemplate:
+    templateType: VELOCITY
+    template: |
+      {
+        "body": {
+          "resourceType": "Parameters",
+          "parameter": [
+            {
+              "name": "pseudonym",
+              "part": [
+                {
+                  "name": "original",
+                  "valueIdentifier": {
+                    "system": "https://ths-greifswald.de/gpas",
+                    "value": "test"
+                  }
+                },
+                {
+                  "name": "target",
+                  "valueIdentifier": {
+                    "system": "https://ths-greifswald.de/gpas",
+                    "value": "benchmark"
+                  }
+                },
+                {
+                  "name": "pseudonym",
+                  "valueIdentifier": {
+                    "system": "https://ths-greifswald.de/gpas",
+                    #set($jsonBody = $json.parse($!request.body))
+                    #set($originalValue = "")
+                    #foreach($parameter in $jsonBody.parameter)
+                        #if($parameter.name == 'original')
+                            #set($originalValue = $parameter.valueString)
+                        #end
+                    #end
+                    "value": "pseuded-$originalValue"
+                  }
+                }
+              ]
+            }
+          ]
+        }
+      }
+- id: entici-pseudonymize
+  httpRequest:
+    method: POST
+    path: /entici/$pseudonymize
+  httpResponseTemplate:
+    templateType: VELOCITY
+    template: |
+      {
+        "body": {
+          "resourceType": "Parameters",
+          "parameter": [
+            {
+              "name": "pseudonym",
+              "valueIdentifier": {
+                "use": "secondary",
+                "system": "urn:fdc:difuture.de:trustcenter.plain",
+                #set($jsonBody = $json.parse($!request.body))
+                  #set($originalValue = "")
+                  #foreach($parameter in $jsonBody.parameter)
+                    #if($parameter.name == 'identifier')
+                      #set($originalValue = $parameter.valueIdentifier.value)
+                    #end
+                  #end
+                "value": "pseuded-$originalValue"
+              }
+            }
+          ]
+        }
+      }

src/FhirPseudonymizer.StressTests/StressTests.cs

@@ -1,27 +1,27 @@
+using System.Security.Cryptography;
+using System.Text;
 using Hl7.Fhir.Model;
 using Hl7.Fhir.Rest;
 using NBomber.Contracts.Stats;
-using Xunit.Abstractions;
+using NBomber.Http;
 using Polly;
 using Polly.Retry;
-using Microsoft.FSharp.Core;
-using NBomber.Http;
-using Task = System.Threading.Tasks.Task;
+using Xunit.Abstractions;
 
 namespace FhirPseudonymizer.StressTests;
 
 public class StressTests
 {
     private readonly string reportFolder;
-    private readonly FhirClient fhirClient;
     private readonly ITestOutputHelper output;
     private readonly AsyncRetryPolicy retryPolicy;
+    private readonly Uri pseudonymizerBaseAddress;
 
     public StressTests(ITestOutputHelper output)
     {
         this.output = output;
 
-        var pseudonymizerBaseAddress = new Uri(
+        pseudonymizerBaseAddress = new Uri(
             Environment.GetEnvironmentVariable("FHIR_PSEUDONYMIZER_BASE_URL")
                 ?? "http://localhost:5000/fhir"
         );
@@ -36,17 +36,14 @@ public StressTests(ITestOutputHelper output)
                 (exception, timeSpan, retryAttempt, context) =>
                 {
                     var stepContext = context["stepContext"] as IScenarioContext;
-                    stepContext?.Logger.Warning(
-                        $"Request failed within retry context: {exception.GetType()}: {exception.Message}. Attempt {retryAttempt}."
-                    );
+                    stepContext
+                        ?.Logger
+                        .Warning(
+                            $"Request failed within retry context: {exception.GetType()}: {exception.Message}. Attempt {retryAttempt}."
+                        );
                 }
             );
 
-        fhirClient = new FhirClient(
-            pseudonymizerBaseAddress,
-            settings: new() { PreferredFormat = ResourceFormat.Json, Timeout = 15_000 }
-        );
-
         _ = bool.TryParse(
             Environment.GetEnvironmentVariable("DOTNET_RUNNING_IN_CONTAINER"),
             out bool isRunningInContainer
@@ -56,13 +53,23 @@ out bool isRunningInContainer
             : "./nbomber-reports";
     }
 
-    private async Task<Response<object>> RunPseudonymizeResource(IScenarioContext scenarioContext)
+    private async Task<Response<object>> RunPseudonymizeResource(
+        IScenarioContext scenarioContext,
+        HttpClient httpClient
+    )
     {
         return await Step.Run(
             "pseudonymize resource",
             scenarioContext,
             run: async () =>
             {
+                // this is basically just a copy-paste of what Vfps does when configured to
+                // use the `Sha256HexEncoded` pseudonymization method
+                var originalRecordNumber = Guid.NewGuid().ToString();
+                var inputAsBytes = Encoding.UTF8.GetBytes(originalRecordNumber);
+                var sha256Bytes = SHA256.HashData(inputAsBytes);
+                var expectedPseudonym = $"stress-{Convert.ToHexString(sha256Bytes)}";
+
                 var resource = new Patient()
                 {
                     Id = Guid.NewGuid().ToString(),
@@ -77,7 +84,7 @@ private async Task<Response<object>> RunPseudonymizeResource(IScenarioContext sc
                     },
                     Identifier = new()
                     {
-                        new("https://fhir.example.com/identifiers/mrn", Guid.NewGuid().ToString())
+                        new("https://fhir.example.com/identifiers/mrn", originalRecordNumber)
                         {
                             Type = new("http://terminology.hl7.org/CodeSystem/v2-0203", "MR"),
                         }
@@ -86,20 +93,45 @@ private async Task<Response<object>> RunPseudonymizeResource(IScenarioContext sc
 
                 var parameters = new Parameters().Add("resource", resource);
 
+                var fhirClient = new FhirClient(
+                    pseudonymizerBaseAddress,
+                    httpClient,
+                    settings: new() { PreferredFormat = ResourceFormat.Json, Timeout = 15_000 }
+                );
+
                 try
                 {
                     var response = await retryPolicy.ExecuteAsync(
                         (ctx) => fhirClient.WholeSystemOperationAsync("de-identify", parameters),
                         new Dictionary<string, object> { ["stepContext"] = scenarioContext }
                     );
 
+                    var pseudonymizedPatient = response as Patient;
+
+                    pseudonymizedPatient?.Should().NotBeNull();
+                    pseudonymizedPatient!.Identifier.Should().HaveCount(1);
+                    pseudonymizedPatient!.Identifier.First().Value.Should().Be(expectedPseudonym);
+
                     return Response.Ok(statusCode: "200");
                 }
                 catch (Exception exc)
+                    when (exc is HttpRequestException
+                        || exc is FhirOperationException
+                        || exc is OperationCanceledException
+                    )
                 {
+                    // catch the retry-able exceptions related to transient errors. Any exceptions thrown by
+                    // the FluentAssertions (Should) will still create test-failing exceptions. Their invariants must
+                    // always hold.
                     scenarioContext.Logger.Error(exc, "Pseudonymization of resource failed");
                     return Response.Fail();
                 }
+                catch (Exception exc)
+                {
+                    scenarioContext.Logger.Error(exc, "Stopping test due to invariant violation.");
+                    scenarioContext.StopCurrentTest(exc.Message);
+                    return Response.Fail("400", exc.Message, 0);
+                }
             }
         );
     }
@@ -110,20 +142,26 @@ public void StressTest_FailurePercentage_ShouldBeLessThanThreshold(
         double failPercentageThreshold
     )
     {
+        using var httpClient = new HttpClient();
         var scenario = Scenario
             .Create(
                 "de-identify",
                 async context =>
                 {
-                    return await RunPseudonymizeResource(context);
+                    return await RunPseudonymizeResource(context, httpClient);
                 }
             )
             .WithInit(async context =>
             {
+                using var fhirClient = new FhirClient(
+                    pseudonymizerBaseAddress,
+                    httpClient,
+                    settings: new() { PreferredFormat = ResourceFormat.Json }
+                );
                 await fhirClient.CapabilityStatementAsync();
                 context.Logger.Information("Completed scenario init.");
             })
-            .WithWarmUpDuration(TimeSpan.FromSeconds(5))
+            .WithWarmUpDuration(TimeSpan.FromSeconds(10))
             .WithLoadSimulations(
                 Simulation.RampingConstant(copies: 10, during: TimeSpan.FromMinutes(5)),
                 Simulation.KeepConstant(copies: 100, during: TimeSpan.FromMinutes(5)),
@@ -146,6 +184,15 @@ double failPercentageThreshold
             )
             .Run();
 
+        var deIdentifyStatusCodes = stats.GetScenarioStats("de-identify").Fail.StatusCodes;
+
+        deIdentifyStatusCodes
+            .Should()
+            .NotContain(
+                statusCodeStats => statusCodeStats.StatusCode == "400",
+                because: "it means that pseudonym validation failed."
+            );
+
         var failPercentage = stats.AllFailCount / (double)stats.AllRequestCount * 100.0;
 
         output.WriteLine(

src/FhirPseudonymizer.StressTests/Usings.cs

@@ -1,4 +1,4 @@
-global using Xunit;
+global using FluentAssertions;
 global using NBomber.Contracts;
 global using NBomber.CSharp;
-global using FluentAssertions;
+global using Xunit;

src/FhirPseudonymizer.Tests/Usings.cs

@@ -1,4 +1,4 @@
-global using Xunit;
-global using FluentAssertions;
 global using FakeItEasy;
+global using FluentAssertions;
+global using Xunit;
 global using Task = System.Threading.Tasks.Task;