Redact PII from request URLs in logs (#1406)

DFE-Digital · Jul 4, 2024 · 0ac5831 · 0ac5831
1 parent 0335897
commit 0ac5831
Show file tree

Hide file tree

Showing 29 changed files with 273 additions and 286 deletions.
diff --git a/TeachingRecordSystem/src/TeachingRecordSystem.Api/HttpRequestExtensions.cs b/TeachingRecordSystem/src/TeachingRecordSystem.Api/HttpRequestExtensions.cs
diff --git a/...chingRecordSystem.Api/Infrastructure/ApplicationInsights/RedactedUrlTelemetryProcessor.cs b/...chingRecordSystem.Api/Infrastructure/ApplicationInsights/RedactedUrlTelemetryProcessor.cs
diff --git a/...rdSystem/src/TeachingRecordSystem.Api/Infrastructure/Logging/RedactQueryParamAttribute.cs b/...rdSystem/src/TeachingRecordSystem.Api/Infrastructure/Logging/RedactQueryParamAttribute.cs
diff --git a/...RecordSystem/src/TeachingRecordSystem.Api/Infrastructure/Logging/RedactedUrlParameters.cs b/...RecordSystem/src/TeachingRecordSystem.Api/Infrastructure/Logging/RedactedUrlParameters.cs
diff --git a/...rc/TeachingRecordSystem.Api/Infrastructure/Logging/RemoveRedactedUrlParametersEnricher.cs b/...rc/TeachingRecordSystem.Api/Infrastructure/Logging/RemoveRedactedUrlParametersEnricher.cs
diff --git a/...chingRecordSystem.Api/Infrastructure/Logging/RemoveRedactedUrlParametersEventProcessor.cs b/...chingRecordSystem.Api/Infrastructure/Logging/RemoveRedactedUrlParametersEventProcessor.cs
diff --git a/TeachingRecordSystem/src/TeachingRecordSystem.Api/Program.cs b/TeachingRecordSystem/src/TeachingRecordSystem.Api/Program.cs
@@ -12,7 +12,6 @@
 using TeachingRecordSystem.Api.Infrastructure.ApplicationModel;
 using TeachingRecordSystem.Api.Infrastructure.Filters;
 using TeachingRecordSystem.Api.Infrastructure.Json;
-using TeachingRecordSystem.Api.Infrastructure.Logging;
 using TeachingRecordSystem.Api.Infrastructure.Mapping;
 using TeachingRecordSystem.Api.Infrastructure.Middleware;
 using TeachingRecordSystem.Api.Infrastructure.ModelBinding;
@@ -28,6 +27,7 @@
 using TeachingRecordSystem.Core.Services.NameSynonyms;
 using TeachingRecordSystem.Core.Services.TrnGenerationApi;
 using TeachingRecordSystem.ServiceDefaults;
+using TeachingRecordSystem.ServiceDefaults.Infrastructure.Logging;
 
 [assembly: ApiController]
 namespace TeachingRecordSystem.Api;

diff --git a/TeachingRecordSystem/src/TeachingRecordSystem.Api/TeachingRecordSystem.Api.csproj b/TeachingRecordSystem/src/TeachingRecordSystem.Api/TeachingRecordSystem.Api.csproj
@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk.Web">
+<Project Sdk="Microsoft.NET.Sdk.Web">
 
   <PropertyGroup>
     <TargetFramework>net8.0</TargetFramework>
@@ -10,13 +10,10 @@
     <PackageReference Include="FluentValidation.AspNetCore" />
     <PackageReference Include="idunno.Authentication.Basic" />
     <PackageReference Include="MediatR" />
-    <PackageReference Include="Microsoft.ApplicationInsights.AspNetCore" />
     <PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" />
     <PackageReference Include="Microsoft.Extensions.Caching.StackExchangeRedis" />
     <PackageReference Include="Moq" />
     <PackageReference Include="RedisRateLimiting.AspNetCore" />
-    <PackageReference Include="Sentry.AspNetCore" />
-    <PackageReference Include="Serilog.AspNetCore" />
     <PackageReference Include="Swashbuckle.AspNetCore" />
     <PackageReference Include="Swashbuckle.AspNetCore.Annotations" />
   </ItemGroup>

diff --git a/TeachingRecordSystem/src/TeachingRecordSystem.Api/V1/Controllers/TeachersController.cs b/TeachingRecordSystem/src/TeachingRecordSystem.Api/V1/Controllers/TeachersController.cs
@@ -2,7 +2,6 @@
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
 using Swashbuckle.AspNetCore.Annotations;
-using TeachingRecordSystem.Api.Infrastructure.Logging;
 using TeachingRecordSystem.Api.Infrastructure.Security;
 using TeachingRecordSystem.Api.V1.Requests;
 using TeachingRecordSystem.Api.V1.Responses;
@@ -28,7 +27,6 @@ public TeachersController(IMediator mediator)
     [ProducesResponseType(typeof(GetTeacherResponse), StatusCodes.Status200OK)]
     [ProducesResponseType(typeof(ProblemDetails), StatusCodes.Status400BadRequest)]
     [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)]
-    [RedactQueryParam("birthdate"), RedactQueryParam("nino")]
     public async Task<IActionResult> GetTeacher([FromRoute] GetTeacherRequest request)
     {
         var response = await _mediator.Send(request);

diff --git a/TeachingRecordSystem/src/TeachingRecordSystem.Api/V2/Controllers/IttOutcomeController.cs b/TeachingRecordSystem/src/TeachingRecordSystem.Api/V2/Controllers/IttOutcomeController.cs
@@ -2,7 +2,6 @@
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
 using Swashbuckle.AspNetCore.Annotations;
-using TeachingRecordSystem.Api.Infrastructure.Logging;
 using TeachingRecordSystem.Api.Infrastructure.Security;
 using TeachingRecordSystem.Api.V2.Requests;
 using TeachingRecordSystem.Api.V2.Responses;
@@ -30,7 +29,6 @@ public IttOutcomeController(IMediator mediator)
     [ProducesResponseType(typeof(ProblemDetails), StatusCodes.Status409Conflict)]
     [MapError(10001, statusCode: StatusCodes.Status404NotFound)]
     [MapError(10002, statusCode: StatusCodes.Status409Conflict)]
-    [RedactQueryParam("birthdate")]
     public async Task<IActionResult> SetIttOutcome([FromBody] SetIttOutcomeRequest request)
     {
         var response = await _mediator.Send(request);

diff --git a/TeachingRecordSystem/src/TeachingRecordSystem.Api/V2/Controllers/TeachersController.cs b/TeachingRecordSystem/src/TeachingRecordSystem.Api/V2/Controllers/TeachersController.cs
@@ -2,7 +2,6 @@
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
 using Swashbuckle.AspNetCore.Annotations;
-using TeachingRecordSystem.Api.Infrastructure.Logging;
 using TeachingRecordSystem.Api.Infrastructure.Security;
 using TeachingRecordSystem.Api.V2.Requests;
 using TeachingRecordSystem.Api.V2.Responses;
@@ -53,7 +52,6 @@ public async Task<IActionResult> GetTeacher([FromRoute] GetTeacherRequest reques
     [ProducesResponseType(StatusCodes.Status204NoContent)]
     [MapError(10001, statusCode: StatusCodes.Status404NotFound)]
     [MapError(10002, statusCode: StatusCodes.Status409Conflict)]
-    [RedactQueryParam("birthdate")]
     [Authorize(Policy = AuthorizationPolicies.UpdatePerson)]
     public async Task<IActionResult> Update([FromBody] UpdateTeacherRequest request)
     {

diff --git a/...ingRecordSystem.AuthorizeAccess/Infrastructure/Logging/WebApplicationBuilderExtensions.cs b/...ingRecordSystem.AuthorizeAccess/Infrastructure/Logging/WebApplicationBuilderExtensions.cs
diff --git a/TeachingRecordSystem/src/TeachingRecordSystem.AuthorizeAccess/Program.cs b/TeachingRecordSystem/src/TeachingRecordSystem.AuthorizeAccess/Program.cs
@@ -16,7 +16,6 @@
 using TeachingRecordSystem.AuthorizeAccess;
 using TeachingRecordSystem.AuthorizeAccess.Infrastructure.Filters;
 using TeachingRecordSystem.AuthorizeAccess.Infrastructure.FormFlow;
-using TeachingRecordSystem.AuthorizeAccess.Infrastructure.Logging;
 using TeachingRecordSystem.AuthorizeAccess.Infrastructure.Middleware;
 using TeachingRecordSystem.AuthorizeAccess.Infrastructure.Oidc;
 using TeachingRecordSystem.AuthorizeAccess.Infrastructure.Security;
@@ -28,6 +27,7 @@
 using TeachingRecordSystem.Core.Services.Files;
 using TeachingRecordSystem.Core.Services.PersonMatching;
 using TeachingRecordSystem.ServiceDefaults;
+using TeachingRecordSystem.ServiceDefaults.Infrastructure.Logging;
 using TeachingRecordSystem.SupportUi.Infrastructure.FormFlow;
 using TeachingRecordSystem.UiCommon.Filters;
 using TeachingRecordSystem.UiCommon.FormFlow;

diff --git a/...stem/src/TeachingRecordSystem.AuthorizeAccess/TeachingRecordSystem.AuthorizeAccess.csproj b/...stem/src/TeachingRecordSystem.AuthorizeAccess/TeachingRecordSystem.AuthorizeAccess.csproj
@@ -14,10 +14,7 @@
     <PackageReference Include="GovUk.OneLogin.AspNetCore" />
     <PackageReference Include="Humanizer.Core" />
     <PackageReference Include="Joonasw.AspNetCore.SecurityHeaders" />
-    <PackageReference Include="Microsoft.ApplicationInsights.AspNetCore" />
     <PackageReference Include="OpenIddict.AspNetCore" />
-    <PackageReference Include="Sentry.AspNetCore" />
-    <PackageReference Include="Serilog.AspNetCore" />
   </ItemGroup>
 
   <ItemGroup>

diff --git a/TeachingRecordSystem/src/TeachingRecordSystem.AuthorizeAccess/appsettings.json b/TeachingRecordSystem/src/TeachingRecordSystem.AuthorizeAccess/appsettings.json
@@ -1,10 +1,7 @@
 {
   "Serilog": {
     "MinimumLevel": {
-      "Default": "Information",
-      "Override": {
-        "Microsoft.AspNetCore": "Warning"
-      }
+      "Default": "Information"
     },
     "Enrich": [ "FromLogContext" ],
     "Using": [ "Serilog.Expressions" ]

diff --git a/TeachingRecordSystem/src/TeachingRecordSystem.ServiceDefaults/Extensions.cs b/TeachingRecordSystem/src/TeachingRecordSystem.ServiceDefaults/Extensions.cs
@@ -7,6 +7,7 @@
 using Microsoft.Extensions.Hosting;
 using Prometheus;
 using TeachingRecordSystem.Core.Infrastructure.Configuration;
+using TeachingRecordSystem.ServiceDefaults.Infrastructure.Logging;
 
 namespace TeachingRecordSystem.ServiceDefaults;
 
@@ -24,6 +25,7 @@ public static IHostApplicationBuilder AddServiceDefaults(
 
         builder.Services.AddHealthChecks().AddNpgSql(builder.Configuration.GetPostgresConnectionString());
         builder.Services.AddDatabaseDeveloperPageExceptionFilter();
+        builder.Services.AddSingleton<UrlRedactor>();
 
         if (builder.Environment.IsProduction())
         {

diff --git a/...ystem.ServiceDefaults/Infrastructure/ApplicationInsights/RedactedUrlTelemetryProcessor.cs b/...ystem.ServiceDefaults/Infrastructure/ApplicationInsights/RedactedUrlTelemetryProcessor.cs
@@ -0,0 +1,19 @@
+using Microsoft.ApplicationInsights.Channel;
+using Microsoft.ApplicationInsights.DataContracts;
+using Microsoft.ApplicationInsights.Extensibility;
+using TeachingRecordSystem.ServiceDefaults.Infrastructure.Logging;
+
+namespace TeachingRecordSystem.ServiceDefaults.Infrastructure.ApplicationInsights;
+
+public class RedactedUrlTelemetryProcessor(ITelemetryProcessor next, UrlRedactor urlRedactor) : ITelemetryProcessor
+{
+    public void Process(ITelemetry item)
+    {
+        if (item is RequestTelemetry requestTelemetry)
+        {
+            requestTelemetry.Url = new Uri(urlRedactor.GetScrubbedRequestUrl());
+        }
+
+        next.Process(item);
+    }
+}