Skip to content

Commit

Permalink
Move DqtReportingService into Worker (#941)
Browse files Browse the repository at this point in the history
  • Loading branch information
gunndabad authored Dec 5, 2023
1 parent 42f5e7e commit 4f9f7a8
Show file tree
Hide file tree
Showing 15 changed files with 206 additions and 129 deletions.
37 changes: 3 additions & 34 deletions TeachingRecordSystem/src/TeachingRecordSystem.Api/Program.cs
Original file line number Diff line number Diff line change
@@ -1,18 +1,13 @@
using System.Security.Claims;
using Azure.Storage.Blobs;
using FluentValidation;
using FluentValidation.AspNetCore;
using Hangfire;
using Hangfire.PostgreSql;
using idunno.Authentication.Basic;
using Medallion.Threading;
using Medallion.Threading.Azure;
using Medallion.Threading.FileSystem;
using Microsoft.AspNetCore.Authentication.JwtBearer;
using Microsoft.AspNetCore.DataProtection;
using Microsoft.AspNetCore.HttpOverrides;
using Microsoft.AspNetCore.Mvc.ApiExplorer;
using Microsoft.Extensions.Azure;
using Microsoft.Extensions.Diagnostics.HealthChecks;
using Microsoft.PowerPlatform.Dataverse.Client;
using Npgsql;
Expand All @@ -30,8 +25,8 @@
using TeachingRecordSystem.Core.DataStore.Postgres;
using TeachingRecordSystem.Core.Dqt;
using TeachingRecordSystem.Core.Dqt.Services.CrmEntityChanges;
using TeachingRecordSystem.Core.Dqt.Services.DqtReporting;
using TeachingRecordSystem.Core.Dqt.Services.TrsDataSync;
using TeachingRecordSystem.Core.Infrastructure;
using TeachingRecordSystem.Core.Infrastructure.Configuration;
using TeachingRecordSystem.Core.Jobs;
using TeachingRecordSystem.Core.Services.AccessYourQualifications;
Expand Down Expand Up @@ -223,30 +218,9 @@ public static void Main(string[] args)

services.AddDatabaseDeveloperPageExceptionFilter();

if (!env.IsUnitTests())
{
services.AddAzureClients(clientBuilder =>
{
clientBuilder.AddBlobServiceClient(configuration.GetRequiredValue("StorageConnectionString"));
});
}

if (env.IsProduction())
{
var containerName = configuration.GetRequiredValue("DistributedLockContainerName");
builder.AddBlobStorage();

services.AddSingleton<IDistributedLockProvider>(sp =>
{
var blobServiceClient = sp.GetRequiredService<BlobServiceClient>();
var blobContainerClient = blobServiceClient.GetBlobContainerClient(containerName);
return new AzureBlobLeaseDistributedSynchronizationProvider(blobContainerClient);
});
}
else
{
var lockFileDirectory = Path.Combine(Path.GetTempPath(), "qtlocks");
services.AddSingleton<IDistributedLockProvider>(new FileDistributedSynchronizationProvider(new DirectoryInfo(lockFileDirectory)));
}
builder.AddDistributedLocks();

if (!env.IsUnitTests() && !env.IsEndToEndTests())
{
Expand All @@ -264,18 +238,13 @@ public static void Main(string[] args)
services.AddAccessYourQualifications(configuration, env);
services.AddCertificateGeneration();
services.AddCrmEntityChanges();
services.AddDqtReporting(builder.Configuration);
services.AddTrsSyncService(builder.Configuration);
services.AddBackgroundJobs(env, configuration);
services.AddEmail(env, configuration);
services.AddCrmQueries();
services.AddSingleton<ReferenceDataCache>();
services.AddSingleton<TrsDataSyncHelper>();

// Filter telemetry emitted by DqtReportingService
services.AddApplicationInsightsTelemetry()
.AddApplicationInsightsTelemetryProcessor<IgnoreDependencyTelemetryProcessor>();

if (!env.IsUnitTests())
{
var crmServiceClient = GetCrmServiceClient();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
<PackageReference Include="AspNetCore.HealthChecks.NpgSql" />
<PackageReference Include="AspNetCore.HealthChecks.Redis" />
<PackageReference Include="Azure.Extensions.AspNetCore.DataProtection.Blobs" />
<PackageReference Include="Azure.Storage.Blobs" />
<PackageReference Include="FluentValidation.AspNetCore" />
<PackageReference Include="Hangfire.AspNetCore" />
<PackageReference Include="Hangfire.PostgreSql" />
Expand All @@ -18,7 +17,6 @@
<PackageReference Include="Microsoft.ApplicationInsights.AspNetCore" />
<PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" />
<PackageReference Include="Microsoft.AspNetCore.Diagnostics.EntityFrameworkCore" />
<PackageReference Include="Microsoft.Extensions.Azure" />
<PackageReference Include="Microsoft.Extensions.Caching.StackExchangeRedis" />
<PackageReference Include="Moq" />
<PackageReference Include="NSwag.AspNetCore" />
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,6 @@
"ApiKey": [ "developer" ]
}
},
"DqtReporting": {
"ProcessAllEntityTypesConcurrently": false
},
"TrsSyncService": {
"ProcessAllEntityTypesConcurrently": false,
"IgnoreInvalidData": true
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,6 @@
}
}
},
"DqtReporting": {
"RunService": true
},
"TrsSyncService": {
"RegisterServiceClient": true,
"RunService": true
Expand Down
63 changes: 0 additions & 63 deletions TeachingRecordSystem/src/TeachingRecordSystem.Api/appsettings.json
Original file line number Diff line number Diff line change
Expand Up @@ -9,69 +9,6 @@
"Enrich": [ "FromLogContext" ]
},
"AllowedHosts": "*",
"DqtReporting": {
"PollIntervalSeconds": 300,
"Entities": [
"account",
"accountleads",
"annotation",
"campaign",
"contact",
"contactleads",
"dfeta_autonumber",
"dfeta_businesseventaudit",
"dfeta_confirmationletter",
"dfeta_country",
"dfeta_countrystate",
"dfeta_disability",
"dfeta_document",
"dfeta_earlyyearsstatus",
"dfeta_employment",
"dfeta_ethnicity",
"dfeta_hequalification",
"dfeta_hesaukprnmapping",
"dfeta_hesubject",
"dfeta_induction",
"dfeta_inductionperiod",
"dfeta_initialteachertraining",
"dfeta_initiativecode",
"dfeta_integrationtransaction",
"dfeta_integrationtransactionrecord",
"dfeta_ittqualification",
"dfeta_ittsubject",
"dfeta_legacyaudit",
"dfeta_mqestablishment",
"dfeta_mrapplication",
"dfeta_mrcourse",
"dfeta_mremployment",
"dfeta_mrtraining",
"dfeta_optionsetmapping",
"dfeta_organisationcategory",
"dfeta_prdeclinestatusreason",
"dfeta_previousname",
"dfeta_profileamendrequest",
"dfeta_qtsregistration",
"dfeta_qualification",
"dfeta_sanction",
"dfeta_sanctioncode",
"dfeta_schooldirectinitiative",
"dfeta_serviceannouncement",
"dfeta_specialism",
"dfeta_teacherstatus",
"dfeta_temporaryqueries",
"dfeta_tsstcohorts",
"dfeta_tsstsubjects",
"dfeta_webactivitylog",
"incident",
"incidentresolution",
"lead",
"subject",
"systemuser",
"transactioncurrency"
],
"ProcessAllEntityTypesConcurrently": true,
"RunService": false
},
"RecurringJobs": {
"Enabled": false
},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,26 +6,24 @@

namespace TeachingRecordSystem.Core.Dqt.Services.DqtReporting;

public static class ServiceCollectionExtensions
public static class HostApplicationBuilderExtensions
{
public static IServiceCollection AddDqtReporting(
this IServiceCollection services,
IConfiguration configuration)
public static IHostApplicationBuilder AddDqtReporting(this IHostApplicationBuilder builder)
{
if (configuration.GetValue<bool>("DqtReporting:RunService"))
if (builder.Configuration.GetValue<bool>("DqtReporting:RunService"))
{
services.AddOptions<DqtReportingOptions>()
.Bind(configuration.GetSection("DqtReporting"))
builder.Services.AddOptions<DqtReportingOptions>()
.Bind(builder.Configuration.GetSection("DqtReporting"))
.ValidateDataAnnotations()
.ValidateOnStart();

services.AddSingleton<IHostedService, DqtReportingService>();
builder.Services.AddSingleton<IHostedService, DqtReportingService>();

services.AddServiceClient(
builder.Services.AddServiceClient(
DqtReportingService.CrmClientName,
sp => new ServiceClient(sp.GetRequiredService<IOptions<DqtReportingOptions>>().Value.CrmConnectionString));
}

return services;
return builder;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
using Azure.Storage.Blobs;
using Medallion.Threading;
using Medallion.Threading.Azure;
using Medallion.Threading.FileSystem;
using Microsoft.Extensions.Azure;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;

namespace TeachingRecordSystem.Core.Infrastructure;

public static class HostApplicationBuilderExtensions
{
public static IHostApplicationBuilder AddDistributedLocks(this IHostApplicationBuilder builder)
{
if (builder.Environment.IsProduction())
{
var containerName = builder.Configuration.GetRequiredValue("DistributedLockContainerName");

builder.Services.AddSingleton<IDistributedLockProvider>(sp =>
{
var blobServiceClient = sp.GetRequiredService<BlobServiceClient>();
var blobContainerClient = blobServiceClient.GetBlobContainerClient(containerName);
return new AzureBlobLeaseDistributedSynchronizationProvider(blobContainerClient);
});
}
else
{
var lockFileDirectory = Path.Combine(Path.GetTempPath(), "qtlocks");
builder.Services.AddSingleton<IDistributedLockProvider>(new FileDistributedSynchronizationProvider(new DirectoryInfo(lockFileDirectory)));
}

return builder;
}

public static IHostApplicationBuilder AddBlobStorage(this IHostApplicationBuilder builder)
{
if (!builder.Environment.IsUnitTests())
{
builder.Services.AddAzureClients(clientBuilder =>
{
clientBuilder.AddBlobServiceClient(builder.Configuration.GetRequiredValue("StorageConnectionString"));
});
}

return builder;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
using Microsoft.Extensions.DependencyInjection;
using TeachingRecordSystem.Core.Dqt;

namespace TeachingRecordSystem.Core;

public static partial class ServiceCollectionExtensions
{
public static IServiceCollection AddTrsBaseServices(this IServiceCollection services)
{
return services
.AddSingleton<IClock, Clock>()
.AddCrmQueries()
.AddSingleton<ReferenceDataCache>();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
</ItemGroup>

<ItemGroup>
<PackageReference Include="Azure.Storage.Blobs" />
<PackageReference Include="dbup-sqlserver" />
<PackageReference Include="DistributedLock.Azure" />
<PackageReference Include="DistributedLock.FileSystem" />
Expand All @@ -54,6 +55,7 @@
<PrivateAssets>all</PrivateAssets>
</PackageReference>
<PackageReference Include="Microsoft.EntityFrameworkCore.Relational" />
<PackageReference Include="Microsoft.Extensions.Azure" />
<PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" />
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" />
<PackageReference Include="Microsoft.Extensions.Configuration.Json" />
Expand Down
46 changes: 46 additions & 0 deletions TeachingRecordSystem/src/TeachingRecordSystem.Worker/Program.cs
Original file line number Diff line number Diff line change
@@ -1,12 +1,58 @@
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.PowerPlatform.Dataverse.Client;
using TeachingRecordSystem;
using TeachingRecordSystem.Core;
using TeachingRecordSystem.Core.DataStore.Postgres;
using TeachingRecordSystem.Core.Dqt.Services.CrmEntityChanges;
using TeachingRecordSystem.Core.Dqt.Services.DqtReporting;
using TeachingRecordSystem.Core.Infrastructure;
using TeachingRecordSystem.Core.Infrastructure.Configuration;
using TeachingRecordSystem.Worker.Infrastructure.Logging;

var builder = Host.CreateApplicationBuilder(args);

if (builder.Environment.IsProduction())
{
builder.Configuration.AddJsonEnvironmentVariable("AppConfig");
}

builder.ConfigureLogging();

builder.Services.Configure<HostOptions>(o => o.BackgroundServiceExceptionBehavior = BackgroundServiceExceptionBehavior.Ignore);

var pgConnectionString = builder.Configuration.GetRequiredValue("ConnectionStrings:DefaultConnection");

builder.Services.AddDbContext<TrsDbContext>(
options => TrsDbContext.ConfigureOptions(options, pgConnectionString),
contextLifetime: ServiceLifetime.Transient,
optionsLifetime: ServiceLifetime.Singleton);

builder.Services.AddDbContextFactory<TrsDbContext>(options => TrsDbContext.ConfigureOptions(options, pgConnectionString));

builder
.AddBlobStorage()
.AddDistributedLocks()
.AddDqtReporting();

var crmServiceClient = new ServiceClient(builder.Configuration.GetRequiredValue("ConnectionStrings:Crm"))
{
DisableCrossThreadSafeties = true,
EnableAffinityCookie = true,
MaxRetryCount = 2,
RetryPauseTime = TimeSpan.FromSeconds(1)
};
builder.Services.AddTransient<IOrganizationServiceAsync>(_ => crmServiceClient.Clone());

builder.Services
.AddTrsBaseServices()
.AddCrmEntityChanges();

// Filter telemetry emitted by DqtReportingService;
// annoyingly we can't put this into the AddDqtReporting extension method since the method for adding Telemetry Processors
// is different depending on whether you're in a Worker app or Web app :-/
builder.Services.AddApplicationInsightsTelemetryWorkerService()
.AddApplicationInsightsTelemetryProcessor<IgnoreDependencyTelemetryProcessor>();

var host = builder.Build();
await host.RunAsync();
Original file line number Diff line number Diff line change
@@ -1,2 +1,6 @@
{
"DqtReporting": {
"ProcessAllEntityTypesConcurrently": false
},
"StorageConnectionString": "UseDevelopmentStorage=true"
}
Original file line number Diff line number Diff line change
Expand Up @@ -6,5 +6,8 @@
"TeachingRecordSystem.Worker": "Warning"
}
}
},
"DqtReporting": {
"RunService": true
}
}
Loading

0 comments on commit 4f9f7a8

Please sign in to comment.