diff --git a/dockerfile b/dockerfile index d04eb9ef1..da052071b 100644 --- a/dockerfile +++ b/dockerfile @@ -1,4 +1,4 @@ -FROM mcr.microsoft.com/dotnet/sdk:6.0-jammy AS build-env +FROM mcr.microsoft.com/dotnet/sdk:8.0-jammy AS build-env WORKDIR /app RUN apt-get update && apt-get install -y g++ curl cmake @@ -12,7 +12,7 @@ RUN dotnet publish ./src/SIL.Machine.Serval.EngineServer/SIL.Machine.Serval.Engi RUN dotnet publish ./src/SIL.Machine.Serval.JobServer/SIL.Machine.Serval.JobServer.csproj -c Release -o out_job_server # Build runtime image -FROM mcr.microsoft.com/dotnet/aspnet:6.0-jammy as production +FROM mcr.microsoft.com/dotnet/aspnet:8.0-jammy as production # libgomp needed for thot RUN apt-get update && apt-get install -y libgomp1 WORKDIR /app diff --git a/dockerfile.development b/dockerfile.development index cd6847185..a395da6dd 100644 --- a/dockerfile.development +++ b/dockerfile.development @@ -1,4 +1,4 @@ -FROM mcr.microsoft.com/dotnet/sdk:6.0-jammy +FROM mcr.microsoft.com/dotnet/sdk:8.0-jammy # libgomp needed for thot RUN apt update && apt install -y unzip libgomp1 && \ curl -sSL https://aka.ms/getvsdbgsh | /bin/sh /dev/stdin -v latest -l /remote_debugger \ No newline at end of file diff --git a/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions.cs b/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions.cs index e5e948573..ade0347d9 100644 --- a/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions.cs +++ b/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions.cs @@ -103,10 +103,13 @@ public static IMachineBuilder AddUnigramTruecaser(this IMachineBuilder builder) public static IMachineBuilder AddClearMLService(this IMachineBuilder builder, string? connectionString = null) { - connectionString ??= builder.Configuration.GetConnectionString("ClearML"); + connectionString ??= builder.Configuration?.GetConnectionString("ClearML"); + if (connectionString is null) + throw new InvalidOperationException("ClearML connection string is required"); + builder.Services .AddHttpClient("ClearML") - .ConfigureHttpClient(httpClient => httpClient.BaseAddress = new Uri(connectionString)) + .ConfigureHttpClient(httpClient => httpClient.BaseAddress = new Uri(connectionString!)) // Add retry policy; fail after approx. 2 + 4 + 8 = 14 seconds .AddTransientHttpErrorPolicy( b => b.WaitAndRetryAsync(3, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt))) @@ -120,8 +123,9 @@ public static IMachineBuilder AddClearMLService(this IMachineBuilder builder, st builder.Services .AddHttpClient("ClearML-NoRetry") - .ConfigureHttpClient(httpClient => httpClient.BaseAddress = new Uri(connectionString)); + .ConfigureHttpClient(httpClient => httpClient.BaseAddress = new Uri(connectionString!)); builder.Services.AddSingleton(); + builder.Services.AddHealthChecks().AddCheck("ClearML Health Check"); return builder; @@ -152,13 +156,17 @@ public static IMachineBuilder AddMongoHangfireJobClient( string? connectionString = null ) { + connectionString ??= builder.Configuration?.GetConnectionString("Hangfire"); + if (connectionString is null) + throw new InvalidOperationException("Hangfire connection string is required"); + builder.Services.AddHangfire( c => c.SetDataCompatibilityLevel(CompatibilityLevel.Version_170) .UseSimpleAssemblyNameTypeSerializer() .UseRecommendedSerializerSettings() .UseMongoStorage( - connectionString ?? builder.Configuration.GetConnectionString("Hangfire"), + connectionString, new MongoStorageOptions { MigrationOptions = new MongoMigrationOptions @@ -183,7 +191,7 @@ public static IMachineBuilder AddHangfireJobServer( { engineTypes ??= builder.Configuration?.GetSection("TranslationEngines").Get() - ?? new[] { TranslationEngineType.SmtTransfer, TranslationEngineType.Nmt }; + ?? [TranslationEngineType.SmtTransfer, TranslationEngineType.Nmt]; var queues = new List(); foreach (TranslationEngineType engineType in engineTypes.Distinct()) { @@ -220,9 +228,11 @@ public static IMachineBuilder AddMemoryDataAccess(this IMachineBuilder builder) public static IMachineBuilder AddMongoDataAccess(this IMachineBuilder builder, string? connectionString = null) { - connectionString ??= builder.Configuration.GetConnectionString("Mongo"); + connectionString ??= builder.Configuration?.GetConnectionString("Mongo"); + if (connectionString is null) + throw new InvalidOperationException("Mongo connection string is required"); builder.Services.AddMongoDataAccess( - connectionString, + connectionString!, "SIL.Machine.AspNetCore.Models", o => { @@ -257,21 +267,22 @@ await c.Indexes.CreateOrUpdateAsync( ); } ); - builder.Services.AddHealthChecks().AddMongoDb(connectionString, name: "Mongo"); + builder.Services.AddHealthChecks().AddMongoDb(connectionString!, name: "Mongo"); return builder; } - public static IMachineBuilder AddServalPlatformService( - this IMachineBuilder builder, - string? connectionString = null - ) + public static IMachineBuilder AddServalPlatformService(this IMachineBuilder builder, string? connectionString = null) { + connectionString ??= builder.Configuration?.GetConnectionString("Serval"); + if (connectionString is null) + throw new InvalidOperationException("Serval connection string is required"); + builder.Services.AddScoped(); builder.Services .AddGrpcClient(o => { - o.Address = new Uri(connectionString ?? builder.Configuration.GetConnectionString("Serval")); + o.Address = new Uri(connectionString); }) .ConfigureChannel(o => { @@ -321,10 +332,10 @@ public static IMachineBuilder AddServalTranslationEngineService( options.Interceptors.Add(); options.Interceptors.Add(); }); - builder.AddServalPlatformService(connectionString ?? builder.Configuration.GetConnectionString("Serval")); + builder.AddServalPlatformService(connectionString); engineTypes ??= builder.Configuration?.GetSection("TranslationEngines").Get() - ?? new[] { TranslationEngineType.SmtTransfer, TranslationEngineType.Nmt }; + ?? [TranslationEngineType.SmtTransfer, TranslationEngineType.Nmt]; foreach (TranslationEngineType engineType in engineTypes.Distinct()) { switch (engineType) @@ -340,7 +351,6 @@ public static IMachineBuilder AddServalTranslationEngineService( break; } } - builder.Services.AddGrpcHealthChecks(); return builder; } @@ -359,8 +369,9 @@ Action configureOptions public static IMachineBuilder AddBuildJobService(this IMachineBuilder builder, IConfiguration config) { builder.Services.Configure(config); - var options = config.Get(); - return builder.AddBuildJobService(options); + var buildJobOptions = new BuildJobOptions(); + config.GetSection(BuildJobOptions.Key).Bind(buildJobOptions); + return builder.AddBuildJobService(buildJobOptions); } public static IMachineBuilder AddBuildJobService(this IMachineBuilder builder) @@ -368,7 +379,24 @@ public static IMachineBuilder AddBuildJobService(this IMachineBuilder builder) if (builder.Configuration is null) builder.AddBuildJobService(o => { }); else + { builder.AddBuildJobService(builder.Configuration.GetSection(BuildJobOptions.Key)); + + var smtTransferEngineOptions = new SmtTransferEngineOptions(); + builder.Configuration.GetSection(SmtTransferEngineOptions.Key).Bind(smtTransferEngineOptions); + string? driveLetter = Path.GetPathRoot(smtTransferEngineOptions.EnginesDir)?[..1]; + if(driveLetter is null) + throw new InvalidOperationException("SMT Engine directory is required"); + // add health check for disk storage capacity + builder.Services + .AddHealthChecks() + .AddDiskStorageHealthCheck( + x => x.AddDrive(driveLetter, 1_000), // 1GB + "SMT Engine Storage Capacity", + HealthStatus.Degraded + ); + } + return builder; } diff --git a/src/SIL.Machine.AspNetCore/SIL.Machine.AspNetCore.csproj b/src/SIL.Machine.AspNetCore/SIL.Machine.AspNetCore.csproj index ab401c602..549e009ff 100644 --- a/src/SIL.Machine.AspNetCore/SIL.Machine.AspNetCore.csproj +++ b/src/SIL.Machine.AspNetCore/SIL.Machine.AspNetCore.csproj @@ -1,7 +1,7 @@  - net6.0 + net8.0 An ASP.NET Core web API middleware for the Machine library. 1591 enable @@ -26,6 +26,7 @@ + diff --git a/src/SIL.Machine.AspNetCore/Services/ClearMLAuthenticationService.cs b/src/SIL.Machine.AspNetCore/Services/ClearMLAuthenticationService.cs index 3efc2a81b..721d68868 100644 --- a/src/SIL.Machine.AspNetCore/Services/ClearMLAuthenticationService.cs +++ b/src/SIL.Machine.AspNetCore/Services/ClearMLAuthenticationService.cs @@ -1,10 +1,15 @@ namespace SIL.Machine.AspNetCore.Services; -public class ClearMLAuthenticationService : RecurrentTask, IClearMLAuthenticationService +public class ClearMLAuthenticationService( + IServiceProvider services, + IHttpClientFactory httpClientFactory, + IOptionsMonitor options, + ILogger logger + ) : RecurrentTask("ClearML authentication service", services, RefreshPeriod, logger), IClearMLAuthenticationService { - private readonly HttpClient _httpClient; - private readonly IOptionsMonitor _options; - private readonly ILogger _logger; + private readonly HttpClient _httpClient = httpClientFactory.CreateClient("ClearML"); + private readonly IOptionsMonitor _options = options; + private readonly ILogger _logger = logger; private readonly AsyncLock _lock = new(); // technically, the token should be good for 30 days, but let's refresh each hour @@ -12,19 +17,6 @@ public class ClearMLAuthenticationService : RecurrentTask, IClearMLAuthenticatio private static readonly TimeSpan RefreshPeriod = TimeSpan.FromSeconds(3600); private string _authToken = ""; - public ClearMLAuthenticationService( - IServiceProvider services, - IHttpClientFactory httpClientFactory, - IOptionsMonitor options, - ILogger logger - ) - : base("ClearML authentication service", services, RefreshPeriod, logger) - { - _httpClient = httpClientFactory.CreateClient("ClearML"); - _options = options; - _logger = logger; - } - public async Task GetAuthTokenAsync(CancellationToken cancellationToken = default) { using (await _lock.LockAsync(cancellationToken)) @@ -48,7 +40,14 @@ protected override async Task DoWorkAsync(IServiceScope scope, CancellationToken } catch (Exception e) { - _logger.LogError(e, "Error occurred while refreshing ClearML authentication token."); + if (_authToken is ""){ + _logger.LogError(e, "Error occurred while aquiring ClearML authentication token for the first time."); + // The ClearML token never was set. We can't continue without it. + throw; + } + else + _logger.LogError(e, "Error occurred while refreshing ClearML authentication token."); + } } @@ -63,7 +62,10 @@ private async Task AuthorizeAsync(CancellationToken cancellationToken) request.Headers.Add("Authorization", $"Basic {base64EncodedAuthenticationString}"); HttpResponseMessage response = await _httpClient.SendAsync(request, cancellationToken); string result = await response.Content.ReadAsStringAsync(cancellationToken); - _authToken = (string)((JsonObject?)JsonNode.Parse(result))?["data"]?["token"]!; + string? refreshedToken = (string?)((JsonObject?)JsonNode.Parse(result))?["data"]?["token"]; + if (refreshedToken is null || refreshedToken is "") + throw new Exception($"ClearML authentication failed - {response.StatusCode}: {response.ReasonPhrase}"); + _authToken = refreshedToken; _logger.LogInformation("ClearML Authentication Token Refresh Successful."); } } diff --git a/src/SIL.Machine.AspNetCore/Services/ServalTranslationEngineServiceV1.cs b/src/SIL.Machine.AspNetCore/Services/ServalTranslationEngineServiceV1.cs index a0b010c61..fb87853c6 100644 --- a/src/SIL.Machine.AspNetCore/Services/ServalTranslationEngineServiceV1.cs +++ b/src/SIL.Machine.AspNetCore/Services/ServalTranslationEngineServiceV1.cs @@ -9,9 +9,15 @@ public class ServalTranslationEngineServiceV1 : TranslationEngineApi.Translation private readonly Dictionary _engineServices; - public ServalTranslationEngineServiceV1(IEnumerable engineServices) + private readonly HealthCheckService _healthCheckService; + + public ServalTranslationEngineServiceV1( + IEnumerable engineServices, + HealthCheckService healthCheckService + ) { _engineServices = engineServices.ToDictionary(es => es.Type); + _healthCheckService = healthCheckService; } public override async Task Create(CreateRequest request, ServerCallContext context) @@ -127,6 +133,13 @@ ServerCallContext context return new GetQueueSizeResponse { Size = await engineService.GetQueueSizeAsync(context.CancellationToken) }; } + public override async Task HealthCheck(Empty request, ServerCallContext context) + { + HealthReport healthReport = await _healthCheckService.CheckHealthAsync(); + HealthCheckResponse healthCheckResponse = WriteGrpcHealthCheckResponse.Generate(healthReport); + return healthCheckResponse; + } + private ITranslationEngineService GetEngineService(string engineTypeStr) { if (_engineServices.TryGetValue(GetEngineType(engineTypeStr), out ITranslationEngineService? service)) diff --git a/src/SIL.Machine.Morphology.HermitCrab.Tool/SIL.Machine.Morphology.HermitCrab.Tool.csproj b/src/SIL.Machine.Morphology.HermitCrab.Tool/SIL.Machine.Morphology.HermitCrab.Tool.csproj index 9517bc14e..4b8769225 100644 --- a/src/SIL.Machine.Morphology.HermitCrab.Tool/SIL.Machine.Morphology.HermitCrab.Tool.csproj +++ b/src/SIL.Machine.Morphology.HermitCrab.Tool/SIL.Machine.Morphology.HermitCrab.Tool.csproj @@ -2,7 +2,7 @@ Exe - net6.0 + net8.0 SIL.Machine.Morphology.HermitCrab true hc diff --git a/src/SIL.Machine.Plugin/SIL.Machine.Plugin.csproj b/src/SIL.Machine.Plugin/SIL.Machine.Plugin.csproj index edc7ad063..7b503e74d 100644 --- a/src/SIL.Machine.Plugin/SIL.Machine.Plugin.csproj +++ b/src/SIL.Machine.Plugin/SIL.Machine.Plugin.csproj @@ -1,7 +1,7 @@ - net6.0 + net8.0 A plugin framework for the Machine library. diff --git a/src/SIL.Machine.Serval.EngineServer/Program.cs b/src/SIL.Machine.Serval.EngineServer/Program.cs index fa843e3ad..50c389b71 100644 --- a/src/SIL.Machine.Serval.EngineServer/Program.cs +++ b/src/SIL.Machine.Serval.EngineServer/Program.cs @@ -29,7 +29,6 @@ app.UseHttpsRedirection(); app.MapServalTranslationEngineService(); -app.MapGrpcHealthChecksService(); app.MapHangfireDashboard(); app.Run(); diff --git a/src/SIL.Machine.Serval.EngineServer/SIL.Machine.Serval.EngineServer.csproj b/src/SIL.Machine.Serval.EngineServer/SIL.Machine.Serval.EngineServer.csproj index 9a6962da2..244245baa 100644 --- a/src/SIL.Machine.Serval.EngineServer/SIL.Machine.Serval.EngineServer.csproj +++ b/src/SIL.Machine.Serval.EngineServer/SIL.Machine.Serval.EngineServer.csproj @@ -1,7 +1,7 @@  - net6.0 + net8.0 enable enable 34e222a9-ef76-48f9-869e-338547f9bd25 @@ -23,7 +23,7 @@ - + icu.net.dll.config diff --git a/src/SIL.Machine.Serval.JobServer/Program.cs b/src/SIL.Machine.Serval.JobServer/Program.cs index 30c2fa593..38b147243 100644 --- a/src/SIL.Machine.Serval.JobServer/Program.cs +++ b/src/SIL.Machine.Serval.JobServer/Program.cs @@ -25,6 +25,4 @@ var app = builder.Build(); -app.MapHealthChecks("/health"); - app.Run(); diff --git a/src/SIL.Machine.Serval.JobServer/SIL.Machine.Serval.JobServer.csproj b/src/SIL.Machine.Serval.JobServer/SIL.Machine.Serval.JobServer.csproj index 0e6059cc0..949f3e0b2 100644 --- a/src/SIL.Machine.Serval.JobServer/SIL.Machine.Serval.JobServer.csproj +++ b/src/SIL.Machine.Serval.JobServer/SIL.Machine.Serval.JobServer.csproj @@ -1,7 +1,7 @@ - net6.0 + net8.0 enable enable aa9e7440-5a04-4de6-ba51-bab9ef4a62e1 @@ -25,7 +25,7 @@ - + icu.net.dll.config diff --git a/src/SIL.Machine.Tool/SIL.Machine.Tool.csproj b/src/SIL.Machine.Tool/SIL.Machine.Tool.csproj index 61a0d22ca..25e5634d7 100644 --- a/src/SIL.Machine.Tool/SIL.Machine.Tool.csproj +++ b/src/SIL.Machine.Tool/SIL.Machine.Tool.csproj @@ -2,7 +2,7 @@ Exe - net6.0 + net8.0 SIL.Machine true machine diff --git a/src/SIL.Machine.Translation.Thot/SimplexModelWeightTuner.cs b/src/SIL.Machine.Translation.Thot/SimplexModelWeightTuner.cs index 32b28a2a4..6880e80eb 100644 --- a/src/SIL.Machine.Translation.Thot/SimplexModelWeightTuner.cs +++ b/src/SIL.Machine.Translation.Thot/SimplexModelWeightTuner.cs @@ -44,7 +44,6 @@ double Evaluate(Vector weights, int evalCount) } return quality; } - ; progress.Report(new ProgressStatus(0, MaxProgressFunctionEvaluations)); var simplex = new NelderMeadSimplex(ConvergenceTolerance, MaxFunctionEvaluations, 1.0); MinimizationResult result = simplex.FindMinimum( diff --git a/tests/SIL.Machine.AspNetCore.Tests/SIL.Machine.AspNetCore.Tests.csproj b/tests/SIL.Machine.AspNetCore.Tests/SIL.Machine.AspNetCore.Tests.csproj index dd09631e7..7a2923cf6 100644 --- a/tests/SIL.Machine.AspNetCore.Tests/SIL.Machine.AspNetCore.Tests.csproj +++ b/tests/SIL.Machine.AspNetCore.Tests/SIL.Machine.AspNetCore.Tests.csproj @@ -1,7 +1,7 @@  - net6.0 + net8.0 SIL.Machine.AspNetCore enable enable diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/SIL.Machine.Morphology.HermitCrab.Tests.csproj b/tests/SIL.Machine.Morphology.HermitCrab.Tests/SIL.Machine.Morphology.HermitCrab.Tests.csproj index dbea327e6..c419d0f9a 100644 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/SIL.Machine.Morphology.HermitCrab.Tests.csproj +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/SIL.Machine.Morphology.HermitCrab.Tests.csproj @@ -1,7 +1,7 @@  - net6.0 + net8.0 SIL.Machine.Morphology.HermitCrab enable enable diff --git a/tests/SIL.Machine.Tests/SIL.Machine.Tests.csproj b/tests/SIL.Machine.Tests/SIL.Machine.Tests.csproj index ed2c113fe..beb8a730b 100644 --- a/tests/SIL.Machine.Tests/SIL.Machine.Tests.csproj +++ b/tests/SIL.Machine.Tests/SIL.Machine.Tests.csproj @@ -1,7 +1,7 @@  - net6.0 + net8.0 SIL.Machine enable enable diff --git a/tests/SIL.Machine.Tokenization.SentencePiece.Tests/SIL.Machine.Tokenization.SentencePiece.Tests.csproj b/tests/SIL.Machine.Tokenization.SentencePiece.Tests/SIL.Machine.Tokenization.SentencePiece.Tests.csproj index 4e875a4b4..7f64c54bf 100644 --- a/tests/SIL.Machine.Tokenization.SentencePiece.Tests/SIL.Machine.Tokenization.SentencePiece.Tests.csproj +++ b/tests/SIL.Machine.Tokenization.SentencePiece.Tests/SIL.Machine.Tokenization.SentencePiece.Tests.csproj @@ -1,7 +1,7 @@ - net6.0 + net8.0 enable enable false diff --git a/tests/SIL.Machine.Translation.Thot.Tests/SIL.Machine.Translation.Thot.Tests.csproj b/tests/SIL.Machine.Translation.Thot.Tests/SIL.Machine.Translation.Thot.Tests.csproj index 4ebd167f9..a79528cc3 100644 --- a/tests/SIL.Machine.Translation.Thot.Tests/SIL.Machine.Translation.Thot.Tests.csproj +++ b/tests/SIL.Machine.Translation.Thot.Tests/SIL.Machine.Translation.Thot.Tests.csproj @@ -1,7 +1,7 @@  - net6.0 + net8.0 SIL.Machine.Translation.Thot enable enable