Skip to content

Commit

Permalink
Work around problems with asynchronous SSL certificate arrival
Browse files Browse the repository at this point in the history 
Work around the problem with ASP.NET crashing because SSL certificates arrive ordered via LetsEncrypt arrive after starting the app: Disable HTTPS for a first start, if we find the certificate is not available. Later, if the HTTPS certificate has arrived, restart the ASP.NET host with the HTTPS URLS.
For discussion of this issue, see:

+ <ffMathy/FluffySpoon.AspNet.EncryptWeMust#151>
+ <natemcmaster/LettuceEncrypt#293>
+ <dotnet/aspnetcore#26258>
+ <dotnet/aspnetcore#45801>

Also, update the LetsEncrypt library and Certes to integrate various  recent upstream improvements.
  • Loading branch information
@Viir
Viir committed Mar 23, 2024
1 parent 6a484c7 commit 338f4fd
Show file tree
Hide file tree
Showing 6 changed files with 151 additions and 40 deletions.
28 changes: 16 additions & 12 deletions implement/elm-time/Platform/WebService/PublicAppState.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@ public PublicAppState(

public WebApplication Build(
WebApplicationBuilder appBuilder,
ILogger logger,
IHostEnvironment env,
IReadOnlyList<string> publicWebHostUrls,
bool? disableLetsEncrypt,
Expand All @@ -99,16 +100,11 @@ public WebApplication Build(
logging.AddDebug();
});

using var loggerFactory = LoggerFactory.Create(logging =>
{
logging.AddConsole();
logging.AddDebug();
});

var logger = loggerFactory.CreateLogger<PublicAppState>();
var enableUseFluffySpoonLetsEncrypt =
serverAndElmAppConfig.ServerConfig?.letsEncryptOptions is not null && !(disableLetsEncrypt ?? false);

var canUseHttps =
serverAndElmAppConfig.ServerConfig?.letsEncryptOptions is not null && !(disableLetsEncrypt ?? false);
enableUseFluffySpoonLetsEncrypt;

var publicWebHostUrlsFilteredForHttps =
canUseHttps && !disableHttps
Expand All @@ -119,9 +115,17 @@ public WebApplication Build(
.Where(url => !url.StartsWith("https://", StringComparison.OrdinalIgnoreCase))
.ToImmutableArray();

logger.LogInformation("disableLetsEncrypt: {disableLetsEncrypt}", disableLetsEncrypt?.ToString() ?? "null");
logger.LogInformation("canUseHttps: {canUseHttps}", canUseHttps);
logger.LogInformation("disableHttps: {disableHttps}", disableHttps);
logger.LogInformation(
"disableLetsEncrypt: {disableLetsEncrypt}", disableLetsEncrypt?.ToString() ?? "null");

logger.LogInformation(
"enableUseFluffySpoonLetsEncrypt: {enableUseFluffySpoonLetsEncrypt}", enableUseFluffySpoonLetsEncrypt);

logger.LogInformation(
"canUseHttps: {canUseHttps}", canUseHttps);

logger.LogInformation(
"disableHttps: {disableHttps}", disableHttps);

var webHostBuilder =
appBuilder.WebHost
Expand All @@ -142,7 +146,7 @@ public WebApplication Build(
app.UseDeveloperExceptionPage();
}

if (canUseHttps)
if (enableUseFluffySpoonLetsEncrypt)
{
app.UseFluffySpoonLetsEncrypt();
}
Expand Down
143 changes: 119 additions & 24 deletions implement/elm-time/Platform/WebService/StartupAdminInterface.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
using ApiRouteMethodConfig =
System.Func<
Microsoft.AspNetCore.Http.HttpContext,
ElmTime.Platform.WebService.StartupAdminInterface.PublicHostConfiguration?,
ElmTime.Platform.WebService.StartupAdminInterface.PublicHostProcess?,
System.Threading.Tasks.Task>;

namespace ElmTime.Platform.WebService;
Expand Down Expand Up @@ -93,9 +93,13 @@ public static void ConfigureServices(IServiceCollection services)
services.AddSingleton(getDateTimeOffset);
}

public record PublicHostConfiguration(
PersistentProcessLiveRepresentation processLiveRepresentation,
IHost webHost);
public record PublicHostProcess(
PersistentProcessLiveRepresentation ProcessLiveRepresentation,
AspHostConfig AspHostConfig,
IHost WebHost);

public record AspHostConfig(
bool DisableHttps);

public void Configure(
IApplicationBuilder app,
Expand Down Expand Up @@ -128,7 +132,7 @@ public void Configure(

var processStoreFileStore = processStoreForFileStore.fileStore;

PublicHostConfiguration? publicAppHost = null;
PublicHostProcess? publicAppHost = null;

void stopPublicApp()
{
Expand All @@ -139,9 +143,9 @@ void stopPublicApp()

logger.LogInformation("Begin to stop the public app.");

publicAppHost?.webHost?.StopAsync(TimeSpan.FromSeconds(10)).Wait();
publicAppHost?.webHost?.Dispose();
publicAppHost?.processLiveRepresentation?.Dispose();
publicAppHost?.WebHost?.StopAsync(TimeSpan.FromSeconds(10)).Wait();
publicAppHost?.WebHost?.Dispose();
publicAppHost?.ProcessLiveRepresentation?.Dispose();
publicAppHost = null;
}
}
Expand All @@ -155,6 +159,85 @@ void stopPublicApp()
processStoreFileStore);

void startPublicApp()
{
var aspHostConfig = BuildCurrentAspHostConfig(logger);

startPublicAppLessRestartForHttps(aspHostConfig);

if (aspHostConfig.DisableHttps)
{
/*
* Workaround for the issue of ASP.NET crashing discovered during the incident in March.
* TODO: Review: Consider moving the certificate out of the versioned web service config.
* */

System.Threading.Tasks.Task.Run(() =>
{
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(100));

logger.LogInformation(
"Last start of public interface had HTTPS disabled. Checking if we can enable HTTPS now...");

var newConfig = BuildCurrentAspHostConfig(logger);

if (!newConfig.DisableHttps)
{
logger.LogInformation(
"Looks like we can enable HTTPS now, restarting the public interface...");

startPublicAppLessRestartForHttps(
aspHostConfig
with
{
DisableHttps = false
});
}
});
}
}

AspHostConfig BuildCurrentAspHostConfig(ILogger? logger)
{
var letsEncryptRenewalServiceCertificateCompleted =
FluffySpoon.AspNet.EncryptWeMust.Certes.LetsEncryptRenewalService.Certificate is { };

logger?.LogInformation(
"letsEncryptRenewalServiceCertificateCompleted: {completed}",
letsEncryptRenewalServiceCertificateCompleted);

var aspWouldCrashOnConfiguringHttpsEndpoint =
/*
* Adapt to crashes observed 2024-03-23:
*
warn: Microsoft.AspNetCore.Hosting.Diagnostics[15]
Overriding HTTP_PORTS '8080' and HTTPS_PORTS ''. Binding to values defined by URLS instead 'http://*;https://*'.
fail: Microsoft.Extensions.Hosting.Internal.Host[11]
Hosting failed to start
System.InvalidOperationException: Unable to configure HTTPS endpoint. No server certificate was specified, and the default developer certificate could not be found or is out of date.
To generate a developer certificate run 'dotnet dev-certs https'. To trust the certificate (Windows and macOS only) run 'dotnet dev-certs https --trust'.
For more information on configuring HTTPS see https://go.microsoft.com/fwlink/?linkid=848054.
at Microsoft.AspNetCore.Hosting.ListenOptionsHttpsExtensions.UseHttps(ListenOptions listenOptions, Action`1 configureOptions)
at Microsoft.AspNetCore.Server.Kestrel.Core.Internal.AddressBinder.AddressesStrategy.BindAsync(AddressBindContext context, CancellationToken cancellationToken)
at Microsoft.AspNetCore.Server.Kestrel.Core.Internal.AddressBinder.BindAsync(ListenOptions[] listenOptions, AddressBindContext context, Func`2 useHttps, CancellationToken cancellationToken)
at Microsoft.AspNetCore.Server.Kestrel.Core.KestrelServerImpl.BindAsync(CancellationToken cancellationToken)
at Microsoft.AspNetCore.Server.Kestrel.Core.KestrelServerImpl.StartAsync[TContext](IHttpApplication`1 application, CancellationToken cancellationToken)
at Microsoft.AspNetCore.Hosting.GenericWebHostService.StartAsync(CancellationToken cancellationToken)
at Microsoft.Extensions.Hosting.Internal.Host.<StartAsync>b__15_1(IHostedService service, CancellationToken token)
at Microsoft.Extensions.Hosting.Internal.Host.ForeachService[T](IEnumerable`1 services, CancellationToken token, Boolean concurrent, Boolean abortOnFirstException, List`1 exceptions, Func`3 operation)
For discussion of crashes following race condition like this, see:
https://github.com/natemcmaster/LettuceEncrypt/issues/293
* */
!letsEncryptRenewalServiceCertificateCompleted;

var aspHostConfig = new AspHostConfig(
DisableHttps: disableHttps || aspWouldCrashOnConfiguringHttpsEndpoint);

return aspHostConfig;
}

void startPublicAppLessRestartForHttps(AspHostConfig aspHostConfig)
{
lock (avoidConcurrencyLock)
{
Expand Down Expand Up @@ -221,6 +304,7 @@ void maintainStoreReductions()

WebApplication buildWebApplication(
ProcessAppConfig processAppConfig,
AspHostConfig aspHostConfig,
IReadOnlyList<string> publicWebHostUrls)
{
var appConfigTree =
Expand Down Expand Up @@ -272,35 +356,46 @@ WebApplication buildWebApplication(

var appBuilder = WebApplication.CreateBuilder();

using var loggerFactory = LoggerFactory.Create(logging =>
{
logging.AddConsole();
logging.AddDebug();
});

var logger = loggerFactory.CreateLogger<PublicAppState>();

var app =
publicAppState.Build(
appBuilder,
logger,
env,
publicWebHostUrls: publicWebHostUrls,
disableLetsEncrypt: disableLetsEncrypt,
disableHttps: disableHttps);
disableHttps: aspHostConfig.DisableHttps);

publicAppState.ProcessEventTimeHasArrived();

return app;
}

if (processLiveRepresentation?.lastAppConfig != null)
if (processLiveRepresentation?.lastAppConfig is { } lastAppConfig)
{
var publicWebHostUrls = configuration.GetSettingPublicWebHostUrls();

var webHost = buildWebApplication(
processLiveRepresentation.lastAppConfig,
lastAppConfig,
aspHostConfig,
publicWebHostUrls: publicWebHostUrls);

webHost.StartAsync(appLifetime.ApplicationStopping).Wait();

logger.LogInformation(
"Started the public app at '{urls}'.", string.Join(",", webHost.Urls));
"Started the public app at '{urls}'.", string.Join(", ", webHost.Urls));

publicAppHost = new PublicHostConfiguration(
processLiveRepresentation: processLiveRepresentation,
webHost: webHost);
publicAppHost = new PublicHostProcess(
ProcessLiveRepresentation: processLiveRepresentation,
AspHostConfig: aspHostConfig,
WebHost: webHost);
}

return 0;
Expand Down Expand Up @@ -342,7 +437,7 @@ private static RequestDelegate AdminInterfaceRun(
IFileStore processStoreFileStore,
IProcessStoreWriter processStoreWriter,
string? adminPassword,
Func<PublicHostConfiguration?> getPublicAppHost,
Func<PublicHostProcess?> getPublicAppHost,
object avoidConcurrencyLock,
Action startPublicApp,
Action stopPublicApp)
Expand Down Expand Up @@ -457,7 +552,7 @@ async System.Threading.Tasks.Task deployElmApp(bool initElmAppState)
return Result<string, IReadOnlyList<StateShim.InterfaceToHost.NamedExposedFunction>>.err(
"No application deployed.");

return publicAppHost.processLiveRepresentation.ListDatabaseFunctions();
return publicAppHost.ProcessLiveRepresentation.ListDatabaseFunctions();
}

Result<string, AdminInterface.ApplyDatabaseFunctionSuccess> applyDatabaseFunction(
Expand All @@ -469,7 +564,7 @@ async System.Threading.Tasks.Task deployElmApp(bool initElmAppState)
return Result<string, AdminInterface.ApplyDatabaseFunctionSuccess>.err(
"No application deployed.");

return publicAppHost.processLiveRepresentation.ApplyFunctionOnMainBranch(storeWriter: processStoreWriter, request);
return publicAppHost.ProcessLiveRepresentation.ApplyFunctionOnMainBranch(storeWriter: processStoreWriter, request);
}
}

Expand Down Expand Up @@ -506,7 +601,7 @@ async System.Threading.Tasks.Task deployElmApp(bool initElmAppState)
methods : ImmutableDictionary<string, ApiRouteMethodConfig>.Empty
.Add("get", async (context, publicAppHost) =>
{
var appConfig = publicAppHost?.processLiveRepresentation?.lastAppConfig.appConfigComponent;
var appConfig = publicAppHost?.ProcessLiveRepresentation?.lastAppConfig.appConfigComponent;

if (appConfig == null)
{
Expand Down Expand Up @@ -548,7 +643,7 @@ async System.Threading.Tasks.Task deployElmApp(bool initElmAppState)
return;
}

var processLiveRepresentation = publicAppHost?.processLiveRepresentation;
var processLiveRepresentation = publicAppHost?.ProcessLiveRepresentation;

var components = new List<PineValue>();

Expand Down Expand Up @@ -592,10 +687,10 @@ async System.Threading.Tasks.Task deployElmApp(bool initElmAppState)
var elmAppStateToSet = await System.Text.Json.JsonSerializer.DeserializeAsync<System.Text.Json.JsonElement>(context.Request.Body);

var setAppStateResult =
Result<string, PublicHostConfiguration?>.ok(publicAppHost)
Result<string, PublicHostProcess?>.ok(publicAppHost)
.AndThen(maybeNull => Maybe.NothingFromNull(maybeNull).ToResult("Not possible because there is no app (state)."))
.AndThen(publicAppHost =>
publicAppHost.processLiveRepresentation.SetStateOnMainBranch(
publicAppHost.ProcessLiveRepresentation.SetStateOnMainBranch(
storeWriter: processStoreWriter,
elmAppStateToSet))
.Map(compositionLogEventAndResponse =>
Expand Down Expand Up @@ -818,7 +913,7 @@ TruncateProcessHistoryReport truncateProcessHistory(TimeSpan productionBlockDura

var storeReductionReport =
getPublicAppHost()
?.processLiveRepresentation?.StoreReductionRecordForCurrentState(processStoreWriter).report;
?.ProcessLiveRepresentation?.StoreReductionRecordForCurrentState(processStoreWriter).report;

storeReductionStopwatch.Stop();

Expand Down Expand Up @@ -960,7 +1055,7 @@ TruncateProcessHistoryReport truncateProcessHistory(TimeSpan productionBlockDura

var storeReductionReport =
getPublicAppHost()
?.processLiveRepresentation?.StoreReductionRecordForCurrentState(processStoreWriter).report;
?.ProcessLiveRepresentation?.StoreReductionRecordForCurrentState(processStoreWriter).report;

storeReductionStopwatch.Stop();

Expand Down
2 changes: 1 addition & 1 deletion implement/elm-time/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ namespace ElmTime;

public class Program
{
public static string AppVersionId => "2024-03-22";
public static string AppVersionId => "2024-03-23";

private static int AdminInterfaceDefaultPort => 4000;

Expand Down
12 changes: 9 additions & 3 deletions implement/elm-time/elm-time.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@
<TargetFramework>net8.0</TargetFramework>
<RootNamespace>ElmTime</RootNamespace>
<AssemblyName>elm-time</AssemblyName>
<AssemblyVersion>2024.0322.0.0</AssemblyVersion>
<FileVersion>2024.0322.0.0</FileVersion>
<AssemblyVersion>2024.0323.0.0</AssemblyVersion>
<FileVersion>2024.0323.0.0</FileVersion>
<Nullable>enable</Nullable>
<GenerateEmbeddedFilesManifest>true</GenerateEmbeddedFilesManifest>
</PropertyGroup>
Expand Down Expand Up @@ -38,7 +38,7 @@
</ItemGroup>

<ItemGroup>
<PackageReference Include="FluffySpoon.AspNet.EncryptWeMust" Version="1.171.0" />
<PackageReference Include="certes" Version="3.0.4" />
<PackageReference Include="JavaScriptEngineSwitcher.V8" Version="3.21.5" />
<PackageReference Include="Jint" Version="3.0.1" />
<PackageReference Include="LibGit2Sharp" Version="0.28.0" />
Expand All @@ -59,4 +59,10 @@
<EmbeddedResource Include="Gui\elm\**" />
</ItemGroup>

<ItemGroup>
<Reference Include="FluffySpoon.AspNet.EncryptWeMust">
<HintPath>./../lib/FluffySpoon.AspNet.EncryptWeMust.dll</HintPath>
</Reference>
</ItemGroup>

</Project>
Binary file added implement/lib/FluffySpoon.AspNet.EncryptWeMust.dll
View file
Binary file not shown.
6 changes: 6 additions & 0 deletions implement/test-elm-time/test-elm-time.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,4 +29,10 @@
<ProjectReference Include="..\elm-time\elm-time.csproj" />
</ItemGroup>

<ItemGroup>
<Reference Include="FluffySpoon.AspNet.EncryptWeMust">
<HintPath>./../lib/FluffySpoon.AspNet.EncryptWeMust.dll</HintPath>
</Reference>
</ItemGroup>

</Project>

0 comments on commit 338f4fd

Please sign in to comment.