From 11aaf72ebf19466cea1b9d04d89b5611f81aae04 Mon Sep 17 00:00:00 2001 From: Mauro Servienti Date: Tue, 18 Feb 2025 12:33:24 +0100 Subject: [PATCH 1/5] Introduce a setting to control the primary instance shutdown timeout --- src/ServiceControl/HostApplicationBuilderExtensions.cs | 2 +- src/ServiceControl/Infrastructure/Settings/Settings.cs | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/ServiceControl/HostApplicationBuilderExtensions.cs b/src/ServiceControl/HostApplicationBuilderExtensions.cs index 0c365fecbc..a1774d4b30 100644 --- a/src/ServiceControl/HostApplicationBuilderExtensions.cs +++ b/src/ServiceControl/HostApplicationBuilderExtensions.cs @@ -51,7 +51,7 @@ public static void AddServiceControl(this IHostApplicationBuilder hostBuilder, S var transportCustomization = TransportFactory.Create(transportSettings); transportCustomization.AddTransportForPrimary(services, transportSettings); - services.Configure(options => options.ShutdownTimeout = TimeSpan.FromSeconds(30)); + services.Configure(options => options.ShutdownTimeout = settings.ShutdownTimeout); services.AddSingleton(); services.AddSingleton(); diff --git a/src/ServiceControl/Infrastructure/Settings/Settings.cs b/src/ServiceControl/Infrastructure/Settings/Settings.cs index 04b7a232bb..5ae680324b 100644 --- a/src/ServiceControl/Infrastructure/Settings/Settings.cs +++ b/src/ServiceControl/Infrastructure/Settings/Settings.cs @@ -65,6 +65,7 @@ public Settings( TimeToRestartErrorIngestionAfterFailure = GetTimeToRestartErrorIngestionAfterFailure(); DisableExternalIntegrationsPublishing = SettingsReader.Read(SettingsRootNamespace, "DisableExternalIntegrationsPublishing", false); TrackInstancesInitialValue = SettingsReader.Read(SettingsRootNamespace, "TrackInstancesInitialValue", true); + ShutdownTimeout = SettingsReader.Read(SettingsRootNamespace, "ShutdownTimeout", ShutdownTimeout); AssemblyLoadContextResolver = static assemblyPath => new PluginAssemblyLoadContext(assemblyPath); } @@ -180,6 +181,11 @@ public TimeSpan HeartbeatGracePeriod public bool DisableHealthChecks { get; set; } + // Windows services allow a maximum of 125 seconds when stopping a service. + // When shutting down or restarting the OS we have no control over the + // shutdown timeout + public TimeSpan ShutdownTimeout { get; set; } = TimeSpan.FromMinutes(2); + public string GetConnectionString() { var settingsValue = SettingsReader.Read(SettingsRootNamespace, "ConnectionString"); From d14d671438c3095e128ffd12cc2a896f3afacb98 Mon Sep 17 00:00:00 2001 From: Mauro Servienti Date: Tue, 18 Feb 2025 12:33:34 +0100 Subject: [PATCH 2/5] Introduce a setting to control the audit instance shutdown timeout --- .../HostApplicationBuilderExtensions.cs | 2 +- .../Infrastructure/Settings/Settings.cs | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs b/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs index 13d630456e..f4ed005435 100644 --- a/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs +++ b/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs @@ -45,7 +45,7 @@ public static void AddServiceControlAudit(this IHostApplicationBuilder builder, var transportCustomization = TransportFactory.Create(transportSettings); transportCustomization.AddTransportForAudit(services, transportSettings); - services.Configure(options => options.ShutdownTimeout = TimeSpan.FromSeconds(30)); + services.Configure(options => options.ShutdownTimeout = settings.ShutdownTimeout); services.AddSingleton(settings); services.AddSingleton(); diff --git a/src/ServiceControl.Audit/Infrastructure/Settings/Settings.cs b/src/ServiceControl.Audit/Infrastructure/Settings/Settings.cs index c0b1dfb892..7e89ace1f1 100644 --- a/src/ServiceControl.Audit/Infrastructure/Settings/Settings.cs +++ b/src/ServiceControl.Audit/Infrastructure/Settings/Settings.cs @@ -49,6 +49,7 @@ public Settings(string transportType = null, string persisterType = null, Loggin ServiceControlQueueAddress = SettingsReader.Read(SettingsRootNamespace, "ServiceControlQueueAddress"); TimeToRestartAuditIngestionAfterFailure = GetTimeToRestartAuditIngestionAfterFailure(); EnableFullTextSearchOnBodies = SettingsReader.Read(SettingsRootNamespace, "EnableFullTextSearchOnBodies", true); + ShutdownTimeout = SettingsReader.Read(SettingsRootNamespace, "ShutdownTimeout", ShutdownTimeout); AssemblyLoadContextResolver = static assemblyPath => new PluginAssemblyLoadContext(assemblyPath); } @@ -152,6 +153,11 @@ public int MaxBodySizeToStore public bool EnableFullTextSearchOnBodies { get; set; } + // Windows services allow a maximum of 125 seconds when stopping a service. + // When shutting down or restarting the OS we have no control over the + // shutdown timeout + public TimeSpan ShutdownTimeout { get; set; } = TimeSpan.FromMinutes(2); + public TransportSettings ToTransportSettings() { var transportSettings = new TransportSettings From 599b508e14145013374f6792563626ee553df188 Mon Sep 17 00:00:00 2001 From: Mauro Servienti Date: Tue, 18 Feb 2025 12:54:57 +0100 Subject: [PATCH 3/5] Approved API --- .../APIApprovals.PlatformSampleSettings.approved.txt | 3 ++- .../APIApprovals.PlatformSampleSettings.approved.txt | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/ServiceControl.Audit.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt b/src/ServiceControl.Audit.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt index 77b2ef7666..97fee174ed 100644 --- a/src/ServiceControl.Audit.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt +++ b/src/ServiceControl.Audit.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt @@ -28,5 +28,6 @@ "MaximumConcurrencyLevel": null, "ServiceControlQueueAddress": "Particular.ServiceControl", "TimeToRestartAuditIngestionAfterFailure": "00:01:00", - "EnableFullTextSearchOnBodies": true + "EnableFullTextSearchOnBodies": true, + "ShutdownTimeout": "00:02:00" } \ No newline at end of file diff --git a/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt b/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt index 98ed016ffb..c3502fee11 100644 --- a/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt +++ b/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt @@ -42,5 +42,6 @@ "MaximumConcurrencyLevel": null, "RetryHistoryDepth": 10, "RemoteInstances": [], - "DisableHealthChecks": false + "DisableHealthChecks": false, + "ShutdownTimeout": "00:02:00" } \ No newline at end of file From 9c0ab387be8ab3e998d07afe96b29b7dc5bfacf4 Mon Sep 17 00:00:00 2001 From: Mauro Servienti Date: Tue, 18 Feb 2025 17:12:58 +0100 Subject: [PATCH 4/5] Set the default shutdown timeout to the max allowed by the most restrictive hosting platform --- src/ServiceControl/Infrastructure/Settings/Settings.cs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/ServiceControl/Infrastructure/Settings/Settings.cs b/src/ServiceControl/Infrastructure/Settings/Settings.cs index 5ae680324b..492c3d0917 100644 --- a/src/ServiceControl/Infrastructure/Settings/Settings.cs +++ b/src/ServiceControl/Infrastructure/Settings/Settings.cs @@ -181,10 +181,9 @@ public TimeSpan HeartbeatGracePeriod public bool DisableHealthChecks { get; set; } - // Windows services allow a maximum of 125 seconds when stopping a service. - // When shutting down or restarting the OS we have no control over the - // shutdown timeout - public TimeSpan ShutdownTimeout { get; set; } = TimeSpan.FromMinutes(2); + // The default value is set to the maximum allowed time by the most restrictive + // hosting platform, which Docker Linux containers. + public TimeSpan ShutdownTimeout { get; set; } = TimeSpan.FromSeconds(10); public string GetConnectionString() { From 2ce99db721763ce06e8ef4c25fe10d64343f4b3f Mon Sep 17 00:00:00 2001 From: Mauro Servienti Date: Tue, 18 Feb 2025 17:13:52 +0100 Subject: [PATCH 5/5] When running the installer engine (via SCMU or PowerShell) we can assume it is Windows, and thus we can set the shutdown timeout to 2 minutes --- .../Configuration/ServiceControl/ServiceControlAppConfig.cs | 6 ++++++ .../Configuration/ServiceControl/SettingsList.cs | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/src/ServiceControlInstaller.Engine/Configuration/ServiceControl/ServiceControlAppConfig.cs b/src/ServiceControlInstaller.Engine/Configuration/ServiceControl/ServiceControlAppConfig.cs index 950058cfa7..c94ca3f13e 100644 --- a/src/ServiceControlInstaller.Engine/Configuration/ServiceControl/ServiceControlAppConfig.cs +++ b/src/ServiceControlInstaller.Engine/Configuration/ServiceControl/ServiceControlAppConfig.cs @@ -32,6 +32,12 @@ protected override void UpdateSettings() settings.Set(ServiceControlSettings.EnableFullTextSearchOnBodies, details.EnableFullTextSearchOnBodies.ToString(), version); settings.Set(ServiceControlSettings.RemoteInstances, RemoteInstanceConverter.ToJson(details.RemoteInstances), version); + // Windows services allow a maximum of 125 seconds when stopping a service. + // When shutting down or restarting the OS we have no control over the + // shutdown timeout. This is by the installer engine that is run _only_ on + // Windows via SCMU or PowerShell + settings.Set(ServiceControlSettings.ShutdownTimeout, "00:02:00"); + // Retired settings settings.RemoveIfRetired(ServiceControlSettings.AuditQueue, version); settings.RemoveIfRetired(ServiceControlSettings.AuditLogQueue, version); diff --git a/src/ServiceControlInstaller.Engine/Configuration/ServiceControl/SettingsList.cs b/src/ServiceControlInstaller.Engine/Configuration/ServiceControl/SettingsList.cs index bd0b5e4c87..5f324db1f8 100644 --- a/src/ServiceControlInstaller.Engine/Configuration/ServiceControl/SettingsList.cs +++ b/src/ServiceControlInstaller.Engine/Configuration/ServiceControl/SettingsList.cs @@ -89,5 +89,11 @@ public static class ServiceControlSettings Name = "ServiceControl/EnableFullTextSearchOnBodies", SupportedFrom = new SemanticVersion(4, 17, 0) }; + + public static SettingInfo ShutdownTimeout = new() + { + Name = "ServiceControl/ShutdownTimeout", + SupportedFrom = new SemanticVersion(6, 4, 0) + }; } } \ No newline at end of file