From b61b89c11cc6b2c77e2eff2b3002466e239591f4 Mon Sep 17 00:00:00 2001 From: JamesChenX Date: Sat, 10 Aug 2024 18:17:54 +0800 Subject: [PATCH] Support phase-based job shutdown (graceful shutdown, aggressive graceful shutdown, forced shutdown) to shut down gracefully and promptly --- .../application/TurmsApplicationContext.java | 13 ++++++++----- .../property/env/common/ShutdownProperties.java | 12 +++++++++--- .../resources/application-full-example.yaml | 8 ++++++-- ...properties-metadata-with-property-value.json | 17 +++++++++++++---- .../resources/turms-properties-metadata.json | 14 +++++++++++--- 5 files changed, 47 insertions(+), 17 deletions(-) diff --git a/turms-server-common/src/main/java/im/turms/server/common/infra/application/TurmsApplicationContext.java b/turms-server-common/src/main/java/im/turms/server/common/infra/application/TurmsApplicationContext.java index 5d247eda71..bce3a96c8c 100644 --- a/turms-server-common/src/main/java/im/turms/server/common/infra/application/TurmsApplicationContext.java +++ b/turms-server-common/src/main/java/im/turms/server/common/infra/application/TurmsApplicationContext.java @@ -85,7 +85,8 @@ public class TurmsApplicationContext { private final String activeEnvProfile; private final BuildProperties buildProperties; - private long shutdownJobTimeoutMillis; + private long shutdownJobGracefulTimeoutMillis; + private long shutdownJobForcedTimeoutMillis; private final TreeMap shutdownHooks = new TreeMap<>(); public TurmsApplicationContext(Environment environment) { @@ -175,7 +176,9 @@ public void handleContextRefreshedEvent(ContextRefreshedEvent event) { .getBean(TurmsPropertiesManager.class); ShutdownProperties properties = propertiesManager.getLocalProperties() .getShutdown(); - shutdownJobTimeoutMillis = properties.getJobTimeoutMillis(); + shutdownJobGracefulTimeoutMillis = properties.getJobGracefulTimeoutMillis(); + shutdownJobForcedTimeoutMillis = + Math.max(properties.getJobForcedTimeoutMillis(), shutdownJobGracefulTimeoutMillis); } @EventListener(classes = ContextClosedEvent.class) @@ -188,18 +191,18 @@ public void handleContextClosedEvent() { String jobName = key.name(); boolean isClosingLogProcessor = key == JobShutdownOrder.CLOSE_LOG_PROCESSOR; Future> shutdownFuture = executor.submit(() -> orderAndJob.getValue() - .run(shutdownJobTimeoutMillis)); + .run(shutdownJobGracefulTimeoutMillis)); try { long time = System.currentTimeMillis(); Mono mono = - shutdownFuture.get(shutdownJobTimeoutMillis, TimeUnit.MILLISECONDS); + shutdownFuture.get(shutdownJobForcedTimeoutMillis, TimeUnit.MILLISECONDS); if (mono == null) { throw new IllegalArgumentException( "The result of the job \"" + jobName + "\" must not be null"); } - time = shutdownJobTimeoutMillis - (System.currentTimeMillis() - time); + time = shutdownJobForcedTimeoutMillis - (System.currentTimeMillis() - time); if (time <= 0) { throw new TimeoutException(); } diff --git a/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/common/ShutdownProperties.java b/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/common/ShutdownProperties.java index 5cb512dcbd..39762e9637 100644 --- a/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/common/ShutdownProperties.java +++ b/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/common/ShutdownProperties.java @@ -35,9 +35,15 @@ @NoArgsConstructor public class ShutdownProperties { - @Description("Wait for a job 2 minutes at most for extreme cases by default. " - + "Though it is a long time, graceful shutdown is usually better than force shutdown.") + @Description("The graceful shutdown timeout in milliseconds. After the timeout, the job will try to shut down gracefully in a more aggressive way " + + "before being forced to be shutdown when reaching \"jobForcedTimeoutMillis\"") @Min(0) - private long jobTimeoutMillis = 120 * 1000; + private long jobGracefulTimeoutMillis = 60 * 1000; + + @Description("The forced shutdown timeout in milliseconds. After the timeout, the job will be forced to be shutdown immediately no matter " + + "whether it is running or not. If this value is equal to or less than \"jobGracefulTimeoutMillis\", the job will be forced to be shutdown when " + + "reaching \"jobGracefulTimeoutMillis\"") + @Min(0) + private long jobForcedTimeoutMillis = 90 * 1000; } \ No newline at end of file diff --git a/turms-server-common/src/test/resources/application-full-example.yaml b/turms-server-common/src/test/resources/application-full-example.yaml index 12e6a0375c..e2fbbfc096 100644 --- a/turms-server-common/src/test/resources/application-full-example.yaml +++ b/turms-server-common/src/test/resources/application-full-example.yaml @@ -2474,10 +2474,14 @@ turms: # mutable property: true ignore-unknown-settings-on-upsert: false shutdown: - # Wait for a job 2 minutes at most for extreme cases by default. Though it is a long time, graceful shutdown is usually better than force shutdown. + # The forced shutdown timeout in milliseconds. After the timeout, the job will be forced to be shutdown immediately no matter whether it is running or not. If this value is equal to or less than "jobGracefulTimeoutMillis", the job will be forced to be shutdown when reaching "jobGracefulTimeoutMillis". # global property: false # mutable property: false - job-timeout-millis: 120000 + job-forced-timeout-millis: 90000 + # The graceful shutdown timeout in milliseconds. After the timeout, the job will try to shut down gracefully in a more aggressive way before being forced to be shutdown when reaching "jobForcedTimeoutMillis". + # global property: false + # mutable property: false + job-graceful-timeout-millis: 60000 user-status: # Whether to cache the user sessions status. # global property: false diff --git a/turms-server-common/src/test/resources/turms-properties-metadata-with-property-value.json b/turms-server-common/src/test/resources/turms-properties-metadata-with-property-value.json index c80b4c1144..c0252d9bfc 100644 --- a/turms-server-common/src/test/resources/turms-properties-metadata-with-property-value.json +++ b/turms-server-common/src/test/resources/turms-properties-metadata-with-property-value.json @@ -4528,7 +4528,7 @@ "type": "boolean", "value": false }, - "roleId": { + "roleIds": { "deprecated": false, "global": false, "mutable": false, @@ -6189,14 +6189,23 @@ } }, "shutdown": { - "jobTimeoutMillis": { + "jobForcedTimeoutMillis": { "deprecated": false, - "description": "Wait for a job 2 minutes at most for extreme cases by default. Though it is a long time, graceful shutdown is usually better than force shutdown.", + "description": "The forced shutdown timeout in milliseconds. After the timeout, the job will be forced to be shutdown immediately no matter whether it is running or not. If this value is equal to or less than \"jobGracefulTimeoutMillis\", the job will be forced to be shutdown when reaching \"jobGracefulTimeoutMillis\"", "global": false, "mutable": false, "sensitive": false, "type": "long", - "value": 120000 + "value": 90000 + }, + "jobGracefulTimeoutMillis": { + "deprecated": false, + "description": "The graceful shutdown timeout in milliseconds. After the timeout, the job will try to shut down gracefully in a more aggressive way before being forced to be shutdown when reaching \"jobForcedTimeoutMillis\"", + "global": false, + "mutable": false, + "sensitive": false, + "type": "long", + "value": 60000 } }, "userStatus": { diff --git a/turms-server-common/src/test/resources/turms-properties-metadata.json b/turms-server-common/src/test/resources/turms-properties-metadata.json index 76134b8800..f6b1d40b30 100644 --- a/turms-server-common/src/test/resources/turms-properties-metadata.json +++ b/turms-server-common/src/test/resources/turms-properties-metadata.json @@ -3972,7 +3972,7 @@ "sensitive": false, "type": "boolean" }, - "roleId": { + "roleIds": { "deprecated": false, "global": false, "mutable": false, @@ -5435,9 +5435,17 @@ } }, "shutdown": { - "jobTimeoutMillis": { + "jobForcedTimeoutMillis": { "deprecated": false, - "description": "Wait for a job 2 minutes at most for extreme cases by default. Though it is a long time, graceful shutdown is usually better than force shutdown.", + "description": "The forced shutdown timeout in milliseconds. After the timeout, the job will be forced to be shutdown immediately no matter whether it is running or not. If this value is equal to or less than \"jobGracefulTimeoutMillis\", the job will be forced to be shutdown when reaching \"jobGracefulTimeoutMillis\"", + "global": false, + "mutable": false, + "sensitive": false, + "type": "long" + }, + "jobGracefulTimeoutMillis": { + "deprecated": false, + "description": "The graceful shutdown timeout in milliseconds. After the timeout, the job will try to shut down gracefully in a more aggressive way before being forced to be shutdown when reaching \"jobForcedTimeoutMillis\"", "global": false, "mutable": false, "sensitive": false,