From 4329a85cf3cea6ad2caa8d1aac443237f57e8d59 Mon Sep 17 00:00:00 2001 From: Fei Wang Date: Sun, 4 Feb 2024 10:03:48 -0800 Subject: [PATCH] async stop --- .../org/apache/kyuubi/engine/spark/SparkSQLEngine.scala | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkSQLEngine.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkSQLEngine.scala index 1339c7193a1..6bb9a148884 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkSQLEngine.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkSQLEngine.scala @@ -175,9 +175,12 @@ case class SparkSQLEngine(spark: SparkSession) extends Serverable("SparkSQLEngin val elapsedTime = System.currentTimeMillis() - getStartTime if (!shutdown.get && elapsedTime > maxLifetime) { if (deregistered.compareAndSet(false, true)) { - info(s"Spark engine has been running for more than $maxLifetime ms," + - s" deregistering from engine discovery space.") - frontendServices.flatMap(_.discoveryService).foreach(_.stop()) + ThreadUtils.runInNewThread("engine-de-register", isDaemon = false) { + // for ETCD, the de-registering process might be blocked, so deregister it async + info(s"Spark engine has been running for more than $maxLifetime ms," + + s" deregistering from engine discovery space.") + frontendServices.flatMap(_.discoveryService).foreach(_.stop()) + } } if (backendService.sessionManager.getOpenSessionCount <= 0) {