Skip to content

Commit

Permalink
[K8S] Support to cleanup the spark driver pod with specified clean up…
Browse files Browse the repository at this point in the history 
… strategy
  • Loading branch information
@liaoyt
liaoyt committed Nov 19, 2023
1 parent 3478fc9 commit 4d970fa
Show file tree
Hide file tree
Showing 3 changed files with 24 additions and 11 deletions.
2 changes: 1 addition & 1 deletion docs/configuration/settings.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -324,7 +324,7 @@ You can configure the Kyuubi properties in `$KYUUBI_HOME/conf/kyuubi-defaults.co
| kyuubi.kubernetes.master.address | <undefined> | The internal Kubernetes master (API server) address to be used for kyuubi. | string | 1.7.0 |
| kyuubi.kubernetes.namespace | default | The namespace that will be used for running the kyuubi pods and find engines. | string | 1.7.0 |
| kyuubi.kubernetes.namespace.allow.list || The allowed kubernetes namespace list, if it is empty, there is no kubernetes namespace limitation. | set | 1.8.0 |
| kyuubi.kubernetes.spark.deleteDriverPodOnTermination.enabled | false | If set to true then Kyuubi server will delete the spark driver pod after the application terminates for kyuubi.kubernetes.terminatedApplicationRetainPeriod. | boolean | 1.8.1 |
| kyuubi.kubernetes.spark.deleteDriverPodOnTermination | NONE | Kyuubi server will delete the spark driver pod after the application terminates for kyuubi.kubernetes.terminatedApplicationRetainPeriod. Available options are NONE, ALL, COMPLETED and default value is None which means none of the pod will be deleted | string | 1.8.1 |
| kyuubi.kubernetes.spark.forciblyRewriteDriverPodName.enabled | false | Whether to forcibly rewrite Spark driver pod name with 'kyuubi-<uuid>-driver'. If disabled, Kyuubi will try to preserve the application name while satisfying K8s' pod name policy, but some vendors may have stricter pod name policies, thus the generated name may become illegal. | boolean | 1.8.1 |
| kyuubi.kubernetes.spark.forciblyRewriteExecutorPodNamePrefix.enabled | false | Whether to forcibly rewrite Spark executor pod name prefix with 'kyuubi-<uuid>'. If disabled, Kyuubi will try to preserve the application name while satisfying K8s' pod name policy, but some vendors may have stricter Pod name policies, thus the generated name may become illegal. | boolean | 1.8.1 |
| kyuubi.kubernetes.terminatedApplicationRetainPeriod | PT5M | The period for which the Kyuubi server retains application information after the application terminates. | duration | 1.7.1 |
Expand Down
19 changes: 13 additions & 6 deletions kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1231,13 +1231,20 @@ object KyuubiConf {
.checkValue(_ > 0, "must be positive number")
.createWithDefault(Duration.ofMinutes(5).toMillis)

val KUBERNETES_SPARK_DELETE_DRIVER_POD_ON_TERMINATION_ENABLED: ConfigEntry[Boolean] =
buildConf("kyuubi.kubernetes.spark.deleteDriverPodOnTermination.enabled")
.doc("If set to true then Kyuubi server will delete the spark driver pod after " +
s"the application terminates for ${KUBERNETES_TERMINATED_APPLICATION_RETAIN_PERIOD.key}.")
val KUBERNETES_SPARK_DELETE_DRIVER_POD_ON_TERMINATION: ConfigEntry[String] =
buildConf("kyuubi.kubernetes.spark.deleteDriverPodOnTermination")
.doc("Kyuubi server will delete the spark driver pod after " +
s"the application terminates for ${KUBERNETES_TERMINATED_APPLICATION_RETAIN_PERIOD.key}. " +
"Available options are NONE, ALL, COMPLETED and " +
"default value is None which means none of the pod will be deleted")
.version("1.8.1")
.booleanConf
.createWithDefault(false)
.stringConf
.createWithDefault(KubernetesDeleteDriverPodStrategy.NONE.toString)

object KubernetesDeleteDriverPodStrategy extends Enumeration {
type KubernetesDeleteDriverPodStrategy = Value
val NONE, ALL, COMPLETED = Value
}

val KUBERNETES_APPLICATION_STATE_CONTAINER: ConfigEntry[String] =
buildConf("kyuubi.kubernetes.application.state.container")
Expand Down
14 changes: 10 additions & 4 deletions kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KubernetesApplicationOperation.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,9 @@ import io.fabric8.kubernetes.client.informers.{ResourceEventHandler, SharedIndex

import org.apache.kyuubi.{KyuubiException, Logging, Utils}
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.config.KyuubiConf.KubernetesApplicationStateSource
import org.apache.kyuubi.config.KyuubiConf.{KubernetesApplicationStateSource, KubernetesDeleteDriverPodStrategy}
import org.apache.kyuubi.config.KyuubiConf.KubernetesApplicationStateSource.KubernetesApplicationStateSource
import org.apache.kyuubi.config.KyuubiConf.KubernetesDeleteDriverPodStrategy.{ALL, COMPLETED, NONE}
import org.apache.kyuubi.engine.ApplicationState.{isTerminated, ApplicationState, FAILED, FINISHED, NOT_FOUND, PENDING, RUNNING, UNKNOWN}
import org.apache.kyuubi.util.KubernetesUtils

Expand Down Expand Up @@ -107,14 +108,19 @@ class KubernetesApplicationOperation extends ApplicationOperation with Logging {
submitTimeout = conf.get(KyuubiConf.ENGINE_KUBERNETES_SUBMIT_TIMEOUT)
// Defer cleaning terminated application information
val retainPeriod = conf.get(KyuubiConf.KUBERNETES_TERMINATED_APPLICATION_RETAIN_PERIOD)
val deleteSparkDriverPodOnTermination =
conf.get(KyuubiConf.KUBERNETES_SPARK_DELETE_DRIVER_POD_ON_TERMINATION_ENABLED)
val deleteDriverPodStrategy = KubernetesDeleteDriverPodStrategy.withName(
conf.get(KyuubiConf.KUBERNETES_SPARK_DELETE_DRIVER_POD_ON_TERMINATION))
cleanupTerminatedAppInfoTrigger = CacheBuilder.newBuilder()
.expireAfterWrite(retainPeriod, TimeUnit.MILLISECONDS)
.removalListener((notification: RemovalNotification[String, ApplicationState]) => {
Option(appInfoStore.remove(notification.getKey)).foreach { case (kubernetesInfo, removed) =>
val appLabel = notification.getKey
if (deleteSparkDriverPodOnTermination) {
val shouldDelete = deleteDriverPodStrategy match {
case NONE => false
case ALL => true
case COMPLETED => !ApplicationState.isFailed(notification.getValue)
}
if (shouldDelete) {
val podName = removed.name
try {
val kubernetesClient = getOrCreateKubernetesClient(kubernetesInfo)
Expand Down

0 comments on commit 4d970fa

Please sign in to comment.