Add pre stop kubernetes support

DRSchlaubi · Dec 7, 2024 · a41dcbb · a41dcbb
1 parent 33b5410
commit a41dcbb
Show file tree

Hide file tree

Showing 12 changed files with 47 additions and 23 deletions.
diff --git a/core/kubernetes/README.md b/core/kubernetes/README.md
@@ -4,22 +4,28 @@ Plugin providing tools for running Mikbot within kubernetes
 
 Also shouteout to @lucsoft, who helped me to set this up
 
-## health endpoint
+## kubernetes hooks
 
 The plugin adds a HTTP endpoint, which you can use as a K8s probe, it will return 200 unless a shard or the DB connection
-is down
+is down.
+It also adds a pre-stop hook, which will call hooks of the [redeploy-hook](../redeploy-hook) plugin
 
 ```yaml
           livenessProbe:
               httpGet:
                   path: /healthz
-                  port: mikbot
+                  port: mikbot-kubernetes # remember to specify this port somewhere, it defaults to 8081
           startupProbe:
               httpGet:
                   path: /healthz
-                  port: mikbot # remember to specify that port name somewhere
+                  port: mikbot-kubernetes
               # let's give our shards time to connect
               initialDelaySeconds: 15
+          lifecycle:
+              preStop:
+                  httpGet:
+                      port: mikbot-kubernetes
+                      path: /kubernetes/pre-stop
 ```
 
 ### Own health checks

diff --git a/core/kubernetes/build.gradle.kts b/core/kubernetes/build.gradle.kts
@@ -15,7 +15,9 @@ repositories {
 }
 
 dependencies {
-    plugin(projects.core.ktor)
+    optionalPlugin(projects.core.redeployHook)
+    implementation(libs.ktor.server.netty)
+    implementation(libs.ktor.server.resources)
 
     implementation(libs.kubernetes.client)
     implementation(libs.kotlin.jsonpatch)

diff --git a/core/kubernetes/src/main/kotlin/dev/schlaubi/mikbot/core/health/Config.kt b/core/kubernetes/src/main/kotlin/dev/schlaubi/mikbot/core/health/Config.kt
@@ -7,6 +7,7 @@ object Config : EnvironmentConfig() {
     val POD_ID by getEnv(transform = String::toInt)
     val SHARDS_PER_POD by getEnv(2, String::toInt)
     val TOTAL_SHARDS by getEnv(transform = String::toInt)
+    val PORT by getEnv(8081, String::toInt)
     val STATEFUL_SET_NAME by this
     val NAMESPACE by getEnv("default")
     val CONTAINER_NAME by this

diff --git a/core/kubernetes/src/main/kotlin/dev/schlaubi/mikbot/core/health/KubernetesAPIServer.kt b/core/kubernetes/src/main/kotlin/dev/schlaubi/mikbot/core/health/KubernetesAPIServer.kt
@@ -1,23 +1,23 @@
 package dev.schlaubi.mikbot.core.health
 
-import dev.kordex.core.koin.KordExKoinComponent
 import dev.schlaubi.mikbot.core.health.check.HealthCheck
 import dev.schlaubi.mikbot.core.health.routes.HealthRoutes
-import dev.schlaubi.mikbot.util_plugins.ktor.api.KtorExtensionPoint
+import dev.schlaubi.mikbot.core.redeploy_hook.api.RedeployExtensionPoint
+import dev.schlaubi.mikbot.plugin.api.PluginContext
+import dev.schlaubi.mikbot.plugin.api.getExtensions
 import io.ktor.http.*
 import io.ktor.server.application.*
+import io.ktor.server.engine.*
+import io.ktor.server.netty.*
 import io.ktor.server.resources.*
 import io.ktor.server.response.*
 import io.ktor.server.routing.*
 import mu.KotlinLogging
-import org.koin.core.component.inject
-import org.pf4j.Extension
 
-@Extension
-class KubernetesAPIServer : KtorExtensionPoint, KordExKoinComponent {
-    private val checks by inject<List<HealthCheck>>()
+fun startServer(checks: List<HealthCheck>, context: PluginContext) =
+    embeddedServer(Netty, Config.PORT) {
+        install(Resources)
 
-    override fun Application.apply() {
         routing {
             get<HealthRoutes.Health> {
                 if (checks.all { it.isSuccessful() }) {
@@ -26,9 +26,15 @@ class KubernetesAPIServer : KtorExtensionPoint, KordExKoinComponent {
                     call.respond(HttpStatusCode.InternalServerError)
                 }
             }
+
+            if (context.pluginWrapper.pluginManager.getPlugin("redeploy-hook") != null) {
+                val redeployHooks = context.pluginSystem.getExtensions<RedeployExtensionPoint>()
+                get<HealthRoutes.PreStop> {
+                    redeployHooks.forEach { it.beforeRedeploy() }
+                }
+            }
         }
-    }
-}
+    }.start(wait = false)
 
 private val logger = KotlinLogging.logger {}
 

diff --git a/core/kubernetes/src/main/kotlin/dev/schlaubi/mikbot/core/health/KubernetesPlugin.kt b/core/kubernetes/src/main/kotlin/dev/schlaubi/mikbot/core/health/KubernetesPlugin.kt
@@ -5,6 +5,7 @@ import dev.kordex.core.builders.ExtensionsBuilder
 import dev.kordex.core.utils.loadModule
 import dev.schlaubi.mikbot.core.health.check.HealthCheck
 import dev.schlaubi.mikbot.core.health.ratelimit.setupDistributedRateLimiter
+import dev.schlaubi.mikbot.core.redeploy_hook.api.RedeployExtensionPoint
 import dev.schlaubi.mikbot.plugin.api.Plugin
 import dev.schlaubi.mikbot.plugin.api.PluginContext
 import dev.schlaubi.mikbot.plugin.api.PluginMain
@@ -18,9 +19,12 @@ private val LOG = KotlinLogging.logger { }
 @PluginMain
 class KubernetesPlugin(context: PluginContext) : Plugin(context) {
     private val healthChecks by lazy<List<HealthCheck>>(context.pluginSystem::getExtensions)
+
     private val logger = KotlinLogging.logger(log)
     override fun start() {
         logger.info { "Registered ${healthChecks.size} health checks available at /healthz" }
+
+        startServer(healthChecks, contextSafe)
     }
 
     override fun ExtensionsBuilder.addExtensions() {

diff --git a/core/kubernetes/src/main/kotlin/dev/schlaubi/mikbot/core/health/routes/HealthRoutes.kt b/core/kubernetes/src/main/kotlin/dev/schlaubi/mikbot/core/health/routes/HealthRoutes.kt
@@ -3,10 +3,12 @@ package dev.schlaubi.mikbot.core.health.routes
 import io.ktor.resources.*
 import kotlinx.serialization.Serializable
 
-@Serializable
 @Resource("/")
 class HealthRoutes {
-    @Serializable
     @Resource("/healthz") // this is not a typo. See https://stackoverflow.com/questions/43380939/where-does-the-convention-of-using-healthz-for-application-health-checks-come-f
     class Health(val health: HealthRoutes)
+
+    @Resource("/kubernetes/pre-stop")
+    @Serializable
+    class PreStop(val parent: HealthRoutes = HealthRoutes())
 }
diff --git a/music/api/server/src/main/kotlin/StateWatcher.kt b/music/api/server/src/main/kotlin/StateWatcher.kt
@@ -49,9 +49,10 @@ class StateWatcher : Extension() {
         val guild = kord.getGuild(guildId)
         val player = musicModule.getCachedMusicPlayer(guildId) ?: return
         val time = (this as? LavalinkPlayerUpdateEvent)?.state?.time ?: System.currentTimeMillis()
+        val channelId = player.lastChannelId ?: return
 
         broadcastEvent(
-            Snowflake(player.lastChannelId!!),
+            Snowflake(channelId),
             PlayerUpdateEvent(player.toPlayerState(guild), player.queuedTracks.mapToAPIQueuedTrack(guild), time, guildId),
         )
     }

diff --git a/music/commands/src/main/kotlin/dev/schlaubi/mikmusic/commands/NextCommand.kt b/music/commands/src/main/kotlin/dev/schlaubi/mikmusic/commands/NextCommand.kt
@@ -2,10 +2,10 @@ package dev.schlaubi.mikmusic.commands
 
 import dev.schlaubi.mikbot.plugin.api.util.translate
 import dev.schlaubi.mikbot.translations.MusicTranslations
+import dev.schlaubi.mikmusic.api.types.ChapterQueuedTrack
 import dev.schlaubi.mikmusic.checks.anyMusicPlaying
 import dev.schlaubi.mikmusic.core.MusicModule
 import dev.schlaubi.mikmusic.core.musicControlContexts
-import dev.schlaubi.mikmusic.player.ChapterQueuedTrack
 
 suspend fun MusicModule.nextCommand() = ephemeralControlSlashCommand {
     name = MusicTranslations.Commands.Next.name

diff --git a/music/commands/src/main/kotlin/dev/schlaubi/mikmusic/commands/QueueCommand.kt b/music/commands/src/main/kotlin/dev/schlaubi/mikmusic/commands/QueueCommand.kt
@@ -5,10 +5,10 @@ import dev.kord.rest.builder.message.embed
 import dev.schlaubi.mikbot.plugin.api.util.forList
 import dev.schlaubi.mikbot.plugin.api.util.translate
 import dev.schlaubi.mikbot.translations.MusicTranslations
+import dev.schlaubi.mikmusic.api.types.QueuedTrack
 import dev.schlaubi.mikmusic.checks.anyMusicPlaying
 import dev.schlaubi.mikmusic.core.MusicModule
 import dev.schlaubi.mikmusic.core.musicControlContexts
-import dev.schlaubi.mikmusic.player.QueuedTrack
 import dev.schlaubi.mikmusic.player.addAutoPlaySongs
 import dev.schlaubi.mikmusic.util.format
 import kotlin.time.Duration.Companion.minutes

diff --git a/music/commands/src/main/kotlin/dev/schlaubi/mikmusic/commands/RadioCommand.kt b/music/commands/src/main/kotlin/dev/schlaubi/mikmusic/commands/RadioCommand.kt
@@ -10,10 +10,10 @@ import dev.schlaubi.lavakord.rest.loadItem
 import dev.schlaubi.mikbot.plugin.api.util.discordError
 import dev.schlaubi.mikbot.plugin.api.util.translate
 import dev.schlaubi.mikbot.translations.MusicTranslations
+import dev.schlaubi.mikmusic.api.types.SimpleQueuedTrack
 import dev.schlaubi.mikmusic.autocomplete.autoCompletedYouTubeQuery
 import dev.schlaubi.mikmusic.checks.joinSameChannelCheck
 import dev.schlaubi.mikmusic.core.MusicModule
-import dev.schlaubi.mikmusic.player.SimpleQueuedTrack
 import dev.schlaubi.mikmusic.player.enableAutoPlay
 import dev.schlaubi.mikmusic.util.spotifyId
 

diff --git a/music/commands/src/main/kotlin/dev/schlaubi/mikmusic/playlist/commands/SaveCommand.kt b/music/commands/src/main/kotlin/dev/schlaubi/mikmusic/playlist/commands/SaveCommand.kt
@@ -6,8 +6,8 @@ import dev.kordex.core.commands.converters.impl.optionalString
 import dev.kordex.core.commands.converters.impl.string
 import dev.schlaubi.mikbot.plugin.api.util.translate
 import dev.schlaubi.mikbot.translations.MusicTranslations
+import dev.schlaubi.mikmusic.api.types.QueuedTrack
 import dev.schlaubi.mikmusic.core.musicControlContexts
-import dev.schlaubi.mikmusic.player.QueuedTrack
 import dev.schlaubi.mikmusic.player.queue.QueueOptions
 import dev.schlaubi.mikmusic.player.queue.findTracks
 import dev.schlaubi.mikmusic.playlist.Playlist

diff --git a/runtime/plugins.txt b/runtime/plugins.txt
@@ -1,3 +1,5 @@
 :core:ktor
+:core:redeploy-hook
+:core:kubernetes
 :music:player
-:music:api:server
+:music:commands