From a41dcbbf0acb7ebcb817ea1c4b3428326ef20c03 Mon Sep 17 00:00:00 2001 From: Michael Rittmeister Date: Sat, 7 Dec 2024 16:16:22 +0100 Subject: [PATCH] Add pre stop kubernetes support --- core/kubernetes/README.md | 14 +++++++--- core/kubernetes/build.gradle.kts | 4 ++- .../dev/schlaubi/mikbot/core/health/Config.kt | 1 + .../mikbot/core/health/KubernetesAPIServer.kt | 26 ++++++++++++------- .../mikbot/core/health/KubernetesPlugin.kt | 4 +++ .../mikbot/core/health/routes/HealthRoutes.kt | 6 +++-- .../server/src/main/kotlin/StateWatcher.kt | 3 ++- .../schlaubi/mikmusic/commands/NextCommand.kt | 2 +- .../mikmusic/commands/QueueCommand.kt | 2 +- .../mikmusic/commands/RadioCommand.kt | 2 +- .../mikmusic/playlist/commands/SaveCommand.kt | 2 +- runtime/plugins.txt | 4 ++- 12 files changed, 47 insertions(+), 23 deletions(-) diff --git a/core/kubernetes/README.md b/core/kubernetes/README.md index cdfec337e..af09bcfaf 100644 --- a/core/kubernetes/README.md +++ b/core/kubernetes/README.md @@ -4,22 +4,28 @@ Plugin providing tools for running Mikbot within kubernetes Also shouteout to @lucsoft, who helped me to set this up -## health endpoint +## kubernetes hooks The plugin adds a HTTP endpoint, which you can use as a K8s probe, it will return 200 unless a shard or the DB connection -is down +is down. +It also adds a pre-stop hook, which will call hooks of the [redeploy-hook](../redeploy-hook) plugin ```yaml livenessProbe: httpGet: path: /healthz - port: mikbot + port: mikbot-kubernetes # remember to specify this port somewhere, it defaults to 8081 startupProbe: httpGet: path: /healthz - port: mikbot # remember to specify that port name somewhere + port: mikbot-kubernetes # let's give our shards time to connect initialDelaySeconds: 15 + lifecycle: + preStop: + httpGet: + port: mikbot-kubernetes + path: /kubernetes/pre-stop ``` ### Own health checks diff --git a/core/kubernetes/build.gradle.kts b/core/kubernetes/build.gradle.kts index f7e71129c..82ae6f935 100644 --- a/core/kubernetes/build.gradle.kts +++ b/core/kubernetes/build.gradle.kts @@ -15,7 +15,9 @@ repositories { } dependencies { - plugin(projects.core.ktor) + optionalPlugin(projects.core.redeployHook) + implementation(libs.ktor.server.netty) + implementation(libs.ktor.server.resources) implementation(libs.kubernetes.client) implementation(libs.kotlin.jsonpatch) diff --git a/core/kubernetes/src/main/kotlin/dev/schlaubi/mikbot/core/health/Config.kt b/core/kubernetes/src/main/kotlin/dev/schlaubi/mikbot/core/health/Config.kt index 3eeaa63ee..2481c7692 100644 --- a/core/kubernetes/src/main/kotlin/dev/schlaubi/mikbot/core/health/Config.kt +++ b/core/kubernetes/src/main/kotlin/dev/schlaubi/mikbot/core/health/Config.kt @@ -7,6 +7,7 @@ object Config : EnvironmentConfig() { val POD_ID by getEnv(transform = String::toInt) val SHARDS_PER_POD by getEnv(2, String::toInt) val TOTAL_SHARDS by getEnv(transform = String::toInt) + val PORT by getEnv(8081, String::toInt) val STATEFUL_SET_NAME by this val NAMESPACE by getEnv("default") val CONTAINER_NAME by this diff --git a/core/kubernetes/src/main/kotlin/dev/schlaubi/mikbot/core/health/KubernetesAPIServer.kt b/core/kubernetes/src/main/kotlin/dev/schlaubi/mikbot/core/health/KubernetesAPIServer.kt index 2568e8e00..6075afa33 100644 --- a/core/kubernetes/src/main/kotlin/dev/schlaubi/mikbot/core/health/KubernetesAPIServer.kt +++ b/core/kubernetes/src/main/kotlin/dev/schlaubi/mikbot/core/health/KubernetesAPIServer.kt @@ -1,23 +1,23 @@ package dev.schlaubi.mikbot.core.health -import dev.kordex.core.koin.KordExKoinComponent import dev.schlaubi.mikbot.core.health.check.HealthCheck import dev.schlaubi.mikbot.core.health.routes.HealthRoutes -import dev.schlaubi.mikbot.util_plugins.ktor.api.KtorExtensionPoint +import dev.schlaubi.mikbot.core.redeploy_hook.api.RedeployExtensionPoint +import dev.schlaubi.mikbot.plugin.api.PluginContext +import dev.schlaubi.mikbot.plugin.api.getExtensions import io.ktor.http.* import io.ktor.server.application.* +import io.ktor.server.engine.* +import io.ktor.server.netty.* import io.ktor.server.resources.* import io.ktor.server.response.* import io.ktor.server.routing.* import mu.KotlinLogging -import org.koin.core.component.inject -import org.pf4j.Extension -@Extension -class KubernetesAPIServer : KtorExtensionPoint, KordExKoinComponent { - private val checks by inject>() +fun startServer(checks: List, context: PluginContext) = + embeddedServer(Netty, Config.PORT) { + install(Resources) - override fun Application.apply() { routing { get { if (checks.all { it.isSuccessful() }) { @@ -26,9 +26,15 @@ class KubernetesAPIServer : KtorExtensionPoint, KordExKoinComponent { call.respond(HttpStatusCode.InternalServerError) } } + + if (context.pluginWrapper.pluginManager.getPlugin("redeploy-hook") != null) { + val redeployHooks = context.pluginSystem.getExtensions() + get { + redeployHooks.forEach { it.beforeRedeploy() } + } + } } - } -} + }.start(wait = false) private val logger = KotlinLogging.logger {} diff --git a/core/kubernetes/src/main/kotlin/dev/schlaubi/mikbot/core/health/KubernetesPlugin.kt b/core/kubernetes/src/main/kotlin/dev/schlaubi/mikbot/core/health/KubernetesPlugin.kt index c920bd21b..7805141cd 100644 --- a/core/kubernetes/src/main/kotlin/dev/schlaubi/mikbot/core/health/KubernetesPlugin.kt +++ b/core/kubernetes/src/main/kotlin/dev/schlaubi/mikbot/core/health/KubernetesPlugin.kt @@ -5,6 +5,7 @@ import dev.kordex.core.builders.ExtensionsBuilder import dev.kordex.core.utils.loadModule import dev.schlaubi.mikbot.core.health.check.HealthCheck import dev.schlaubi.mikbot.core.health.ratelimit.setupDistributedRateLimiter +import dev.schlaubi.mikbot.core.redeploy_hook.api.RedeployExtensionPoint import dev.schlaubi.mikbot.plugin.api.Plugin import dev.schlaubi.mikbot.plugin.api.PluginContext import dev.schlaubi.mikbot.plugin.api.PluginMain @@ -18,9 +19,12 @@ private val LOG = KotlinLogging.logger { } @PluginMain class KubernetesPlugin(context: PluginContext) : Plugin(context) { private val healthChecks by lazy>(context.pluginSystem::getExtensions) + private val logger = KotlinLogging.logger(log) override fun start() { logger.info { "Registered ${healthChecks.size} health checks available at /healthz" } + + startServer(healthChecks, contextSafe) } override fun ExtensionsBuilder.addExtensions() { diff --git a/core/kubernetes/src/main/kotlin/dev/schlaubi/mikbot/core/health/routes/HealthRoutes.kt b/core/kubernetes/src/main/kotlin/dev/schlaubi/mikbot/core/health/routes/HealthRoutes.kt index b7cf20432..0f43cc417 100644 --- a/core/kubernetes/src/main/kotlin/dev/schlaubi/mikbot/core/health/routes/HealthRoutes.kt +++ b/core/kubernetes/src/main/kotlin/dev/schlaubi/mikbot/core/health/routes/HealthRoutes.kt @@ -3,10 +3,12 @@ package dev.schlaubi.mikbot.core.health.routes import io.ktor.resources.* import kotlinx.serialization.Serializable -@Serializable @Resource("/") class HealthRoutes { - @Serializable @Resource("/healthz") // this is not a typo. See https://stackoverflow.com/questions/43380939/where-does-the-convention-of-using-healthz-for-application-health-checks-come-f class Health(val health: HealthRoutes) + + @Resource("/kubernetes/pre-stop") + @Serializable + class PreStop(val parent: HealthRoutes = HealthRoutes()) } diff --git a/music/api/server/src/main/kotlin/StateWatcher.kt b/music/api/server/src/main/kotlin/StateWatcher.kt index 999e58f6a..a6954c200 100644 --- a/music/api/server/src/main/kotlin/StateWatcher.kt +++ b/music/api/server/src/main/kotlin/StateWatcher.kt @@ -49,9 +49,10 @@ class StateWatcher : Extension() { val guild = kord.getGuild(guildId) val player = musicModule.getCachedMusicPlayer(guildId) ?: return val time = (this as? LavalinkPlayerUpdateEvent)?.state?.time ?: System.currentTimeMillis() + val channelId = player.lastChannelId ?: return broadcastEvent( - Snowflake(player.lastChannelId!!), + Snowflake(channelId), PlayerUpdateEvent(player.toPlayerState(guild), player.queuedTracks.mapToAPIQueuedTrack(guild), time, guildId), ) } diff --git a/music/commands/src/main/kotlin/dev/schlaubi/mikmusic/commands/NextCommand.kt b/music/commands/src/main/kotlin/dev/schlaubi/mikmusic/commands/NextCommand.kt index 221e8bcdf..54d47a04c 100644 --- a/music/commands/src/main/kotlin/dev/schlaubi/mikmusic/commands/NextCommand.kt +++ b/music/commands/src/main/kotlin/dev/schlaubi/mikmusic/commands/NextCommand.kt @@ -2,10 +2,10 @@ package dev.schlaubi.mikmusic.commands import dev.schlaubi.mikbot.plugin.api.util.translate import dev.schlaubi.mikbot.translations.MusicTranslations +import dev.schlaubi.mikmusic.api.types.ChapterQueuedTrack import dev.schlaubi.mikmusic.checks.anyMusicPlaying import dev.schlaubi.mikmusic.core.MusicModule import dev.schlaubi.mikmusic.core.musicControlContexts -import dev.schlaubi.mikmusic.player.ChapterQueuedTrack suspend fun MusicModule.nextCommand() = ephemeralControlSlashCommand { name = MusicTranslations.Commands.Next.name diff --git a/music/commands/src/main/kotlin/dev/schlaubi/mikmusic/commands/QueueCommand.kt b/music/commands/src/main/kotlin/dev/schlaubi/mikmusic/commands/QueueCommand.kt index fc1b86014..26bafa126 100644 --- a/music/commands/src/main/kotlin/dev/schlaubi/mikmusic/commands/QueueCommand.kt +++ b/music/commands/src/main/kotlin/dev/schlaubi/mikmusic/commands/QueueCommand.kt @@ -5,10 +5,10 @@ import dev.kord.rest.builder.message.embed import dev.schlaubi.mikbot.plugin.api.util.forList import dev.schlaubi.mikbot.plugin.api.util.translate import dev.schlaubi.mikbot.translations.MusicTranslations +import dev.schlaubi.mikmusic.api.types.QueuedTrack import dev.schlaubi.mikmusic.checks.anyMusicPlaying import dev.schlaubi.mikmusic.core.MusicModule import dev.schlaubi.mikmusic.core.musicControlContexts -import dev.schlaubi.mikmusic.player.QueuedTrack import dev.schlaubi.mikmusic.player.addAutoPlaySongs import dev.schlaubi.mikmusic.util.format import kotlin.time.Duration.Companion.minutes diff --git a/music/commands/src/main/kotlin/dev/schlaubi/mikmusic/commands/RadioCommand.kt b/music/commands/src/main/kotlin/dev/schlaubi/mikmusic/commands/RadioCommand.kt index 7c43d660c..38bcd1fb9 100644 --- a/music/commands/src/main/kotlin/dev/schlaubi/mikmusic/commands/RadioCommand.kt +++ b/music/commands/src/main/kotlin/dev/schlaubi/mikmusic/commands/RadioCommand.kt @@ -10,10 +10,10 @@ import dev.schlaubi.lavakord.rest.loadItem import dev.schlaubi.mikbot.plugin.api.util.discordError import dev.schlaubi.mikbot.plugin.api.util.translate import dev.schlaubi.mikbot.translations.MusicTranslations +import dev.schlaubi.mikmusic.api.types.SimpleQueuedTrack import dev.schlaubi.mikmusic.autocomplete.autoCompletedYouTubeQuery import dev.schlaubi.mikmusic.checks.joinSameChannelCheck import dev.schlaubi.mikmusic.core.MusicModule -import dev.schlaubi.mikmusic.player.SimpleQueuedTrack import dev.schlaubi.mikmusic.player.enableAutoPlay import dev.schlaubi.mikmusic.util.spotifyId diff --git a/music/commands/src/main/kotlin/dev/schlaubi/mikmusic/playlist/commands/SaveCommand.kt b/music/commands/src/main/kotlin/dev/schlaubi/mikmusic/playlist/commands/SaveCommand.kt index 28f61cc6a..e54c1ef87 100644 --- a/music/commands/src/main/kotlin/dev/schlaubi/mikmusic/playlist/commands/SaveCommand.kt +++ b/music/commands/src/main/kotlin/dev/schlaubi/mikmusic/playlist/commands/SaveCommand.kt @@ -6,8 +6,8 @@ import dev.kordex.core.commands.converters.impl.optionalString import dev.kordex.core.commands.converters.impl.string import dev.schlaubi.mikbot.plugin.api.util.translate import dev.schlaubi.mikbot.translations.MusicTranslations +import dev.schlaubi.mikmusic.api.types.QueuedTrack import dev.schlaubi.mikmusic.core.musicControlContexts -import dev.schlaubi.mikmusic.player.QueuedTrack import dev.schlaubi.mikmusic.player.queue.QueueOptions import dev.schlaubi.mikmusic.player.queue.findTracks import dev.schlaubi.mikmusic.playlist.Playlist diff --git a/runtime/plugins.txt b/runtime/plugins.txt index 101e6b947..419e768ac 100644 --- a/runtime/plugins.txt +++ b/runtime/plugins.txt @@ -1,3 +1,5 @@ :core:ktor +:core:redeploy-hook +:core:kubernetes :music:player -:music:api:server +:music:commands