From 093418c840fde2a913a8faa68ccfb3874f84c5c0 Mon Sep 17 00:00:00 2001 From: Laszlo Bende Date: Tue, 8 Oct 2024 20:55:56 +0100 Subject: [PATCH] logging the kafka-cluster name in every log line Signed-off-by: Laszlo Bende --- ...tlin-application-common-conventions.gradle.kts | 3 +++ ...ogic.kotlin-application-conventions.gradle.kts | 4 ---- .../com/mwam/kafkakewl/common/config/Config.kt | 15 +++++++++++++++ .../mwam/kafkakewl/common/plugins/Configure.kt | 5 +++++ .../com/mwam/kafkakewl/deploy/Application.kt | 14 +++----------- .../src/main/resources/application.yaml | 1 + kafkakewl-deploy/src/main/resources/logback.xml | 5 +++++ .../com/mwam/kafkakewl/metrics/Application.kt | 11 +++-------- .../src/main/resources/application.yaml | 1 + kafkakewl-metrics/src/main/resources/logback.xml | 5 +++++ 10 files changed, 41 insertions(+), 23 deletions(-) diff --git a/buildSrc/src/main/kotlin/buildlogic.kotlin-application-common-conventions.gradle.kts b/buildSrc/src/main/kotlin/buildlogic.kotlin-application-common-conventions.gradle.kts index fc03c50..f039d30 100644 --- a/buildSrc/src/main/kotlin/buildlogic.kotlin-application-common-conventions.gradle.kts +++ b/buildSrc/src/main/kotlin/buildlogic.kotlin-application-common-conventions.gradle.kts @@ -9,6 +9,7 @@ val prometheus_version: String by project val koin_version: String by project val kotlin_kafka_version: String by project val kafka_version: String by project +val hoplite_version: String by project plugins { // Apply the common convention plugin for shared build configuration between library and application projects. @@ -39,4 +40,6 @@ dependencies { implementation("io.github.nomisrev:kotlin-kafka:$kotlin_kafka_version") implementation("org.apache.kafka:kafka-clients:$kafka_version") testImplementation("io.ktor:ktor-server-test-host-jvm:$ktor_version") + implementation("com.sksamuel.hoplite:hoplite-core:$hoplite_version") + implementation("com.sksamuel.hoplite:hoplite-yaml:$hoplite_version") } diff --git a/buildSrc/src/main/kotlin/buildlogic.kotlin-application-conventions.gradle.kts b/buildSrc/src/main/kotlin/buildlogic.kotlin-application-conventions.gradle.kts index 325a068..33efca4 100644 --- a/buildSrc/src/main/kotlin/buildlogic.kotlin-application-conventions.gradle.kts +++ b/buildSrc/src/main/kotlin/buildlogic.kotlin-application-conventions.gradle.kts @@ -3,8 +3,6 @@ * This file was generated by the Gradle 'init' task. */ -val hoplite_version: String by project - plugins { // Apply the application-common convention plugin for shared build configuration between application projects. id("buildlogic.kotlin-application-common-conventions") @@ -21,6 +19,4 @@ application { } dependencies { - implementation("com.sksamuel.hoplite:hoplite-core:$hoplite_version") - implementation("com.sksamuel.hoplite:hoplite-yaml:$hoplite_version") } \ No newline at end of file diff --git a/kafkakewl-common/src/main/kotlin/com/mwam/kafkakewl/common/config/Config.kt b/kafkakewl-common/src/main/kotlin/com/mwam/kafkakewl/common/config/Config.kt index 9df0181..5fbac4b 100644 --- a/kafkakewl-common/src/main/kotlin/com/mwam/kafkakewl/common/config/Config.kt +++ b/kafkakewl-common/src/main/kotlin/com/mwam/kafkakewl/common/config/Config.kt @@ -6,6 +6,10 @@ package com.mwam.kafkakewl.common.config +import com.sksamuel.hoplite.ConfigLoaderBuilder +import com.sksamuel.hoplite.addFileSource +import com.sksamuel.hoplite.addResourceSource + data class HttpConfig( val host: String = "0.0.0.0", val port: Int = 8080 @@ -29,5 +33,16 @@ data class KafkaClientConfig( ) data class KafkaClusterConfig( + val name: String, val client: KafkaClientConfig ) + +inline fun loadConfig(overrideFileName: String, applicationConfigResource: String = "/application.yaml"): A { + return ConfigLoaderBuilder.default() + // TODO this fails currently because there are weird env vars on my linux box + //.addEnvironmentSource(useUnderscoresAsSeparator = true) + .addFileSource(overrideFileName) + .addResourceSource(applicationConfigResource) + .build() + .loadConfigOrThrow() +} diff --git a/kafkakewl-common/src/main/kotlin/com/mwam/kafkakewl/common/plugins/Configure.kt b/kafkakewl-common/src/main/kotlin/com/mwam/kafkakewl/common/plugins/Configure.kt index d6ce46b..bac6e63 100644 --- a/kafkakewl-common/src/main/kotlin/com/mwam/kafkakewl/common/plugins/Configure.kt +++ b/kafkakewl-common/src/main/kotlin/com/mwam/kafkakewl/common/plugins/Configure.kt @@ -28,6 +28,11 @@ import io.micrometer.prometheus.* import kotlinx.serialization.json.Json import org.slf4j.event.Level +/** initializes the logging, sets some logback variables */ +fun initializeLogging(kafkaClusterName: String) { + System.setProperty("LOGBACK_KAFKA_CLUSTER", kafkaClusterName) +} + fun Application.configureMonitoring() { val appMicrometerRegistry = PrometheusMeterRegistry(PrometheusConfig.DEFAULT) diff --git a/kafkakewl-deploy/src/main/kotlin/com/mwam/kafkakewl/deploy/Application.kt b/kafkakewl-deploy/src/main/kotlin/com/mwam/kafkakewl/deploy/Application.kt index e4cabcb..cc9c4e7 100644 --- a/kafkakewl-deploy/src/main/kotlin/com/mwam/kafkakewl/deploy/Application.kt +++ b/kafkakewl-deploy/src/main/kotlin/com/mwam/kafkakewl/deploy/Application.kt @@ -6,24 +6,16 @@ package com.mwam.kafkakewl.deploy +import com.mwam.kafkakewl.common.config.loadConfig import com.mwam.kafkakewl.common.plugins.* import com.mwam.kafkakewl.deploy.plugins.* -import com.mwam.kafkakewl.deploy.services.TopologyDeploymentsService -import com.sksamuel.hoplite.* import io.ktor.server.application.* import io.ktor.server.engine.* import io.ktor.server.netty.* -import org.koin.ktor.ext.inject -import kotlin.getValue fun main() { - val config = ConfigLoaderBuilder.default() - // TODO this fails currently because there are weird env vars on my linux box - //.addEnvironmentSource(useUnderscoresAsSeparator = true) - .addFileSource(".kafkakewl-deploy-overrides-application.yaml") - .addResourceSource("/application.yaml") - .build() - .loadConfigOrThrow() + val config = loadConfig(".kafkakewl-deploy-overrides-application.yaml", "/application.yaml") + initializeLogging(config.kafkaCluster.name) embeddedServer(Netty, port = config.http.port, host = config.http.host, module = { module(config) }) .start(wait = true) diff --git a/kafkakewl-deploy/src/main/resources/application.yaml b/kafkakewl-deploy/src/main/resources/application.yaml index dbdbb2a..0a1e7e3 100644 --- a/kafkakewl-deploy/src/main/resources/application.yaml +++ b/kafkakewl-deploy/src/main/resources/application.yaml @@ -3,6 +3,7 @@ http: port: 8080 kafkaCluster: + name: test client: brokers: broker1 config: {} diff --git a/kafkakewl-deploy/src/main/resources/logback.xml b/kafkakewl-deploy/src/main/resources/logback.xml index c02ab37..629339e 100644 --- a/kafkakewl-deploy/src/main/resources/logback.xml +++ b/kafkakewl-deploy/src/main/resources/logback.xml @@ -3,6 +3,7 @@ + @@ -26,6 +27,10 @@ ${MW_APP_TAG_VERSION:-0.0.0} true + + kafkaCluster + ${LOGBACK_KAFKA_CLUSTER} + stacktrace %ex diff --git a/kafkakewl-metrics/src/main/kotlin/com/mwam/kafkakewl/metrics/Application.kt b/kafkakewl-metrics/src/main/kotlin/com/mwam/kafkakewl/metrics/Application.kt index 7b41221..ce3424c 100644 --- a/kafkakewl-metrics/src/main/kotlin/com/mwam/kafkakewl/metrics/Application.kt +++ b/kafkakewl-metrics/src/main/kotlin/com/mwam/kafkakewl/metrics/Application.kt @@ -6,10 +6,10 @@ package com.mwam.kafkakewl.metrics +import com.mwam.kafkakewl.common.config.loadConfig import com.mwam.kafkakewl.common.plugins.* import com.mwam.kafkakewl.metrics.plugins.* import com.mwam.kafkakewl.metrics.services.KafkaTopicInfoSource -import com.sksamuel.hoplite.* import io.ktor.server.application.* import io.ktor.server.engine.* import io.ktor.server.netty.* @@ -17,13 +17,8 @@ import org.koin.ktor.ext.inject import kotlin.getValue fun main() { - val config = ConfigLoaderBuilder.default() - // TODO this fails currently because there are weird env vars on my linux box - //.addEnvironmentSource(useUnderscoresAsSeparator = true) - .addFileSource(".kafkakewl-metrics-overrides-application.yaml") - .addResourceSource("/application.yaml") - .build() - .loadConfigOrThrow() + val config = loadConfig(".kafkakewl-metrics-overrides-application.yaml", "/application.yaml") + initializeLogging(config.kafkaCluster.name) embeddedServer(Netty, port = config.http.port, host = config.http.host, module = { module(config) }) .start(wait = true) diff --git a/kafkakewl-metrics/src/main/resources/application.yaml b/kafkakewl-metrics/src/main/resources/application.yaml index 4446ecb..e0cfd6d 100644 --- a/kafkakewl-metrics/src/main/resources/application.yaml +++ b/kafkakewl-metrics/src/main/resources/application.yaml @@ -3,6 +3,7 @@ http: port: 8080 kafkaCluster: + name: test client: brokers: broker1 config: {} diff --git a/kafkakewl-metrics/src/main/resources/logback.xml b/kafkakewl-metrics/src/main/resources/logback.xml index 7f2ba45..32b2b4c 100644 --- a/kafkakewl-metrics/src/main/resources/logback.xml +++ b/kafkakewl-metrics/src/main/resources/logback.xml @@ -3,6 +3,7 @@ + @@ -26,6 +27,10 @@ ${MW_APP_TAG_VERSION:-0.0.0} true + + kafkaCluster + ${LOGBACK_KAFKA_CLUSTER} + stacktrace %ex