Merge pull request #62 from MarshallWace/logging-cluster

logging the kafka-cluster name in every log line
MarshallWace · Oct 8, 2024 · 20030ee · 20030ee
2 parents 191ffcd + 093418c
commit 20030ee
Show file tree

Hide file tree

Showing 10 changed files with 41 additions and 23 deletions.
diff --git a/buildSrc/src/main/kotlin/buildlogic.kotlin-application-common-conventions.gradle.kts b/buildSrc/src/main/kotlin/buildlogic.kotlin-application-common-conventions.gradle.kts
@@ -9,6 +9,7 @@ val prometheus_version: String by project
 val koin_version: String by project
 val kotlin_kafka_version: String by project
 val kafka_version: String by project
+val hoplite_version: String by project
 
 plugins {
     // Apply the common convention plugin for shared build configuration between library and application projects.
@@ -39,4 +40,6 @@ dependencies {
     implementation("io.github.nomisrev:kotlin-kafka:$kotlin_kafka_version")
     implementation("org.apache.kafka:kafka-clients:$kafka_version")
     testImplementation("io.ktor:ktor-server-test-host-jvm:$ktor_version")
+    implementation("com.sksamuel.hoplite:hoplite-core:$hoplite_version")
+    implementation("com.sksamuel.hoplite:hoplite-yaml:$hoplite_version")
 }
diff --git a/buildSrc/src/main/kotlin/buildlogic.kotlin-application-conventions.gradle.kts b/buildSrc/src/main/kotlin/buildlogic.kotlin-application-conventions.gradle.kts
@@ -3,8 +3,6 @@
  * This file was generated by the Gradle 'init' task.
  */
 
-val hoplite_version: String by project
-
 plugins {
     // Apply the application-common convention plugin for shared build configuration between application projects.
     id("buildlogic.kotlin-application-common-conventions")
@@ -21,6 +19,4 @@ application {
 }
 
 dependencies {
-    implementation("com.sksamuel.hoplite:hoplite-core:$hoplite_version")
-    implementation("com.sksamuel.hoplite:hoplite-yaml:$hoplite_version")
 }
diff --git a/kafkakewl-common/src/main/kotlin/com/mwam/kafkakewl/common/config/Config.kt b/kafkakewl-common/src/main/kotlin/com/mwam/kafkakewl/common/config/Config.kt
@@ -6,6 +6,10 @@
 
 package com.mwam.kafkakewl.common.config
 
+import com.sksamuel.hoplite.ConfigLoaderBuilder
+import com.sksamuel.hoplite.addFileSource
+import com.sksamuel.hoplite.addResourceSource
+
 data class HttpConfig(
     val host: String = "0.0.0.0",
     val port: Int = 8080
@@ -29,5 +33,16 @@ data class KafkaClientConfig(
 )
 
 data class KafkaClusterConfig(
+    val name: String,
     val client: KafkaClientConfig
 )
+
+inline fun <reified A : Any> loadConfig(overrideFileName: String, applicationConfigResource: String = "/application.yaml"): A {
+    return ConfigLoaderBuilder.default()
+        // TODO this fails currently because there are weird env vars on my linux box
+        //.addEnvironmentSource(useUnderscoresAsSeparator = true)
+        .addFileSource(overrideFileName)
+        .addResourceSource(applicationConfigResource)
+        .build()
+        .loadConfigOrThrow<A>()
+}
diff --git a/kafkakewl-common/src/main/kotlin/com/mwam/kafkakewl/common/plugins/Configure.kt b/kafkakewl-common/src/main/kotlin/com/mwam/kafkakewl/common/plugins/Configure.kt
@@ -28,6 +28,11 @@ import io.micrometer.prometheus.*
 import kotlinx.serialization.json.Json
 import org.slf4j.event.Level
 
+/** initializes the logging, sets some logback variables */
+fun initializeLogging(kafkaClusterName: String) {
+    System.setProperty("LOGBACK_KAFKA_CLUSTER", kafkaClusterName)
+}
+
 fun Application.configureMonitoring() {
     val appMicrometerRegistry = PrometheusMeterRegistry(PrometheusConfig.DEFAULT)
 

diff --git a/kafkakewl-deploy/src/main/kotlin/com/mwam/kafkakewl/deploy/Application.kt b/kafkakewl-deploy/src/main/kotlin/com/mwam/kafkakewl/deploy/Application.kt
@@ -6,24 +6,16 @@
 
 package com.mwam.kafkakewl.deploy
 
+import com.mwam.kafkakewl.common.config.loadConfig
 import com.mwam.kafkakewl.common.plugins.*
 import com.mwam.kafkakewl.deploy.plugins.*
-import com.mwam.kafkakewl.deploy.services.TopologyDeploymentsService
-import com.sksamuel.hoplite.*
 import io.ktor.server.application.*
 import io.ktor.server.engine.*
 import io.ktor.server.netty.*
-import org.koin.ktor.ext.inject
-import kotlin.getValue
 
 fun main() {
-    val config = ConfigLoaderBuilder.default()
-        // TODO this fails currently because there are weird env vars on my linux box
-        //.addEnvironmentSource(useUnderscoresAsSeparator = true)
-        .addFileSource(".kafkakewl-deploy-overrides-application.yaml")
-        .addResourceSource("/application.yaml")
-        .build()
-        .loadConfigOrThrow<Config>()
+    val config = loadConfig<Config>(".kafkakewl-deploy-overrides-application.yaml", "/application.yaml")
+    initializeLogging(config.kafkaCluster.name)
 
     embeddedServer(Netty, port = config.http.port, host = config.http.host, module = { module(config) })
         .start(wait = true)

diff --git a/kafkakewl-deploy/src/main/resources/application.yaml b/kafkakewl-deploy/src/main/resources/application.yaml
@@ -3,6 +3,7 @@ http:
   port: 8080
 
 kafkaCluster:
+  name: test
   client:
     brokers: broker1
     config: {}

diff --git a/kafkakewl-deploy/src/main/resources/logback.xml b/kafkakewl-deploy/src/main/resources/logback.xml
@@ -3,6 +3,7 @@
     <variable name="LOGBACK_STDOUT_APPENDER" value="${LOGBACK_STDOUT_APPENDER:-stdout-json}" />
     <variable name="LOGBACK_LOCALFILE_ROOT" value="${LOGBACK_LOCALFILE_ROOT:-./logs}" />
     <variable name="LOGBACK_LOCALFILE_NAME" value="${LOGBACK_LOCALFILE_NAME:-./com.mwam.kafkakewl.deploy}" />
+    <variable name="LOGBACK_KAFKA_CLUSTER" value="${LOGBACK_KAFKA_CLUSTER:-na}" />
 
     <!-- changing a kafka-log messages' levels, etc... to reduce warn/error logs when it's unnecessary -->
     <turboFilter class="com.mwam.kafkakewl.utils.logging.LogFilter">
@@ -26,6 +27,10 @@
                 <version>${MW_APP_TAG_VERSION:-0.0.0}</version>
                 <includeMDC>true</includeMDC>
                 <customFields>
+                    <field>
+                        <name>kafkaCluster</name>
+                        <value>${LOGBACK_KAFKA_CLUSTER}</value>
+                    </field>
                     <field>
                         <name>stacktrace</name>
                         <value>%ex</value>

diff --git a/kafkakewl-metrics/src/main/kotlin/com/mwam/kafkakewl/metrics/Application.kt b/kafkakewl-metrics/src/main/kotlin/com/mwam/kafkakewl/metrics/Application.kt
@@ -6,24 +6,19 @@
 
 package com.mwam.kafkakewl.metrics
 
+import com.mwam.kafkakewl.common.config.loadConfig
 import com.mwam.kafkakewl.common.plugins.*
 import com.mwam.kafkakewl.metrics.plugins.*
 import com.mwam.kafkakewl.metrics.services.KafkaTopicInfoSource
-import com.sksamuel.hoplite.*
 import io.ktor.server.application.*
 import io.ktor.server.engine.*
 import io.ktor.server.netty.*
 import org.koin.ktor.ext.inject
 import kotlin.getValue
 
 fun main() {
-    val config = ConfigLoaderBuilder.default()
-        // TODO this fails currently because there are weird env vars on my linux box
-        //.addEnvironmentSource(useUnderscoresAsSeparator = true)
-        .addFileSource(".kafkakewl-metrics-overrides-application.yaml")
-        .addResourceSource("/application.yaml")
-        .build()
-        .loadConfigOrThrow<Config>()
+    val config = loadConfig<Config>(".kafkakewl-metrics-overrides-application.yaml", "/application.yaml")
+    initializeLogging(config.kafkaCluster.name)
 
     embeddedServer(Netty, port = config.http.port, host = config.http.host, module = { module(config) })
         .start(wait = true)

diff --git a/kafkakewl-metrics/src/main/resources/application.yaml b/kafkakewl-metrics/src/main/resources/application.yaml
@@ -3,6 +3,7 @@ http:
   port: 8080
 
 kafkaCluster:
+  name: test
   client:
     brokers: broker1
     config: {}

diff --git a/kafkakewl-metrics/src/main/resources/logback.xml b/kafkakewl-metrics/src/main/resources/logback.xml
@@ -3,6 +3,7 @@
     <variable name="LOGBACK_STDOUT_APPENDER" value="${LOGBACK_STDOUT_APPENDER:-stdout-json}" />
     <variable name="LOGBACK_LOCALFILE_ROOT" value="${LOGBACK_LOCALFILE_ROOT:-./logs}" />
     <variable name="LOGBACK_LOCALFILE_NAME" value="${LOGBACK_LOCALFILE_NAME:-./com.mwam.kafkakewl.metrics}" />
+    <variable name="LOGBACK_KAFKA_CLUSTER" value="${LOGBACK_KAFKA_CLUSTER:-na}" />
 
     <!-- changing a kafka-log messages' levels, etc... to reduce warn/error logs when it's unnecessary -->
     <turboFilter class="com.mwam.kafkakewl.utils.logging.LogFilter">
@@ -26,6 +27,10 @@
                 <version>${MW_APP_TAG_VERSION:-0.0.0}</version>
                 <includeMDC>true</includeMDC>
                 <customFields>
+                    <field>
+                        <name>kafkaCluster</name>
+                        <value>${LOGBACK_KAFKA_CLUSTER}</value>
+                    </field>
                     <field>
                         <name>stacktrace</name>
                         <value>%ex</value>