diff --git a/LICENSE-binary b/LICENSE-binary index 6b4c15fa772..1e30e9d8f28 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -262,6 +262,7 @@ org.eclipse.jetty:jetty-proxy org.apache.logging.log4j:log4j-1.2-api org.apache.logging.log4j:log4j-api org.apache.logging.log4j:log4j-core +org.apache.logging.log4j:log4j-layout-template-json org.apache.logging.log4j:log4j-slf4j-impl org.yaml:snakeyaml io.dropwizard.metrics:metrics-core diff --git a/conf/log4j2.xml.template b/conf/log4j2.xml.template index 1f186bc67e2..3d7c9123878 100644 --- a/conf/log4j2.xml.template +++ b/conf/log4j2.xml.template @@ -16,7 +16,7 @@ ~ limitations under the License. --> - + @@ -57,6 +57,17 @@ + diff --git a/dev/dependencyList b/dev/dependencyList index ab6a0499b7a..6a4056125f1 100644 --- a/dev/dependencyList +++ b/dev/dependencyList @@ -126,6 +126,7 @@ kubernetes-model-storageclass/6.13.1//kubernetes-model-storageclass-6.13.1.jar log4j-1.2-api/2.24.2//log4j-1.2-api-2.24.2.jar log4j-api/2.24.2//log4j-api-2.24.2.jar log4j-core/2.24.2//log4j-core-2.24.2.jar +log4j-layout-template-json/2.24.2//log4j-layout-template-json-2.24.2.jar log4j-slf4j-impl/2.24.2//log4j-slf4j-impl-2.24.2.jar logging-interceptor/3.12.12//logging-interceptor-3.12.12.jar metrics-core/4.2.26//metrics-core-4.2.26.jar diff --git a/docs/monitor/logging.md b/docs/monitor/logging.md index d4e84c75757..d459734306d 100644 --- a/docs/monitor/logging.md +++ b/docs/monitor/logging.md @@ -130,6 +130,55 @@ For example, we can disable the console appender and enable the file appender li Then everything goes to `log/dummy.log`. +#### Sending Structured Logs to Kafka + +The Log4j2 has a built-in [KafkaAppender](https://logging.apache.org/log4j/2.x/manual/appenders/message-queue.html#KafkaAppender) +which allows sending log messages to an Apache Kafka topic with a few configurations, and it also provides a built-in +[JSON Template Layout](https://logging.apache.org/log4j/2.x/manual/json-template-layout.html) that supports encoding +`LogEvents` to structured JSON messages according to the structure described by the provided template. + +For example, we can configure the Kyuubi server to send the structured logs to Kafka `ecs-json-logs` topic, + +```xml + + + + + + + + + + + + + + + + + + +``` + +And each structured log message looks like, + +```json +{ + "@timestamp": "2024-12-24T18:53:01.030Z", + "ecs.version": "1.2.0", + "log.level": "INFO", + "message": "Service[KyuubiServer] is started.", + "process.thread.name": "main", + "log.logger": "org.apache.kyuubi.server.KyuubiServer", + "app": "kyuubi", + "cluster": "kyuubi-cluster", + "host": "hadoop-master1.orb.local" +} +``` + +Note: this feature may require additional jars to work. Please read the Log4j2 docs and ensure those jars are +on the Kyuubi server's classpath before enabling it. + ## Logs of Spark SQL Engine Spark SQL Engine is one type of Kyuubi Engines and also a typical Spark application. diff --git a/kyuubi-assembly/pom.xml b/kyuubi-assembly/pom.xml index f72d2a95ba2..7a45450b0ee 100644 --- a/kyuubi-assembly/pom.xml +++ b/kyuubi-assembly/pom.xml @@ -119,5 +119,10 @@ org.apache.logging.log4j log4j-1.2-api + + + org.apache.logging.log4j + log4j-layout-template-json + diff --git a/pom.xml b/pom.xml index f9ecb5ae30f..49440884975 100644 --- a/pom.xml +++ b/pom.xml @@ -663,6 +663,11 @@ log4j-1.2-api ${log4j.version} + + org.apache.logging.log4j + log4j-layout-template-json + ${log4j.version} + io.dropwizard.metrics