From dae40e1a21a62eb6b928fb485776f4d25663a0b7 Mon Sep 17 00:00:00 2001 From: hezhao2 Date: Thu, 28 Sep 2023 21:31:46 +0800 Subject: [PATCH 1/6] change default metrics reporter to prometheus --- docs/configuration/settings.md | 20 +++++++++---------- docs/deployment/migration-guide.md | 3 +++ .../apache/kyuubi/metrics/MetricsConf.scala | 2 +- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/docs/configuration/settings.md b/docs/configuration/settings.md index 832099764c2..6e890ba585e 100644 --- a/docs/configuration/settings.md +++ b/docs/configuration/settings.md @@ -352,16 +352,16 @@ You can configure the Kyuubi properties in `$KYUUBI_HOME/conf/kyuubi-defaults.co ### Metrics -| Key | Default | Meaning | Type | Since | -|---------------------------------|----------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|-------| -| kyuubi.metrics.console.interval | PT5S | How often should report metrics to console | duration | 1.2.0 | -| kyuubi.metrics.enabled | true | Set to true to enable kyuubi metrics system | boolean | 1.2.0 | -| kyuubi.metrics.json.interval | PT5S | How often should report metrics to JSON file | duration | 1.2.0 | -| kyuubi.metrics.json.location | metrics | Where the JSON metrics file located | string | 1.2.0 | -| kyuubi.metrics.prometheus.path | /metrics | URI context path of prometheus metrics HTTP server | string | 1.2.0 | -| kyuubi.metrics.prometheus.port | 10019 | Prometheus metrics HTTP server port | int | 1.2.0 | -| kyuubi.metrics.reporters | JSON | A comma-separated list for all metrics reporters | set | 1.2.0 | -| kyuubi.metrics.slf4j.interval | PT5S | How often should report metrics to SLF4J logger | duration | 1.2.0 | +| Key | Default | Meaning | Type | Since | +|---------------------------------|------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|-------| +| kyuubi.metrics.console.interval | PT5S | How often should report metrics to console | duration | 1.2.0 | +| kyuubi.metrics.enabled | true | Set to true to enable kyuubi metrics system | boolean | 1.2.0 | +| kyuubi.metrics.json.interval | PT5S | How often should report metrics to JSON file | duration | 1.2.0 | +| kyuubi.metrics.json.location | metrics | Where the JSON metrics file located | string | 1.2.0 | +| kyuubi.metrics.prometheus.path | /metrics | URI context path of prometheus metrics HTTP server | string | 1.2.0 | +| kyuubi.metrics.prometheus.port | 10019 | Prometheus metrics HTTP server port | int | 1.2.0 | +| kyuubi.metrics.reporters | PROMETHEUS | A comma-separated list for all metrics reporters | set | 1.8.0 | +| kyuubi.metrics.slf4j.interval | PT5S | How often should report metrics to SLF4J logger | duration | 1.2.0 | ### Operation diff --git a/docs/deployment/migration-guide.md b/docs/deployment/migration-guide.md index 27dad2aba92..58df0fcc629 100644 --- a/docs/deployment/migration-guide.md +++ b/docs/deployment/migration-guide.md @@ -24,6 +24,9 @@ To restore previous behavior, set `kyuubi.metadata.store.jdbc.database.type=DERBY` and `kyuubi.metadata.store.jdbc.url=jdbc:derby:memory:kyuubi_state_store_db;create=true`. +* Since Kyuubi 1.8, PROMETHEUS is changed as the default metrics reporter. To restore previous behavior, + set `kyuubi.metrics.reporters=JSON`. + ## Upgrading from Kyuubi 1.7.1 to 1.7.2 * Since Kyuubi 1.7.2, for Kyuubi BeeLine, please use `--python-mode` option to run python code or script. diff --git a/kyuubi-metrics/src/main/scala/org/apache/kyuubi/metrics/MetricsConf.scala b/kyuubi-metrics/src/main/scala/org/apache/kyuubi/metrics/MetricsConf.scala index fe11f6eb18b..9bc2e63243a 100644 --- a/kyuubi-metrics/src/main/scala/org/apache/kyuubi/metrics/MetricsConf.scala +++ b/kyuubi-metrics/src/main/scala/org/apache/kyuubi/metrics/MetricsConf.scala @@ -46,7 +46,7 @@ object MetricsConf { .transformToUpperCase .toSet() .checkValues(ReporterType) - .createWithDefault(Set(JSON.toString)) + .createWithDefault(Set(PROMETHEUS.toString)) val METRICS_CONSOLE_INTERVAL: ConfigEntry[Long] = buildConf("kyuubi.metrics.console.interval") .doc("How often should report metrics to console") From f1a4d28617048180a1aa21ffc79fa59f7eead41a Mon Sep 17 00:00:00 2001 From: hezhao2 Date: Thu, 28 Sep 2023 21:44:20 +0800 Subject: [PATCH 2/6] restore version number for kyuubi.metrics.reporters in doc --- docs/configuration/settings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration/settings.md b/docs/configuration/settings.md index 6e890ba585e..2869a59cd92 100644 --- a/docs/configuration/settings.md +++ b/docs/configuration/settings.md @@ -360,7 +360,7 @@ You can configure the Kyuubi properties in `$KYUUBI_HOME/conf/kyuubi-defaults.co | kyuubi.metrics.json.location | metrics | Where the JSON metrics file located | string | 1.2.0 | | kyuubi.metrics.prometheus.path | /metrics | URI context path of prometheus metrics HTTP server | string | 1.2.0 | | kyuubi.metrics.prometheus.port | 10019 | Prometheus metrics HTTP server port | int | 1.2.0 | -| kyuubi.metrics.reporters | PROMETHEUS | A comma-separated list for all metrics reporters | set | 1.8.0 | +| kyuubi.metrics.reporters | PROMETHEUS | A comma-separated list for all metrics reporters | set | 1.2.0 | | kyuubi.metrics.slf4j.interval | PT5S | How often should report metrics to SLF4J logger | duration | 1.2.0 | ### Operation From a3605b62648cbc4558e3775edd040908e359e059 Mon Sep 17 00:00:00 2001 From: hezhao2 Date: Fri, 29 Sep 2023 11:38:46 +0800 Subject: [PATCH 3/6] set METRICS_PROMETHEUS_PORT to 0 for test cases --- .../events/handler/ServerJsonLoggingEventHandlerSuite.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/events/handler/ServerJsonLoggingEventHandlerSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/events/handler/ServerJsonLoggingEventHandlerSuite.scala index 2f794ed4819..5fae375de70 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/events/handler/ServerJsonLoggingEventHandlerSuite.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/events/handler/ServerJsonLoggingEventHandlerSuite.scala @@ -34,6 +34,7 @@ import org.apache.kyuubi._ import org.apache.kyuubi.client.util.BatchUtils._ import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.events.ServerEventHandlerRegister +import org.apache.kyuubi.metrics.MetricsConf.METRICS_PROMETHEUS_PORT import org.apache.kyuubi.operation.HiveJDBCTestHelper import org.apache.kyuubi.operation.OperationState._ import org.apache.kyuubi.server.KyuubiServer @@ -56,6 +57,7 @@ class ServerJsonLoggingEventHandlerSuite extends WithKyuubiServer with HiveJDBCT .set(KyuubiConf.SERVER_EVENT_JSON_LOG_PATH, serverLogRoot) .set(KyuubiConf.ENGINE_SPARK_EVENT_LOGGERS, Seq("JSON")) .set(KyuubiConf.ENGINE_EVENT_JSON_LOG_PATH, engineLogRoot) + .set(METRICS_PROMETHEUS_PORT, 0) } override protected def jdbcUrl: String = getJdbcUrl From 86165a6fef0849faaa10d6f0aea6e661457c3f6a Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Tue, 10 Oct 2023 21:47:41 +0800 Subject: [PATCH 4/6] Update kyuubi-server/src/test/scala/org/apache/kyuubi/events/handler/ServerJsonLoggingEventHandlerSuite.scala --- .../events/handler/ServerJsonLoggingEventHandlerSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/events/handler/ServerJsonLoggingEventHandlerSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/events/handler/ServerJsonLoggingEventHandlerSuite.scala index 5fae375de70..aae775c44b2 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/events/handler/ServerJsonLoggingEventHandlerSuite.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/events/handler/ServerJsonLoggingEventHandlerSuite.scala @@ -57,7 +57,7 @@ class ServerJsonLoggingEventHandlerSuite extends WithKyuubiServer with HiveJDBCT .set(KyuubiConf.SERVER_EVENT_JSON_LOG_PATH, serverLogRoot) .set(KyuubiConf.ENGINE_SPARK_EVENT_LOGGERS, Seq("JSON")) .set(KyuubiConf.ENGINE_EVENT_JSON_LOG_PATH, engineLogRoot) - .set(METRICS_PROMETHEUS_PORT, 0) + .set(KyuubiConf.METRICS_REPORTERS, Seq.empty) } override protected def jdbcUrl: String = getJdbcUrl From b9ee5f711694ae6155686cdc8f2ab7efca15bc10 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Tue, 10 Oct 2023 21:48:06 +0800 Subject: [PATCH 5/6] Update kyuubi-server/src/test/scala/org/apache/kyuubi/events/handler/ServerJsonLoggingEventHandlerSuite.scala --- .../events/handler/ServerJsonLoggingEventHandlerSuite.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/events/handler/ServerJsonLoggingEventHandlerSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/events/handler/ServerJsonLoggingEventHandlerSuite.scala index aae775c44b2..968b3fc17f8 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/events/handler/ServerJsonLoggingEventHandlerSuite.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/events/handler/ServerJsonLoggingEventHandlerSuite.scala @@ -34,7 +34,6 @@ import org.apache.kyuubi._ import org.apache.kyuubi.client.util.BatchUtils._ import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.events.ServerEventHandlerRegister -import org.apache.kyuubi.metrics.MetricsConf.METRICS_PROMETHEUS_PORT import org.apache.kyuubi.operation.HiveJDBCTestHelper import org.apache.kyuubi.operation.OperationState._ import org.apache.kyuubi.server.KyuubiServer From 84f4c8208204be1536412c96065edaa9dfe01e57 Mon Sep 17 00:00:00 2001 From: hezhao2 Date: Wed, 11 Oct 2023 13:59:37 +0800 Subject: [PATCH 6/6] reset METRICS_REPORTERS for test case --- .../events/handler/ServerJsonLoggingEventHandlerSuite.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/events/handler/ServerJsonLoggingEventHandlerSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/events/handler/ServerJsonLoggingEventHandlerSuite.scala index 968b3fc17f8..1dc24aeec94 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/events/handler/ServerJsonLoggingEventHandlerSuite.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/events/handler/ServerJsonLoggingEventHandlerSuite.scala @@ -34,6 +34,7 @@ import org.apache.kyuubi._ import org.apache.kyuubi.client.util.BatchUtils._ import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.events.ServerEventHandlerRegister +import org.apache.kyuubi.metrics.MetricsConf import org.apache.kyuubi.operation.HiveJDBCTestHelper import org.apache.kyuubi.operation.OperationState._ import org.apache.kyuubi.server.KyuubiServer @@ -56,7 +57,7 @@ class ServerJsonLoggingEventHandlerSuite extends WithKyuubiServer with HiveJDBCT .set(KyuubiConf.SERVER_EVENT_JSON_LOG_PATH, serverLogRoot) .set(KyuubiConf.ENGINE_SPARK_EVENT_LOGGERS, Seq("JSON")) .set(KyuubiConf.ENGINE_EVENT_JSON_LOG_PATH, engineLogRoot) - .set(KyuubiConf.METRICS_REPORTERS, Seq.empty) + .set(MetricsConf.METRICS_REPORTERS, Set.empty[String]) } override protected def jdbcUrl: String = getJdbcUrl