From 7a59ff33c0e46dd30109a0618d46629eaa6b6279 Mon Sep 17 00:00:00 2001 From: v-zhuravlev Date: Thu, 15 Aug 2024 04:21:25 +0800 Subject: [PATCH] Kafka update related to JVM metrics and legend fix (#1306) * Update options related to kafka JVM metrics There are three options used prometheus(jmx_exporter+javaagent mode with ver>=1.0.1) prometheus_old(jmx_exporter+javaagent mode with ver<1.0.1) jmx_exporter(jmx_exporter+http mode or javaagent with the extra snippet in the config) * Fix BytesInPerSec metrics for prometheus * Fix topic dashboard(was not showing any consumergroups) * Improve aggLegend: show only single label Legend could have been too long when there multiple group/instance labels * Fix topicBytesInPerSec/topicBytesOutPerSec metrics * Move memoryUsedVirtual to sep panel Only resident memory is important enough. * Update zookeeper legends * Update legends in kafka-observ-lib --- common-lib/common/signal/README.md | 2 +- common-lib/common/signal/base.libsonnet | 4 ++-- jvm-observ-lib/README.md | 20 +++++++++++++---- kafka-mixin/mixin.libsonnet | 1 + kafka-observ-lib/README.md | 22 +++++++++++++++++-- kafka-observ-lib/config.libsonnet | 8 ++++++- kafka-observ-lib/dashboards.libsonnet | 8 ++++++- kafka-observ-lib/main.libsonnet | 2 +- kafka-observ-lib/signals/broker.libsonnet | 10 ++------- kafka-observ-lib/signals/cluster.libsonnet | 16 +++++--------- kafka-observ-lib/signals/topic.libsonnet | 4 ++-- process-observ-lib/panels.libsonnet | 5 ++++- .../signals/cluster.libsonnet | 6 ++--- 13 files changed, 71 insertions(+), 37 deletions(-) diff --git a/common-lib/common/signal/README.md b/common-lib/common/signal/README.md index cf663ee2b..1f0f1bb5a 100644 --- a/common-lib/common/signal/README.md +++ b/common-lib/common/signal/README.md @@ -78,7 +78,7 @@ The following is supported in expressions and legends: - `%(groupLabels)s` - expands to groupLabels list - `%(instanceLabels)s` - expands to instanceLabels list - `%(agg)s` - expands to list of labels according to `aggLevel` and `aggKeepLabels` choosen -- `%(aggLegend)s` - expands to list of labels in legend format (i.e. `{{}}/{{label2}}`) according to `aggLevel` and `aggKeepLabels` choosen +- `%(aggLegend)s` - expands to label in legend format (i.e. `{{}}`) according to `aggLevel` and `aggKeepLabels` choosen - `%(aggFunction)s` - expands to aggregation function - `%(interval)s` - expands to `interval` value - `%(alertsInterval)s` - expands to `interval` value diff --git a/common-lib/common/signal/base.libsonnet b/common-lib/common/signal/base.libsonnet index 7640b0d10..c13369c87 100644 --- a/common-lib/common/signal/base.libsonnet +++ b/common-lib/common/signal/base.libsonnet @@ -1,7 +1,7 @@ local g = import '../g.libsonnet'; local utils = import '../utils.libsonnet'; local signalUtils = import './utils.libsonnet'; - +local xtd = import 'github.com/jsonnet-libs/xtd/main.libsonnet'; { new( signalName, @@ -54,7 +54,7 @@ local signalUtils = import './utils.libsonnet'; if std.length(aggKeepLabels) > 0 then aggKeepLabels else [] ), - aggLegend: utils.labelsToPanelLegend(legendLabels), + aggLegend: utils.labelsToPanelLegend(xtd.array.slice(legendLabels, -1)), aggFunction: aggFunction, }, diff --git a/jvm-observ-lib/README.md b/jvm-observ-lib/README.md index 19db520ec..5db26d025 100644 --- a/jvm-observ-lib/README.md +++ b/jvm-observ-lib/README.md @@ -4,12 +4,24 @@ This lib can be used to generate dashboards, rows, panels, and alerts for JVM mo Supports the following sources: -- `prometheus` (https://prometheus.github.io/client_java/instrumentation/jvm/#jvm-memory-metrics) +- `prometheus` (https://prometheus.github.io/client_java/instrumentation/jvm/#jvm-memory-metrics). This also works for jmx_exporter (javaagent mode) starting from 1.0.1 release. - `otel` (https://github.com/open-telemetry/opentelemetry-java-contrib/blob/main/jmx-metrics/docs/target-systems/jvm.md) - `java_micrometer` (springboot) (https://github.com/micrometer-metrics/micrometer/blob/main/micrometer-core/src/main/java/io/micrometer/core/instrument/binder/jvm/JvmMemoryMetrics.java) -- `prometheus_old` client_java instrumentation prior to 1.0.0 release: (https://github.com/prometheus/client_java/releases/tag/v1.0.0-alpha-4) -- `jmx_exporter` https://github.com/prometheus/jmx_exporter/blob/main/collector/src/test/java/io/prometheus/jmx/JmxCollectorTest.java#L195 -` +- `prometheus_old` client_java instrumentation prior to 1.0.0 release: (https://github.com/prometheus/client_java/releases/tag/v1.0.0-alpha-4). This also works for jmx_exporter (javaagent mode) prior to 1.0.1 release. +- `jmx_exporter`. Works with jmx_exporter (both http and javaagent modes) and the folllowing snippet: + +```yaml +lowercaseOutputName: true +lowercaseOutputLabelNames: true +rules: + - pattern: java.lang<(.+)>(\w+) + name: java_lang_$1_$4_$3_$2 + - pattern: java.lang<>(\w+) + name: java_lang_$1_$3_$2 + - pattern : java.lang +``` + + ## Import ```sh diff --git a/kafka-mixin/mixin.libsonnet b/kafka-mixin/mixin.libsonnet index 22a4274c9..ecb270f7b 100644 --- a/kafka-mixin/mixin.libsonnet +++ b/kafka-mixin/mixin.libsonnet @@ -11,6 +11,7 @@ local kafka = instanceLabels: config.instanceLabels, dashboardTags: config.dashboardTags, metricsSource: 'grafanacloud', + jvmMetricsSource: 'prometheus_old', } ); diff --git a/kafka-observ-lib/README.md b/kafka-observ-lib/README.md index aeb0ff50b..eb237c231 100644 --- a/kafka-observ-lib/README.md +++ b/kafka-observ-lib/README.md @@ -2,10 +2,28 @@ This lib can be used to generate dashboards, rows, panels for Kafka signals. -The library supports two metricsSources: +The library supports two metrics sources (`metricsSource`): - `prometheus`: JMX configs from [prometheus/jmx_exporter](https://github.com/prometheus/jmx_exporter/blob/main/example_configs/kafka-2_0_0.yml) and [kafka_exporter](https://github.com/danielqsj/kafka_exporter) -- `grafanacloud`: JMX configs from [kafka-mixin](../kafka-mixin/jmx) and [kafka_exporter fork](https://github.com/grafana/kafka_exporter) (used in grafana-agent/alloy) +- `grafanacloud`: JMX configs from [kafka-mixin](../kafka-mixin/jmx) and [kafka_exporter fork](https://github.com/grafana/kafka_exporter) (used in grafana-agent/alloy). + +You can also pick on the three JVM metrics sources (`jvmMetricsSource`): + - `jmx_exporter` if you use jmx_exporter in http mode or javaagent mode with the additional config snippet (see README) + - `prometheus_old` if you use jmx_exporter in javaagent mode and version prior to 1.0.1 + - `prometheus` if you use jmx_exporter in javaagent mode and version 1.0.1 or newer + +If you pick `jmx_exporter` option, make sure you add the following snippet to your jmx_exporter config: + +```yaml +lowercaseOutputName: true +lowercaseOutputLabelNames: true +rules: + - pattern: java.lang<(.+)>(\w+) + name: java_lang_$1_$4_$3_$2 + - pattern: java.lang<>(\w+) + name: java_lang_$1_$3_$2 + - pattern : java.lang +``` ## Import diff --git a/kafka-observ-lib/config.libsonnet b/kafka-observ-lib/config.libsonnet index 5f5ae488a..76e73caa5 100644 --- a/kafka-observ-lib/config.libsonnet +++ b/kafka-observ-lib/config.libsonnet @@ -9,7 +9,13 @@ dashboardTags: ['kafka'], alertKafkaLagTooHighThreshold: '100', alertKafkaLagTooHighSeverity: 'critical', - metricsSource: 'prometheus', //or grafanacloud + metricsSource: 'prometheus', //or grafanacloud. See README + + // 'jmx_exporter' if you use jmx_exporter in http mode or javaagent mode with the additional config snippet (see README) + // 'prometheus_old' if you use jmx_exporter in javaagent mode and version prior to 1.0.1 + // 'prometheus' if you use jmx_exporter in javaagent mode and version 1.0.1 or newer + jvmMetricsSource: 'prometheus_old', + //Can be regex: topicsIgnoreSelector: '__consumer_offsets', zookeeperEnabled: true, diff --git a/kafka-observ-lib/dashboards.libsonnet b/kafka-observ-lib/dashboards.libsonnet index 678921e91..a0e160793 100644 --- a/kafka-observ-lib/dashboards.libsonnet +++ b/kafka-observ-lib/dashboards.libsonnet @@ -4,7 +4,13 @@ local g = import './g.libsonnet'; { 'kafka-topic-dashboard.json': g.dashboard.new(this.config.dashboardNamePrefix + 'Kafka topic overview') - + g.dashboard.withVariables(this.signals.topic.getVariablesMultiChoice()) + + g.dashboard.withVariables( + std.setUnion( + this.signals.topic.getVariablesMultiChoice(), + this.signals.consumerGroup.getVariablesMultiChoice(), + keyF=function(x) x.name + ) + ) + g.dashboard.withTags(this.config.dashboardTags) + g.dashboard.withUid(this.config.uid + '-kafka-topic-dashboard') + g.dashboard.withLinks(this.grafana.links.otherDashboards) diff --git a/kafka-observ-lib/main.libsonnet b/kafka-observ-lib/main.libsonnet index c748ed96d..cdd4ff7f7 100644 --- a/kafka-observ-lib/main.libsonnet +++ b/kafka-observ-lib/main.libsonnet @@ -17,7 +17,7 @@ local zookeeperlib = import 'zookeeper-observ-lib/main.libsonnet'; uid: this.config.uid, dashboardNamePrefix: this.config.dashboardNamePrefix, dashboardTags: this.config.dashboardTags, - metricsSource: 'jmx_exporter', + metricsSource: this.config.jvmMetricsSource, } ), zookeeper:: diff --git a/kafka-observ-lib/signals/broker.libsonnet b/kafka-observ-lib/signals/broker.libsonnet index d667b6cbe..041353484 100644 --- a/kafka-observ-lib/signals/broker.libsonnet +++ b/kafka-observ-lib/signals/broker.libsonnet @@ -19,11 +19,9 @@ function(this) unit: 'mps', sources: { prometheus: { - legendCustomTemplate: '{{ %s }}: messages in' % this.instanceLabels[0], expr: 'kafka_server_brokertopicmetrics_messagesin_total{%(queriesSelector)s}', }, grafanacloud: { - legendCustomTemplate: '{{ %s }}: messages in' % this.instanceLabels[0], expr: 'kafka_server_brokertopicmetrics_messagesinpersec{%(queriesSelector)s}', }, }, @@ -35,11 +33,9 @@ function(this) unit: 'Bps', sources: { prometheus: { - legendCustomTemplate: '{{ %s }}: bytes in' % this.instanceLabels[0], - expr: 'kafka_server_brokertopicmetrics_bytesinpersec_count{%(queriesSelector)s}', + expr: 'kafka_server_brokertopicmetrics_bytesin_total{%(queriesSelector)s}', }, grafanacloud: { - legendCustomTemplate: '{{ %s }}: bytes in' % this.instanceLabels[0], expr: 'kafka_server_brokertopicmetrics_bytesinpersec{%(queriesSelector)s}', }, }, @@ -51,11 +47,9 @@ function(this) unit: 'Bps', sources: { prometheus: { - legendCustomTemplate: '{{ %s }}: bytes out' % this.instanceLabels[0], - expr: 'kafka_server_brokertopicmetrics_bytesoutpersec_count{%(queriesSelector)s}', + expr: 'kafka_server_brokertopicmetrics_bytesout_total{%(queriesSelector)s}', }, grafanacloud: { - legendCustomTemplate: '{{ %s }}: bytes out' % this.instanceLabels[0], expr: 'kafka_server_brokertopicmetrics_bytesoutpersec{%(queriesSelector)s}', }, }, diff --git a/kafka-observ-lib/signals/cluster.libsonnet b/kafka-observ-lib/signals/cluster.libsonnet index f30a267a2..9f2fd3525 100644 --- a/kafka-observ-lib/signals/cluster.libsonnet +++ b/kafka-observ-lib/signals/cluster.libsonnet @@ -1,5 +1,5 @@ local commonlib = import 'common-lib/common/main.libsonnet'; - +local xtd = import 'github.com/jsonnet-libs/xtd/main.libsonnet'; function(this) { filteringSelector: this.filteringSelector, @@ -45,7 +45,7 @@ function(this) grafanacloud: { expr: 'kafka_controller_kafkacontroller_activecontrollercount{%(queriesSelector)s}', - legendCustomTemplate: '{{ %s }}' % this.instanceLabels[0], + legendCustomTemplate: '{{ %s }}' % xtd.array.slice(this.instanceLabels, -1), aggKeepLabels: this.instanceLabels, valueMappings: [{ type: 'value', @@ -67,7 +67,7 @@ function(this) { expr: 'kafka_controller_kafkacontroller_activecontrollercount{%(queriesSelector)s}', aggKeepLabels: this.instanceLabels, - legendCustomTemplate: '{{ %s }}' % this.instanceLabels[0], + legendCustomTemplate: '{{ %s }}' % xtd.array.slice(this.instanceLabels, -1), valueMappings: [ { type: 'value', @@ -115,11 +115,9 @@ function(this) unit: 'mps', sources: { prometheus: { - legendCustomTemplate: '%s: messages in' % commonlib.utils.labelsToPanelLegend(this.groupLabels), expr: 'kafka_server_brokertopicmetrics_messagesin_total{%(queriesSelector)s}', }, grafanacloud: { - legendCustomTemplate: '%s: messages in' % commonlib.utils.labelsToPanelLegend(this.groupLabels), expr: 'kafka_server_brokertopicmetrics_messagesinpersec{%(queriesSelector)s}', }, }, @@ -131,11 +129,9 @@ function(this) unit: 'Bps', sources: { prometheus: { - legendCustomTemplate: '%s: bytes in' % commonlib.utils.labelsToPanelLegend(this.groupLabels), - expr: 'kafka_server_brokertopicmetrics_bytesinpersec_count{%(queriesSelector)s}', + expr: 'kafka_server_brokertopicmetrics_bytesin_total{%(queriesSelector)s}', }, grafanacloud: { - legendCustomTemplate: '%s: bytes in' % commonlib.utils.labelsToPanelLegend(this.groupLabels), expr: 'kafka_server_brokertopicmetrics_bytesinpersec{%(queriesSelector)s}', }, }, @@ -147,11 +143,9 @@ function(this) unit: 'Bps', sources: { prometheus: { - legendCustomTemplate: '%s: bytes out' % commonlib.utils.labelsToPanelLegend(this.groupLabels), - expr: 'kafka_server_brokertopicmetrics_bytesoutpersec_count{%(queriesSelector)s}', + expr: 'kafka_server_brokertopicmetrics_bytesout_total{%(queriesSelector)s}', }, grafanacloud: { - legendCustomTemplate: '%s: bytes out' % commonlib.utils.labelsToPanelLegend(this.groupLabels), expr: 'kafka_server_brokertopicmetrics_bytesoutpersec{%(queriesSelector)s}', }, }, diff --git a/kafka-observ-lib/signals/topic.libsonnet b/kafka-observ-lib/signals/topic.libsonnet index 915101ec7..300121716 100644 --- a/kafka-observ-lib/signals/topic.libsonnet +++ b/kafka-observ-lib/signals/topic.libsonnet @@ -50,7 +50,7 @@ function(this) sources: { prometheus: { aggKeepLabels: ['topic'], - expr: 'kafka_server_brokertopicmetrics_bytesinpersec_count{%(queriesSelector)s}', + expr: 'kafka_server_brokertopicmetrics_bytesin_total{%(queriesSelector)s}', }, grafanacloud: { aggKeepLabels: ['topic'], @@ -66,7 +66,7 @@ function(this) sources: { prometheus: { aggKeepLabels: ['topic'], - expr: 'kafka_server_brokertopicmetrics_bytesoutpersec_count{%(queriesSelector)s}', + expr: 'kafka_server_brokertopicmetrics_bytesout_total{%(queriesSelector)s}', }, grafanacloud: { aggKeepLabels: ['topic'], diff --git a/process-observ-lib/panels.libsonnet b/process-observ-lib/panels.libsonnet index 9e9cd3217..0ec9d3f0f 100644 --- a/process-observ-lib/panels.libsonnet +++ b/process-observ-lib/panels.libsonnet @@ -22,7 +22,10 @@ local commonlib = import 'common-lib/common/main.libsonnet'; memoryUsage: signals.process.memoryUsedResident.asTimeSeries() - + signals.process.memoryUsedVirtual.asPanelMixin() + + commonlib.panels.memory.timeSeries.usageBytes.stylize(), + + memoryUsageVirtual: + signals.process.memoryUsedVirtual.asTimeSeries() + commonlib.panels.memory.timeSeries.usageBytes.stylize(), filesUsed: diff --git a/zookeeper-observ-lib/signals/cluster.libsonnet b/zookeeper-observ-lib/signals/cluster.libsonnet index ab4f4b040..5833aac30 100644 --- a/zookeeper-observ-lib/signals/cluster.libsonnet +++ b/zookeeper-observ-lib/signals/cluster.libsonnet @@ -1,5 +1,5 @@ local commonlib = import 'common-lib/common/main.libsonnet'; - +local xtd = import 'github.com/jsonnet-libs/xtd/main.libsonnet'; function(this) { filteringSelector: this.filteringSelector, @@ -25,7 +25,7 @@ function(this) grafanacloud: { expr: 'clamp_max(zookeeper_leaderuptime{%(queriesSelector)s}, 1) or clamp_max(zookeeper_numaliveconnections{%(queriesSelector)s}, 0)', - legendCustomTemplate: '{{ %s }}' % this.instanceLabels[0], + legendCustomTemplate: '{{ %s }}' % xtd.array.slice(this.instanceLabels, -1), aggKeepLabels: this.instanceLabels, valueMappings: [ { @@ -49,7 +49,7 @@ function(this) { expr: 'clamp_max(leader_uptime{%(queriesSelector)s}, 1) or clamp_max(num_alive_connections{%(queriesSelector)s}, 0)', aggKeepLabels: this.instanceLabels, - legendCustomTemplate: '{{ %s }}' % this.instanceLabels[0], + legendCustomTemplate: '{{ %s }}' % xtd.array.slice(this.instanceLabels, -1), valueMappings: [ { type: 'value',