diff --git a/CHANGELOG.md b/CHANGELOG.md index a61ebcf7..e3bca8af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ * [CHANGE] Increased `CortexIngesterReachingSeriesLimit` critical alert threshold from 80% to 85%. #363 * [CHANGE] Decreased `-server.grpc-max-concurrent-streams` from 100k to 10k. #369 * [CHANGE] Decreased blocks storage ingesters graceful termination period from 80m to 20m. #369 +* [ENHANCEMENT] Writes dashboard: fix HA-tracker KV panels; add elections panel and ingester state panel. #371 * [ENHANCEMENT] cortex-mixin: Make `cluster_namespace_deployment:kube_pod_container_resource_requests_{cpu_cores,memory_bytes}:sum` backwards compatible with `kube-state-metrics` v2.0.0. #317 * [ENHANCEMENT] Cortex-mixin: Include `cortex-gw-internal` naming variation in default `gateway` job names. #328 * [ENHANCEMENT] Ruler dashboard: added object storage metrics. #354 diff --git a/cortex-mixin/dashboards/writes.libsonnet b/cortex-mixin/dashboards/writes.libsonnet index 999603be..bf383ab2 100644 --- a/cortex-mixin/dashboards/writes.libsonnet +++ b/cortex-mixin/dashboards/writes.libsonnet @@ -110,6 +110,16 @@ local utils = import 'mixin-utils/utils.libsonnet'; $.panel('Latency') + utils.latencyRecordingRulePanel('cortex_kv_request_duration_seconds', $.jobSelector($._config.job_names.distributor) + [utils.selector.eq('kv_name', 'distributor-hatracker')]) ) + .addPanel( + $.panel('Elected replica changes / min') + + $.queryPanel([ + 'max by(exported_cluster, user)(increase(cortex_ha_tracker_elected_replica_changes_total{%s}[1m])) >0' % $.jobMatcher($._config.job_names.distributor), + ], [ + '{{user}}/{{exported_cluster}}', + ]) + + $.stack + { + yaxes: $.yaxes('cpm'), + }, ) ) .addRow( @@ -138,8 +148,15 @@ local utils = import 'mixin-utils/utils.libsonnet'; ) .addPanel( $.panel('Latency') + - utils.latencyRecordingRulePanel('cortex_kv_request_duration_seconds', $.jobSelector($._config.job_names.ingester)+ [utils.selector.eq('kv_name', 'ingester-lifecycler')]) + utils.latencyRecordingRulePanel('cortex_kv_request_duration_seconds', $.jobSelector($._config.job_names.ingester) + [utils.selector.eq('kv_name', 'ingester-lifecycler')]) ) + .addPanel( + $.panel('Ingester status') + + $.queryPanel([ + 'max by (state)(cortex_ring_members{%s}) >0' % $.jobMatcher($._config.job_names.distributor), + ], [ + '{{state}}', + ]) ) ) .addRowIf(