diff --git a/examples/dashboards/receive-controller.json b/examples/dashboards/receive-controller.json index 7d827221..61f72555 100644 --- a/examples/dashboards/receive-controller.json +++ b/examples/dashboards/receive-controller.json @@ -47,7 +47,6 @@ { "expr": "sum by (namespace, job) (rate(thanos_receive_controller_reconcile_attempts_total{namespace=\"$namespace\", job=\"$job\"}[$interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "rate", "legendLink": null } @@ -122,7 +121,6 @@ { "expr": "sum by (namespace, job, type) (rate(thanos_receive_controller_reconcile_errors_total{namespace=\"$namespace\", job=\"$job\"}[$interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{type}}", "legendLink": null } @@ -209,7 +207,6 @@ { "expr": "sum by (namespace, job) (rate(thanos_receive_controller_configmap_change_attempts_total{namespace=\"$namespace\", job=\"$job\"}[$interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "rate", "legendLink": null } @@ -284,7 +281,6 @@ { "expr": "sum by (namespace, job, type) (rate(thanos_receive_controller_configmap_change_errors_total{namespace=\"$namespace\", job=\"$job\"}[$interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{type}}", "legendLink": null } @@ -371,7 +367,6 @@ { "expr": "sum by (namespace, job) (rate(thanos_receive_hashrings_file_changes_total{job=~\"thanos-receive-default.*\"}[$interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "all", "legendLink": null } @@ -536,14 +531,12 @@ { "expr": "avg by (namespace, job, name) (thanos_receive_controller_hashring_nodes{namespace=\"$namespace\", job=\"$job\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "receive controller {{name}}", "legendLink": null }, { "expr": "avg by (namespace, job, name) (thanos_receive_hashring_nodes{job=~\"thanos-receive-default.*\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "receive {{name}}", "legendLink": null } @@ -618,14 +611,12 @@ { "expr": "avg by (namespace, job, name) (thanos_receive_controller_hashring_tenants{namespace=\"$namespace\", job=\"$job\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "receive controller {{name}}", "legendLink": null }, { "expr": "avg by (namespace, job, name) (thanos_receive_hashring_tenants{job=~\"thanos-receive-default.*\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "receive {{name}}", "legendLink": null } @@ -716,7 +707,6 @@ "expr": "time() - max by (namespace, job) (thanos_receive_controller_configmap_last_reload_success_timestamp_seconds{namespace=\"$namespace\", job=\"$job\"})", "format": "time_series", "instant": true, - "intervalFactor": 2, "refId": "A" } ], @@ -794,7 +784,6 @@ "expr": "time() - max by (namespace, job) (thanos_receive_config_last_reload_success_timestamp_seconds{namespace=\"$namespace\", job=\"$job\"})", "format": "time_series", "instant": true, - "intervalFactor": 2, "refId": "A" } ], @@ -857,7 +846,7 @@ "value": "default" }, "hide": 0, - "label": "Data Source", + "label": "Data source", "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/main.go b/main.go index 79500e89..835dc94a 100644 --- a/main.go +++ b/main.go @@ -551,6 +551,8 @@ func (c *controller) sync(ctx context.Context) { statefulsets := make(map[string]*appsv1.StatefulSet) + var matchingStatefulSet bool + for _, obj := range c.ssetInf.GetStore().List() { sts, ok := obj.(*appsv1.StatefulSet) if !ok { @@ -560,6 +562,8 @@ func (c *controller) sync(ctx context.Context) { hashring, ok := sts.Labels[hashringLabelKey] if !ok { continue + } else { + matchingStatefulSet = true } // If there's an increase in replicas we poll for the new replicas to be ready @@ -584,6 +588,10 @@ func (c *controller) sync(ctx context.Context) { time.Sleep(c.options.scaleTimeout) // Give some time for all replicas before they receive hundreds req/s } + if !matchingStatefulSet { + level.Warn(c.logger).Log("msg", "could not find a statefulset with the label key "+hashringLabelKey) + } + c.populate(ctx, hashrings, statefulsets) err = c.saveHashring(ctx, hashrings, cm)