diff --git a/metrics/grafana/pd.json b/metrics/grafana/pd.json index abfe049b905..ced79c5b9e0 100644 --- a/metrics/grafana/pd.json +++ b/metrics/grafana/pd.json @@ -1655,12 +1655,20 @@ ], "targets": [ { - "expr": "pd_cluster_placement_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}", + "expr": "sum(pd_cluster_placement_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}) by (name)", "format": "time_series", "hide": false, "intervalFactor": 2, "legendFormat": "{{name}}", "refId": "A" + }, + { + "expr": "pd_cluster_placement_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}", + "format": "time_series", + "hide": true, + "intervalFactor": 2, + "legendFormat": "{{name}}--{{store}}", + "refId": "B" } ], "timeFrom": "1s", diff --git a/pkg/statistics/metrics.go b/pkg/statistics/metrics.go index a5ea07f4f55..68cbf142479 100644 --- a/pkg/statistics/metrics.go +++ b/pkg/statistics/metrics.go @@ -55,7 +55,7 @@ var ( Subsystem: "cluster", Name: "placement_status", Help: "Status of the cluster placement.", - }, []string{"type", "name"}) + }, []string{"type", "name", "store"}) configStatusGauge = prometheus.NewGaugeVec( prometheus.GaugeOpts{ diff --git a/pkg/statistics/store_collection.go b/pkg/statistics/store_collection.go index 4f76ffb0b5f..6d5df0bda62 100644 --- a/pkg/statistics/store_collection.go +++ b/pkg/statistics/store_collection.go @@ -47,7 +47,7 @@ type storeStatistics struct { LeaderCount int LearnerCount int WitnessCount int - LabelCounter map[string]int + LabelCounter map[string][]uint64 Preparing int Serving int Removing int @@ -57,7 +57,7 @@ type storeStatistics struct { func newStoreStatistics(opt config.ConfProvider) *storeStatistics { return &storeStatistics{ opt: opt, - LabelCounter: make(map[string]int), + LabelCounter: make(map[string][]uint64), } } @@ -70,7 +70,7 @@ func (s *storeStatistics) Observe(store *core.StoreInfo) { key := fmt.Sprintf("%s:%s", k, v) // exclude tombstone if !store.IsRemoved() { - s.LabelCounter[key]++ + s.LabelCounter[key] = append(s.LabelCounter[key], store.GetID()) } } storeAddress := store.GetAddress() @@ -249,8 +249,10 @@ func (s *storeStatistics) Collect() { configStatusGauge.WithLabelValues(typ).Set(value) } - for name, value := range s.LabelCounter { - placementStatusGauge.WithLabelValues(labelType, name).Set(float64(value)) + for name, stores := range s.LabelCounter { + for _, storeID := range stores { + placementStatusGauge.WithLabelValues(labelType, name, strconv.FormatUint(storeID, 10)).Set(1) + } } for storeID, limit := range s.opt.GetStoresLimit() { diff --git a/pkg/statistics/store_collection_test.go b/pkg/statistics/store_collection_test.go index 64a02a54bb4..e9fd1bba1fb 100644 --- a/pkg/statistics/store_collection_test.go +++ b/pkg/statistics/store_collection_test.go @@ -85,12 +85,14 @@ func TestStoreStatistics(t *testing.T) { re.Equal(0, stats.Disconnect) re.Equal(1, stats.Tombstone) re.Equal(1, stats.LowSpace) - re.Equal(2, stats.LabelCounter["zone:z1"]) - re.Equal(2, stats.LabelCounter["zone:z2"]) - re.Equal(2, stats.LabelCounter["zone:z3"]) - re.Equal(4, stats.LabelCounter["host:h1"]) - re.Equal(4, stats.LabelCounter["host:h2"]) - re.Equal(2, stats.LabelCounter["zone:unknown"]) + re.Len(stats.LabelCounter["zone:z1"], 2) + re.Equal([]uint64{1, 2}, stats.LabelCounter["zone:z1"]) + re.Len(stats.LabelCounter["zone:z2"], 2) + re.Len(stats.LabelCounter["zone:z3"], 2) + re.Len(stats.LabelCounter["host:h1"], 4) + re.Equal([]uint64{1, 3, 5, 7}, stats.LabelCounter["host:h1"]) + re.Len(stats.LabelCounter["host:h2"], 4) + re.Len(stats.LabelCounter["zone:unknown"], 2) } func TestSummaryStoreInfos(t *testing.T) {