fix metrics

Signed-off-by: Ryan Leung <[email protected]>
tikv · Sep 21, 2023 · 074797e · 074797e
1 parent 6376783
commit 074797e
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 14 deletions.
diff --git a/metrics/grafana/pd.json b/metrics/grafana/pd.json
@@ -1663,7 +1663,7 @@
           "tableColumn": "idalloc",
           "targets": [
             {
-              "expr": "pd_cluster_id{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"idalloc\"}",
+              "expr": "max(pd_cluster_id{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"idalloc\"})",
               "format": "time_series",
               "hide": false,
               "instant": true,
@@ -2202,7 +2202,7 @@
           "tableColumn": "tso",
           "targets": [
             {
-              "expr": "pd_cluster_tso{type=\"tso\", dc=\"global\"}",
+              "expr": "max(pd_cluster_tso{type=\"tso\", dc=\"global\"})",
               "format": "time_series",
               "instant": true,
               "interval": "",
@@ -2508,7 +2508,7 @@
           "tableColumn": "tso",
           "targets": [
             {
-              "expr": "pd_cluster_tso{type=\"tso\", dc=\"global\"}",
+              "expr": "max(pd_cluster_tso{type=\"tso\", dc=\"global\"})",
               "format": "time_series",
               "instant": true,
               "interval": "",
@@ -7814,7 +7814,7 @@
           "tableColumn": "",
           "targets": [
             {
-              "expr": "pd_checker_patrol_regions_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}",
+              "expr": "pd_checker_patrol_regions_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"} != 0",
               "format": "time_series",
               "intervalFactor": 1,
               "refId": "A"
@@ -9105,7 +9105,9 @@
             "show": true,
             "sideWidth": 300,
             "total": false,
-            "values": true
+            "values": true,
+            "hideEmpty": true,
+            "hideZero": true
           },
           "lines": true,
           "linewidth": 1,
@@ -9200,7 +9202,9 @@
             "show": true,
             "sideWidth": 300,
             "total": false,
-            "values": true
+            "values": true,
+            "hideEmpty": true,
+            "hideZero": true
           },
           "lines": true,
           "linewidth": 1,
@@ -9526,7 +9530,9 @@
             "rightSide": true,
             "show": true,
             "total": false,
-            "values": true
+            "values": true,
+            "hideEmpty": true,
+            "hideZero": true
           },
           "lines": true,
           "linewidth": 1,
@@ -9723,7 +9729,9 @@
             "rightSide": true,
             "show": true,
             "total": false,
-            "values": true
+            "values": true,
+            "hideEmpty": true,
+            "hideZero": true
           },
           "lines": true,
           "linewidth": 1,

diff --git a/pkg/mcs/scheduling/server/cluster.go b/pkg/mcs/scheduling/server/cluster.go
@@ -7,7 +7,6 @@ import (
 	"time"
 
 	"github.com/pingcap/errors"
-	"github.com/pingcap/failpoint"
 	"github.com/pingcap/kvproto/pkg/pdpb"
 	"github.com/pingcap/kvproto/pkg/schedulingpb"
 	"github.com/pingcap/log"
@@ -400,11 +399,6 @@ func (c *Cluster) runMetricsCollectionJob() {
 	defer c.wg.Done()
 
 	ticker := time.NewTicker(10 * time.Second)
-	failpoint.Inject("highFrequencyClusterJobs", func() {
-		ticker.Stop()
-		ticker = time.NewTicker(time.Microsecond)
-	})
-
 	defer ticker.Stop()
 
 	for {

diff --git a/pkg/schedule/coordinator.go b/pkg/schedule/coordinator.go
@@ -156,6 +156,7 @@ func (c *Coordinator) PatrolRegions() {
 			// Note: we reset the ticker here to support updating configuration dynamically.
 			ticker.Reset(c.cluster.GetCheckerConfig().GetPatrolRegionInterval())
 		case <-c.ctx.Done():
+			patrolCheckRegionsGauge.Set(0)
 			log.Info("patrol regions has been stopped")
 			return
 		}