Skip to content

Commit

Permalink
*: add pending task metrics in runner and add templ (#8138)
Browse files Browse the repository at this point in the history 
ref #7897

Signed-off-by: nolouch <[email protected]>
  • Loading branch information
@nolouch
nolouch committed May 10, 2024
1 parent a674e66 commit 78cb081
Show file tree
Hide file tree
Showing 4 changed files with 257 additions and 18 deletions.
257 changes: 243 additions & 14 deletions metrics/grafana/pd.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1218,7 +1218,6 @@
},
"yaxes": [
{
"$$hashKey": "object:192",
"format": "short",
"label": null,
"logBase": 1,
Expand All @@ -1227,7 +1226,6 @@
"show": true
},
{
"$$hashKey": "object:193",
"format": "short",
"label": null,
"logBase": 1,
Expand Down Expand Up @@ -11431,15 +11429,13 @@
"renderer": "flot",
"seriesOverrides": [
{
"$$hashKey": "object:1147",
"alias": "WaitRegionsLock",
"bars": false,
"lines": true,
"linewidth": 2,
"stack": false
},
{
"$$hashKey": "object:1251",
"alias": "WaitSubRegionsLock",
"bars": false,
"lines": true,
Expand Down Expand Up @@ -11486,14 +11482,12 @@
},
"yaxes": [
{
"$$hashKey": "object:322",
"format": "s",
"logBase": 1,
"min": "0",
"show": true
},
{
"$$hashKey": "object:323",
"format": "s",
"logBase": 1,
"show": true
Expand Down Expand Up @@ -11606,17 +11600,252 @@
"dashLength": 10,
"dashes": false,
"datasource": "${DS_TEST-CLUSTER}",
"description": "The count of the corresponding schedule commands which PD sends to each TiKV instance",
"description": "The count of the heartbeats which pending in the task queue.",
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"grid": {},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 39
},
"hiddenSeries": false,
"id": 1608,
"legend": {
"alignAsTable": true,
"avg": true,
"current": true,
"hideEmpty": true,
"hideZero": true,
"max": true,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null as zero",
"options": {
"alertThreshold": true
},
"paceLength": 10,
"percentage": false,
"pluginVersion": "7.5.17",
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"exemplar": true,
"expr": "pd_ratelimit_runner_task_pending_tasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}",
"format": "time_series",
"hide": false,
"interval": "",
"intervalFactor": 2,
"legendFormat": "{{task_type}}_({{runner_name}})",
"refId": "A",
"step": 4
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Heartbeat Runner Pending Task",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "opm",
"label": null,
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"format": "s",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "${DS_TEST-CLUSTER}",
"description": "The count of the heartbeats which faileds in the task queue.",
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"grid": {},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 47
},
"hiddenSeries": false,
"id": 1609,
"legend": {
"alignAsTable": true,
"avg": true,
"current": true,
"hideEmpty": true,
"hideZero": true,
"max": true,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null as zero",
"options": {
"alertThreshold": true
},
"paceLength": 10,
"percentage": false,
"pluginVersion": "7.5.17",
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"alias": "/max-wait-duration.*/",
"bars": true,
"lines": false,
"transform": "negative-Y",
"yaxis": 2
}
],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"exemplar": true,
"expr": "rate(pd_ratelimit_runner_task_failed_tasks_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])*60",
"format": "time_series",
"hide": false,
"interval": "",
"intervalFactor": 2,
"legendFormat": "failed-tasks-({{runner_name}})",
"refId": "A",
"step": 4
},
{
"exemplar": true,
"expr": "pd_ratelimit_runner_task_max_waiting_duration_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}",
"hide": false,
"interval": "",
"legendFormat": "max-wait-duration-({{runner_name}})",
"refId": "B"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Concurrent Runner Failed Task",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"decimals": null,
"format": "opm",
"label": "",
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"format": "s",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "${DS_TEST-CLUSTER}",
"description": "The count of the corresponding schedule commands which PD sends to each TiKV instance",
"editable": true,
"error": false,
"fill": 0,
"grid": {},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 47
},
"id": 1305,
"legend": {
"alignAsTable": true,
Expand Down Expand Up @@ -11709,7 +11938,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 47
"y": 55
},
"id": 1306,
"legend": {
Expand Down Expand Up @@ -11799,7 +12028,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 47
"y": 55
},
"id": 1307,
"legend": {
Expand Down Expand Up @@ -11892,7 +12121,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 55
"y": 63
},
"id": 1308,
"legend": {
Expand Down Expand Up @@ -11989,7 +12218,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 55
"y": 63
},
"id": 1309,
"legend": {
Expand Down Expand Up @@ -12086,7 +12315,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 63
"y": 71
},
"id": 1310,
"legend": {
Expand Down Expand Up @@ -12183,7 +12412,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 63
"y": 71
},
"id": 1311,
"legend": {
Expand Down Expand Up @@ -12280,7 +12509,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 71
"y": 79
},
"id": 1312,
"legend": {
Expand Down
7 changes: 5 additions & 2 deletions pkg/ratelimit/metrics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,10 @@ import (
"github.com/prometheus/client_golang/prometheus"
)

const nameStr = "runner_name"
const (
nameStr = "runner_name"
taskStr = "task_type"
)

var (
RunnerTaskMaxWaitingDuration = prometheus.NewGaugeVec(
Expand All @@ -35,7 +38,7 @@ var (
Subsystem: "ratelimit",
Name: "runner_task_pending_tasks",
Help: "The number of pending tasks in the runner.",
}, []string{nameStr})
}, []string{nameStr, taskStr})
RunnerTaskFailedTasks = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: "pd",
Expand Down
Loading

0 comments on commit 78cb081

Please sign in to comment.