From 841f318116cc447364b2ef78f1ec3314e7136045 Mon Sep 17 00:00:00 2001
From: Christopher Bartz <christopher.bartz@canonical.com>
Date: Wed, 28 Aug 2024 10:32:30 +0200
Subject: [PATCH] add max job queue duration panel (#355)

---
 src/grafana_dashboards/metrics.json | 128 ++++++++++++++++++++++++----
 1 file changed, 112 insertions(+), 16 deletions(-)

diff --git a/src/grafana_dashboards/metrics.json b/src/grafana_dashboards/metrics.json
index 6d1dd36cc..8582b77ef 100644
--- a/src/grafana_dashboards/metrics.json
+++ b/src/grafana_dashboards/metrics.json
@@ -24,7 +24,7 @@
   "editable": true,
   "fiscalYearStartMonth": 0,
   "graphTooltip": 0,
-  "id": 293,
+  "id": 537,
   "links": [],
   "liveNow": false,
   "panels": [
@@ -405,7 +405,7 @@
         "type": "loki",
         "uid": "${lokids}"
       },
-      "description": "This panel totals the number of idle runners reported by the charm units after a reconciliation event. Note that there may be active runners in the time between reconciliations which will not be shown in this panel.",
+      "description": "All aggregations are based on a 1-hour time period.",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -441,7 +441,6 @@
               "mode": "off"
             }
           },
-          "decimals": 0,
           "mappings": [],
           "thresholds": {
             "mode": "absolute",
@@ -456,7 +455,7 @@
               }
             ]
           },
-          "unit": "none"
+          "unit": "s"
         },
         "overrides": []
       },
@@ -466,13 +465,13 @@
         "x": 12,
         "y": 9
       },
-      "id": 7,
+      "id": 23,
       "options": {
         "legend": {
           "calcs": [],
           "displayMode": "list",
           "placement": "bottom",
-          "showLegend": false
+          "showLegend": true
         },
         "tooltip": {
           "mode": "single",
@@ -485,16 +484,16 @@
             "type": "loki",
             "uid": "${lokids}"
           },
-          "editorMode": "code",
-          "expr": "sum by(filename)(last_over_time({filename=\"/var/log/github-runner-metrics.log\", juju_application=~\"$juju_application\", juju_model=~\"$juju_model\", juju_model_uuid=~\"$juju_model_uuid\", juju_unit=~\"$juju_unit\"} | json event=\"event\",idle_runners=\"idle_runners\",flavor=\"flavor\" | event=\"reconciliation\" | flavor=~\"$flavor\" | unwrap idle_runners[60m]))",
+          "editorMode": "builder",
+          "expr": "max_over_time({filename=\"/var/log/github-runner-metrics.log\", juju_application=~\"$juju_application\", juju_model=~\"$juju_model\", juju_model_uuid=~\"$juju_model_uuid\", juju_unit=~\"$juju_unit\"} | json event=\"event\", duration=\"queue_duration\", flavor=\"flavor\" | __error__=`` | event = `runner_start` | flavor =~ `$flavor` | unwrap duration [1h]) by (flavor)",
           "hide": false,
-          "legendFormat": "Idle",
+          "key": "Q-9302bc4d-cce0-4674-bad5-353257fdd2f4-0",
+          "legendFormat": "{{flavor}}",
           "queryType": "range",
-          "refId": "D"
+          "refId": "E"
         }
       ],
-      "title": "Idle Runners after Reconciliation",
-      "transformations": [],
+      "title": "Max Job Queue Duration By Flavour",
       "type": "timeseries"
     },
     {
@@ -631,6 +630,103 @@
       "title": "Runner Installation Duration (Percentile)",
       "type": "timeseries"
     },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "${lokids}"
+      },
+      "description": "This panel totals the number of idle runners reported by the charm units after a reconciliation event. Note that there may be active runners in the time between reconciliations which will not be shown in this panel.",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "decimals": 0,
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 17
+      },
+      "id": 7,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": false
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "${lokids}"
+          },
+          "editorMode": "code",
+          "expr": "sum by(filename)(last_over_time({filename=\"/var/log/github-runner-metrics.log\", juju_application=~\"$juju_application\", juju_model=~\"$juju_model\", juju_model_uuid=~\"$juju_model_uuid\", juju_unit=~\"$juju_unit\"} | json event=\"event\",idle_runners=\"idle_runners\",flavor=\"flavor\" | event=\"reconciliation\" | flavor=~\"$flavor\" | unwrap idle_runners[60m]))",
+          "hide": false,
+          "legendFormat": "Idle",
+          "queryType": "range",
+          "refId": "D"
+        }
+      ],
+      "title": "Idle Runners after Reconciliation",
+      "transformations": [],
+      "type": "timeseries"
+    },
     {
       "datasource": {
         "type": "loki",
@@ -694,7 +790,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 17
+        "y": 25
       },
       "id": 4,
       "options": {
@@ -828,7 +924,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 25
+        "y": 33
       },
       "id": 6,
       "options": {
@@ -905,7 +1001,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 33
+        "y": 41
       },
       "id": 12,
       "panels": [
@@ -1828,6 +1924,6 @@
   "timepicker": {},
   "timezone": "",
   "title": "GitHub Self-Hosted Runner Metrics",
-  "version": 13,
+  "version": 14,
   "weekStart": ""
 }