Add HTTP route without any arguments to make the plugin usable in Kub…

…ernetes (#29) Why? In a Kubernetes context, Prometheus scrape configuration are given using annotations like this: metadata: annotations: prometheus.io/scrape: "true" prometheus.io/path: /_/metrics prometheus.io/port: "7512" spec: ... These annotations don't support the params configuration making the plugin unusable. How? Add an HTTP route that doesn't require HTTP query parameters to fetch Prometheus formatted metrics. Test it You should now be able to fetch Prometheus formatted metrics using two different way: GET "http://localhost:7512/_metrics?format=prometheus" and the new one: GET "http://localhost:7512/_/metrics" Other changes Update documentation and add a Kubernetes section Add an improvement on the Kuzzle demo Grafana dashboard
kuzzleio · Feb 10, 2022 · 3077982 · 3077982
1 parent 4bed636
commit 3077982
Show file tree

Hide file tree

Showing 10 changed files with 147 additions and 122 deletions.
diff --git a/README.md b/README.md
@@ -19,14 +19,15 @@
 - [About](#about)
   - [Kuzzle Prometheus Plugin](#kuzzle-prometheus-plugin)
   - [Kuzzle](#kuzzle)
-  - [Compatibility matrice](#compatibility-matrice)
+  - [Compatibility matrix](#compatibility-matrix)
 - [Installation](#installation)
 - [Configuration](#configuration)
   - [Plugin](#plugin)
   - [Prometheus](#prometheus)
     - [With only one Kuzzle node](#with-only-one-kuzzle-node)
     - [With an authentified user](#with-an-authentified-user)
     - [With multiple Kuzzle nodes and using Docker Compose](#with-multiple-kuzzle-nodes-and-using-docker-compose)
+    - [Using Kubernetes annotations](#using-kubernetes-annotations)
   - [Dashboards](#dashboards)
     - [Features](#features)
     - [Screenshots](#screenshots)
@@ -200,6 +201,21 @@ scrape_configs:
         - 'kuzzle-plugin-prometheus-kuzzle-3:7512'
 ```
 
+### Using Kubernetes annotations
+
+If your Prometheus inside a Kubernetes cluster, you must use the helper HTTP route `/_/metrics` since Prometheus `params` configuration is not supported.
+Your Pods annotations should look like this:
+
+```yaml
+metadata:
+  annotations:
+    prometheus.io/scrape: "true"
+    prometheus.io/path: /_/metrics
+    prometheus.io/port: "7512"
+spec:
+...
+```
+
 ## Dashboards
 
 ### Features

diff --git a/config/grafana/dashboards/demo.json b/config/grafana/dashboards/demo.json
@@ -22,7 +22,7 @@
   "editable": true,
   "fiscalYearStartMonth": 0,
   "graphTooltip": 0,
-  "iteration": 1642671164685,
+  "iteration": 1643633313636,
   "links": [],
   "liveNow": false,
   "panels": [
@@ -90,107 +90,11 @@
         "overrides": []
       },
       "gridPos": {
-        "h": 8,
-        "w": 8,
+        "h": 11,
+        "w": 12,
         "x": 0,
         "y": 0
       },
-      "id": 30,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "hidden",
-          "placement": "bottom"
-        },
-        "tooltip": {
-          "mode": "single"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "Prometheus"
-          },
-          "exemplar": true,
-          "expr": "rate(kuzzle_api_request_duration_ms_count{status=~\"5.+\",nodeId=~\"$nodeId\"}[$__rate_interval])",
-          "interval": "",
-          "legendFormat": "",
-          "refId": "A"
-        }
-      ],
-      "title": "Status code 500",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "continuous-RdYlGr"
-          },
-          "custom": {
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 5,
-            "gradientMode": "hue",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "lineInterpolation": "smooth",
-            "lineStyle": {
-              "dash": [
-                0,
-                3,
-                3
-              ],
-              "fill": "dot"
-            },
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "never",
-            "spanNulls": true,
-            "stacking": {
-              "group": "A",
-              "mode": "normal"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 8,
-        "x": 8,
-        "y": 0
-      },
       "id": 28,
       "options": {
         "legend": {
@@ -213,7 +117,7 @@
             "uid": "Prometheus"
           },
           "exemplar": true,
-          "expr": "rate(kuzzle_api_request_duration_ms_count{status=~\"5.+\",nodeId=~\"$nodeId\"}[$__rate_interval])",
+          "expr": "sum by(nodeId) (rate(kuzzle_api_request_duration_ms_count{status=~\"5.+\",nodeId=~\"$nodeId\"}[$__rate_interval]) * 50)",
           "interval": "",
           "legendFormat": "{{nodeId}}",
           "refId": "A"
@@ -286,9 +190,9 @@
         "overrides": []
       },
       "gridPos": {
-        "h": 8,
-        "w": 8,
-        "x": 16,
+        "h": 11,
+        "w": 12,
+        "x": 12,
         "y": 0
       },
       "id": 32,
@@ -313,7 +217,7 @@
             "uid": "Prometheus"
           },
           "exemplar": true,
-          "expr": "rate(kuzzle_api_request_duration_ms_count{status=~\"5.+\",nodeId=~\"$nodeId\"}[$__rate_interval])",
+          "expr": "sum by(action, controller) (rate(kuzzle_api_request_duration_ms_count{status=~\"5.+\",nodeId=~\"$nodeId\"}[$__rate_interval]) * 50)",
           "interval": "",
           "legendFormat": "{{controller}} - {{action}}",
           "refId": "A"
@@ -389,7 +293,7 @@
         "h": 12,
         "w": 12,
         "x": 0,
-        "y": 8
+        "y": 11
       },
       "id": 22,
       "options": {
@@ -413,7 +317,7 @@
             "uid": "Prometheus"
           },
           "exemplar": true,
-          "expr": "kuzzle_network_connections{nodeId=~\"$nodeId\"}",
+          "expr": "sum by(nodeId) (kuzzle_network_connections{nodeId=~\"$nodeId\"})",
           "interval": "",
           "legendFormat": "{{nodeId}}",
           "refId": "A"
@@ -491,7 +395,7 @@
         "h": 12,
         "w": 12,
         "x": 12,
-        "y": 8
+        "y": 11
       },
       "id": 18,
       "options": {
@@ -591,7 +495,7 @@
         "h": 8,
         "w": 8,
         "x": 0,
-        "y": 20
+        "y": 23
       },
       "id": 16,
       "options": {
@@ -691,7 +595,7 @@
         "h": 8,
         "w": 8,
         "x": 8,
-        "y": 20
+        "y": 23
       },
       "id": 24,
       "options": {
@@ -791,7 +695,7 @@
         "h": 8,
         "w": 8,
         "x": 16,
-        "y": 20
+        "y": 23
       },
       "id": 26,
       "options": {
@@ -815,7 +719,7 @@
             "uid": "Prometheus"
           },
           "exemplar": true,
-          "expr": "kuzzle_realtime_subscriptions{nodeId=~\"$nodeId\"}",
+          "expr": "sum by(nodeId) (kuzzle_realtime_subscriptions{nodeId=~\"$nodeId\"})",
           "interval": "",
           "legendFormat": "{{nodeId}}",
           "refId": "A"
@@ -893,7 +797,7 @@
         "h": 13,
         "w": 12,
         "x": 0,
-        "y": 28
+        "y": 31
       },
       "id": 10,
       "interval": "3s",
@@ -996,7 +900,7 @@
         "h": 13,
         "w": 12,
         "x": 12,
-        "y": 28
+        "y": 31
       },
       "hideTimeOverride": false,
       "id": 4,
@@ -1042,7 +946,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 41
+        "y": 44
       },
       "id": 14,
       "panels": [],
@@ -1117,7 +1021,7 @@
         "h": 13,
         "w": 12,
         "x": 0,
-        "y": 42
+        "y": 45
       },
       "id": 20,
       "options": {
@@ -1219,7 +1123,7 @@
         "h": 13,
         "w": 12,
         "x": 12,
-        "y": 42
+        "y": 45
       },
       "hideTimeOverride": false,
       "id": 2,
@@ -1272,7 +1176,7 @@
     "list": [
       {
         "current": {
-          "selected": true,
+          "selected": false,
           "text": [
             "All"
           ],

diff --git a/features/Metrics.feature b/features/Metrics.feature
@@ -1,4 +1,4 @@
-Feature: Prometheus metrics fetching
+Feature: Prometheus metrics fetching using server:metrics
   Scenario: Fetching Prometheus formatted metrics from server:metrics with the format parameter set to "prometheus"
     Given A running Kuzzle instance at "localhost:7512"
     When I send a HTTP request to "/_metrics?format=prometheus"

diff --git a/features/PrometheusMetrics.feature b/features/PrometheusMetrics.feature
@@ -0,0 +1,12 @@
+Feature: Prometheus metrics fetching using prometheus:metrics
+  Scenario: Fetching Prometheus formatted metrics from prometheus:metrics"
+    Given A running Kuzzle instance at "localhost:7512"
+    When I send a HTTP request to "/_/metrics"
+    Then The HTTP response should be a Prometheus formatted metrics containing:
+      | kuzzle_api_concurrent_requests    | 1 |
+      | process_start_time_seconds |   |
+
+  Scenario: Trying to fetch Prometheus formatted metrics from prometheus:metrics through WebSocket
+    Given A running Kuzzle instance at "localhost:7512"
+    When I send a WebSocket request to "prometheus":"metrics" with the format parameter set to "prometheus"
+    Then The WebSocket response should be a JSON object with "200" status code and a "" property
diff --git a/features/step_definitions/plugin.steps.ts b/features/step_definitions/plugin.steps.ts
@@ -51,7 +51,9 @@ export class PluginSteps {
   public async thenTheResponseShouldBeAJSONObjectWithAProperty(status: string, property: string) {
     const json = JSON.parse(this.result);
     assert(json.status === parseInt(status));
-    assert(_.get(json, property) !== undefined);
+    if (property !== null) {
+      assert(_.get(json, property) !== undefined);
+    }
   }
 
   @then(/The HTTP response should be a JSON object/)

diff --git a/lib/PrometheusPlugin.ts b/lib/PrometheusPlugin.ts
@@ -135,6 +135,17 @@ export class PrometheusPlugin extends Plugin {
       'request:onError': this.recordRequest.bind(this),
     };
 
+    this.api = {
+      prometheus: {
+        actions: {
+          metrics: {
+            handler: (request: KuzzleRequest) => this.metrics(request),
+            http: [{ verb: 'get', path: 'metrics' }],
+          },
+        },
+      }
+    }
+
     this.metricService = new MetricService(this.config);
   }
 
@@ -176,4 +187,29 @@ export class PrometheusPlugin extends Plugin {
       }
     );
   }
+
+  /**
+   * Return the metrics in Prometheus format
+   * NOTE: This is an HTTP route for Prometheus installations that do not support HTTP arguments
+   * @param {KuzzleRequest} request - Kuzzle request
+   * @returns {Promise<string>}
+   */
+  async metrics (request: KuzzleRequest): Promise<string> {
+    if (request.context.connection.protocol === 'http') {
+      const responsePayload = await this.context.accessors.sdk.query({
+        controller: 'server',
+        action: 'metrics',
+      });
+      this.metricService.updateCoreMetrics(responsePayload.result);
+
+      request.response.configure({
+        headers: {
+          'Content-Type': this.metricService.getPrometheusContentType()
+        },
+        format: 'raw',
+      });
+
+      return await this.metricService.getMetrics();
+    }
+  }
 }
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "kuzzle-plugin-prometheus",
-  "version": "4.0.0",
+  "version": "4.1.0",
   "description": "Kuzzle plugin: monitoring Kuzzle using Prometheus",
   "author": {
     "name": "The Kuzzle Team <[email protected]>"