From 3077982d3b3036a5ec7e484f36ea8aa3e6b10e17 Mon Sep 17 00:00:00 2001 From: Alexandre Bouthinon Date: Thu, 10 Feb 2022 15:49:08 +0100 Subject: [PATCH] Add HTTP route without any arguments to make the plugin usable in Kubernetes (#29) Why? In a Kubernetes context, Prometheus scrape configuration are given using annotations like this: metadata: annotations: prometheus.io/scrape: "true" prometheus.io/path: /_/metrics prometheus.io/port: "7512" spec: ... These annotations don't support the params configuration making the plugin unusable. How? Add an HTTP route that doesn't require HTTP query parameters to fetch Prometheus formatted metrics. Test it You should now be able to fetch Prometheus formatted metrics using two different way: GET "http://localhost:7512/_metrics?format=prometheus" and the new one: GET "http://localhost:7512/_/metrics" Other changes Update documentation and add a Kubernetes section Add an improvement on the Kuzzle demo Grafana dashboard --- README.md | 18 ++- config/grafana/dashboards/demo.json | 138 ++++------------------ features/Metrics.feature | 2 +- features/PrometheusMetrics.feature | 12 ++ features/step_definitions/plugin.steps.ts | 4 +- lib/PrometheusPlugin.ts | 36 ++++++ package-lock.json | 2 +- package.json | 2 +- test/PrometheusPlugin.spec.ts | 50 ++++++++ test/mocks/context.mock.ts | 5 + 10 files changed, 147 insertions(+), 122 deletions(-) create mode 100644 features/PrometheusMetrics.feature diff --git a/README.md b/README.md index 59798b9..de5490f 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ - [About](#about) - [Kuzzle Prometheus Plugin](#kuzzle-prometheus-plugin) - [Kuzzle](#kuzzle) - - [Compatibility matrice](#compatibility-matrice) + - [Compatibility matrix](#compatibility-matrix) - [Installation](#installation) - [Configuration](#configuration) - [Plugin](#plugin) @@ -27,6 +27,7 @@ - [With only one Kuzzle node](#with-only-one-kuzzle-node) - [With an authentified user](#with-an-authentified-user) - [With multiple Kuzzle nodes and using Docker Compose](#with-multiple-kuzzle-nodes-and-using-docker-compose) + - [Using Kubernetes annotations](#using-kubernetes-annotations) - [Dashboards](#dashboards) - [Features](#features) - [Screenshots](#screenshots) @@ -200,6 +201,21 @@ scrape_configs: - 'kuzzle-plugin-prometheus-kuzzle-3:7512' ``` +### Using Kubernetes annotations + +If your Prometheus inside a Kubernetes cluster, you must use the helper HTTP route `/_/metrics` since Prometheus `params` configuration is not supported. +Your Pods annotations should look like this: + +```yaml +metadata: + annotations: + prometheus.io/scrape: "true" + prometheus.io/path: /_/metrics + prometheus.io/port: "7512" +spec: +... +``` + ## Dashboards ### Features diff --git a/config/grafana/dashboards/demo.json b/config/grafana/dashboards/demo.json index 3b989a9..35703f2 100644 --- a/config/grafana/dashboards/demo.json +++ b/config/grafana/dashboards/demo.json @@ -22,7 +22,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "iteration": 1642671164685, + "iteration": 1643633313636, "links": [], "liveNow": false, "panels": [ @@ -90,107 +90,11 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 8, + "h": 11, + "w": 12, "x": 0, "y": 0 }, - "id": 30, - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "exemplar": true, - "expr": "rate(kuzzle_api_request_duration_ms_count{status=~\"5.+\",nodeId=~\"$nodeId\"}[$__rate_interval])", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Status code 500", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-RdYlGr" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 5, - "gradientMode": "hue", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineStyle": { - "dash": [ - 0, - 3, - 3 - ], - "fill": "dot" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 0 - }, "id": 28, "options": { "legend": { @@ -213,7 +117,7 @@ "uid": "Prometheus" }, "exemplar": true, - "expr": "rate(kuzzle_api_request_duration_ms_count{status=~\"5.+\",nodeId=~\"$nodeId\"}[$__rate_interval])", + "expr": "sum by(nodeId) (rate(kuzzle_api_request_duration_ms_count{status=~\"5.+\",nodeId=~\"$nodeId\"}[$__rate_interval]) * 50)", "interval": "", "legendFormat": "{{nodeId}}", "refId": "A" @@ -286,9 +190,9 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 8, - "x": 16, + "h": 11, + "w": 12, + "x": 12, "y": 0 }, "id": 32, @@ -313,7 +217,7 @@ "uid": "Prometheus" }, "exemplar": true, - "expr": "rate(kuzzle_api_request_duration_ms_count{status=~\"5.+\",nodeId=~\"$nodeId\"}[$__rate_interval])", + "expr": "sum by(action, controller) (rate(kuzzle_api_request_duration_ms_count{status=~\"5.+\",nodeId=~\"$nodeId\"}[$__rate_interval]) * 50)", "interval": "", "legendFormat": "{{controller}} - {{action}}", "refId": "A" @@ -389,7 +293,7 @@ "h": 12, "w": 12, "x": 0, - "y": 8 + "y": 11 }, "id": 22, "options": { @@ -413,7 +317,7 @@ "uid": "Prometheus" }, "exemplar": true, - "expr": "kuzzle_network_connections{nodeId=~\"$nodeId\"}", + "expr": "sum by(nodeId) (kuzzle_network_connections{nodeId=~\"$nodeId\"})", "interval": "", "legendFormat": "{{nodeId}}", "refId": "A" @@ -491,7 +395,7 @@ "h": 12, "w": 12, "x": 12, - "y": 8 + "y": 11 }, "id": 18, "options": { @@ -591,7 +495,7 @@ "h": 8, "w": 8, "x": 0, - "y": 20 + "y": 23 }, "id": 16, "options": { @@ -691,7 +595,7 @@ "h": 8, "w": 8, "x": 8, - "y": 20 + "y": 23 }, "id": 24, "options": { @@ -791,7 +695,7 @@ "h": 8, "w": 8, "x": 16, - "y": 20 + "y": 23 }, "id": 26, "options": { @@ -815,7 +719,7 @@ "uid": "Prometheus" }, "exemplar": true, - "expr": "kuzzle_realtime_subscriptions{nodeId=~\"$nodeId\"}", + "expr": "sum by(nodeId) (kuzzle_realtime_subscriptions{nodeId=~\"$nodeId\"})", "interval": "", "legendFormat": "{{nodeId}}", "refId": "A" @@ -893,7 +797,7 @@ "h": 13, "w": 12, "x": 0, - "y": 28 + "y": 31 }, "id": 10, "interval": "3s", @@ -996,7 +900,7 @@ "h": 13, "w": 12, "x": 12, - "y": 28 + "y": 31 }, "hideTimeOverride": false, "id": 4, @@ -1042,7 +946,7 @@ "h": 1, "w": 24, "x": 0, - "y": 41 + "y": 44 }, "id": 14, "panels": [], @@ -1117,7 +1021,7 @@ "h": 13, "w": 12, "x": 0, - "y": 42 + "y": 45 }, "id": 20, "options": { @@ -1219,7 +1123,7 @@ "h": 13, "w": 12, "x": 12, - "y": 42 + "y": 45 }, "hideTimeOverride": false, "id": 2, @@ -1272,7 +1176,7 @@ "list": [ { "current": { - "selected": true, + "selected": false, "text": [ "All" ], diff --git a/features/Metrics.feature b/features/Metrics.feature index 6e935bd..bba0fa1 100644 --- a/features/Metrics.feature +++ b/features/Metrics.feature @@ -1,4 +1,4 @@ -Feature: Prometheus metrics fetching +Feature: Prometheus metrics fetching using server:metrics Scenario: Fetching Prometheus formatted metrics from server:metrics with the format parameter set to "prometheus" Given A running Kuzzle instance at "localhost:7512" When I send a HTTP request to "/_metrics?format=prometheus" diff --git a/features/PrometheusMetrics.feature b/features/PrometheusMetrics.feature new file mode 100644 index 0000000..7da7c75 --- /dev/null +++ b/features/PrometheusMetrics.feature @@ -0,0 +1,12 @@ +Feature: Prometheus metrics fetching using prometheus:metrics + Scenario: Fetching Prometheus formatted metrics from prometheus:metrics" + Given A running Kuzzle instance at "localhost:7512" + When I send a HTTP request to "/_/metrics" + Then The HTTP response should be a Prometheus formatted metrics containing: + | kuzzle_api_concurrent_requests | 1 | + | process_start_time_seconds | | + + Scenario: Trying to fetch Prometheus formatted metrics from prometheus:metrics through WebSocket + Given A running Kuzzle instance at "localhost:7512" + When I send a WebSocket request to "prometheus":"metrics" with the format parameter set to "prometheus" + Then The WebSocket response should be a JSON object with "200" status code and a "" property diff --git a/features/step_definitions/plugin.steps.ts b/features/step_definitions/plugin.steps.ts index f7ec264..a8f9948 100644 --- a/features/step_definitions/plugin.steps.ts +++ b/features/step_definitions/plugin.steps.ts @@ -51,7 +51,9 @@ export class PluginSteps { public async thenTheResponseShouldBeAJSONObjectWithAProperty(status: string, property: string) { const json = JSON.parse(this.result); assert(json.status === parseInt(status)); - assert(_.get(json, property) !== undefined); + if (property !== null) { + assert(_.get(json, property) !== undefined); + } } @then(/The HTTP response should be a JSON object/) diff --git a/lib/PrometheusPlugin.ts b/lib/PrometheusPlugin.ts index 45e6b97..fd3d130 100644 --- a/lib/PrometheusPlugin.ts +++ b/lib/PrometheusPlugin.ts @@ -135,6 +135,17 @@ export class PrometheusPlugin extends Plugin { 'request:onError': this.recordRequest.bind(this), }; + this.api = { + prometheus: { + actions: { + metrics: { + handler: (request: KuzzleRequest) => this.metrics(request), + http: [{ verb: 'get', path: 'metrics' }], + }, + }, + } + } + this.metricService = new MetricService(this.config); } @@ -176,4 +187,29 @@ export class PrometheusPlugin extends Plugin { } ); } + + /** + * Return the metrics in Prometheus format + * NOTE: This is an HTTP route for Prometheus installations that do not support HTTP arguments + * @param {KuzzleRequest} request - Kuzzle request + * @returns {Promise} + */ + async metrics (request: KuzzleRequest): Promise { + if (request.context.connection.protocol === 'http') { + const responsePayload = await this.context.accessors.sdk.query({ + controller: 'server', + action: 'metrics', + }); + this.metricService.updateCoreMetrics(responsePayload.result); + + request.response.configure({ + headers: { + 'Content-Type': this.metricService.getPrometheusContentType() + }, + format: 'raw', + }); + + return await this.metricService.getMetrics(); + } + } } diff --git a/package-lock.json b/package-lock.json index 8f1aa76..1c24992 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "kuzzle-plugin-prometheus", - "version": "4.0.0", + "version": "4.1.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 9b674b7..f1f35d2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "kuzzle-plugin-prometheus", - "version": "4.0.0", + "version": "4.1.0", "description": "Kuzzle plugin: monitoring Kuzzle using Prometheus", "author": { "name": "The Kuzzle Team " diff --git a/test/PrometheusPlugin.spec.ts b/test/PrometheusPlugin.spec.ts index 5fd6a10..d5c425f 100644 --- a/test/PrometheusPlugin.spec.ts +++ b/test/PrometheusPlugin.spec.ts @@ -149,4 +149,54 @@ describe('PrometheusPlugin', () => { }); }); }); + + describe('#metrics', () => { + it('should format the metrics and send them to the client as Prometheus format', async () => { + // We need to override global Kuzzle getter to manipulate the request response + Reflect.defineProperty(global, 'kuzzle', { + get () { + return { + id: 'kuzzle', + }; + }, + }); + + plugin.init(undefined, context); + const request = new KuzzleRequest({ + controller: 'prometheus', + action: 'metrics', + }, {protocol: 'http'}); + + const getMetricsStub = sandbox.stub(plugin.metricService, 'getMetrics').returns('fake metrics'); + const getPrometheusContentTypeStub = sandbox.stub(plugin.metricService, 'getPrometheusContentType').returns('text/plain'); + const updateCoreMetricsStub = sandbox.stub(plugin.metricService, 'updateCoreMetrics').returns(); + plugin.context.accessors.sdk.query.returns({ result: 'fake metrics' }); + + const formattedRequest = await plugin.metrics(request); + + expect(plugin.context.accessors.sdk.query.calledOnce).to.be.true; + expect(getMetricsStub.calledOnce).to.be.true; + expect(getPrometheusContentTypeStub.calledOnce).to.be.true; + expect(updateCoreMetricsStub.calledOnce).to.be.true; + expect(formattedRequest).to.contains('fake metrics'); + }); + + it('should not format the metrics if protocol is not HTTP', async () => { + plugin.init(undefined, context); + const request = new KuzzleRequest({ + controller: 'prometheus', + action: 'metrics', + }, {protocol: 'foo'}); + + const getMetricsSpy = sandbox.spy(plugin.metricService, 'getMetrics'); + const getPrometheusContentTypeSpy = sandbox.spy(plugin.metricService, 'getPrometheusContentType'); + const updateCoreMetricsSpy = sandbox.spy(plugin.metricService, 'updateCoreMetrics'); + + await plugin.pipeFormatMetrics(request); + + expect(getMetricsSpy.called).to.be.false; + expect(getPrometheusContentTypeSpy.called).to.be.false; + expect(updateCoreMetricsSpy.called).to.be.false; + }); + }); }); \ No newline at end of file diff --git a/test/mocks/context.mock.ts b/test/mocks/context.mock.ts index 1eee3cc..fa063e6 100644 --- a/test/mocks/context.mock.ts +++ b/test/mocks/context.mock.ts @@ -17,5 +17,10 @@ export class ContextMock { warn: sinon.stub(), error: sinon.stub() }; + this.accessors = { + sdk: { + query: sinon.stub(), + }, + }; } } \ No newline at end of file