diff --git a/README.md b/README.md index 59798b9..de5490f 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ - [About](#about) - [Kuzzle Prometheus Plugin](#kuzzle-prometheus-plugin) - [Kuzzle](#kuzzle) - - [Compatibility matrice](#compatibility-matrice) + - [Compatibility matrix](#compatibility-matrix) - [Installation](#installation) - [Configuration](#configuration) - [Plugin](#plugin) @@ -27,6 +27,7 @@ - [With only one Kuzzle node](#with-only-one-kuzzle-node) - [With an authentified user](#with-an-authentified-user) - [With multiple Kuzzle nodes and using Docker Compose](#with-multiple-kuzzle-nodes-and-using-docker-compose) + - [Using Kubernetes annotations](#using-kubernetes-annotations) - [Dashboards](#dashboards) - [Features](#features) - [Screenshots](#screenshots) @@ -200,6 +201,21 @@ scrape_configs: - 'kuzzle-plugin-prometheus-kuzzle-3:7512' ``` +### Using Kubernetes annotations + +If your Prometheus inside a Kubernetes cluster, you must use the helper HTTP route `/_/metrics` since Prometheus `params` configuration is not supported. +Your Pods annotations should look like this: + +```yaml +metadata: + annotations: + prometheus.io/scrape: "true" + prometheus.io/path: /_/metrics + prometheus.io/port: "7512" +spec: +... +``` + ## Dashboards ### Features diff --git a/config/grafana/dashboards/demo.json b/config/grafana/dashboards/demo.json index 3b989a9..35703f2 100644 --- a/config/grafana/dashboards/demo.json +++ b/config/grafana/dashboards/demo.json @@ -22,7 +22,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "iteration": 1642671164685, + "iteration": 1643633313636, "links": [], "liveNow": false, "panels": [ @@ -90,107 +90,11 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 8, + "h": 11, + "w": 12, "x": 0, "y": 0 }, - "id": 30, - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "exemplar": true, - "expr": "rate(kuzzle_api_request_duration_ms_count{status=~\"5.+\",nodeId=~\"$nodeId\"}[$__rate_interval])", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Status code 500", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-RdYlGr" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 5, - "gradientMode": "hue", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineStyle": { - "dash": [ - 0, - 3, - 3 - ], - "fill": "dot" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 0 - }, "id": 28, "options": { "legend": { @@ -213,7 +117,7 @@ "uid": "Prometheus" }, "exemplar": true, - "expr": "rate(kuzzle_api_request_duration_ms_count{status=~\"5.+\",nodeId=~\"$nodeId\"}[$__rate_interval])", + "expr": "sum by(nodeId) (rate(kuzzle_api_request_duration_ms_count{status=~\"5.+\",nodeId=~\"$nodeId\"}[$__rate_interval]) * 50)", "interval": "", "legendFormat": "{{nodeId}}", "refId": "A" @@ -286,9 +190,9 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 8, - "x": 16, + "h": 11, + "w": 12, + "x": 12, "y": 0 }, "id": 32, @@ -313,7 +217,7 @@ "uid": "Prometheus" }, "exemplar": true, - "expr": "rate(kuzzle_api_request_duration_ms_count{status=~\"5.+\",nodeId=~\"$nodeId\"}[$__rate_interval])", + "expr": "sum by(action, controller) (rate(kuzzle_api_request_duration_ms_count{status=~\"5.+\",nodeId=~\"$nodeId\"}[$__rate_interval]) * 50)", "interval": "", "legendFormat": "{{controller}} - {{action}}", "refId": "A" @@ -389,7 +293,7 @@ "h": 12, "w": 12, "x": 0, - "y": 8 + "y": 11 }, "id": 22, "options": { @@ -413,7 +317,7 @@ "uid": "Prometheus" }, "exemplar": true, - "expr": "kuzzle_network_connections{nodeId=~\"$nodeId\"}", + "expr": "sum by(nodeId) (kuzzle_network_connections{nodeId=~\"$nodeId\"})", "interval": "", "legendFormat": "{{nodeId}}", "refId": "A" @@ -491,7 +395,7 @@ "h": 12, "w": 12, "x": 12, - "y": 8 + "y": 11 }, "id": 18, "options": { @@ -591,7 +495,7 @@ "h": 8, "w": 8, "x": 0, - "y": 20 + "y": 23 }, "id": 16, "options": { @@ -691,7 +595,7 @@ "h": 8, "w": 8, "x": 8, - "y": 20 + "y": 23 }, "id": 24, "options": { @@ -791,7 +695,7 @@ "h": 8, "w": 8, "x": 16, - "y": 20 + "y": 23 }, "id": 26, "options": { @@ -815,7 +719,7 @@ "uid": "Prometheus" }, "exemplar": true, - "expr": "kuzzle_realtime_subscriptions{nodeId=~\"$nodeId\"}", + "expr": "sum by(nodeId) (kuzzle_realtime_subscriptions{nodeId=~\"$nodeId\"})", "interval": "", "legendFormat": "{{nodeId}}", "refId": "A" @@ -893,7 +797,7 @@ "h": 13, "w": 12, "x": 0, - "y": 28 + "y": 31 }, "id": 10, "interval": "3s", @@ -996,7 +900,7 @@ "h": 13, "w": 12, "x": 12, - "y": 28 + "y": 31 }, "hideTimeOverride": false, "id": 4, @@ -1042,7 +946,7 @@ "h": 1, "w": 24, "x": 0, - "y": 41 + "y": 44 }, "id": 14, "panels": [], @@ -1117,7 +1021,7 @@ "h": 13, "w": 12, "x": 0, - "y": 42 + "y": 45 }, "id": 20, "options": { @@ -1219,7 +1123,7 @@ "h": 13, "w": 12, "x": 12, - "y": 42 + "y": 45 }, "hideTimeOverride": false, "id": 2, @@ -1272,7 +1176,7 @@ "list": [ { "current": { - "selected": true, + "selected": false, "text": [ "All" ], diff --git a/features/Metrics.feature b/features/Metrics.feature index 6e935bd..bba0fa1 100644 --- a/features/Metrics.feature +++ b/features/Metrics.feature @@ -1,4 +1,4 @@ -Feature: Prometheus metrics fetching +Feature: Prometheus metrics fetching using server:metrics Scenario: Fetching Prometheus formatted metrics from server:metrics with the format parameter set to "prometheus" Given A running Kuzzle instance at "localhost:7512" When I send a HTTP request to "/_metrics?format=prometheus" diff --git a/features/PrometheusMetrics.feature b/features/PrometheusMetrics.feature new file mode 100644 index 0000000..7da7c75 --- /dev/null +++ b/features/PrometheusMetrics.feature @@ -0,0 +1,12 @@ +Feature: Prometheus metrics fetching using prometheus:metrics + Scenario: Fetching Prometheus formatted metrics from prometheus:metrics" + Given A running Kuzzle instance at "localhost:7512" + When I send a HTTP request to "/_/metrics" + Then The HTTP response should be a Prometheus formatted metrics containing: + | kuzzle_api_concurrent_requests | 1 | + | process_start_time_seconds | | + + Scenario: Trying to fetch Prometheus formatted metrics from prometheus:metrics through WebSocket + Given A running Kuzzle instance at "localhost:7512" + When I send a WebSocket request to "prometheus":"metrics" with the format parameter set to "prometheus" + Then The WebSocket response should be a JSON object with "200" status code and a "" property diff --git a/features/step_definitions/plugin.steps.ts b/features/step_definitions/plugin.steps.ts index f7ec264..a8f9948 100644 --- a/features/step_definitions/plugin.steps.ts +++ b/features/step_definitions/plugin.steps.ts @@ -51,7 +51,9 @@ export class PluginSteps { public async thenTheResponseShouldBeAJSONObjectWithAProperty(status: string, property: string) { const json = JSON.parse(this.result); assert(json.status === parseInt(status)); - assert(_.get(json, property) !== undefined); + if (property !== null) { + assert(_.get(json, property) !== undefined); + } } @then(/The HTTP response should be a JSON object/) diff --git a/lib/PrometheusPlugin.ts b/lib/PrometheusPlugin.ts index 45e6b97..fd3d130 100644 --- a/lib/PrometheusPlugin.ts +++ b/lib/PrometheusPlugin.ts @@ -135,6 +135,17 @@ export class PrometheusPlugin extends Plugin { 'request:onError': this.recordRequest.bind(this), }; + this.api = { + prometheus: { + actions: { + metrics: { + handler: (request: KuzzleRequest) => this.metrics(request), + http: [{ verb: 'get', path: 'metrics' }], + }, + }, + } + } + this.metricService = new MetricService(this.config); } @@ -176,4 +187,29 @@ export class PrometheusPlugin extends Plugin { } ); } + + /** + * Return the metrics in Prometheus format + * NOTE: This is an HTTP route for Prometheus installations that do not support HTTP arguments + * @param {KuzzleRequest} request - Kuzzle request + * @returns {Promise} + */ + async metrics (request: KuzzleRequest): Promise { + if (request.context.connection.protocol === 'http') { + const responsePayload = await this.context.accessors.sdk.query({ + controller: 'server', + action: 'metrics', + }); + this.metricService.updateCoreMetrics(responsePayload.result); + + request.response.configure({ + headers: { + 'Content-Type': this.metricService.getPrometheusContentType() + }, + format: 'raw', + }); + + return await this.metricService.getMetrics(); + } + } } diff --git a/package-lock.json b/package-lock.json index 8f1aa76..1c24992 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "kuzzle-plugin-prometheus", - "version": "4.0.0", + "version": "4.1.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 9b674b7..f1f35d2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "kuzzle-plugin-prometheus", - "version": "4.0.0", + "version": "4.1.0", "description": "Kuzzle plugin: monitoring Kuzzle using Prometheus", "author": { "name": "The Kuzzle Team " diff --git a/test/PrometheusPlugin.spec.ts b/test/PrometheusPlugin.spec.ts index 5fd6a10..d5c425f 100644 --- a/test/PrometheusPlugin.spec.ts +++ b/test/PrometheusPlugin.spec.ts @@ -149,4 +149,54 @@ describe('PrometheusPlugin', () => { }); }); }); + + describe('#metrics', () => { + it('should format the metrics and send them to the client as Prometheus format', async () => { + // We need to override global Kuzzle getter to manipulate the request response + Reflect.defineProperty(global, 'kuzzle', { + get () { + return { + id: 'kuzzle', + }; + }, + }); + + plugin.init(undefined, context); + const request = new KuzzleRequest({ + controller: 'prometheus', + action: 'metrics', + }, {protocol: 'http'}); + + const getMetricsStub = sandbox.stub(plugin.metricService, 'getMetrics').returns('fake metrics'); + const getPrometheusContentTypeStub = sandbox.stub(plugin.metricService, 'getPrometheusContentType').returns('text/plain'); + const updateCoreMetricsStub = sandbox.stub(plugin.metricService, 'updateCoreMetrics').returns(); + plugin.context.accessors.sdk.query.returns({ result: 'fake metrics' }); + + const formattedRequest = await plugin.metrics(request); + + expect(plugin.context.accessors.sdk.query.calledOnce).to.be.true; + expect(getMetricsStub.calledOnce).to.be.true; + expect(getPrometheusContentTypeStub.calledOnce).to.be.true; + expect(updateCoreMetricsStub.calledOnce).to.be.true; + expect(formattedRequest).to.contains('fake metrics'); + }); + + it('should not format the metrics if protocol is not HTTP', async () => { + plugin.init(undefined, context); + const request = new KuzzleRequest({ + controller: 'prometheus', + action: 'metrics', + }, {protocol: 'foo'}); + + const getMetricsSpy = sandbox.spy(plugin.metricService, 'getMetrics'); + const getPrometheusContentTypeSpy = sandbox.spy(plugin.metricService, 'getPrometheusContentType'); + const updateCoreMetricsSpy = sandbox.spy(plugin.metricService, 'updateCoreMetrics'); + + await plugin.pipeFormatMetrics(request); + + expect(getMetricsSpy.called).to.be.false; + expect(getPrometheusContentTypeSpy.called).to.be.false; + expect(updateCoreMetricsSpy.called).to.be.false; + }); + }); }); \ No newline at end of file diff --git a/test/mocks/context.mock.ts b/test/mocks/context.mock.ts index 1eee3cc..fa063e6 100644 --- a/test/mocks/context.mock.ts +++ b/test/mocks/context.mock.ts @@ -17,5 +17,10 @@ export class ContextMock { warn: sinon.stub(), error: sinon.stub() }; + this.accessors = { + sdk: { + query: sinon.stub(), + }, + }; } } \ No newline at end of file