From 3ed315ba9287621566b59bca955ae4f638228d98 Mon Sep 17 00:00:00 2001 From: Timo Rolfsmeier Date: Thu, 11 Apr 2024 16:23:00 +0200 Subject: [PATCH 1/7] feat: make agent configurable for underlying http(s) request --- index.d.ts | 4 ++++ index.js | 4 +++- src/batcher.js | 2 +- src/requests.js | 6 ++++-- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/index.d.ts b/index.d.ts index 539eb70..13543b4 100644 --- a/index.d.ts +++ b/index.d.ts @@ -1,4 +1,6 @@ import TransportStream from "winston-transport"; +import http from 'http'; +import https from 'https'; declare interface LokiTransportOptions extends TransportStream.TransportStreamOptions{ host: string; @@ -12,6 +14,8 @@ declare interface LokiTransportOptions extends TransportStream.TransportStreamOp replaceTimestamp?: boolean, gracefulShutdown?: boolean, timeout?: number, + httpAgent?: http.Agent | boolean; + httpsAgent?: https.Agent | boolean; onConnectionError?(error: unknown): void } diff --git a/index.js b/index.js index e1c7605..d0589ad 100644 --- a/index.js +++ b/index.js @@ -29,7 +29,9 @@ class LokiTransport extends Transport { onConnectionError: options.onConnectionError, replaceTimestamp: options.replaceTimestamp !== false, gracefulShutdown: options.gracefulShutdown !== false, - timeout: options.timeout + timeout: options.timeout, + httpAgent: options.httpAgent, + httpsAgent: options.httpsAgent }) this.useCustomFormat = options.format !== undefined diff --git a/src/batcher.js b/src/batcher.js index 5659320..82becc4 100644 --- a/src/batcher.js +++ b/src/batcher.js @@ -251,7 +251,7 @@ class Batcher { } // Send the data to Grafana Loki - req.post(this.url, this.contentType, this.options.headers, reqBody, this.options.timeout) + req.post(this.url, this.contentType, this.options.headers, reqBody, this.options.timeout, this.options.httpAgent, this.options.httpsAgent) .then(() => { // No need to clear the batch if batching is disabled logEntry === undefined && this.clearBatch() diff --git a/src/requests.js b/src/requests.js index 5f57fdc..6ada7bc 100644 --- a/src/requests.js +++ b/src/requests.js @@ -1,7 +1,7 @@ const http = require('http') const https = require('https') -const post = async (lokiUrl, contentType, headers = {}, data = '', timeout) => { +const post = async (lokiUrl, contentType, headers = {}, data = '', timeout, httpAgent, httpsAgent) => { // Construct a buffer from the data string to have deterministic data size const dataBuffer = Buffer.from(data, 'utf8') @@ -14,6 +14,7 @@ const post = async (lokiUrl, contentType, headers = {}, data = '', timeout) => { return new Promise((resolve, reject) => { // Decide which http library to use based on the url const lib = lokiUrl.protocol === 'https:' ? https : http + const agent = lokiUrl.protocol === 'https:' ? httpsAgent : httpAgent // Construct the node request options const options = { @@ -22,7 +23,8 @@ const post = async (lokiUrl, contentType, headers = {}, data = '', timeout) => { path: lokiUrl.pathname, method: 'POST', headers: Object.assign(defaultHeaders, headers), - timeout: timeout + timeout: timeout, + agent: agent } // Construct the request From e9c453f2fb51286daa21e50e36729357c87a006c Mon Sep 17 00:00:00 2001 From: Jani Anttonen Date: Mon, 15 Apr 2024 20:15:12 +0300 Subject: [PATCH 2/7] Update package.json --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 66fc4e0..b179fdf 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "winston-loki", - "version": "6.1.0", + "version": "6.1.1", "description": "A Winston transport for Grafana Loki", "keywords": [ "winston", From 468e72d72d648d25f9ea934138dd98243732a951 Mon Sep 17 00:00:00 2001 From: MasterIO02 <40836390+MasterIO02@users.noreply.github.com> Date: Tue, 16 Apr 2024 17:05:05 +0200 Subject: [PATCH 3/7] Add useWinstonMetaAsLabels option --- README.md | 1 + index.d.ts | 1 + index.js | 11 ++++++++--- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 1973af9..f92de55 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,7 @@ LokiTransport() takes a Javascript object as an input. These are the options tha | `timeout` | timeout for requests to grafana loki in ms | 30000 | undefined | | `basicAuth` | basic authentication credentials to access Loki over HTTP | username:password | undefined | | `onConnectionError`| Loki error connection handler | (err) => console.error(err) | undefined | +| `useWinstonMetaAsLabels` | Use Winston's "meta" (such as defaultMeta values) as Loki labels | true | false | ### Example With default formatting: diff --git a/index.d.ts b/index.d.ts index 13543b4..9d137f1 100644 --- a/index.d.ts +++ b/index.d.ts @@ -16,6 +16,7 @@ declare interface LokiTransportOptions extends TransportStream.TransportStreamOp timeout?: number, httpAgent?: http.Agent | boolean; httpsAgent?: https.Agent | boolean; + useWinstonMetaAsLabels?: boolean; onConnectionError?(error: unknown): void } diff --git a/index.js b/index.js index d0589ad..b3d0c0a 100644 --- a/index.js +++ b/index.js @@ -36,6 +36,7 @@ class LokiTransport extends Transport { this.useCustomFormat = options.format !== undefined this.labels = options.labels + this.useWinstonMetaAsLabels = options.useWinstonMetaAsLabels } /** @@ -60,10 +61,14 @@ class LokiTransport extends Transport { // build custom labels if provided let lokiLabels = { level: level } - if (this.labels) { - lokiLabels = Object.assign(lokiLabels, this.labels) + if (this.useWinstonMetaAsLabels) { + lokiLabels = Object.assign(lokiLabels, rest) } else { - lokiLabels.job = label + if (this.labels) { + lokiLabels = Object.assign(lokiLabels, this.labels) + } else { + lokiLabels.job = label + } } lokiLabels = Object.assign(lokiLabels, labels) From d638ed52909307411255a2cb78b937057b3ed83c Mon Sep 17 00:00:00 2001 From: MasterIO02 <40836390+MasterIO02@users.noreply.github.com> Date: Thu, 18 Apr 2024 21:14:03 +0200 Subject: [PATCH 4/7] Add ignoredMeta option --- README.md | 1 + index.d.ts | 1 + index.js | 13 ++++++++----- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index f92de55..15e9089 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,7 @@ LokiTransport() takes a Javascript object as an input. These are the options tha | `basicAuth` | basic authentication credentials to access Loki over HTTP | username:password | undefined | | `onConnectionError`| Loki error connection handler | (err) => console.error(err) | undefined | | `useWinstonMetaAsLabels` | Use Winston's "meta" (such as defaultMeta values) as Loki labels | true | false | +| `ignoredMeta` | When useWinstonMetaAsLabels is enabled, a list of meta values to ignore | ["error_description"] | undefined | ### Example With default formatting: diff --git a/index.d.ts b/index.d.ts index 9d137f1..c550bec 100644 --- a/index.d.ts +++ b/index.d.ts @@ -17,6 +17,7 @@ declare interface LokiTransportOptions extends TransportStream.TransportStreamOp httpAgent?: http.Agent | boolean; httpsAgent?: https.Agent | boolean; useWinstonMetaAsLabels?: boolean; + ignoredMeta?: Array; onConnectionError?(error: unknown): void } diff --git a/index.js b/index.js index b3d0c0a..941d3f5 100644 --- a/index.js +++ b/index.js @@ -37,6 +37,7 @@ class LokiTransport extends Transport { this.useCustomFormat = options.format !== undefined this.labels = options.labels this.useWinstonMetaAsLabels = options.useWinstonMetaAsLabels + this.ignoredMeta = options.ignoredMeta || [] } /** @@ -62,13 +63,15 @@ class LokiTransport extends Transport { let lokiLabels = { level: level } if (this.useWinstonMetaAsLabels) { + // deleting the keys (labels) that we want to ignore from Winston's meta + for (const [key, _] of Object.entries(rest)) { + if (this.ignoredMeta.includes(key)) delete rest[key] + } lokiLabels = Object.assign(lokiLabels, rest) + } else if (this.labels) { + lokiLabels = Object.assign(lokiLabels, this.labels) } else { - if (this.labels) { - lokiLabels = Object.assign(lokiLabels, this.labels) - } else { - lokiLabels.job = label - } + lokiLabels.job = label } lokiLabels = Object.assign(lokiLabels, labels) From 2834a1908741f774751138a99969906947fd11b2 Mon Sep 17 00:00:00 2001 From: Jani Anttonen Date: Wed, 24 Apr 2024 13:57:49 +0300 Subject: [PATCH 5/7] Bump the version number --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index b179fdf..ccd8945 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "winston-loki", - "version": "6.1.1", + "version": "6.1.2", "description": "A Winston transport for Grafana Loki", "keywords": [ "winston", From 3438f364625286047fee31764f48b7bb92164e14 Mon Sep 17 00:00:00 2001 From: moxious Date: Thu, 19 Sep 2024 19:54:23 -0400 Subject: [PATCH 6/7] example for connecting to Grafana Cloud --- README.md | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 15e9089..659717c 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ LokiTransport() takes a Javascript object as an input. These are the options tha | `useWinstonMetaAsLabels` | Use Winston's "meta" (such as defaultMeta values) as Loki labels | true | false | | `ignoredMeta` | When useWinstonMetaAsLabels is enabled, a list of meta values to ignore | ["error_description"] | undefined | -### Example +### Example (Running Loki Locally) With default formatting: ```js const { createLogger, transports } = require("winston"); @@ -58,6 +58,37 @@ logger.debug({ message: 'test', labels: { 'key': 'value' } }) TODO: Add custom formatting example +### Example (Grafana Cloud Loki) + +**Important**: this snippet requires the following values, here are the instructions for how you can find them. + +* `LOKI_HOST`: find this in your Grafana Cloud instance by checking Connections > Data Sources, find the right Loki connection, and copy its URL, which may look like `https://logs-prod-006.grafana.net` +* `USER_ID`: the user number in the same data source definition, it will be a multi-digit number like `372040` +* `GRAFANA_CLOUD_TOKEN`: In Grafana Cloud, search for Cloud Access Policies. Create a new Cloud Access Policy, ensuring its scopes include `logs:write`. Generate a token for this cloud access policy, and use this value here. + +```js +const { createLogger, transports } = require("winston"); +const LokiTransport = require("winston-loki"); +const options = { + ..., + transports: [ + new LokiTransport({ + host: 'LOKI_HOST', + labels: { app: 'my-app' }, + json: true, + basicAuth: 'USER_ID:GRAFANA_CLOUD_TOKEN', + format: winston.format.json(), + replaceTimestamp: true, + onConnectionError: (err) => console.error(err), + }) + ] + ... +}; +const logger = createLogger(options); +logger.debug({ message: 'test', labels: { 'key': 'value' } }) +``` + + ## Developing ### Requirements Running a local Loki for testing is probably required, and the easiest way to do that is to follow this guide: https://github.com/grafana/loki/tree/master/production#run-locally-using-docker. After that, Grafana Loki instance is available at `http://localhost:3100`, with a Grafana instance running at `http://localhost:3000`. Username `admin`, password `admin`. Add the Loki source with the URL `http://loki:3100`, and the explorer should work. From 802f029af621a4fc29d5bf0f35e85e1411b09287 Mon Sep 17 00:00:00 2001 From: moxious Date: Thu, 19 Sep 2024 19:54:46 -0400 Subject: [PATCH 7/7] support username/password passing in the URL directly --- src/batcher.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/batcher.js b/src/batcher.js index 82becc4..af50c40 100644 --- a/src/batcher.js +++ b/src/batcher.js @@ -42,11 +42,14 @@ class Batcher { const URL = this.loadUrl() this.url = new URL(this.options.host + '/loki/api/v1/push') + const btoa = require('btoa') // Parse basic auth parameters if given if (options.basicAuth) { - const btoa = require('btoa') const basicAuth = 'Basic ' + btoa(options.basicAuth) this.options.headers = Object.assign(this.options.headers, { Authorization: basicAuth }) + } else if(this.url.username && this.url.password) { + const basicAuth = 'Basic ' + btoa(this.url.username + ':' + this.url.password) + this.options.headers = Object.assign(this.options.headers, { Authorization: basicAuth }) } // Define the batching intervals