From b86df9a6276f92ad634a1d3061ead2e0be09ddcb Mon Sep 17 00:00:00 2001 From: Phenix Rizen Date: Tue, 26 Oct 2021 13:22:28 -0500 Subject: [PATCH 1/2] add otel metrics exporter --- lib/kayvee.ts | 7 +- lib/logger/logger.ts | 84 +++++++-- package-lock.json | 420 +++++++++++++++++++++++++++++++++++++++++-- package.json | 7 +- 4 files changed, 487 insertions(+), 31 deletions(-) diff --git a/lib/kayvee.ts b/lib/kayvee.ts index dfd0794..2bbdd78 100644 --- a/lib/kayvee.ts +++ b/lib/kayvee.ts @@ -70,4 +70,9 @@ const LOG_LEVELS = { TRACE: "trace", }; -_.extend(module.exports, LOG_LEVELS); +const METRICS_OUTPUT = { + LOG: "log", + OTEL: "otel", +}; + +_.extend(module.exports, LOG_LEVELS, METRICS_OUTPUT); diff --git a/lib/logger/logger.ts b/lib/logger/logger.ts index 5112ce5..6f0a830 100644 --- a/lib/logger/logger.ts +++ b/lib/logger/logger.ts @@ -2,6 +2,12 @@ var _ = require("underscore"); var kv = require("../kayvee"); var router = require("../router"); +const { DiagConsoleLogger, DiagLogLevel, diag } = require("@opentelemetry/api"); +const { OTLPMetricExporter } = require("@opentelemetry/exporter-otlp-grpc"); +const { MeterProvider } = require("@opentelemetry/sdk-metrics-base"); +const { Resource } = require("@opentelemetry/resources"); +const { SemanticResourceAttributes } = require("@opentelemetry/semantic-conventions"); + var LEVELS = { Trace: "trace", Debug: "debug", @@ -20,6 +26,11 @@ var LOG_LEVEL_ENUM = { critical: 5, }; +var METRICS_OUTPUT_ENUM = { + log: 0, + otel: 1, +}; + const assign = Object.assign || _.assign; // Use the faster Object.assign if possible let globalRouter; @@ -40,6 +51,7 @@ class Logger { globals = null; logWriter = null; logRouter = null; + metricsOutput = null; constructor( source, @@ -52,6 +64,7 @@ class Logger { this.globals = {}; this.globals.source = source; this.logWriter = output; + this.metricsOutput = null; if (process.env._TEAM_OWNER) { this.globals.team = process.env._TEAM_OWNER; @@ -59,6 +72,9 @@ class Logger { if (process.env._DEPLOY_ENV) { this.globals.deploy_env = process.env._DEPLOY_ENV; } + if (process.env._BUILD_ID) { + this.globals.build_id = process.env._BUILD_ID; + } if (process.env._EXECUTION_NAME) { this.globals.wf_id = process.env._EXECUTION_NAME; } @@ -118,6 +134,28 @@ class Logger { return this.logWriter; } + setMetricsOutput(output) { + if (output in METRICS_OUTPUT_ENUM && output === METRICS_OUTPUT.OTEL) { + var metricExporter = new OTLPMetricExporter({ + url: "http://localhost:4318/v1/metrics", + }); + + this.metricsOutput = new MeterProvider({ + exporter: metricExporter, + interval: 1000, + resource: new Resource({ + [SemanticResourceAttributes.SERVICE_NAME]: this.globals.source, + [SemanticResourceAttributes.SERVICE_VERSION]: this.globals.build_id, + [SemanticResourceAttributes.DEPLOYMENT_ENVIRONMENT]: this.globals.deploy_env, + }), + }).getMeter(this.globals.source); + } else { + this.warn("invalid metrics output specified, falling back to log output"); + this.metricsOutput = METRICS_OUTPUT.LOG; + } + return this.metricsOutput; + } + trace(title) { this.traceD(title, {}); } @@ -211,27 +249,37 @@ class Logger { } counterD(title, value, data) { - this._logWithLevel( - LEVELS.Info, - { - title, - value, - type: "counter", - }, - data, - ); + if (this.metricsOutput == null) { + this._logWithLevel( + LEVELS.Info, + { + title, + value, + type: "counter", + }, + data, + ); + } else { + var counter = this.metricsOutput.meter.createUpDownCounter(title); + counter.bind(data).add(value); + } } gaugeD(title, value, data) { - this._logWithLevel( - LEVELS.Info, - { - title, - value, - type: "gauge", - }, - data, - ); + if (this.metricsOutput == null) { + this._logWithLevel( + LEVELS.Info, + { + title, + value, + type: "gauge", + }, + data, + ); + } else { + var gauge = this.metricsOutput.meter.createHistogram(title); + gauge.bind(data).record(value); + } } _logWithLevel(logLvl, metadata, userdata) { diff --git a/package-lock.json b/package-lock.json index 71e20c1..87d79b8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,6 +10,167 @@ "integrity": "sha512-kWQUUXgyP+KvjXvjrK+FOD+RIXlcP3sFmArXDowgHtMolv+COMkdlUsgu5/+x1fNymLa+JmWVnXk2P8STllSVA==", "dev": true }, + "@grpc/grpc-js": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.4.1.tgz", + "integrity": "sha512-/chkA48TdAvATHA7RXJPeHQLdfFhpu51974s8htjO/XTDHA41j5+SkR5Io+lr9XsLmkZD6HxLyRAFGmA9wjO2w==", + "requires": { + "@grpc/proto-loader": "^0.6.4", + "@types/node": ">=12.12.47" + } + }, + "@grpc/proto-loader": { + "version": "0.6.6", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.6.6.tgz", + "integrity": "sha512-cdMaPZ8AiFz6ua6PUbP+LKbhwJbFXnrQ/mlnKGUyzDUZ3wp7vPLksnmLCBX6SHgSmjX7CbNVNLFYD5GmmjO4GQ==", + "requires": { + "@types/long": "^4.0.1", + "lodash.camelcase": "^4.3.0", + "long": "^4.0.0", + "protobufjs": "^6.10.0", + "yargs": "^16.1.1" + } + }, + "@opentelemetry/api": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.0.3.tgz", + "integrity": "sha512-puWxACExDe9nxbBB3lOymQFrLYml2dVOrd7USiVRnSbgXE+KwBu+HxFvxrzfqsiSda9IWsXJG1ef7C1O2/GmKQ==" + }, + "@opentelemetry/api-metrics": { + "version": "0.26.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api-metrics/-/api-metrics-0.26.0.tgz", + "integrity": "sha512-idDSUTx+LRwJiHhVHhdh45SWow5u9lKNDROKu5AMzsIVPI29utH5FfT9vor8qMM6blxWWvlT22HUNdNMWqUQfQ==" + }, + "@opentelemetry/core": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-1.0.0.tgz", + "integrity": "sha512-1+qvKilADnSFW4PiXy+f7D22pvfGVxepZ69GcbF8cTcbQTUt7w63xEBWn5f5j92x9I3c0sqbW1RUx5/a4wgzxA==", + "requires": { + "@opentelemetry/semantic-conventions": "1.0.0", + "semver": "^7.3.5" + } + }, + "@opentelemetry/exporter-otlp-grpc": { + "version": "0.26.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-otlp-grpc/-/exporter-otlp-grpc-0.26.0.tgz", + "integrity": "sha512-64VPck7CbGhI7c2bj54xaGGHCy4mP+rMnrKBAruNBmUUnN9OgihddNcMsIiHyddUcyC1I+hXS3JLW1G6AvlAmg==", + "requires": { + "@grpc/grpc-js": "^1.3.7", + "@grpc/proto-loader": "^0.6.4", + "@opentelemetry/core": "1.0.0", + "@opentelemetry/exporter-otlp-http": "0.26.0", + "@opentelemetry/resources": "1.0.0", + "@opentelemetry/sdk-metrics-base": "0.26.0", + "@opentelemetry/sdk-trace-base": "1.0.0" + } + }, + "@opentelemetry/exporter-otlp-http": { + "version": "0.26.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-otlp-http/-/exporter-otlp-http-0.26.0.tgz", + "integrity": "sha512-V3FcUEIVDZ66b3/6vjSBjwwozf/XV5eUXuELNzN8PAvGZH4mw36vaWlaxnGEV8HaZb2hbu2KbRpcOzqxx3tFDA==", + "requires": { + "@opentelemetry/api-metrics": "0.26.0", + "@opentelemetry/core": "1.0.0", + "@opentelemetry/resources": "1.0.0", + "@opentelemetry/sdk-metrics-base": "0.26.0", + "@opentelemetry/sdk-trace-base": "1.0.0" + } + }, + "@opentelemetry/resources": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-1.0.0.tgz", + "integrity": "sha512-ORP8F2LLcJEm5M3H24RmdlMdiDc70ySPushpkrAW34KZGdZXwkrFoFXZhhs5MUxPT+fLrTuBafXxZVr8eHtFuQ==", + "requires": { + "@opentelemetry/core": "1.0.0", + "@opentelemetry/semantic-conventions": "1.0.0" + } + }, + "@opentelemetry/sdk-metrics-base": { + "version": "0.26.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics-base/-/sdk-metrics-base-0.26.0.tgz", + "integrity": "sha512-PbJsso7Vy/CLATAOyXbt/VP7ZQ2QYnvlq28lhOWaLPw8aqLogMBvidNGRrt7rF4/hfzLT6pMgpAAcit2C/nUMA==", + "requires": { + "@opentelemetry/api-metrics": "0.26.0", + "@opentelemetry/core": "1.0.0", + "@opentelemetry/resources": "1.0.0", + "lodash.merge": "^4.6.2" + } + }, + "@opentelemetry/sdk-trace-base": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-1.0.0.tgz", + "integrity": "sha512-/rXoyQlDlJTJ4SOVAbP0Gpj89B8oZ2hJApYG2Dq5klkgFAtDifN8271TIzwtM8/ET8HUhgx9eyoUJi42LhIesg==", + "requires": { + "@opentelemetry/core": "1.0.0", + "@opentelemetry/resources": "1.0.0", + "@opentelemetry/semantic-conventions": "1.0.0", + "lodash.merge": "^4.6.2" + } + }, + "@opentelemetry/semantic-conventions": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.0.0.tgz", + "integrity": "sha512-XCZ6ZSmc8FOspxKUU+Ow9UtJeSSRcS5rFBYGpjzix02U2v+X9ofjOjgNRnpvxlSvkccYIhdTuwcvNskmZ46SeA==" + }, + "@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha1-m4sMxmPWaafY9vXQiToU00jzD78=" + }, + "@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==" + }, + "@protobufjs/codegen": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", + "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==" + }, + "@protobufjs/eventemitter": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", + "integrity": "sha1-NVy8mLr61ZePntCV85diHx0Ga3A=" + }, + "@protobufjs/fetch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", + "integrity": "sha1-upn7WYYUr2VwDBYZ/wbUVLDYTEU=", + "requires": { + "@protobufjs/aspromise": "^1.1.1", + "@protobufjs/inquire": "^1.1.0" + } + }, + "@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha1-Xp4avctz/Ap8uLKR33jIy9l7h9E=" + }, + "@protobufjs/inquire": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", + "integrity": "sha1-/yAOPnzyQp4tyvwRQIKOjMY48Ik=" + }, + "@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha1-bMKyDFya1q0NzP0hynZz2Nf79o0=" + }, + "@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha1-Cf0V8tbTq/qbZbw2ZQbWrXhG/1Q=" + }, + "@protobufjs/utf8": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", + "integrity": "sha1-p3c2C1s5oaLlEG+OhY8v0tBgxXA=" + }, + "@types/long": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.1.tgz", + "integrity": "sha512-5tXH6Bx/kNGd3MgffdmP4dy2Z+G4eaXw0SE81Tq3BNadtnMR5/ySMzX4SLEzHJzSmPNn4HIdpQsBvXMUykr58w==" + }, "@types/mocha": { "version": "8.0.3", "resolved": "https://registry.npmjs.org/@types/mocha/-/mocha-8.0.3.tgz", @@ -19,8 +180,7 @@ "@types/node": { "version": "12.12.68", "resolved": "https://registry.npmjs.org/@types/node/-/node-12.12.68.tgz", - "integrity": "sha512-3RW2s24ewB7F9dAHvgb9FRvNHn6nO9IK6Eaknbz7HTOe2a5GVne5XbUh5+YA+kcCn67glyHhClUUdFP73LWrgQ==", - "dev": true + "integrity": "sha512-3RW2s24ewB7F9dAHvgb9FRvNHn6nO9IK6Eaknbz7HTOe2a5GVne5XbUh5+YA+kcCn67glyHhClUUdFP73LWrgQ==" }, "accepts": { "version": "1.3.7", @@ -341,6 +501,46 @@ "integrity": "sha512-GRMWDxpOB6Dgk2E5Uo+3eEBvtOOlimMmpbFiKuLFnQzYDavtLFY3K5ona41jgN/WdRZtG7utuVSVTL4HbZHGkw==", "dev": true }, + "cliui": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", + "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", + "requires": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.0", + "wrap-ansi": "^7.0.0" + }, + "dependencies": { + "ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==" + }, + "is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" + }, + "string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "requires": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + } + }, + "strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "requires": { + "ansi-regex": "^5.0.1" + } + } + } + }, "co": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", @@ -353,6 +553,19 @@ "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=", "dev": true }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, "colors": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/colors/-/colors-1.4.0.tgz", @@ -517,6 +730,11 @@ "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" }, + "emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + }, "encodeurl": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", @@ -636,6 +854,11 @@ "es6-symbol": "^3.1.1" } }, + "escalade": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", + "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==" + }, "escape-html": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", @@ -1011,6 +1234,11 @@ "is-property": "^1.0.0" } }, + "get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" + }, "glob": { "version": "7.1.6", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", @@ -1349,9 +1577,9 @@ } }, "lodash": { - "version": "4.17.20", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", - "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==", + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", "dev": true }, "lodash._baseassign": { @@ -1394,6 +1622,11 @@ "integrity": "sha1-DZnzzNem0mHRm9rrkkUAXShYCOc=", "dev": true }, + "lodash.camelcase": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", + "integrity": "sha1-soqmKIorn8ZRA1x3EfZathkDMaY=" + }, "lodash.create": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/lodash.create/-/lodash.create-3.1.1.tgz", @@ -1428,6 +1661,11 @@ "lodash.isarray": "^3.0.0" } }, + "lodash.merge": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==" + }, "lodash.pickby": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/lodash.pickby/-/lodash.pickby-4.6.0.tgz", @@ -1440,6 +1678,11 @@ "integrity": "sha1-fD2mL/yzDw9agKJWbKJORdigHzE=", "dev": true }, + "long": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", + "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==" + }, "loose-envify": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", @@ -1449,6 +1692,14 @@ "js-tokens": "^3.0.0 || ^4.0.0" } }, + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "requires": { + "yallist": "^4.0.0" + } + }, "make-error": { "version": "1.3.6", "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", @@ -1765,9 +2016,9 @@ "dev": true }, "path-parse": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz", - "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==", + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", "dev": true }, "path-to-regexp": { @@ -1812,6 +2063,33 @@ "integrity": "sha1-4mDHj2Fhzdmw5WzD4Khd4Xx6V74=", "dev": true }, + "protobufjs": { + "version": "6.11.2", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.11.2.tgz", + "integrity": "sha512-4BQJoPooKJl2G9j3XftkIXjoC9C0Av2NOrWmbLWT1vH32GcSUHjM0Arra6UfTsVyfMAuFzaLucXn1sadxJydAw==", + "requires": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/long": "^4.0.1", + "@types/node": ">=13.7.0", + "long": "^4.0.0" + }, + "dependencies": { + "@types/node": { + "version": "16.11.6", + "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.6.tgz", + "integrity": "sha512-ua7PgUoeQFjmWPcoo9khiPum3Pd60k4/2ZGXt18sm2Slk0W0xZTqt5Y0Ny1NyBiN1EVQ/+FaF9NcY4Qe6rwk5w==" + } + } + }, "proxy-addr": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.6.tgz", @@ -1883,6 +2161,11 @@ "integrity": "sha512-MguG95oij0fC3QV3URf4V2SDYGJhJnJGqvIIgdECeODCT98wSWDAJ94SSuVpYQUoTcGUIL6L4yNB7j1DFFHSBg==", "dev": true }, + "require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=" + }, "require-uncached": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/require-uncached/-/require-uncached-1.0.3.tgz", @@ -1959,6 +2242,14 @@ "integrity": "sha1-vsEf3IOp/aBjQBIQ5AF2wwJNFWc=", "dev": true }, + "semver": { + "version": "7.3.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz", + "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==", + "requires": { + "lru-cache": "^6.0.0" + } + }, "send": { "version": "0.17.1", "resolved": "https://registry.npmjs.org/send/-/send-0.17.1.tgz", @@ -2351,9 +2642,9 @@ "dev": true }, "underscore": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.11.0.tgz", - "integrity": "sha512-xY96SsN3NA461qIRKZ/+qox37YXPtSBswMGfiNptr+wrt6ds4HaMw23TP612fEyGekRE6LNRiLYr/aqbHXNedw==" + "version": "1.13.1", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.1.tgz", + "integrity": "sha512-hzSoAVtJF+3ZtiFX0VgfFPHEDRm7Y/QPjGyNo4TVdnDTdft3tr8hEkD25a1jC+TjTuE7tkHGKkhwCgs9dgBB2g==" }, "underscore.string": { "version": "3.3.5", @@ -2438,6 +2729,54 @@ "integrity": "sha1-o9XabNXAvAAI03I0u68b7WMFkQc=", "dev": true }, + "wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "requires": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "dependencies": { + "ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==" + }, + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "requires": { + "color-convert": "^2.0.1" + } + }, + "is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" + }, + "string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "requires": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + } + }, + "strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "requires": { + "ansi-regex": "^5.0.1" + } + } + } + }, "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", @@ -2459,6 +2798,65 @@ "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", "dev": true }, + "y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==" + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + }, + "yargs": { + "version": "16.2.0", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", + "integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==", + "requires": { + "cliui": "^7.0.2", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.0", + "y18n": "^5.0.5", + "yargs-parser": "^20.2.2" + }, + "dependencies": { + "ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==" + }, + "is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" + }, + "string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "requires": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + } + }, + "strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "requires": { + "ansi-regex": "^5.0.1" + } + } + } + }, + "yargs-parser": { + "version": "20.2.9", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz", + "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==" + }, "yn": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", diff --git a/package.json b/package.json index 0e31717..d7d1c73 100644 --- a/package.json +++ b/package.json @@ -8,12 +8,17 @@ "url": "git://github.com/Clever/kayvee-js" }, "dependencies": { + "@opentelemetry/api": "^1.0.3", + "@opentelemetry/exporter-otlp-grpc": "^0.26.0", + "@opentelemetry/resources": "^1.0.0", + "@opentelemetry/sdk-metrics-base": "^0.26.0", + "@opentelemetry/semantic-conventions": "^1.0.0", "js-yaml": "^3.6.1", "jsonschema": "^1.1.0", "morgan": "^1.7.0", "qs": "^6.9.4", "split": "^1.0.0", - "underscore": "^1.8.3" + "underscore": "^1.13.1" }, "devDependencies": { "@clever/prettier-config": "1.0.0", From bfac791cf30ac5a0e78bb037495fdf37f3e65af7 Mon Sep 17 00:00:00 2001 From: Phenix Rizen Date: Thu, 28 Oct 2021 11:30:37 -0500 Subject: [PATCH 2/2] change logic to store metrics and update readme --- README.md | 13 +++++++++++ lib/kayvee.ts | 7 +----- lib/logger/logger.ts | 55 ++++++++++++++++++++++++++++++++------------ tsconfig.json | 8 +++++-- 4 files changed, 60 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 910e64d..ca56a82 100644 --- a/README.md +++ b/README.md @@ -179,6 +179,19 @@ Title + Metadata: * `log.counterD("counter-with-data", 2, {extra: "info"})` * `log.gaugeD("gauge-with-data", 2, {extra: "info"})` +#### kayvee/logger opentelemetry metrics +```js +var kv = require("../kayvee-js"); +var log = new kv.logger("test-logger"); + +log.setMetricsOutput(kv.Logger.METRICS_OUTPUT_OTEL); +log.counterD("counter-name", 1, {"foo": "bar"}); +log.gaugeD("gauge-name", 1.02, {"foo": "bar"}); + +// required to sync metrics with otel collector +log.shutdown(); +``` + ### Formatters #### format diff --git a/lib/kayvee.ts b/lib/kayvee.ts index 2bbdd78..dfd0794 100644 --- a/lib/kayvee.ts +++ b/lib/kayvee.ts @@ -70,9 +70,4 @@ const LOG_LEVELS = { TRACE: "trace", }; -const METRICS_OUTPUT = { - LOG: "log", - OTEL: "otel", -}; - -_.extend(module.exports, LOG_LEVELS, METRICS_OUTPUT); +_.extend(module.exports, LOG_LEVELS); diff --git a/lib/logger/logger.ts b/lib/logger/logger.ts index 6f0a830..4675793 100644 --- a/lib/logger/logger.ts +++ b/lib/logger/logger.ts @@ -2,7 +2,6 @@ var _ = require("underscore"); var kv = require("../kayvee"); var router = require("../router"); -const { DiagConsoleLogger, DiagLogLevel, diag } = require("@opentelemetry/api"); const { OTLPMetricExporter } = require("@opentelemetry/exporter-otlp-grpc"); const { MeterProvider } = require("@opentelemetry/sdk-metrics-base"); const { Resource } = require("@opentelemetry/resources"); @@ -26,7 +25,12 @@ var LOG_LEVEL_ENUM = { critical: 5, }; -var METRICS_OUTPUT_ENUM = { +const METRICS_OUTPUT = { + LOG: "log", + OTEL: "otel", +}; + +const METRICS_OUTPUT_ENUM = { log: 0, otel: 1, }; @@ -51,7 +55,8 @@ class Logger { globals = null; logWriter = null; logRouter = null; - metricsOutput = null; + metricsProvider = null; + metrics = null; constructor( source, @@ -64,7 +69,8 @@ class Logger { this.globals = {}; this.globals.source = source; this.logWriter = output; - this.metricsOutput = null; + this.metricsProvider = null; + this.metrics = {}; if (process.env._TEAM_OWNER) { this.globals.team = process.env._TEAM_OWNER; @@ -137,10 +143,10 @@ class Logger { setMetricsOutput(output) { if (output in METRICS_OUTPUT_ENUM && output === METRICS_OUTPUT.OTEL) { var metricExporter = new OTLPMetricExporter({ - url: "http://localhost:4318/v1/metrics", + concurrencyLimit: 1, }); - this.metricsOutput = new MeterProvider({ + this.metricsProvider = new MeterProvider({ exporter: metricExporter, interval: 1000, resource: new Resource({ @@ -150,10 +156,12 @@ class Logger { }), }).getMeter(this.globals.source); } else { - this.warn("invalid metrics output specified, falling back to log output"); - this.metricsOutput = METRICS_OUTPUT.LOG; + this.errorD("set-metrics-ouput", { + err: "invalid metrics output specified, falling back to log output", + }); + this.metricsProvider = null; } - return this.metricsOutput; + return this.metricsProvider; } trace(title) { @@ -249,7 +257,7 @@ class Logger { } counterD(title, value, data) { - if (this.metricsOutput == null) { + if (this.metricsProvider == null) { this._logWithLevel( LEVELS.Info, { @@ -260,13 +268,15 @@ class Logger { data, ); } else { - var counter = this.metricsOutput.meter.createUpDownCounter(title); - counter.bind(data).add(value); + if (this.metrics[title] === undefined) { + this.metrics[title] = this.metricsProvider.createUpDownCounter(title); + } + this.metrics[title].add(value, data); } } gaugeD(title, value, data) { - if (this.metricsOutput == null) { + if (this.metricsProvider == null) { this._logWithLevel( LEVELS.Info, { @@ -277,9 +287,22 @@ class Logger { data, ); } else { - var gauge = this.metricsOutput.meter.createHistogram(title); - gauge.bind(data).record(value); + // use a histogram post v0.26.0 of metrics sdk + if (this.metrics[title] === undefined) { + this.metrics[title] = this.metricsProvider.createValueRecorder(title); + } + this.metrics[title].record(value, data); + } + } + + shutdown() { + if (this.metricsProvider != null) { + return this.metricsProvider.shutdown().catch((err) => { + console.error(err); + return err; + }); } + return null; } _logWithLevel(logLvl, metadata, userdata) { @@ -299,6 +322,8 @@ class Logger { module.exports = Logger; module.exports.setGlobalRouting = setGlobalRouting; module.exports.getGlobalRouter = getGlobalRouter; +module.exports.METRICS_OUTPUT_OTEL = METRICS_OUTPUT.OTEL; +module.exports.METRICS_OUTPUT_LOG = METRICS_OUTPUT.LOG; module.exports.mockRouting = (cb) => { const _logWithLevel: any = Logger.prototype._logWithLevel; diff --git a/tsconfig.json b/tsconfig.json index aab1c05..8ce91c4 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -2,9 +2,13 @@ "compilerOptions": { "module": "commonjs", "target": "es5", - "noImplicitAny": false + "noImplicitAny": false, + "allowJs": true, }, "exclude": [ "node_modules" - ] + ], + "typingOptions": { + "enableAutoDiscovery": true + } }