From fa093e3ed59682c51e5803507bacc8447c284af2 Mon Sep 17 00:00:00 2001 From: Golda Velez Date: Wed, 24 Apr 2024 14:26:12 -0700 Subject: [PATCH] add capability to set an instance identifier which labels all metrics (#1198) * use latest version of observability with otlp updates, reduce export interval * for cas server, set an instance identifier from the ecs container * tell the linter that it will be a string, for reals * the linter wants to be very careful about regexp logic maybe * fix typos * do not escape / inside character class * add test coverage for some lines * add a test for presence of the instanceIdentifier * set the instance identifier in the deployment, use in the code * make tests reflect expected reality --- Dockerfile | 4 +++ config/default.json | 1 + config/env/dev.json | 1 + config/env/prod.json | 1 + config/env/test.json | 1 + entrypoint.sh | 14 ++++++++ package-lock.json | 14 ++++---- package.json | 2 +- src/__tests__/ceramic_integration.test.ts | 39 +++++++++++++++++++++++ src/app.ts | 3 ++ 10 files changed, 72 insertions(+), 8 deletions(-) create mode 100644 entrypoint.sh diff --git a/Dockerfile b/Dockerfile index 21f51ea3e..23fe06027 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,6 +43,10 @@ RUN npm install WORKDIR / +COPY entrypoint.sh . +RUN chmod +x ./entrypoint.sh + COPY runner.sh . +ENTRYPOINT ["./entrypoint.sh"] CMD [ "./runner.sh" ] diff --git a/config/default.json b/config/default.json index b483b2fdc..987fbbe42 100644 --- a/config/default.json +++ b/config/default.json @@ -55,6 +55,7 @@ "collectorHost": "", "exportIntervalMillis": 30000, "exportTimeoutMillis": 20000, + "instanceIdentifier": "", "prometheusPort": 0 }, "db": { diff --git a/config/env/dev.json b/config/env/dev.json index 15264ab6e..688b5c49d 100644 --- a/config/env/dev.json +++ b/config/env/dev.json @@ -56,6 +56,7 @@ "traceRatio": "@@METRICS_TRACE_RATIO", "exportIntervalMillis": "@@METRICS_EXPORT_INTERVAL_MS", "exportTimeoutMillis": "@@METRICS_EXPORT_TIMEOUT_MS", + "instanceIdentifier": "@@INSTANCE_IDENTIFIER", "prometheusPort": "@@METRICS_PORT" }, "db": { diff --git a/config/env/prod.json b/config/env/prod.json index 15264ab6e..688b5c49d 100644 --- a/config/env/prod.json +++ b/config/env/prod.json @@ -56,6 +56,7 @@ "traceRatio": "@@METRICS_TRACE_RATIO", "exportIntervalMillis": "@@METRICS_EXPORT_INTERVAL_MS", "exportTimeoutMillis": "@@METRICS_EXPORT_TIMEOUT_MS", + "instanceIdentifier": "@@INSTANCE_IDENTIFIER", "prometheusPort": "@@METRICS_PORT" }, "db": { diff --git a/config/env/test.json b/config/env/test.json index a3c072bc9..3db670787 100644 --- a/config/env/test.json +++ b/config/env/test.json @@ -37,6 +37,7 @@ "traceRatio": "@@METRICS_TRACE_RATIO", "exportIntervalMillis": "@@METRICS_EXPORT_INTERVAL_MS", "exportTimeoutMillis": "@@METRICS_EXPORT_TIMEOUT_MS", + "instanceIdentifier": "@@INSTANCE_IDENTIFIER", "prometheusPort": "@@METRICS_PORT" }, "db": { diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100644 index 000000000..5fffa861f --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +instance_uri=$ECS_CONTAINER_METADATA_URI + +# Parse the last portion of the URI +instance_id=${instance_uri##*/} + +# Remove everything after the hyphen to get the task ID +instance_id=${instance_id%%-*} + +# if present, will be the task ID, otherwise empty string +export INSTANCE_IDENTIFIER=$instance_id + +exec "$@" diff --git a/package-lock.json b/package-lock.json index 50c9e877f..5bd5ffdd4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,7 +16,7 @@ "@ceramicnetwork/common": "^2.28.0-rc.0", "@ceramicnetwork/core": "^2.35.0-rc.0", "@ceramicnetwork/logger": "^2.5.0", - "@ceramicnetwork/observability": "^1.4.6", + "@ceramicnetwork/observability": "^1.5.0", "@ceramicnetwork/streamid": "^2.15.0-rc.0", "@ceramicnetwork/wasm-bloom-filter": "^0.1.0", "@overnightjs/core": "^1.7.6", @@ -3884,9 +3884,9 @@ } }, "node_modules/@ceramicnetwork/observability": { - "version": "1.4.7", - "resolved": "https://registry.npmjs.org/@ceramicnetwork/observability/-/observability-1.4.7.tgz", - "integrity": "sha512-bNPP9zDPYetMZ/Qp6jFdAjqZTUtw0gqztvfxvfzt6cMaX5hcD8zakKyabGQx5HbIMgKlqxXGLOXMTDDKXjwFaw==", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@ceramicnetwork/observability/-/observability-1.5.0.tgz", + "integrity": "sha512-fAcdw0OrAl0LBTan0d9ixER6p558cKPyJkyz7l3/JtBXLjIf3U6Rbf+V4Pf662VNH2Gn69bTnQkA1s0KUV4qhw==", "dependencies": { "@opentelemetry/api": "^1.8.0", "@opentelemetry/exporter-metrics-otlp-http": "^0.50.0", @@ -29744,9 +29744,9 @@ } }, "@ceramicnetwork/observability": { - "version": "1.4.7", - "resolved": "https://registry.npmjs.org/@ceramicnetwork/observability/-/observability-1.4.7.tgz", - "integrity": "sha512-bNPP9zDPYetMZ/Qp6jFdAjqZTUtw0gqztvfxvfzt6cMaX5hcD8zakKyabGQx5HbIMgKlqxXGLOXMTDDKXjwFaw==", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@ceramicnetwork/observability/-/observability-1.5.0.tgz", + "integrity": "sha512-fAcdw0OrAl0LBTan0d9ixER6p558cKPyJkyz7l3/JtBXLjIf3U6Rbf+V4Pf662VNH2Gn69bTnQkA1s0KUV4qhw==", "requires": { "@opentelemetry/api": "^1.8.0", "@opentelemetry/exporter-metrics-otlp-http": "^0.50.0", diff --git a/package.json b/package.json index fe0e0ad90..e169b6dc7 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,7 @@ "@ceramicnetwork/common": "^2.28.0-rc.0", "@ceramicnetwork/core": "^2.35.0-rc.0", "@ceramicnetwork/logger": "^2.5.0", - "@ceramicnetwork/observability": "^1.4.6", + "@ceramicnetwork/observability": "^1.5.0", "@ceramicnetwork/streamid": "^2.15.0-rc.0", "@ceramicnetwork/wasm-bloom-filter": "^0.1.0", "@overnightjs/core": "^1.7.6", diff --git a/src/__tests__/ceramic_integration.test.ts b/src/__tests__/ceramic_integration.test.ts index a9420aa58..a387c3e47 100644 --- a/src/__tests__/ceramic_integration.test.ts +++ b/src/__tests__/ceramic_integration.test.ts @@ -584,3 +584,42 @@ describe('CAR file', () => { await casIPFS.stop() }) }) + +describe('Metrics Options', () => { + test('cas starts with a typical instance identifier', async () => { + const ipfsApiPort = await getPort() + const casIPFS = await createIPFS(ipfsApiPort) + const ganacheServer = await makeGanache() + const dbConnection = await createDbConnection() + const casPort = await getPort() + const cas = await makeCAS(createInjector(), dbConnection, { + mode: 'server', + ipfsPort: ipfsApiPort, + ganachePort: ganacheServer.port, + port: casPort, + useSmartContractAnchors: true, + metrics: { + instanceIdentifier: '234fffffffffffffffffffffffffffffffff9726129' + } + }) + await cas.start() + // Teardown + await cas.stop() + + const cas2 = await makeCAS(createInjector(), dbConnection, { + mode: 'server', + ipfsPort: ipfsApiPort, + ganachePort: ganacheServer.port, + port: casPort, + useSmartContractAnchors: true, + metrics: { + instanceIdentifier: '' + } + }) + await cas2.start() + await cas2.stop() + + await ganacheServer.close() + await casIPFS.stop() + }) +}) diff --git a/src/app.ts b/src/app.ts index ad5445d13..13f0a5469 100644 --- a/src/app.ts +++ b/src/app.ts @@ -129,6 +129,9 @@ export class CeramicAnchorApp { ) Metrics.count('HELLO', 1) logger.imp('Metrics exporter started') + if (this.config.metrics.instanceIdentifier) { + Metrics.setInstanceIdentifier(this.config.metrics.instanceIdentifier) + } } catch (e: any) { logger.imp('ERROR: Metrics exporter failed to start. Continuing anyway.') logger.err(e)