diff --git a/src/controllers/opentelemetry.ts b/src/controllers/opentelemetry.ts index 252b286..77b81e3 100644 --- a/src/controllers/opentelemetry.ts +++ b/src/controllers/opentelemetry.ts @@ -1,5 +1,5 @@ import { NodeSDK } from "@opentelemetry/sdk-node"; -import { SimpleSpanProcessor } from "@opentelemetry/sdk-trace-node"; +import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-node"; import { HttpInstrumentation } from "@opentelemetry/instrumentation-http"; import { Resource } from "@opentelemetry/resources"; import Debug from "debug"; @@ -10,9 +10,11 @@ import { ATTR_SERVICE_NAME } from '@opentelemetry/semantic-conventions'; const jaegerHost = process.env.JAEGER_HOST ?? 'jaeger-otel-collector.sri'; const jaegerPort = process.env.JAEGER_PORT ?? 4318; const jaegerResName = process.env.JAEGER_RES_NAME ?? '/v1/traces'; + const traceExporter = new OTLPTraceExporter({ url: `http://${jaegerHost}:${jaegerPort}${jaegerResName}` }); +const spanProcessor = new BatchSpanProcessor(traceExporter); debug("Initializing Opentelemetry instrumentation..."); const sdk = new NodeSDK({ @@ -22,10 +24,15 @@ const sdk = new NodeSDK({ resource: new Resource({ [ATTR_SERVICE_NAME]: "biothings-explorer", }), - // use simple span processor to avoid losing data when the forked process exits (taskHandler) - // @ts-ignore - fix from MetinSeylan/Nestjs-OpenTelemetry#63 - spanProcessors: [new SimpleSpanProcessor(traceExporter)], + // @ts-ignore because MetinSeylan/Nestjs-OpenTelemetry#63 + spanProcessors: [spanProcessor], }); debug(`OTel URL http://${jaegerHost}:${jaegerPort}${jaegerResName}`); sdk.start(); debug("Opentelemetry instrumentation initialized."); + +export async function flushRemainingSpans(): Promise { + // avoid losing any spans in the buffer when taskHandler exits + debug("Flushing remaining spans..."); + await spanProcessor.forceFlush(); +} diff --git a/src/controllers/threading/taskHandler.ts b/src/controllers/threading/taskHandler.ts index 1a42dbd..b22ae41 100644 --- a/src/controllers/threading/taskHandler.ts +++ b/src/controllers/threading/taskHandler.ts @@ -17,6 +17,7 @@ import { ProfilingIntegration } from "@sentry/profiling-node"; import { Span, trace, context, propagation, Context, Tracer } from "@opentelemetry/api"; import { Telemetry } from "@biothings-explorer/utils"; import { InnerTaskData } from "@biothings-explorer/types"; +import { flushRemainingSpans } from "../opentelemetry"; // use SENTRY_DSN environment variable try { @@ -113,6 +114,7 @@ async function runTask({ transaction.finish(); span.end(); Telemetry.removeOtelSpan(); + await flushRemainingSpans(); } catch (error) { debug("Sentry/OpenTelemetry transaction finish error. This does not affect execution."); debug(error);