diff --git a/.changeset/thick-snails-explain.md b/.changeset/thick-snails-explain.md new file mode 100644 index 00000000..f389b89a --- /dev/null +++ b/.changeset/thick-snails-explain.md @@ -0,0 +1,5 @@ +--- +'@hyperdx/node-opentelemetry': patch +--- + +Add the networkHeadersCapture and networkBodyCapture flags. Users can decide whether to capture the headers or body information through these two flags. diff --git a/examples/hello-node-express/index.js b/examples/hello-node-express/index.js index 0b0f3ea0..88591c36 100644 --- a/examples/hello-node-express/index.js +++ b/examples/hello-node-express/index.js @@ -3,6 +3,8 @@ const { context, metrics, propagation, trace } = require('@opentelemetry/api'); initSDK({ // advancedNetworkCapture: true, + networkBodyCapture: false, + networkHeadersCapture: false, // betaMode: true, // consoleCapture: true, }); diff --git a/packages/node-opentelemetry/README.md b/packages/node-opentelemetry/README.md index 3ab0f477..ad41ee14 100644 --- a/packages/node-opentelemetry/README.md +++ b/packages/node-opentelemetry/README.md @@ -192,10 +192,11 @@ export HDX_NODE_BETA_MODE=1 By enabling advanced network capture, the SDK will additionally capture full HTTP request/response headers and bodies for all inbound/outbound HTTP requests, to help with more in-depth request debugging. -This can be accomplished by setting `HDX_NODE_ADVANCED_NETWORK_CAPTURE` environment variable to 1. +This can be accomplished by setting `HDX_NODE_NETWORK_HEADERS_CAPTURE` and `HDX_NODE_NETWORK_BODY_CAPTURE` environment variables to 1. ```sh -export HDX_NODE_ADVANCED_NETWORK_CAPTURE=1 +export HDX_NODE_NETWORK_HEADERS_CAPTURE=1 +export HDX_NODE_NETWORK_BODY_CAPTURE=1 ``` By default, all request/response headers will be captured. You can specify a custom list of headers to capture diff --git a/packages/node-opentelemetry/__tests__/packageJsonComp.test.ts b/packages/node-opentelemetry/__tests__/packageJsonComp.test.ts new file mode 100644 index 00000000..5d0fa55f --- /dev/null +++ b/packages/node-opentelemetry/__tests__/packageJsonComp.test.ts @@ -0,0 +1,51 @@ +import { comparePackageVersions } from '../utils/comparison'; + +describe('comparePackageJsonVersions', () => { + test('equal versions return true for ==', () => { + expect(comparePackageVersions('1.0.0', '==', '1.0.0')).toBe(true); + }); + + test('different versions return false for ==', () => { + expect(comparePackageVersions('1.0.0', '==', '1.0.1')).toBe(false); + }); + + test('higher version returns true for >', () => { + expect(comparePackageVersions('1.0.1', '>', '1.0.0')).toBe(true); + }); + + test('lower version returns false for >', () => { + expect(comparePackageVersions('1.0.0', '>', '1.0.1')).toBe(false); + }); + + test('higher version returns true for >= when versions are equal', () => { + expect(comparePackageVersions('1.0.0', '>=', '1.0.0')).toBe(true); + }); + + test('higher version returns true for >=', () => { + expect(comparePackageVersions('1.0.1', '>=', '1.0.0')).toBe(true); + }); + + test('lower version returns false for >=', () => { + expect(comparePackageVersions('1.0.0', '>=', '1.0.1')).toBe(false); + }); + + test('lower version returns true for <', () => { + expect(comparePackageVersions('1.0.0', '<', '1.0.1')).toBe(true); + }); + + test('higher version returns false for <', () => { + expect(comparePackageVersions('1.0.1', '<', '1.0.0')).toBe(false); + }); + + test('lower version returns true for <= when versions are equal', () => { + expect(comparePackageVersions('1.0.0', '<=', '1.0.0')).toBe(true); + }); + + test('lower version returns true for <=', () => { + expect(comparePackageVersions('1.0.0', '<=', '1.0.1')).toBe(true); + }); + + test('higher version returns false for <=', () => { + expect(comparePackageVersions('1.0.1', '<=', '1.0.0')).toBe(false); + }); +}); diff --git a/packages/node-opentelemetry/src/instrumentations/http.ts b/packages/node-opentelemetry/src/instrumentations/http.ts index 751acdf9..d96fdf1d 100644 --- a/packages/node-opentelemetry/src/instrumentations/http.ts +++ b/packages/node-opentelemetry/src/instrumentations/http.ts @@ -80,240 +80,270 @@ export const _handleHttpOutgoingClientRequest = ( request: http.ClientRequest, span: Span, shouldRecordBody: (body: string) => boolean, + networkHeadersCapture: boolean, + networkBodyCapture: boolean, httpCaptureHeadersClientRequest?: string, ) => { /* Capture Headers */ - try { - const headers = - splitCommaSeparatedStrings(httpCaptureHeadersClientRequest) ?? - request.getRawHeaderNames(); - headerCapture('request', headers)(span, (header) => - request.getHeader(header), - ); - } catch (e) { - hdx(`error parsing outgoing-request headers in requestHook: ${e}`); - } - - /* Capture Body */ - const chunks = []; - const oldWrite = request.write.bind(request); - request.write = (data: any) => { + if (networkHeadersCapture) { try { - if (typeof data === 'string') { - chunks.push(Buffer.from(data)); - } else { - chunks.push(data); - } + const headers = + splitCommaSeparatedStrings(httpCaptureHeadersClientRequest) ?? + request.getRawHeaderNames(); + headerCapture('request', headers)(span, (header) => + request.getHeader(header), + ); } catch (e) { - hdx(`error in request.write: ${e}`); + hdx(`error parsing outgoing-request headers in requestHook: ${e}`); } - return oldWrite(data); - }; - const oldEnd = request.end.bind(request); - request.end = (data: any) => { - try { - if (data) { + } + + /* Capture Body */ + if (networkBodyCapture) { + const chunks = []; + const oldWrite = request.write.bind(request); + request.write = (data: any) => { + try { if (typeof data === 'string') { chunks.push(Buffer.from(data)); } else { chunks.push(data); } + } catch (e) { + hdx(`error in request.write: ${e}`); } - if (chunks.length > 0) { - const body = Buffer.concat(chunks).toString('utf8'); - if (shouldRecordBody(body)) { - span.setAttribute('http.request.body', body); - } else { - span.setAttribute('http.request.body', SENSITIVE_DATA_SUBSTITUTE); + return oldWrite(data); + }; + const oldEnd = request.end.bind(request); + request.end = (data: any) => { + try { + if (data) { + if (typeof data === 'string') { + chunks.push(Buffer.from(data)); + } else { + chunks.push(data); + } + } + if (chunks.length > 0) { + const body = Buffer.concat(chunks).toString('utf8'); + if (shouldRecordBody(body)) { + span.setAttribute('http.request.body', body); + } else { + span.setAttribute('http.request.body', SENSITIVE_DATA_SUBSTITUTE); + } } + } catch (e) { + hdx(`error in request.end: ${e}`); } - } catch (e) { - hdx(`error in request.end: ${e}`); - } - return oldEnd(data); - }; + return oldEnd(data); + }; + } }; export const _handleHttpIncomingServerRequest = ( request: http.IncomingMessage, span: Span, shouldRecordBody: (body: string) => boolean, + networkHeadersCapture: boolean, + networkBodyCapture: boolean, httpCaptureHeadersServerRequest?: string, ) => { /* Capture Headers */ - try { - const headers = - splitCommaSeparatedStrings(httpCaptureHeadersServerRequest) ?? - request.headers; - headerCapture('request', Object.keys(headers))( - span, - (header) => headers[header], - ); - } catch (e) { - hdx(`error parsing incoming-request headers in requestHook: ${e}`); - } - - /* Capture Body */ - const chunks = []; - const pt = new PassThrough(); - pt.on('data', (chunk) => { + if (networkHeadersCapture) { try { - if (typeof chunk === 'string') { - chunks.push(Buffer.from(chunk)); - } else { - chunks.push(chunk); - } + const headers = + splitCommaSeparatedStrings(httpCaptureHeadersServerRequest) ?? + request.headers; + headerCapture('request', Object.keys(headers))( + span, + (header) => headers[header], + ); } catch (e) { - hdx(`error in request.on('data'): ${e}`); + hdx(`error parsing incoming-request headers in requestHook: ${e}`); } - }).on('end', () => { - try { - if (chunks.length > 0) { - const body = Buffer.concat(chunks).toString('utf8'); - if (shouldRecordBody(body)) { - span.setAttribute('http.request.body', body); + } + + /* Capture Body */ + if (networkBodyCapture) { + const chunks = []; + const pt = new PassThrough(); + pt.on('data', (chunk) => { + try { + if (typeof chunk === 'string') { + chunks.push(Buffer.from(chunk)); } else { - span.setAttribute('http.request.body', SENSITIVE_DATA_SUBSTITUTE); + chunks.push(chunk); } + } catch (e) { + hdx(`error in request.on('data'): ${e}`); } - } catch (e) { - hdx(`error in request.on('end'): ${e}`); - } - }); - interceptReadableStream(request, pt); + }).on('end', () => { + try { + if (chunks.length > 0) { + const body = Buffer.concat(chunks).toString('utf8'); + if (shouldRecordBody(body)) { + span.setAttribute('http.request.body', body); + } else { + span.setAttribute('http.request.body', SENSITIVE_DATA_SUBSTITUTE); + } + } + } catch (e) { + hdx(`error in request.on('end'): ${e}`); + } + }); + interceptReadableStream(request, pt); + } }; export const _handleHttpIncomingServerResponse = ( response: http.ServerResponse, span: Span, shouldRecordBody: (body: string) => boolean, + networkHeadersCapture: boolean, + networkBodyCapture: boolean, httpCaptureHeadersServerResponse?: string, ) => { - /* Capture Body */ - const chunks = []; - const oldWrite = response.write.bind(response); - response.write = (data: any) => { + /* Capture Headers */ + if (networkHeadersCapture) { try { - if (typeof data === 'string') { - chunks.push(Buffer.from(data)); - } else { - chunks.push(data); - } + const headers = + splitCommaSeparatedStrings(httpCaptureHeadersServerResponse) ?? + response.getHeaderNames(); + headerCapture('response', headers)(span, (header) => + response.getHeader(header), + ); } catch (e) { - hdx(`error in response.write: ${e}`); + hdx(`error parsing incoming-response headers in responseHook: ${e}`); } - return oldWrite(data); - }; - const oldEnd = response.end.bind(response); - response.end = (data: any) => { - try { - if (data) { + } + + /* Capture Body */ + if (networkBodyCapture) { + const chunks = []; + const oldWrite = response.write.bind(response); + response.write = (data: any) => { + try { if (typeof data === 'string') { chunks.push(Buffer.from(data)); } else { chunks.push(data); } + } catch (e) { + hdx(`error in response.write: ${e}`); } - if (chunks.length > 0) { - const buffers = Buffer.concat(chunks); - let body = buffers.toString('utf8'); - const isGzip = response.getHeader('content-encoding') === 'gzip'; - if (isGzip) { - body = zlib.gunzipSync(buffers).toString('utf8'); + return oldWrite(data); + }; + const oldEnd = response.end.bind(response); + response.end = (data: any) => { + try { + if (data) { + if (typeof data === 'string') { + chunks.push(Buffer.from(data)); + } else { + chunks.push(data); + } } - if (shouldRecordBody(body)) { - span.setAttribute('http.response.body', body); - } else { - span.setAttribute('http.response.body', SENSITIVE_DATA_SUBSTITUTE); + if (chunks.length > 0) { + const buffers = Buffer.concat(chunks); + let body = buffers.toString('utf8'); + const isGzip = response.getHeader('content-encoding') === 'gzip'; + if (isGzip) { + body = zlib.gunzipSync(buffers).toString('utf8'); + } + if (shouldRecordBody(body)) { + span.setAttribute('http.response.body', body); + } else { + span.setAttribute('http.response.body', SENSITIVE_DATA_SUBSTITUTE); + } } + } catch (e) { + hdx(`error in response.end: ${e}`); } - } catch (e) { - hdx(`error in response.end: ${e}`); - } - - /* Capture Headers */ - try { - const headers = - splitCommaSeparatedStrings(httpCaptureHeadersServerResponse) ?? - response.getHeaderNames(); - headerCapture('response', headers)(span, (header) => - response.getHeader(header), - ); - } catch (e) { - hdx(`error parsing incoming-response headers in responseHook: ${e}`); - } - return oldEnd(data); - }; + return oldEnd(data); + }; + } }; export const _handleHttpOutgoingClientResponse = ( response: http.IncomingMessage, span: Span, shouldRecordBody: (body: string) => boolean, + networkHeadersCapture: boolean, + networkBodyCapture: boolean, httpCaptureHeadersClientResponse?: string, ) => { /* Capture Headers */ - try { - const headers = - splitCommaSeparatedStrings(httpCaptureHeadersClientResponse) ?? - response.headers; - headerCapture('response', Object.keys(headers))( - span, - (header) => headers[header], - ); - } catch (e) { - hdx(`error parsing outgoing-response headers in responseHook: ${e}`); - } - - /* Capture Body */ - const chunks = []; - const pt = new PassThrough(); - pt.on('data', (chunk) => { + if (networkHeadersCapture) { try { - if (typeof chunk === 'string') { - chunks.push(Buffer.from(chunk)); - } else { - chunks.push(chunk); - } + const headers = + splitCommaSeparatedStrings(httpCaptureHeadersClientResponse) ?? + response.headers; + headerCapture('response', Object.keys(headers))( + span, + (header) => headers[header], + ); } catch (e) { - hdx(`error in response.on('data'): ${e}`); + hdx(`error parsing outgoing-response headers in responseHook: ${e}`); } - }).on('end', () => { - try { - if (chunks.length > 0) { - const buffers = Buffer.concat(chunks); - let body = buffers.toString('utf8'); - const isGzip = response.headers['content-encoding'] === 'gzip'; - if (isGzip) { - body = zlib.gunzipSync(buffers).toString('utf8'); - } - if (shouldRecordBody(body)) { - span.setAttribute('http.response.body', body); + } + + /* Capture Body */ + if (networkBodyCapture) { + const chunks = []; + const pt = new PassThrough(); + pt.on('data', (chunk) => { + try { + if (typeof chunk === 'string') { + chunks.push(Buffer.from(chunk)); } else { - span.setAttribute('http.response.body', SENSITIVE_DATA_SUBSTITUTE); + chunks.push(chunk); } + } catch (e) { + hdx(`error in response.on('data'): ${e}`); } - } catch (e) { - hdx(`error in response.on('end'): ${e}`); - } - }); - interceptReadableStream(response, pt); + }).on('end', () => { + try { + if (chunks.length > 0) { + const buffers = Buffer.concat(chunks); + let body = buffers.toString('utf8'); + const isGzip = response.headers['content-encoding'] === 'gzip'; + if (isGzip) { + body = zlib.gunzipSync(buffers).toString('utf8'); + } + if (shouldRecordBody(body)) { + span.setAttribute('http.response.body', body); + } else { + span.setAttribute('http.response.body', SENSITIVE_DATA_SUBSTITUTE); + } + } + } catch (e) { + hdx(`error in response.on('end'): ${e}`); + } + }); + interceptReadableStream(response, pt); + } }; export const getHyperDXHTTPInstrumentationConfig = ({ + networkHeadersCapture, + networkBodyCapture, httpCaptureBodyKeywordsFilter, httpCaptureHeadersClientRequest, httpCaptureHeadersClientResponse, httpCaptureHeadersServerRequest, httpCaptureHeadersServerResponse, }: { + networkHeadersCapture: boolean; + networkBodyCapture: boolean; httpCaptureBodyKeywordsFilter?: string; httpCaptureHeadersClientRequest?: string; httpCaptureHeadersClientResponse?: string; httpCaptureHeadersServerRequest?: string; httpCaptureHeadersServerResponse?: string; }) => { - const shouldRecordBody = getShouldRecordBody(httpCaptureBodyKeywordsFilter); + const shouldRecordBody = networkBodyCapture + ? getShouldRecordBody(httpCaptureBodyKeywordsFilter) + : (body: string) => false; return { requestHook: ( span: Span, @@ -325,6 +355,8 @@ export const getHyperDXHTTPInstrumentationConfig = ({ request, span, shouldRecordBody, + networkHeadersCapture, + networkBodyCapture, httpCaptureHeadersClientRequest, ); } else { @@ -333,6 +365,8 @@ export const getHyperDXHTTPInstrumentationConfig = ({ request, span, shouldRecordBody, + networkHeadersCapture, + networkBodyCapture, httpCaptureHeadersServerRequest, ); } @@ -347,6 +381,8 @@ export const getHyperDXHTTPInstrumentationConfig = ({ response, span, shouldRecordBody, + networkHeadersCapture, + networkBodyCapture, httpCaptureHeadersServerResponse, ); } else { @@ -355,6 +391,8 @@ export const getHyperDXHTTPInstrumentationConfig = ({ response, span, shouldRecordBody, + networkHeadersCapture, + networkBodyCapture, httpCaptureHeadersClientResponse, ); } diff --git a/packages/node-opentelemetry/src/otel.ts b/packages/node-opentelemetry/src/otel.ts index 3fb098e9..82a1295d 100644 --- a/packages/node-opentelemetry/src/otel.ts +++ b/packages/node-opentelemetry/src/otel.ts @@ -16,6 +16,7 @@ import hdx, { import { getHyperDXHTTPInstrumentationConfig } from './instrumentations/http'; import { hyperDXGlobalContext } from './context'; import { version as PKG_VERSION } from '../package.json'; +import { comparePackageVersions } from '../utils/comparison'; const LOG_PREFIX = `⚠️ ${_LOG_PREFIX}`; @@ -23,6 +24,8 @@ const env = process.env; export type SDKConfig = { advancedNetworkCapture?: boolean; + networkHeadersCapture?: boolean; + networkBodyCapture?: boolean; betaMode?: boolean; consoleCapture?: boolean; instrumentations?: InstrumentationConfigMap; @@ -68,6 +71,13 @@ export const initSDK = (config: SDKConfig) => { service: env.OTEL_SERVICE_NAME, }); + if (config.advancedNetworkCapture) { + if (comparePackageVersions(PKG_VERSION, '>', '0.6.1')) { + throw new Error( + `${LOG_PREFIX} The "advancedNetworkCapture" flag is no longer available. Please use "networkHeadersCapture" and "networkBodyCapture" instead.`, + ); + } + } sdk = new NodeSDK({ resource: new Resource({ 'hyperdx.distro.version': PKG_VERSION, @@ -89,18 +99,23 @@ export const initSDK = (config: SDKConfig) => { }), instrumentations: [ getNodeAutoInstrumentations({ - '@opentelemetry/instrumentation-http': config.advancedNetworkCapture - ? getHyperDXHTTPInstrumentationConfig({ - httpCaptureHeadersClientRequest: - env.OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_CLIENT_REQUEST, - httpCaptureHeadersClientResponse: - env.OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_CLIENT_RESPONSE, - httpCaptureHeadersServerRequest: - env.OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST, - httpCaptureHeadersServerResponse: - env.OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE, - }) - : { enabled: true }, + '@opentelemetry/instrumentation-http': + config.networkBodyCapture || + config.networkHeadersCapture || + config.advancedNetworkCapture + ? getHyperDXHTTPInstrumentationConfig({ + networkHeadersCapture: config.networkHeadersCapture, + networkBodyCapture: config.networkBodyCapture, + httpCaptureHeadersClientRequest: + env.OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_CLIENT_REQUEST, + httpCaptureHeadersClientResponse: + env.OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_CLIENT_RESPONSE, + httpCaptureHeadersServerRequest: + env.OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST, + httpCaptureHeadersServerResponse: + env.OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE, + }) + : { enabled: true }, // FIXME: issue detected with fs instrumentation (infinite loop) '@opentelemetry/instrumentation-fs': { enabled: false, @@ -116,6 +131,8 @@ export const initSDK = (config: SDKConfig) => { `${LOG_PREFIX} Tracing is enabled with configs (${JSON.stringify( { advancedNetworkCapture: config.advancedNetworkCapture, + networkHeadersCapture: config.networkHeadersCapture, + networkBodyCapture: config.networkBodyCapture, betaMode: config.betaMode, consoleCapture: consoleInstrumentationEnabled, endpoint: env.OTEL_EXPORTER_OTLP_ENDPOINT, diff --git a/packages/node-opentelemetry/src/tracing.ts b/packages/node-opentelemetry/src/tracing.ts index d5e40cc9..a1fed1d9 100644 --- a/packages/node-opentelemetry/src/tracing.ts +++ b/packages/node-opentelemetry/src/tracing.ts @@ -8,9 +8,8 @@ const env = process.env; initSDK({ betaMode: stringToBoolean(env.HDX_NODE_BETA_MODE), consoleCapture: stringToBoolean(env.HDX_NODE_CONSOLE_CAPTURE), - advancedNetworkCapture: stringToBoolean( - env.HDX_NODE_ADVANCED_NETWORK_CAPTURE, - ), + networkHeadersCapture: stringToBoolean(env.HDX_NODE_NETWORK_HEADERS_CAPTURE), + networkBodyCapture: stringToBoolean(env.HDX_NODE_NETWORK_BODY_CAPTURE), stopOnTerminationSignals: stringToBoolean(env.HDX_NODE_STOP_ON_TERMINATION_SIGNALS) ?? true, }); diff --git a/packages/node-opentelemetry/utils/comparison.ts b/packages/node-opentelemetry/utils/comparison.ts new file mode 100644 index 00000000..8600d957 --- /dev/null +++ b/packages/node-opentelemetry/utils/comparison.ts @@ -0,0 +1,24 @@ +/** + * Compare the version of package.json + */ +export function comparePackageVersions( + v1: string, + comparator: string, + v2: string, +): boolean { + const v1parts = v1.split('.').map(Number); + const v2parts = v2.split('.').map(Number); + const maxLength = Math.max(v1parts.length, v2parts.length); + + for (let i = 0; i < maxLength; i++) { + const v1part = v1parts[i] || 0; + const v2part = v2parts[i] || 0; + + if (v1part === v2part) continue; + + if (v1part > v2part) return comparator === '>' || comparator === '>='; + if (v1part < v2part) return comparator === '<' || comparator === '<='; + } + + return comparator === '==' || comparator === '>=' || comparator === '<='; +}