From ab0f7411711a431a11b862ad8a7f637c126f5290 Mon Sep 17 00:00:00 2001 From: Mochamad Noor Syamsu <9563873+syamsudotdev@users.noreply.github.com> Date: Fri, 15 Mar 2024 10:11:29 +0700 Subject: [PATCH] feat(native fetch): Handle Node.js fetch errors (#1252) * handle fetch errors based on Undici * document error codes --------- Co-authored-by: Denny Pradipta --- dev/prometheus/docker-compose.yaml | 2 +- dev/prometheus/prometheus.yaml | 4 +- docs/src/pages/guides/probes.md | 45 +- src/components/probe/prober/http/request.ts | 445 ++++++++++++++++---- 4 files changed, 384 insertions(+), 112 deletions(-) diff --git a/dev/prometheus/docker-compose.yaml b/dev/prometheus/docker-compose.yaml index d18a85b6b..d539eea94 100644 --- a/dev/prometheus/docker-compose.yaml +++ b/dev/prometheus/docker-compose.yaml @@ -3,7 +3,7 @@ services: image: prom/prometheus container_name: monika_prometheus volumes: - - ./prometheus.yaml:/etc/prometheus/prometheus.yml + - ./prometheus.yaml:/etc/prometheus/prometheus.yml ports: - 9090:9090 network_mode: host diff --git a/dev/prometheus/prometheus.yaml b/dev/prometheus/prometheus.yaml index 366a11d43..b454653bd 100644 --- a/dev/prometheus/prometheus.yaml +++ b/dev/prometheus/prometheus.yaml @@ -2,7 +2,7 @@ global: scrape_interval: 1s scrape_configs: - - job_name: "monika" + - job_name: 'monika' static_configs: - - targets: ["localhost:3001"] + - targets: ['localhost:3001'] diff --git a/docs/src/pages/guides/probes.md b/docs/src/pages/guides/probes.md index fb346ccb1..3ec3380a9 100644 --- a/docs/src/pages/guides/probes.md +++ b/docs/src/pages/guides/probes.md @@ -307,24 +307,33 @@ Probe response data could be used for [Request Chaining](https://hyperjumptech.g To make it easier to troubleshoot HTTP requests, we have mapped low-level errors returned by the HTTP library to numbers between 0 and 99. These custom errors are returned as the HTTP status code and can be used to trigger alerts in the same way as regular HTTP status codes. -| Code | Error | -| :--- | -------------------- | -| 0 | Connection not found | -| 1 | Connection reset | -| 2 | Connection refused | -| 3 | Too many redirects | -| 4 | Bad option value | -| 5 | Bad option | -| 6 | Timed out | -| 7 | Network error | -| 8 | Deprecated | -| 9 | Bad response | -| 11 | Bad request | -| 12 | Canceled | -| 13 | Not Supported | -| 14 | Invalid URL | -| 99 | Others | -| 599 | Connection aborted | +| Code | Error | +| :--- | ----------------------------------------------------------------- | +| 0 | Connection not found | +| 1 | Connection reset | +| 2 | Connection refused | +| 3 | Too many redirects | +| 4 | Bad option value | +| 5 | Bad option | +| 6 | Timed out | +| 7 | Network error | +| 8 | Deprecated | +| 9 | Bad response | +| 11 | Bad request | +| 12 | Canceled | +| 13 | Not Supported | +| 14 | Invalid URL | +| 18 | Header / response size limit exceeded | +| 19 | HTTP status code returns >= 400 | +| 20 | Invalid HTTP arguments | +| 21 | Unexpected HTTP response to handle | +| 22 | Connection closed unexpectedly | +| 23 | Unsupported HTTP functionality | +| 24 | Request / response size mismatch with Content-Length header value | +| 25 | Missing HTTP client pool | +| 26 | Expected error, exact reason is shown on runtime | +| 99 | Others | +| 599 | Connection aborted | ## Execution order diff --git a/src/components/probe/prober/http/request.ts b/src/components/probe/prober/http/request.ts index d3750139b..ae49bdf18 100644 --- a/src/components/probe/prober/http/request.ts +++ b/src/components/probe/prober/http/request.ts @@ -40,6 +40,10 @@ import registerFakes from '../../../../utils/fakes' import { sendHttpRequest, sendHttpRequestFetch } from '../../../../utils/http' import { log } from '../../../../utils/pino' import { AxiosError } from 'axios' +import { MonikaFlags } from 'src/flag' +import { getErrorMessage } from '../../../../utils/catch-error-handler' +import { errors as undiciErrors } from 'undici' +import Joi from 'joi' // Register Handlebars helpers registerFakes(Handlebars) @@ -49,6 +53,10 @@ type probingParams = { responses: Array // an array of previous responses } +const UndiciErrorValidator = Joi.object({ + cause: Joi.object({ name: Joi.string(), code: Joi.string() }), +}) + /** * probing() is the heart of monika requests generation * @param {obj} parameter as input object @@ -76,111 +84,45 @@ export async function httpRequest({ newReq.headers = newHeaders newReq.body = newBody - const requestStartedAt = Date.now() + const startTime = Date.now() try { // is this a request for ping? if (newReq.ping === true) { return icmpRequest({ host: renderedURL }) } + // Do the request using compiled URL and compiled headers (if exists) if (flags['native-fetch']) { - if (flags.verbose) log.info(`Probing ${renderedURL} with Node.js fetch`) - const response = await sendHttpRequestFetch({ - ...newReq, - allowUnauthorizedSsl: allowUnauthorized, - keepalive: true, - url: renderedURL, - maxRedirects: flags['follow-redirects'], - body: - typeof newReq.body === 'string' - ? newReq.body - : JSON.stringify(newReq.body), + return await probeHttpFetch({ + startTime, + flags, + renderedURL, + requestParams: newReq, + allowUnauthorized, }) - - const responseTime = Date.now() - requestStartedAt - let responseHeaders: Record | undefined - if (response.headers) { - responseHeaders = {} - for (const [key, value] of Object.entries(response.headers)) { - responseHeaders[key] = value - } - } - - const responseBody = response.headers - .get('Content-Type') - ?.includes('application/json') - ? response.json() - : response.text() - - return { - requestType: 'HTTP', - data: responseBody, - body: responseBody, - status: response.status, - headers: responseHeaders || '', - responseTime, - result: probeRequestResult.success, - } } - // Do the request using compiled URL and compiled headers (if exists) - const resp = await sendHttpRequest({ - ...newReq, - allowUnauthorizedSsl: allowUnauthorized, - keepalive: true, - url: renderedURL, - maxRedirects: flags['follow-redirects'], - body: - typeof newReq.body === 'string' - ? newReq.body - : JSON.stringify(newReq.body), + return await probeHttpAxios({ + startTime, + flags, + renderedURL, + requestParams: newReq, + allowUnauthorized, }) - - const responseTime = Date.now() - requestStartedAt - const { data, headers, status } = resp - - return { - requestType: 'HTTP', - data, - body: data, - status, - headers, - responseTime, - result: probeRequestResult.success, - } } catch (error: unknown) { - const responseTime = Date.now() - requestStartedAt + const responseTime = Date.now() - startTime if (error instanceof AxiosError) { - // The request was made and the server responded with a status code - // 400, 500 get here - if (error?.response) { - return { - data: '', - body: '', - status: error?.response?.status, - headers: error?.response?.headers, - responseTime, - result: probeRequestResult.success, - error: error?.response?.data, - } - } + return handleAxiosError(responseTime, error) + } - // The request was made but no response was received - // timeout is here, ECONNABORTED, ENOTFOUND, ECONNRESET, ECONNREFUSED - if (error?.request) { - const { status, description } = getErrorStatusWithExplanation(error) - - return { - data: '', - body: '', - status, - headers: '', - responseTime, - result: probeRequestResult.failed, - error: description, - } - } + const { value, error: undiciErrorValidator } = + UndiciErrorValidator.validate(error, { + allowUnknown: true, + }) + + if (!undiciErrorValidator) { + return handleUndiciError(responseTime, value.cause) } // other errors @@ -191,7 +133,7 @@ export async function httpRequest({ headers: '', responseTime, result: probeRequestResult.failed, - error: (error as Error).message, + error: getErrorMessage(error), } } } @@ -280,6 +222,109 @@ function compileBody( return { headers: newHeaders, body: newBody } } +async function probeHttpFetch({ + startTime, + flags, + renderedURL, + requestParams, + allowUnauthorized, +}: { + startTime: number + flags: MonikaFlags + renderedURL: string + allowUnauthorized: boolean | undefined + requestParams: { + method: string | undefined + headers: object | undefined + timeout: number + body: string | object + ping: boolean | undefined + } +}): Promise { + if (flags.verbose) log.info(`Probing ${renderedURL} with Node.js fetch`) + const response = await sendHttpRequestFetch({ + ...requestParams, + allowUnauthorizedSsl: allowUnauthorized, + keepalive: true, + url: renderedURL, + maxRedirects: flags['follow-redirects'], + body: + typeof requestParams.body === 'string' + ? requestParams.body + : JSON.stringify(requestParams.body), + }) + + const responseTime = Date.now() - startTime + let responseHeaders: Record | undefined + if (response.headers) { + responseHeaders = {} + for (const [key, value] of Object.entries(response.headers)) { + responseHeaders[key] = value + } + } + + const responseBody = response.headers + .get('Content-Type') + ?.includes('application/json') + ? response.json() + : response.text() + + return { + requestType: 'HTTP', + data: responseBody, + body: responseBody, + status: response.status, + headers: responseHeaders || '', + responseTime, + result: probeRequestResult.success, + } +} + +async function probeHttpAxios({ + startTime, + flags, + renderedURL, + requestParams, + allowUnauthorized, +}: { + startTime: number + flags: MonikaFlags + renderedURL: string + allowUnauthorized: boolean | undefined + requestParams: { + method: string | undefined + headers: object | undefined + timeout: number + body: string | object + ping: boolean | undefined + } +}): Promise { + const resp = await sendHttpRequest({ + ...requestParams, + allowUnauthorizedSsl: allowUnauthorized, + keepalive: true, + url: renderedURL, + maxRedirects: flags['follow-redirects'], + body: + typeof requestParams.body === 'string' + ? requestParams.body + : JSON.stringify(requestParams.body), + }) + + const responseTime = Date.now() - startTime + const { data, headers, status } = resp + + return { + requestType: 'HTTP', + data, + body: data, + status, + headers, + responseTime, + result: probeRequestResult.success, + } +} + export function generateRequestChainingBody( body: object | string, responses: ProbeRequestResponse[] @@ -326,6 +371,50 @@ function transformContentByType( } } +function handleAxiosError( + responseTime: number, + error: AxiosError +): ProbeRequestResponse { + // The request was made and the server responded with a status code + // 400, 500 get here + if (error?.response) { + return { + data: '', + body: '', + status: error?.response?.status, + headers: error?.response?.headers, + responseTime, + result: probeRequestResult.success, + error: error?.response?.data as string, + } + } + + // The request was made but no response was received + // timeout is here, ECONNABORTED, ENOTFOUND, ECONNRESET, ECONNREFUSED + if (error?.request) { + const { status, description } = getErrorStatusWithExplanation(error) + return { + data: '', + body: '', + status, + headers: '', + responseTime, + result: probeRequestResult.failed, + error: description, + } + } + + return { + data: '', + body: '', + status: 99, + headers: '', + responseTime, + result: probeRequestResult.failed, + error: getErrorMessage(error), + } +} + function getErrorStatusWithExplanation(error: unknown): { status: number description: string @@ -494,3 +583,177 @@ function getErrorStatusWithExplanation(error: unknown): { } // in the event an unlikely unknown error, send here } } + +function handleUndiciError( + responseTime: number, + error: undiciErrors.UndiciError +): ProbeRequestResponse { + if ( + error instanceof undiciErrors.BodyTimeoutError || + error instanceof undiciErrors.ConnectTimeoutError || + error instanceof undiciErrors.HeadersTimeoutError + ) { + return { + data: '', + body: '', + status: 6, + headers: '', + responseTime, + result: probeRequestResult.failed, + error: 'ETIMEDOUT: Connection attempt has timed out.', + } + } + + if (error instanceof undiciErrors.RequestAbortedError) { + // https://httpstatuses.com/599 + return { + data: '', + body: '', + status: 599, + headers: '', + responseTime, + result: probeRequestResult.failed, + error: + 'ECONNABORTED: The connection was unexpectedly terminated, often due to server issues, network problems, or timeouts.', + } + } + + // BEGIN Node.js fetch error status code outside Axios' error handler range (code >= 18) + // fetch's client maxResponseSize and maxHeaderSize is set, limit exceeded + if ( + error instanceof undiciErrors.HeadersOverflowError || + error instanceof undiciErrors.ResponseExceededMaxSizeError + ) { + return { + data: '', + body: '', + status: 18, + headers: '', + responseTime, + result: probeRequestResult.failed, + error: 'ECONNOVERFLOW: Header / response max size exceeded.', + } + } + + // fetch throwOnError is set to true, got HTTP status code >= 400 + if (error instanceof undiciErrors.ResponseStatusCodeError) { + return { + data: '', + body: '', + status: 19, + headers: '', + responseTime, + result: probeRequestResult.failed, + error: 'ERESPONSESTATUSCODE: HTTP status code returns >= 400.', + } + } + + // invalid fetch argument passed + if (error instanceof undiciErrors.InvalidArgumentError) { + return { + data: '', + body: '', + status: 20, + headers: '', + responseTime, + result: probeRequestResult.failed, + error: 'EINVALIDARGUMENT: Invalid HTTP arguments.', + } + } + + // fetch failed to handle return value + if (error instanceof undiciErrors.InvalidReturnValueError) { + return { + data: '', + body: '', + status: 21, + headers: '', + responseTime, + result: probeRequestResult.failed, + error: 'EINVALIDRETURN: Unexpected HTTP response to handle.', + } + } + + if ( + error instanceof undiciErrors.ClientClosedError || + error instanceof undiciErrors.ClientDestroyedError || + error instanceof undiciErrors.SocketError + ) { + return { + data: '', + body: '', + status: 22, + headers: '', + responseTime, + result: probeRequestResult.failed, + error: 'ECONNCLOSED: HTTP client closed unexpectedly.', + } + } + + if (error instanceof undiciErrors.NotSupportedError) { + return { + data: '', + body: '', + status: 23, + headers: '', + responseTime, + result: probeRequestResult.failed, + error: 'ESUPPORT: Unsupported HTTP functionality.', + } + } + + if ( + error instanceof undiciErrors.RequestContentLengthMismatchError || + error instanceof undiciErrors.ResponseContentLengthMismatchError + ) { + return { + data: '', + body: '', + status: 24, + headers: '', + responseTime, + result: probeRequestResult.failed, + error: + 'ECONTENTLENGTH: Request / response content length mismatch with Content-Length header value.', + } + } + + // inline docs in Undici state that this would never happen, + // but they declare and throw this condition anyway + if (error instanceof undiciErrors.BalancedPoolMissingUpstreamError) { + return { + data: '', + body: '', + status: 25, + headers: '', + responseTime, + result: probeRequestResult.failed, + error: `EMISSINGPOOL: Missing HTTP client pool.`, + } + } + + // expected error from fetch, but exact reason is in the message string + // error messages are unpredictable + // reference https://github.com/search?q=repo:nodejs/undici+new+InformationalError(&type=code + if (error instanceof undiciErrors.InformationalError) { + return { + data: '', + body: '', + status: 26, + headers: '', + responseTime, + result: probeRequestResult.failed, + error: `EINFORMATIONAL: ${error.message}.`, + } + } + + return { + data: '', + body: '', + status: 99, + headers: '', + responseTime, + result: probeRequestResult.failed, + error: getErrorMessage(error), + } +}