From 178baa8468b3252bbebb6074baf0e59c9916d1a3 Mon Sep 17 00:00:00 2001 From: Bharat Pasupula <123897612+bhapas@users.noreply.github.com> Date: Thu, 5 Dec 2024 20:18:08 +0100 Subject: [PATCH] [Automatic Import] Restrict unsupported log formats (#202994) --- .../common/api/generation_error.ts | 1 + .../common/api/model/common_attributes.gen.ts | 2 ++ .../api/model/common_attributes.schema.yaml | 2 ++ .../steps/data_stream_step/translations.ts | 18 +++++++--- .../graphs/log_type_detection/prompts.ts | 2 ++ .../server/lib/errors/unsupported_error.ts | 35 ++++++++++++++++--- .../server/routes/analyze_logs_routes.ts | 16 +++++++-- 7 files changed, 64 insertions(+), 12 deletions(-) diff --git a/x-pack/plugins/integration_assistant/common/api/generation_error.ts b/x-pack/plugins/integration_assistant/common/api/generation_error.ts index 03f01e96bee53..9c7fe4888712c 100644 --- a/x-pack/plugins/integration_assistant/common/api/generation_error.ts +++ b/x-pack/plugins/integration_assistant/common/api/generation_error.ts @@ -34,6 +34,7 @@ export function isGenerationErrorBody(obj: unknown | undefined): obj is Generati export interface GenerationErrorAttributes { errorCode: GenerationErrorCode; underlyingMessages?: string[] | undefined; + logFormat?: string | undefined; errorMessageWithLink?: ErrorMessageWithLink | undefined; } diff --git a/x-pack/plugins/integration_assistant/common/api/model/common_attributes.gen.ts b/x-pack/plugins/integration_assistant/common/api/model/common_attributes.gen.ts index 3b8dac7af22ca..a70610aeaaaab 100644 --- a/x-pack/plugins/integration_assistant/common/api/model/common_attributes.gen.ts +++ b/x-pack/plugins/integration_assistant/common/api/model/common_attributes.gen.ts @@ -85,6 +85,8 @@ export const SamplesFormatName = z.enum([ 'unstructured', 'unsupported', 'cef', + 'leef', + 'fix', ]); export type SamplesFormatNameEnum = typeof SamplesFormatName.enum; export const SamplesFormatNameEnum = SamplesFormatName.enum; diff --git a/x-pack/plugins/integration_assistant/common/api/model/common_attributes.schema.yaml b/x-pack/plugins/integration_assistant/common/api/model/common_attributes.schema.yaml index 23ad137d8d83a..35bd6b7ad6755 100644 --- a/x-pack/plugins/integration_assistant/common/api/model/common_attributes.schema.yaml +++ b/x-pack/plugins/integration_assistant/common/api/model/common_attributes.schema.yaml @@ -65,6 +65,8 @@ components: - unstructured - unsupported - cef + - leef + - fix SamplesFormat: type: object diff --git a/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/translations.ts b/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/translations.ts index 8f2aab49622f0..cbabd8aa0d62b 100644 --- a/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/translations.ts +++ b/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/translations.ts @@ -212,12 +212,20 @@ export const GENERATION_ERROR_TRANSLATION: Record< defaultMessage: 'Max attempts exceeded. Please try again.', } ), - [GenerationErrorCode.UNSUPPORTED_LOG_SAMPLES_FORMAT]: i18n.translate( - 'xpack.integrationAssistant.errors.unsupportedLogSamples', - { - defaultMessage: 'Unsupported log format in the samples.', + [GenerationErrorCode.UNSUPPORTED_LOG_SAMPLES_FORMAT]: (attributes) => { + if (attributes.logFormat !== undefined && attributes.logFormat?.length !== 0) { + return i18n.translate('xpack.integrationAssistant.errors.uparseableCSV.withReason', { + values: { + format: attributes.logFormat, + }, + defaultMessage: `Unsupported log format in the samples (format: {format}).`, + }); + } else { + return i18n.translate('xpack.integrationAssistant.errors.unsupportedLogSamples', { + defaultMessage: `Unsupported log format in the samples.`, + }); } - ), + }, [GenerationErrorCode.CEF_ERROR]: i18n.translate('xpack.integrationAssistant.errors.cefError', { // This is a default error message if the linking does not work. defaultMessage: diff --git a/x-pack/plugins/integration_assistant/server/graphs/log_type_detection/prompts.ts b/x-pack/plugins/integration_assistant/server/graphs/log_type_detection/prompts.ts index 09a7249a3786c..cc30e90605897 100644 --- a/x-pack/plugins/integration_assistant/server/graphs/log_type_detection/prompts.ts +++ b/x-pack/plugins/integration_assistant/server/graphs/log_type_detection/prompts.ts @@ -24,6 +24,8 @@ Follow these steps to do this: * 'structured': If the log samples have structured message body with key-value pairs then classify it as "name: structured". Look for a flat list of key-value pairs, often separated by some delimiters. Consider variations in formatting, such as quotes around values ("key=value", key="value"), special characters in keys or values, or escape sequences. * 'unstructured': If the log samples have unstructured body like a free-form text then classify it as "name: unstructured". * 'cef': If the log samples have Common Event Format (CEF) then classify it as "name: cef". + * 'leef': If the log samples have Log Event Extended Format (LEEF) then classify it as "name: leef". + * 'fix': If the log samples have Financial Information eXchange (FIX) then classify it as "name: fix". * 'unsupported': If you cannot put the format into any of the above categories then classify it with "name: unsupported". 2. Header: for structured and unstructured format: - if the samples have any or all of priority, timestamp, loglevel, hostname, ipAddress, messageId in the beginning information then set "header: true". diff --git a/x-pack/plugins/integration_assistant/server/lib/errors/unsupported_error.ts b/x-pack/plugins/integration_assistant/server/lib/errors/unsupported_error.ts index 7ab4e0569ca83..0d21faefa4432 100644 --- a/x-pack/plugins/integration_assistant/server/lib/errors/unsupported_error.ts +++ b/x-pack/plugins/integration_assistant/server/lib/errors/unsupported_error.ts @@ -9,18 +9,45 @@ import { KibanaResponseFactory } from '@kbn/core/server'; import { ErrorThatHandlesItsOwnResponse } from './types'; import { GenerationErrorCode } from '../../../common/constants'; +interface UnsupportedLogFormat { + message: string; + logFormat?: string; +} + +interface UnsupportedLogFormatResponseBody { + message: string; + attributes: { + errorCode: string; + logFormat?: string; + }; +} + export class UnsupportedLogFormatError extends Error implements ErrorThatHandlesItsOwnResponse { private readonly errorCode: string = GenerationErrorCode.UNSUPPORTED_LOG_SAMPLES_FORMAT; + private logFormat: string | undefined; - // eslint-disable-next-line @typescript-eslint/no-useless-constructor - constructor(message: string) { - super(message); + constructor(unsupportedLogFormat: UnsupportedLogFormat) { + super(unsupportedLogFormat.message); + if (unsupportedLogFormat.logFormat) { + this.logFormat = unsupportedLogFormat.logFormat; + } } public sendResponse(res: KibanaResponseFactory) { + const responseBody: UnsupportedLogFormatResponseBody = { + message: this.message, + attributes: { + errorCode: this.errorCode, + }, + }; + + if (this.logFormat) { + responseBody.attributes.logFormat = this.logFormat; + } + return res.customError({ statusCode: 501, - body: { message: this.message, attributes: { errorCode: this.errorCode } }, + body: responseBody, }); } } diff --git a/x-pack/plugins/integration_assistant/server/routes/analyze_logs_routes.ts b/x-pack/plugins/integration_assistant/server/routes/analyze_logs_routes.ts index 37926dac19156..93ac55f6f712c 100644 --- a/x-pack/plugins/integration_assistant/server/routes/analyze_logs_routes.ts +++ b/x-pack/plugins/integration_assistant/server/routes/analyze_logs_routes.ts @@ -106,11 +106,21 @@ export function registerAnalyzeLogsRoutes( switch (graphLogFormat) { case 'unsupported': - throw new UnsupportedLogFormatError( - GenerationErrorCode.UNSUPPORTED_LOG_SAMPLES_FORMAT - ); + throw new UnsupportedLogFormatError({ + message: GenerationErrorCode.UNSUPPORTED_LOG_SAMPLES_FORMAT, + }); case 'cef': throw new CefError(GenerationErrorCode.CEF_ERROR); + case 'leef': + throw new UnsupportedLogFormatError({ + message: GenerationErrorCode.UNSUPPORTED_LOG_SAMPLES_FORMAT, + logFormat: 'Log Event Extended Format (LEEF)', + }); + case 'fix': + throw new UnsupportedLogFormatError({ + message: GenerationErrorCode.UNSUPPORTED_LOG_SAMPLES_FORMAT, + logFormat: 'Financial Information eXchange (FIX)', + }); } return res.ok({ body: AnalyzeLogsResponse.parse(graphResults) });