From 750827669aa1de72430ea9785815ea1884b22189 Mon Sep 17 00:00:00 2001 From: Hanna Tamoudi Date: Fri, 27 Dec 2024 18:24:02 +0100 Subject: [PATCH] [Automatic Import] add timestamp to ECS constants (#204931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Ensure mapping contains `@timestamp` field whenever possible. https://github.com/elastic/kibana/issues/196040 Tested cases: | test case | has `@timestamp` | is expected result | |---|---|---| | sample logs with datetime value for `time` and `expires` fields | yes (picked `time` as `@timestamp`) | ✅ | | sample logs with datetime value for `expires` field only | no | ✅ | | sample logs with no datetime values | no | ✅ | | sample logs with `created_at` field that does not contain datetime value | no | ✅ | Tested values for `time` field: | value | match correctly `@timestamp` | |---|---| | `2024-02-24T06:56:50.648137154Z` | ✅ | | `10/01/2023 12:34:56` | ✅ | | `01-10-2023 12:34:56` | ✅ | | `Thu, 25 December 2023 10:15:00GMT` | ✅ --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Ilya Nikokoshev (cherry picked from commit 4cc6952c83de1bd7eacd95c458bbe6c281b364b0) --- .../server/graphs/ecs/constants.ts | 1 + .../server/graphs/ecs/prompts.ts | 1 + .../server/graphs/ecs/validate.test.ts | 48 +++++++++++++++++++ .../integration_assistant/tsconfig.json | 2 +- 4 files changed, 51 insertions(+), 1 deletion(-) diff --git a/x-pack/platform/plugins/shared/integration_assistant/server/graphs/ecs/constants.ts b/x-pack/platform/plugins/shared/integration_assistant/server/graphs/ecs/constants.ts index 48cac5626f9fe..ea39910b5a4ee 100644 --- a/x-pack/platform/plugins/shared/integration_assistant/server/graphs/ecs/constants.ts +++ b/x-pack/platform/plugins/shared/integration_assistant/server/graphs/ecs/constants.ts @@ -1681,6 +1681,7 @@ export const ECS_TYPES: EcsFields = { }; export const ECS_FIELDS: EcsFields = { + '@timestamp': 'Date/time when the event originated.', 'as.number': 'Unique number allocated to the autonomous system.', 'as.organization.name': 'Organization name of the autonomous system.', 'client.address': 'Client network address.', diff --git a/x-pack/platform/plugins/shared/integration_assistant/server/graphs/ecs/prompts.ts b/x-pack/platform/plugins/shared/integration_assistant/server/graphs/ecs/prompts.ts index fab18e0decdbd..ed1962a856f11 100644 --- a/x-pack/platform/plugins/shared/integration_assistant/server/graphs/ecs/prompts.ts +++ b/x-pack/platform/plugins/shared/integration_assistant/server/graphs/ecs/prompts.ts @@ -37,6 +37,7 @@ Go through each value step by step and modify it with the following process: 9. When you want to use an ECS field as a value for a target, but another field already has the same ECS field as its target, try to find another fitting ECS field. If none is found then the one you are least confident about should have the object replaced with null. 10. If you are not confident for a specific field, you should always set the value to null. 11. These {package_name} log samples are based on source and destination type data, prioritize these compared to other related ECS fields like host.* and observer.*. +12. Whenever possible, map the @timestamp field to the relevant field that contains the event creation date. You ALWAYS follow these guidelines when writing your response: diff --git a/x-pack/platform/plugins/shared/integration_assistant/server/graphs/ecs/validate.test.ts b/x-pack/platform/plugins/shared/integration_assistant/server/graphs/ecs/validate.test.ts index 39c4e3ac4bab3..a63819b4c83f9 100644 --- a/x-pack/platform/plugins/shared/integration_assistant/server/graphs/ecs/validate.test.ts +++ b/x-pack/platform/plugins/shared/integration_assistant/server/graphs/ecs/validate.test.ts @@ -5,12 +5,15 @@ * 2.0. */ +import { ecsTestState } from '../../../__jest__/fixtures/ecs_mapping'; import { ECS_RESERVED } from './constants'; +import { EcsMappingState } from '../../types'; import { extractECSMapping, findDuplicateFields, findInvalidEcsFields, + handleValidateMappings, removeReservedFields, } from './validate'; @@ -286,3 +289,48 @@ describe('removeReservedFields', () => { expect(ecsMapping).not.toEqual(result); }); }); + +describe('handleValidateMappings', () => { + it('should return empty missing fields if none found', () => { + const state: EcsMappingState = ecsTestState; + state.currentMapping = { + test: { + test: { + event: { target: 'event.action', confidence: 0.95, type: 'string' }, + }, + }, + }; + state.combinedSamples = JSON.stringify({ + test: { + test: { + event: 'cert.create', + }, + }, + }); + const { missingKeys } = handleValidateMappings({ state }); + + expect(missingKeys).toEqual([]); + }); + + it('should return missing fields list if any', () => { + const state: EcsMappingState = ecsTestState; + state.currentMapping = { + test: { + test: { + event: { target: 'event.action', confidence: 0.95, type: 'string' }, + }, + }, + }; + state.combinedSamples = JSON.stringify({ + test: { + test: { + event: 'cert.create', + version: '1', + }, + }, + }); + const { missingKeys } = handleValidateMappings({ state }); + + expect(missingKeys).toEqual(['test.test.version']); + }); +}); diff --git a/x-pack/platform/plugins/shared/integration_assistant/tsconfig.json b/x-pack/platform/plugins/shared/integration_assistant/tsconfig.json index 94214e0e8fa4b..6af7a008e5093 100644 --- a/x-pack/platform/plugins/shared/integration_assistant/tsconfig.json +++ b/x-pack/platform/plugins/shared/integration_assistant/tsconfig.json @@ -43,6 +43,6 @@ "@kbn/kibana-utils-plugin", "@kbn/utils", "@kbn/zod", - "@kbn/tooling-log" + "@kbn/tooling-log", ] }