From 8c7883fd9830acdbef4b575071b73ada3e27cef2 Mon Sep 17 00:00:00 2001 From: Sergi Massaneda Date: Thu, 12 Dec 2024 17:12:18 +0100 Subject: [PATCH] [SecuritySolution][siem migrations] Onboarding UI flyout macros input (#203483) ## Summary From: https://github.com/elastic/security-team/issues/10667 This is the part 2 of the issue - The macros input Implementation of the Onboarding card to create migrations using the flyout. > [!NOTE] > This feature needs `siemMigrationsEnabled` experimental flag enabled to work. Otherwise only the default topic will be available and the topic selector won't be displayed. ### Screenshots Macros step loading done #### To do in part 3: - Implement missing steps in the flyout: Lookups ### Test Enable experimental flag Rule file: [rules_test.json](https://github.com/user-attachments/files/18082165/rules_test.json) Macros file: [macros_test.json](https://github.com/user-attachments/files/18082169/macros_test.json) --------- Co-authored-by: Elastic Machine Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../common/api/quickstart_client.gen.ts | 23 ++ .../common/siem_migrations/constants.ts | 2 + .../model/api/rules/rule_migration.gen.ts | 20 ++ .../api/rules/rule_migration.schema.yaml | 38 +++ .../model/rule_migration.gen.ts | 2 +- .../model/rule_migration.schema.yaml | 1 - .../siem_migrations/rules/resources/index.ts | 74 ++++- .../rules/resources/splunk/index.ts | 19 ++ .../splunk/splunk_identifier.test.ts | 135 ++++++++++ .../resources/splunk/splunk_identifier.ts | 56 ++++ .../rules/resources/splunk_identifier.test.ts | 89 ------ .../rules/resources/splunk_identifier.ts | 48 ---- .../siem_migrations/rules/resources/types.ts | 18 +- .../start_migration/start_migration_card.tsx | 22 +- .../start_migration_check_complete.ts | 5 +- .../start_migration/upload_rules_panels.tsx | 2 +- .../public/siem_migrations/rules/api/index.ts | 42 +++ .../components/data_input_flyout/constants.ts | 15 +- .../data_input_flyout/data_input_flyout.tsx | 83 ++++-- .../steps/common/get_status.ts | 18 ++ .../steps/common/sub_step_wrapper.tsx | 4 +- .../steps/common/use_parse_file_input.ts | 117 ++++++++ .../steps/macros/macros_data_input.tsx | 140 ++++++++++ .../sub_steps/check_resources/index.tsx | 54 ++++ .../sub_steps/check_resources/translations.ts | 20 ++ .../copy_export_query/copy_export_query.tsx | 53 ++++ .../sub_steps/copy_export_query/index.tsx | 26 ++ .../copy_export_query/translations.ts | 18 ++ .../sub_steps/macros_file_upload/index.tsx | 94 +++++++ .../macros_file_upload/macros_file_upload.tsx | 89 ++++++ .../macros_file_upload/translations.ts | 26 ++ .../steps/macros/translations.ts | 13 + .../steps/rules/rules_data_input.tsx | 134 +++++---- .../rules/sub_steps/check_resources/index.tsx | 33 ++- .../copy_export_query/translations.ts | 2 +- .../sub_steps/rules_file_upload/index.tsx | 5 +- .../rules_file_upload/parse_rules_file.ts | 79 ------ .../rules_file_upload/rules_file_upload.tsx | 100 +++---- .../rules_file_upload/translations.ts | 46 +--- .../data_input_flyout/translations.ts | 52 ++++ .../components/data_input_flyout/types.ts | 14 +- .../service/hooks/use_create_migration.ts | 16 +- .../hooks/use_get_missing_resources.ts | 48 ++++ .../rules/service/hooks/use_latest_stats.ts | 8 +- .../service/hooks/use_upsert_resources.ts | 51 ++++ .../rules/service/rule_migrations_service.ts | 26 +- .../lib/siem_migrations/rules/api/create.ts | 16 ++ .../lib/siem_migrations/rules/api/index.ts | 2 + .../rules/api/resources/get.ts | 9 +- .../rules/api/resources/missing.ts | 67 +++++ .../rules/api/resources/upsert.ts | 22 +- .../rules/data/__mocks__/mocks.ts | 8 + .../data/rule_migrations_data_base_client.ts | 53 +++- .../rule_migrations_data_resources_client.ts | 103 ++++++- .../data/rule_migrations_data_rules_client.ts | 25 +- .../retrievers/rule_migrations_retriever.ts | 31 ++- .../rule_resource_retriever.test.ts | 254 +++++++----------- .../retrievers/rule_resource_retriever.ts | 133 ++++----- .../rules/task/rule_migrations_task_client.ts | 125 ++++----- .../lib/siem_migrations/rules/task/types.ts | 15 +- .../services/security_solution_api.gen.ts | 25 ++ 61 files changed, 2102 insertions(+), 766 deletions(-) create mode 100644 x-pack/plugins/security_solution/common/siem_migrations/rules/resources/splunk/index.ts create mode 100644 x-pack/plugins/security_solution/common/siem_migrations/rules/resources/splunk/splunk_identifier.test.ts create mode 100644 x-pack/plugins/security_solution/common/siem_migrations/rules/resources/splunk/splunk_identifier.ts delete mode 100644 x-pack/plugins/security_solution/common/siem_migrations/rules/resources/splunk_identifier.test.ts delete mode 100644 x-pack/plugins/security_solution/common/siem_migrations/rules/resources/splunk_identifier.ts create mode 100644 x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/common/get_status.ts create mode 100644 x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/common/use_parse_file_input.ts create mode 100644 x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/macros/macros_data_input.tsx create mode 100644 x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/macros/sub_steps/check_resources/index.tsx create mode 100644 x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/macros/sub_steps/check_resources/translations.ts create mode 100644 x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/macros/sub_steps/copy_export_query/copy_export_query.tsx create mode 100644 x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/macros/sub_steps/copy_export_query/index.tsx create mode 100644 x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/macros/sub_steps/copy_export_query/translations.ts create mode 100644 x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/macros/sub_steps/macros_file_upload/index.tsx create mode 100644 x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/macros/sub_steps/macros_file_upload/macros_file_upload.tsx create mode 100644 x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/macros/sub_steps/macros_file_upload/translations.ts create mode 100644 x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/macros/translations.ts delete mode 100644 x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/rules/sub_steps/rules_file_upload/parse_rules_file.ts create mode 100644 x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/translations.ts create mode 100644 x-pack/plugins/security_solution/public/siem_migrations/rules/service/hooks/use_get_missing_resources.ts create mode 100644 x-pack/plugins/security_solution/public/siem_migrations/rules/service/hooks/use_upsert_resources.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/resources/missing.ts diff --git a/x-pack/plugins/security_solution/common/api/quickstart_client.gen.ts b/x-pack/plugins/security_solution/common/api/quickstart_client.gen.ts index b5d72fc1ef207..3487fdf81c0c9 100644 --- a/x-pack/plugins/security_solution/common/api/quickstart_client.gen.ts +++ b/x-pack/plugins/security_solution/common/api/quickstart_client.gen.ts @@ -368,6 +368,8 @@ import type { GetRuleMigrationResourcesRequestQueryInput, GetRuleMigrationResourcesRequestParamsInput, GetRuleMigrationResourcesResponse, + GetRuleMigrationResourcesMissingRequestParamsInput, + GetRuleMigrationResourcesMissingResponse, GetRuleMigrationStatsRequestParamsInput, GetRuleMigrationStatsResponse, GetRuleMigrationTranslationStatsRequestParamsInput, @@ -1471,6 +1473,24 @@ finalize it. }) .catch(catchAxiosErrorFormatAndThrow); } + /** + * Identifies missing resources from all the rules of an existing SIEM rules migration + */ + async getRuleMigrationResourcesMissing(props: GetRuleMigrationResourcesMissingProps) { + this.log.info(`${new Date().toISOString()} Calling API GetRuleMigrationResourcesMissing`); + return this.kbnClient + .request({ + path: replaceParams( + '/internal/siem_migrations/rules/{migration_id}/resources/missing', + props.params + ), + headers: { + [ELASTIC_HTTP_VERSION_HEADER]: '1', + }, + method: 'GET', + }) + .catch(catchAxiosErrorFormatAndThrow); + } /** * Retrieves the stats of a SIEM rules migration using the migration id provided */ @@ -2423,6 +2443,9 @@ export interface GetRuleMigrationResourcesProps { query: GetRuleMigrationResourcesRequestQueryInput; params: GetRuleMigrationResourcesRequestParamsInput; } +export interface GetRuleMigrationResourcesMissingProps { + params: GetRuleMigrationResourcesMissingRequestParamsInput; +} export interface GetRuleMigrationStatsProps { params: GetRuleMigrationStatsRequestParamsInput; } diff --git a/x-pack/plugins/security_solution/common/siem_migrations/constants.ts b/x-pack/plugins/security_solution/common/siem_migrations/constants.ts index e947dda4bbcc2..531669608ed8b 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/constants.ts +++ b/x-pack/plugins/security_solution/common/siem_migrations/constants.ts @@ -27,6 +27,8 @@ export const SIEM_RULE_MIGRATIONS_PREBUILT_RULES_PATH = `${SIEM_RULE_MIGRATION_PATH}/prebuilt_rules` as const; export const SIEM_RULE_MIGRATION_RESOURCES_PATH = `${SIEM_RULE_MIGRATION_PATH}/resources` as const; +export const SIEM_RULE_MIGRATION_RESOURCES_MISSING_PATH = + `${SIEM_RULE_MIGRATION_RESOURCES_PATH}/missing` as const; export enum SiemMigrationTaskStatus { READY = 'ready', diff --git a/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.gen.ts b/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.gen.ts index 95a81d4436d8a..df89c8d7f1c4e 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.gen.ts +++ b/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.gen.ts @@ -103,6 +103,8 @@ export type GetRuleMigrationResourcesRequestQuery = z.infer< export const GetRuleMigrationResourcesRequestQuery = z.object({ type: RuleMigrationResourceType.optional(), names: ArrayFromString(z.string()).optional(), + from: z.coerce.number().optional(), + size: z.coerce.number().optional(), }); export type GetRuleMigrationResourcesRequestQueryInput = z.input< typeof GetRuleMigrationResourcesRequestQuery @@ -121,6 +123,24 @@ export type GetRuleMigrationResourcesRequestParamsInput = z.input< export type GetRuleMigrationResourcesResponse = z.infer; export const GetRuleMigrationResourcesResponse = z.array(RuleMigrationResource); +export type GetRuleMigrationResourcesMissingRequestParams = z.infer< + typeof GetRuleMigrationResourcesMissingRequestParams +>; +export const GetRuleMigrationResourcesMissingRequestParams = z.object({ + migration_id: NonEmptyString, +}); +export type GetRuleMigrationResourcesMissingRequestParamsInput = z.input< + typeof GetRuleMigrationResourcesMissingRequestParams +>; + +/** + * The identified resources missing + */ +export type GetRuleMigrationResourcesMissingResponse = z.infer< + typeof GetRuleMigrationResourcesMissingResponse +>; +export const GetRuleMigrationResourcesMissingResponse = z.array(RuleMigrationResourceData); + export type GetRuleMigrationStatsRequestParams = z.infer; export const GetRuleMigrationStatsRequestParams = z.object({ migration_id: NonEmptyString, diff --git a/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.schema.yaml b/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.schema.yaml index b7e495e2ea898..fce14a2ac87b1 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.schema.yaml +++ b/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.schema.yaml @@ -473,6 +473,16 @@ paths: description: The names of the resource to retrieve items: type: string + - name: from + in: query + required: false + schema: + type: number + - name: size + in: query + required: false + schema: + type: number responses: 200: description: Indicates migration resources have been retrieved correctly @@ -482,3 +492,31 @@ paths: type: array items: $ref: '../../rule_migration.schema.yaml#/components/schemas/RuleMigrationResource' + + /internal/siem_migrations/rules/{migration_id}/resources/missing: + get: + summary: Gets missing rule migration resources for a migration + operationId: GetRuleMigrationResourcesMissing + x-codegen-enabled: true + x-internal: true + description: Identifies missing resources from all the rules of an existing SIEM rules migration + tags: + - SIEM Rule Migrations + - Resources + parameters: + - name: migration_id + in: path + required: true + schema: + description: The migration id to attach the resources + $ref: '../../../../../common/api/model/primitives.schema.yaml#/components/schemas/NonEmptyString' + responses: + 200: + description: Indicates missing migration resources have been retrieved correctly + content: + application/json: + schema: + type: array + description: The identified resources missing + items: + $ref: '../../rule_migration.schema.yaml#/components/schemas/RuleMigrationResourceData' diff --git a/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts b/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts index 60220bf054a12..9fd3876e141a8 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts +++ b/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts @@ -324,7 +324,7 @@ export const RuleMigrationResourceData = z.object({ /** * The resource content value. */ - content: z.string(), + content: z.string().optional(), /** * The resource arbitrary metadata. */ diff --git a/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml b/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml index e13e9b1d0ed75..0a99bd5ce701f 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml +++ b/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml @@ -291,7 +291,6 @@ components: required: - type - name - - content properties: type: $ref: '#/components/schemas/RuleMigrationResourceType' diff --git a/x-pack/plugins/security_solution/common/siem_migrations/rules/resources/index.ts b/x-pack/plugins/security_solution/common/siem_migrations/rules/resources/index.ts index ffe4b3aca4076..8ec7adf050bf3 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/rules/resources/index.ts +++ b/x-pack/plugins/security_solution/common/siem_migrations/rules/resources/index.ts @@ -5,18 +5,72 @@ * 2.0. */ -import type { OriginalRule, OriginalRuleVendor } from '../../model/rule_migration.gen'; -import type { QueryResourceIdentifier, RuleResourceCollection } from './types'; -import { splResourceIdentifier } from './splunk_identifier'; +import type { + OriginalRule, + OriginalRuleVendor, + RuleMigrationResourceData, +} from '../../model/rule_migration.gen'; +import type { ResourceIdentifiers, RuleResource } from './types'; +import { splResourceIdentifiers } from './splunk'; -export const getRuleResourceIdentifier = (rule: OriginalRule): QueryResourceIdentifier => { - return ruleResourceIdentifiers[rule.vendor]; +const ruleResourceIdentifiers: Record = { + splunk: splResourceIdentifiers, }; -export const identifyRuleResources = (rule: OriginalRule): RuleResourceCollection => { - return getRuleResourceIdentifier(rule)(rule.query); +export const getRuleResourceIdentifier = (vendor: OriginalRuleVendor): ResourceIdentifiers => { + return ruleResourceIdentifiers[vendor]; }; -const ruleResourceIdentifiers: Record = { - splunk: splResourceIdentifier, -}; +export class ResourceIdentifier { + private identifiers: ResourceIdentifiers; + + constructor(vendor: OriginalRuleVendor) { + // The constructor may need query_language as an argument for other vendors + this.identifiers = ruleResourceIdentifiers[vendor]; + } + + public fromOriginalRule(originalRule: OriginalRule): RuleResource[] { + return this.identifiers.fromOriginalRule(originalRule); + } + + public fromResource(resource: RuleMigrationResourceData): RuleResource[] { + return this.identifiers.fromResource(resource); + } + + public fromOriginalRules(originalRules: OriginalRule[]): RuleResource[] { + const lists = new Set(); + const macros = new Set(); + originalRules.forEach((rule) => { + const resources = this.identifiers.fromOriginalRule(rule); + resources.forEach((resource) => { + if (resource.type === 'macro') { + macros.add(resource.name); + } else if (resource.type === 'list') { + lists.add(resource.name); + } + }); + }); + return [ + ...Array.from(macros).map((name) => ({ type: 'macro', name })), + ...Array.from(lists).map((name) => ({ type: 'list', name })), + ]; + } + + public fromResources(resources: RuleMigrationResourceData[]): RuleResource[] { + const lists = new Set(); + const macros = new Set(); + resources.forEach((resource) => { + this.identifiers.fromResource(resource).forEach((identifiedResource) => { + if (identifiedResource.type === 'macro') { + macros.add(identifiedResource.name); + } else if (identifiedResource.type === 'list') { + lists.add(identifiedResource.name); + } + }); + }); + return [ + ...Array.from(macros).map((name) => ({ type: 'macro', name })), + ...Array.from(lists).map((name) => ({ type: 'list', name })), + ]; + } +} diff --git a/x-pack/plugins/security_solution/common/siem_migrations/rules/resources/splunk/index.ts b/x-pack/plugins/security_solution/common/siem_migrations/rules/resources/splunk/index.ts new file mode 100644 index 0000000000000..a16c328da947a --- /dev/null +++ b/x-pack/plugins/security_solution/common/siem_migrations/rules/resources/splunk/index.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ResourceIdentifiers } from '../types'; +import { splResourceIdentifier } from './splunk_identifier'; + +export const splResourceIdentifiers: ResourceIdentifiers = { + fromOriginalRule: (originalRule) => splResourceIdentifier(originalRule.query), + fromResource: (resource) => { + if (resource.type === 'macro' && resource.content) { + return splResourceIdentifier(resource.content); + } + return []; + }, +}; diff --git a/x-pack/plugins/security_solution/common/siem_migrations/rules/resources/splunk/splunk_identifier.test.ts b/x-pack/plugins/security_solution/common/siem_migrations/rules/resources/splunk/splunk_identifier.test.ts new file mode 100644 index 0000000000000..5d144e5b8a38f --- /dev/null +++ b/x-pack/plugins/security_solution/common/siem_migrations/rules/resources/splunk/splunk_identifier.test.ts @@ -0,0 +1,135 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { splResourceIdentifier } from './splunk_identifier'; + +describe('splResourceIdentifier', () => { + it('should extract macros correctly', () => { + const query = + '`macro_zero`, `macro_one(arg1)`, some search command `macro_two(arg1, arg2)` another command `macro_three(arg1, arg2, arg3)`'; + + const result = splResourceIdentifier(query); + expect(result).toEqual([ + { type: 'macro', name: 'macro_zero' }, + { type: 'macro', name: 'macro_one(1)' }, + { type: 'macro', name: 'macro_two(2)' }, + { type: 'macro', name: 'macro_three(3)' }, + ]); + }); + + it('should extract macros with double quotes parameters correctly', () => { + const query = '| `macro_one("90","2")` | `macro_two("20")`'; + + const result = splResourceIdentifier(query); + expect(result).toEqual([ + { type: 'macro', name: 'macro_one(2)' }, + { type: 'macro', name: 'macro_two(1)' }, + ]); + }); + + it('should extract macros with single quotes parameters correctly', () => { + const query = "| `macro_one('90','2')` | `macro_two('20')`"; + + const result = splResourceIdentifier(query); + expect(result).toEqual([ + { type: 'macro', name: 'macro_one(2)' }, + { type: 'macro', name: 'macro_two(1)' }, + ]); + }); + + it('should extract lookup tables correctly', () => { + const query = + 'search ... | lookup my_lookup_table field AS alias OUTPUT new_field | lookup other_lookup_list | lookup third_lookup'; + + const result = splResourceIdentifier(query); + expect(result).toEqual([ + { type: 'list', name: 'my_lookup_table' }, + { type: 'list', name: 'other_lookup_list' }, + { type: 'list', name: 'third_lookup' }, + ]); + }); + + it('should extract both macros and lookup tables correctly', () => { + const query = + '`macro_one` some search command | lookup my_lookup_table field AS alias OUTPUT new_field | lookup other_lookup_list | lookup third_lookup'; + + const result = splResourceIdentifier(query); + expect(result).toEqual([ + { type: 'macro', name: 'macro_one' }, + { type: 'list', name: 'my_lookup_table' }, + { type: 'list', name: 'other_lookup_list' }, + { type: 'list', name: 'third_lookup' }, + ]); + }); + + it('should extract lookup correctly when there are modifiers', () => { + const query = + 'lookup my_lookup_1 field AS alias OUTPUT new_field | lookup local=true my_lookup_2 | lookup update=true my_lookup_3 | lookup local=true update=true my_lookup_4 | lookup update=false local=true my_lookup_5'; + + const result = splResourceIdentifier(query); + expect(result).toEqual([ + { type: 'list', name: 'my_lookup_1' }, + { type: 'list', name: 'my_lookup_2' }, + { type: 'list', name: 'my_lookup_3' }, + { type: 'list', name: 'my_lookup_4' }, + { type: 'list', name: 'my_lookup_5' }, + ]); + }); + + it('should return empty arrays if no macros or lookup tables are found', () => { + const query = 'search | stats count'; + + const result = splResourceIdentifier(query); + expect(result).toEqual([]); + }); + + it('should handle queries with both macros and lookup tables mixed with other commands', () => { + const query = + 'search `macro_one` | `my_lookup_table` field AS alias myfakelookup new_field | lookup real_lookup_list | `third_macro`'; + + const result = splResourceIdentifier(query); + expect(result).toEqual([ + { type: 'macro', name: 'macro_one' }, + { type: 'macro', name: 'my_lookup_table' }, + { type: 'macro', name: 'third_macro' }, + { type: 'list', name: 'real_lookup_list' }, + ]); + }); + + it('should ignore macros or lookup tables inside string literals with double quotes', () => { + const query = + '`macro_one` | lookup my_lookup_table | search title="`macro_two` and lookup another_table"'; + + const result = splResourceIdentifier(query); + expect(result).toEqual([ + { type: 'macro', name: 'macro_one' }, + { type: 'list', name: 'my_lookup_table' }, + ]); + }); + + it('should ignore macros or lookup tables inside string literals with single quotes', () => { + const query = + "`macro_one` | lookup my_lookup_table | search title='`macro_two` and lookup another_table'"; + + const result = splResourceIdentifier(query); + expect(result).toEqual([ + { type: 'macro', name: 'macro_one' }, + { type: 'list', name: 'my_lookup_table' }, + ]); + }); + + it('should ignore macros or lookup tables inside comments wrapped by ```', () => { + const query = + '`macro_one` ```this is a comment with `macro_two` and lookup another_table``` | lookup my_lookup_table ```this is another comment```'; + + const result = splResourceIdentifier(query); + expect(result).toEqual([ + { type: 'macro', name: 'macro_one' }, + { type: 'list', name: 'my_lookup_table' }, + ]); + }); +}); diff --git a/x-pack/plugins/security_solution/common/siem_migrations/rules/resources/splunk/splunk_identifier.ts b/x-pack/plugins/security_solution/common/siem_migrations/rules/resources/splunk/splunk_identifier.ts new file mode 100644 index 0000000000000..2ecc43321b11f --- /dev/null +++ b/x-pack/plugins/security_solution/common/siem_migrations/rules/resources/splunk/splunk_identifier.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * Important: + * This library uses regular expressions that are executed against arbitrary user input, they need to be safe from ReDoS attacks. + * Please make sure to test all regular expressions them before using them. + * At the time of writing, this tool can be used to test it: https://devina.io/redos-checker + */ + +import type { ResourceIdentifier, RuleResource } from '../types'; + +const listRegex = /\b(?:lookup)\s+([\w-]+)\b/g; // Captures only the lookup name +const macrosRegex = /`([\w-]+)(?:\(([^`]*?)\))?`/g; // Captures only the macro name and arguments + +export const splResourceIdentifier: ResourceIdentifier = (input) => { + // sanitize the query to avoid mismatching macro and list names inside comments or literal strings + const sanitizedInput = sanitizeInput(input); + + const resources: RuleResource[] = []; + let macroMatch; + while ((macroMatch = macrosRegex.exec(sanitizedInput)) !== null) { + const macroName = macroMatch[1] as string; + const args = macroMatch[2] as string; // This captures the content inside the parentheses + const argCount = args ? args.split(',').length : 0; // Count arguments if present + const macroWithArgs = argCount > 0 ? `${macroName}(${argCount})` : macroName; + resources.push({ type: 'macro', name: macroWithArgs }); + } + + let listMatch; + while ((listMatch = listRegex.exec(sanitizedInput)) !== null) { + resources.push({ type: 'list', name: listMatch[1] }); + } + + return resources; +}; + +// Comments should be removed before processing the query to avoid matching macro and list names inside them +const commentRegex = /```.*?```/g; +// Literal strings should be replaced with a placeholder to avoid matching macro and list names inside them +const doubleQuoteStrRegex = /".*?"/g; +const singleQuoteStrRegex = /'.*?'/g; +// lookup operator can have modifiers like local=true or update=false before the lookup name, we need to remove them +const lookupModifiers = /\blookup\b\s+((local|update)=\s*(?:true|false)\s*)+/gi; + +const sanitizeInput = (query: string) => { + return query + .replaceAll(commentRegex, '') + .replaceAll(doubleQuoteStrRegex, '"literal"') + .replaceAll(singleQuoteStrRegex, "'literal'") + .replaceAll(lookupModifiers, 'lookup '); +}; diff --git a/x-pack/plugins/security_solution/common/siem_migrations/rules/resources/splunk_identifier.test.ts b/x-pack/plugins/security_solution/common/siem_migrations/rules/resources/splunk_identifier.test.ts deleted file mode 100644 index 2fa3be223aa67..0000000000000 --- a/x-pack/plugins/security_solution/common/siem_migrations/rules/resources/splunk_identifier.test.ts +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { splResourceIdentifier } from './splunk_identifier'; - -describe('splResourceIdentifier', () => { - it('should extract macros correctly', () => { - const query = - '`macro_zero`, `macro_one(arg1)`, some search command `macro_two(arg1, arg2)` another command `macro_three(arg1, arg2, arg3)`'; - - const result = splResourceIdentifier(query); - - expect(result.macro).toEqual(['macro_zero', 'macro_one(1)', 'macro_two(2)', 'macro_three(3)']); - expect(result.list).toEqual([]); - }); - - it('should extract lookup tables correctly', () => { - const query = - 'search ... | lookup my_lookup_table field AS alias OUTPUT new_field | inputlookup other_lookup_list | lookup third_lookup'; - - const result = splResourceIdentifier(query); - - expect(result.macro).toEqual([]); - expect(result.list).toEqual(['my_lookup_table', 'other_lookup_list', 'third_lookup']); - }); - - it('should extract both macros and lookup tables correctly', () => { - const query = - '`macro_one` some search command | lookup my_lookup_table field AS alias OUTPUT new_field | inputlookup other_lookup_list | lookup third_lookup'; - - const result = splResourceIdentifier(query); - - expect(result.macro).toEqual(['macro_one']); - expect(result.list).toEqual(['my_lookup_table', 'other_lookup_list', 'third_lookup']); - }); - - it('should return empty arrays if no macros or lookup tables are found', () => { - const query = 'search | stats count'; - - const result = splResourceIdentifier(query); - - expect(result.macro).toEqual([]); - expect(result.list).toEqual([]); - }); - - it('should handle queries with both macros and lookup tables mixed with other commands', () => { - const query = - 'search `macro_one` | `my_lookup_table` field AS alias myfakelookup new_field | inputlookup real_lookup_list | `third_macro`'; - - const result = splResourceIdentifier(query); - - expect(result.macro).toEqual(['macro_one', 'my_lookup_table', 'third_macro']); - expect(result.list).toEqual(['real_lookup_list']); - }); - - it('should ignore macros or lookup tables inside string literals with double quotes', () => { - const query = - '`macro_one` | lookup my_lookup_table | search title="`macro_two` and lookup another_table"'; - - const result = splResourceIdentifier(query); - - expect(result.macro).toEqual(['macro_one']); - expect(result.list).toEqual(['my_lookup_table']); - }); - - it('should ignore macros or lookup tables inside string literals with single quotes', () => { - const query = - "`macro_one` | lookup my_lookup_table | search title='`macro_two` and lookup another_table'"; - - const result = splResourceIdentifier(query); - - expect(result.macro).toEqual(['macro_one']); - expect(result.list).toEqual(['my_lookup_table']); - }); - - it('should ignore macros or lookup tables inside comments wrapped by ```', () => { - const query = - '`macro_one` | ```this is a comment with `macro_two` and lookup another_table``` lookup my_lookup_table'; - - const result = splResourceIdentifier(query); - - expect(result.macro).toEqual(['macro_one']); - expect(result.list).toEqual(['my_lookup_table']); - }); -}); diff --git a/x-pack/plugins/security_solution/common/siem_migrations/rules/resources/splunk_identifier.ts b/x-pack/plugins/security_solution/common/siem_migrations/rules/resources/splunk_identifier.ts deleted file mode 100644 index fa46fff941c6b..0000000000000 --- a/x-pack/plugins/security_solution/common/siem_migrations/rules/resources/splunk_identifier.ts +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -/** - * Important: - * This library uses regular expressions that are executed against arbitrary user input, they need to be safe from ReDoS attacks. - * Please make sure to test them before using them in production. - * At the time of writing, this tool can be used to test it: https://devina.io/redos-checker - */ - -import type { QueryResourceIdentifier } from './types'; - -const listRegex = /\b(?:lookup|inputlookup)\s+([\w-]+)\b/g; // Captures only the lookup table name -const macrosRegex = /`([\w-]+)(?:\(([^`]*?)\))?`/g; // Captures only the macro name and arguments - -const commentRegex = /```.*```/g; -const doubleQuoteStrRegex = /".*"/g; -const singleQuoteStrRegex = /'.*'/g; - -export const splResourceIdentifier: QueryResourceIdentifier = (query) => { - // sanitize the query to avoid mismatching macro and list names inside comments or literal strings - const sanitizedQuery = query - .replaceAll(commentRegex, '') - .replaceAll(doubleQuoteStrRegex, '"literal"') - .replaceAll(singleQuoteStrRegex, "'literal'"); - - const macro = []; - let macroMatch; - while ((macroMatch = macrosRegex.exec(sanitizedQuery)) !== null) { - const macroName = macroMatch[1]; - const args = macroMatch[2]; // This captures the content inside the parentheses - const argCount = args ? args.split(',').length : 0; // Count arguments if present - const macroWithArgs = argCount > 0 ? `${macroName}(${argCount})` : macroName; - macro.push(macroWithArgs); - } - - const list = []; - let listMatch; - while ((listMatch = listRegex.exec(sanitizedQuery)) !== null) { - list.push(listMatch[1]); - } - - return { macro, list }; -}; diff --git a/x-pack/plugins/security_solution/common/siem_migrations/rules/resources/types.ts b/x-pack/plugins/security_solution/common/siem_migrations/rules/resources/types.ts index 93f6f3ad3db17..70c6e3b72124f 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/rules/resources/types.ts +++ b/x-pack/plugins/security_solution/common/siem_migrations/rules/resources/types.ts @@ -5,7 +5,19 @@ * 2.0. */ -import type { RuleMigrationResourceType } from '../../model/rule_migration.gen'; +import type { + OriginalRule, + RuleMigrationResourceData, + RuleMigrationResourceType, +} from '../../model/rule_migration.gen'; -export type RuleResourceCollection = Record; -export type QueryResourceIdentifier = (query: string) => RuleResourceCollection; +export interface RuleResource { + type: RuleMigrationResourceType; + name: string; +} +export type ResourceIdentifier = (input: string) => RuleResource[]; + +export interface ResourceIdentifiers { + fromOriginalRule: (originalRule: OriginalRule) => RuleResource[]; + fromResource: (resource: RuleMigrationResourceData) => RuleResource[]; +} diff --git a/x-pack/plugins/security_solution/public/onboarding/components/onboarding_body/cards/siem_migrations/start_migration/start_migration_card.tsx b/x-pack/plugins/security_solution/public/onboarding/components/onboarding_body/cards/siem_migrations/start_migration/start_migration_card.tsx index c1e7539c8e101..a8d7aa78d0c93 100644 --- a/x-pack/plugins/security_solution/public/onboarding/components/onboarding_body/cards/siem_migrations/start_migration/start_migration_card.tsx +++ b/x-pack/plugins/security_solution/public/onboarding/components/onboarding_body/cards/siem_migrations/start_migration/start_migration_card.tsx @@ -5,8 +5,9 @@ * 2.0. */ -import React, { useCallback, useState } from 'react'; +import React, { useCallback, useEffect, useState } from 'react'; import { EuiSpacer, EuiText } from '@elastic/eui'; +import { SiemMigrationTaskStatus } from '../../../../../../../common/siem_migrations/constants'; import { OnboardingCardId } from '../../../../../constants'; import type { RuleMigrationTaskStats } from '../../../../../../../common/siem_migrations/model/rule_migration.gen'; import { useLatestStats } from '../../../../../../siem_migrations/rules/service/hooks/use_latest_stats'; @@ -21,22 +22,29 @@ import * as i18n from './translations'; import { MissingAIConnectorCallout } from './missing_ai_connector_callout'; export const StartMigrationCard: OnboardingCardComponent = React.memo( - ({ checkComplete, isCardComplete, setExpandedCardId }) => { + ({ setComplete, isCardComplete, setExpandedCardId }) => { const styles = useStyles(); - const { data: migrationsStats, isLoading } = useLatestStats(); + const { data: migrationsStats, isLoading, refreshStats } = useLatestStats(); const [isFlyoutOpen, setIsFlyoutOpen] = useState(); const [flyoutMigrationStats, setFlyoutMigrationStats] = useState< RuleMigrationTaskStats | undefined >(); + useEffect(() => { + // Set card complete if any migration is finished + if (!isCardComplete(OnboardingCardId.siemMigrationsStart) && migrationsStats) { + if (migrationsStats.some(({ status }) => status === SiemMigrationTaskStatus.FINISHED)) { + setComplete(true); + } + } + }, [isCardComplete, migrationsStats, setComplete]); + const closeFlyout = useCallback(() => { setIsFlyoutOpen(false); setFlyoutMigrationStats(undefined); - if (!isCardComplete(OnboardingCardId.siemMigrationsStart)) { - checkComplete(); - } - }, [checkComplete, isCardComplete]); + refreshStats(); + }, [refreshStats]); const openFlyout = useCallback((migrationStats?: RuleMigrationTaskStats) => { setFlyoutMigrationStats(migrationStats); diff --git a/x-pack/plugins/security_solution/public/onboarding/components/onboarding_body/cards/siem_migrations/start_migration/start_migration_check_complete.ts b/x-pack/plugins/security_solution/public/onboarding/components/onboarding_body/cards/siem_migrations/start_migration/start_migration_check_complete.ts index 41e65352d4bc3..79a7238b9554e 100644 --- a/x-pack/plugins/security_solution/public/onboarding/components/onboarding_body/cards/siem_migrations/start_migration/start_migration_check_complete.ts +++ b/x-pack/plugins/security_solution/public/onboarding/components/onboarding_body/cards/siem_migrations/start_migration/start_migration_check_complete.ts @@ -5,12 +5,15 @@ * 2.0. */ +import { SiemMigrationTaskStatus } from '../../../../../../../common/siem_migrations/constants'; import type { OnboardingCardCheckComplete } from '../../../../../types'; export const checkStartMigrationCardComplete: OnboardingCardCheckComplete = async ({ siemMigrations, }) => { const migrationsStats = await siemMigrations.rules.getRuleMigrationsStats(); - const isComplete = migrationsStats.length > 0; + const isComplete = migrationsStats.some( + (migrationStats) => migrationStats.status === SiemMigrationTaskStatus.FINISHED + ); return isComplete; }; diff --git a/x-pack/plugins/security_solution/public/onboarding/components/onboarding_body/cards/siem_migrations/start_migration/upload_rules_panels.tsx b/x-pack/plugins/security_solution/public/onboarding/components/onboarding_body/cards/siem_migrations/start_migration/upload_rules_panels.tsx index 95b53d921fd1f..6d011fc5fbb5b 100644 --- a/x-pack/plugins/security_solution/public/onboarding/components/onboarding_body/cards/siem_migrations/start_migration/upload_rules_panels.tsx +++ b/x-pack/plugins/security_solution/public/onboarding/components/onboarding_body/cards/siem_migrations/start_migration/upload_rules_panels.tsx @@ -28,7 +28,7 @@ export const UploadRulesPanels = React.memo(({ migration {migrationsStats.map((migrationStats) => ( - + {migrationStats.status === SiemMigrationTaskStatus.READY && ( )} diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/api/index.ts b/x-pack/plugins/security_solution/public/siem_migrations/rules/api/index.ts index 57fb5d0422093..17aa00dd0f01e 100644 --- a/x-pack/plugins/security_solution/public/siem_migrations/rules/api/index.ts +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/api/index.ts @@ -19,6 +19,8 @@ import { SIEM_RULE_MIGRATION_START_PATH, SIEM_RULE_MIGRATION_STATS_PATH, SIEM_RULE_MIGRATION_TRANSLATION_STATS_PATH, + SIEM_RULE_MIGRATION_RESOURCES_MISSING_PATH, + SIEM_RULE_MIGRATION_RESOURCES_PATH, SIEM_RULE_MIGRATIONS_PREBUILT_RULES_PATH, } from '../../../../common/siem_migrations/constants'; import type { @@ -31,6 +33,9 @@ import type { InstallMigrationRulesResponse, StartRuleMigrationRequestBody, GetRuleMigrationStatsResponse, + GetRuleMigrationResourcesMissingResponse, + UpsertRuleMigrationResourcesRequestBody, + UpsertRuleMigrationResourcesResponse, GetRuleMigrationPrebuiltRulesResponse, } from '../../../../common/siem_migrations/model/api/rules/rule_migration.gen'; @@ -86,6 +91,43 @@ export const createRuleMigration = async ({ ); }; +export interface GetRuleMigrationMissingResourcesParams { + /** `id` of the migration to get missing resources for */ + migrationId: string; + /** Optional AbortSignal for cancelling request */ + signal?: AbortSignal; +} +/** Retrieves all missing resources of a specific migration. */ +export const getMissingResources = async ({ + migrationId, + signal, +}: GetRuleMigrationMissingResourcesParams): Promise => { + return KibanaServices.get().http.get( + replaceParams(SIEM_RULE_MIGRATION_RESOURCES_MISSING_PATH, { migration_id: migrationId }), + { version: '1', signal } + ); +}; + +export interface UpsertResourcesParams { + /** Optional `id` of migration to add the resources to. */ + migrationId: string; + /** The body containing the `connectorId` to use for the migration */ + body: UpsertRuleMigrationResourcesRequestBody; + /** Optional AbortSignal for cancelling request */ + signal?: AbortSignal; +} +/** Updates or creates resources for a specific migration. */ +export const upsertMigrationResources = async ({ + migrationId, + body, + signal, +}: UpsertResourcesParams): Promise => { + return KibanaServices.get().http.post( + replaceParams(SIEM_RULE_MIGRATION_RESOURCES_PATH, { migration_id: migrationId }), + { body: JSON.stringify(body), version: '1', signal } + ); +}; + export interface StartRuleMigrationParams { /** `id` of the migration to start */ migrationId: string; diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/constants.ts b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/constants.ts index aa331bf17c832..d390b395ed98f 100644 --- a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/constants.ts +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/constants.ts @@ -5,13 +5,7 @@ * 2.0. */ -export enum DataInputStep { - rules = 'rules', - macros = 'macros', - lookups = 'lookups', -} - -export const SPL_RULES_COLUMNS = [ +export const SPLUNK_RULES_COLUMNS = [ 'id', 'title', 'search', @@ -23,4 +17,9 @@ export const SPL_RULES_COLUMNS = [ export const RULES_SPLUNK_QUERY = `| rest /servicesNS/-/-/saved/searches | search action.correlationsearch.enabled = "1" OR (eai:acl.app = "Splunk_Security_Essentials" AND is_scheduled=1) | where disabled=0 -| table ${SPL_RULES_COLUMNS.join(', ')}`; +| table ${SPLUNK_RULES_COLUMNS.join(', ')}`; + +export const SPLUNK_MACROS_COLUMNS = ['title', 'definition'] as const; + +export const MACROS_SPLUNK_QUERY = `| rest /servicesNS/-/-/admin/macros count=0 +| table ${SPLUNK_MACROS_COLUMNS.join(', ')}`; diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/data_input_flyout.tsx b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/data_input_flyout.tsx index 6a4916a5e54b3..ffc40c59d495a 100644 --- a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/data_input_flyout.tsx +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/data_input_flyout.tsx @@ -17,10 +17,19 @@ import { EuiButton, } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n-react'; -import type { RuleMigrationTaskStats } from '../../../../../common/siem_migrations/model/rule_migration.gen'; -import { DataInputStep } from './constants'; +import type { + RuleMigrationResourceData, + RuleMigrationTaskStats, +} from '../../../../../common/siem_migrations/model/rule_migration.gen'; import { RulesDataInput } from './steps/rules/rules_data_input'; import { useStartMigration } from '../../service/hooks/use_start_migration'; +import { DataInputStep } from './types'; +import { MacrosDataInput } from './steps/macros/macros_data_input'; + +interface MissingResourcesIndexed { + macros: string[]; + lookups: string[]; +} export interface MigrationDataInputFlyoutProps { onClose: () => void; @@ -31,6 +40,9 @@ export const MigrationDataInputFlyout = React.memo( initialMigrationSats ); + const [missingResourcesIndexed, setMissingResourcesIndexed] = useState< + MissingResourcesIndexed | undefined + >(); const { startMigration, isLoading: isStartLoading } = useStartMigration(onClose); const onStartMigration = useCallback(() => { @@ -39,23 +51,43 @@ export const MigrationDataInputFlyout = React.memo(() => { - if (migrationStats) { - return DataInputStep.macros; - } - return DataInputStep.rules; - }); + const [dataInputStep, setDataInputStep] = useState(DataInputStep.Rules); - const onMigrationCreated = useCallback( - (createdMigrationStats: RuleMigrationTaskStats) => { - if (createdMigrationStats) { - setMigrationStats(createdMigrationStats); - setDataInputStep(DataInputStep.macros); + const onMigrationCreated = useCallback((createdMigrationStats: RuleMigrationTaskStats) => { + setMigrationStats(createdMigrationStats); + }, []); + + const onMissingResourcesFetched = useCallback( + (missingResources: RuleMigrationResourceData[]) => { + const newMissingResourcesIndexed = missingResources.reduce( + (acc, { type, name }) => { + if (type === 'macro') { + acc.macros.push(name); + } else if (type === 'list') { + acc.lookups.push(name); + } + return acc; + }, + { macros: [], lookups: [] } + ); + setMissingResourcesIndexed(newMissingResourcesIndexed); + if (newMissingResourcesIndexed.macros.length) { + setDataInputStep(DataInputStep.Macros); + return; + } + if (newMissingResourcesIndexed.lookups.length) { + setDataInputStep(DataInputStep.Lookups); + return; } + setDataInputStep(DataInputStep.End); }, - [setDataInputStep] + [] ); + const onMacrosCreated = useCallback(() => { + setDataInputStep(DataInputStep.Lookups); + }, []); + return ( - + + + + + + + + diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/common/get_status.ts b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/common/get_status.ts new file mode 100644 index 0000000000000..f9f14298d00be --- /dev/null +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/common/get_status.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { EuiStepStatus } from '@elastic/eui'; + +export const getStatus = (step: number, currentStep: number): EuiStepStatus => { + if (step === currentStep) { + return 'current'; + } + if (step < currentStep) { + return 'complete'; + } + return 'incomplete'; +}; diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/common/sub_step_wrapper.tsx b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/common/sub_step_wrapper.tsx index 438134b0ad99a..fc0bd0e8c3b44 100644 --- a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/common/sub_step_wrapper.tsx +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/common/sub_step_wrapper.tsx @@ -16,11 +16,11 @@ const style = css` } `; -export const SubStepWrapper = React.memo>(({ children }) => { +export const SubStepsWrapper = React.memo>(({ children }) => { return ( {children} ); }); -SubStepWrapper.displayName = 'SubStepWrapper'; +SubStepsWrapper.displayName = 'SubStepsWrapper'; diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/common/use_parse_file_input.ts b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/common/use_parse_file_input.ts new file mode 100644 index 0000000000000..b99cf826194f9 --- /dev/null +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/common/use_parse_file_input.ts @@ -0,0 +1,117 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useState, useCallback } from 'react'; +import { FILE_UPLOAD_ERROR } from '../../translations'; + +export interface SplunkRow { + result: T; +} + +export type OnFileParsed = (content: SplunkRow[]) => void; + +export const useParseFileInput = (onFileParsed: OnFileParsed) => { + const [isParsing, setIsParsing] = useState(false); + const [error, setError] = useState(); + + const parseFile = useCallback( + (files: FileList | null) => { + if (!files) { + return; + } + + setError(undefined); + + const rulesFile = files[0]; + const reader = new FileReader(); + + reader.onloadstart = () => setIsParsing(true); + reader.onloadend = () => setIsParsing(false); + + reader.onload = function (e) { + // We can safely cast to string since we call `readAsText` to load the file. + const fileContent = e.target?.result as string | undefined; + + if (fileContent == null) { + setError(FILE_UPLOAD_ERROR.CAN_NOT_READ); + return; + } + + if (fileContent === '' && e.loaded > 100000) { + // V8-based browsers can't handle large files and return an empty string + // instead of an error; see https://stackoverflow.com/a/61316641 + setError(FILE_UPLOAD_ERROR.TOO_LARGE_TO_PARSE); + return; + } + + try { + const parsedData = parseContent(fileContent); + onFileParsed(parsedData); + } catch (err) { + setError(err.message); + } + }; + + const handleReaderError = function () { + const message = reader.error?.message; + if (message) { + setError(FILE_UPLOAD_ERROR.CAN_NOT_READ_WITH_REASON(message)); + } else { + setError(FILE_UPLOAD_ERROR.CAN_NOT_READ); + } + }; + + reader.onerror = handleReaderError; + reader.onabort = handleReaderError; + + reader.readAsText(rulesFile); + }, + [onFileParsed] + ); + + return { parseFile, isParsing, error }; +}; + +const parseContent = (fileContent: string): SplunkRow[] => { + const trimmedContent = fileContent.trim(); + let arrayContent: SplunkRow[]; + if (trimmedContent.startsWith('[')) { + arrayContent = parseJSONArray(trimmedContent); + } else { + arrayContent = parseNDJSON(trimmedContent); + } + if (arrayContent.length === 0) { + throw new Error(FILE_UPLOAD_ERROR.EMPTY); + } + return arrayContent; +}; + +const parseNDJSON = (fileContent: string): SplunkRow[] => { + return fileContent + .split(/\n(?=\{)/) // split at newline followed by '{'. + .filter((entry) => entry.trim() !== '') // Remove empty entries. + .map(parseJSON); // Parse each entry as JSON. +}; + +const parseJSONArray = (fileContent: string): SplunkRow[] => { + const parsedContent = parseJSON(fileContent); + if (!Array.isArray(parsedContent)) { + throw new Error(FILE_UPLOAD_ERROR.NOT_ARRAY); + } + return parsedContent; +}; + +const parseJSON = (fileContent: string) => { + try { + return JSON.parse(fileContent); + } catch (error) { + if (error instanceof RangeError) { + throw new Error(FILE_UPLOAD_ERROR.TOO_LARGE_TO_PARSE); + } + throw new Error(FILE_UPLOAD_ERROR.CAN_NOT_PARSE); + } +}; diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/macros/macros_data_input.tsx b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/macros/macros_data_input.tsx new file mode 100644 index 0000000000000..f19e704b96710 --- /dev/null +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/macros/macros_data_input.tsx @@ -0,0 +1,140 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { EuiStepProps } from '@elastic/eui'; +import { + EuiFlexGroup, + EuiFlexItem, + EuiPanel, + EuiStepNumber, + EuiSteps, + EuiTitle, +} from '@elastic/eui'; +import React, { useCallback, useMemo, useState } from 'react'; +import type { RuleMigrationTaskStats } from '../../../../../../../common/siem_migrations/model/rule_migration.gen'; +import { SubStepsWrapper } from '../common/sub_step_wrapper'; +import type { OnResourcesCreated, OnMissingResourcesFetched, DataInputStep } from '../../types'; +import { getStatus } from '../common/get_status'; +import { useCopyExportQueryStep } from './sub_steps/copy_export_query'; +import { useMacrosFileUploadStep } from './sub_steps/macros_file_upload'; +import * as i18n from './translations'; +import { useCheckResourcesStep } from './sub_steps/check_resources'; + +const DataInputStepNumber: DataInputStep = 2; + +interface MacrosDataInputSubStepsProps { + migrationStats: RuleMigrationTaskStats; + missingMacros: string[]; + onMacrosCreated: OnResourcesCreated; + onMissingResourcesFetched: OnMissingResourcesFetched; +} +interface MacrosDataInputProps + extends Omit { + dataInputStep: DataInputStep; + migrationStats?: RuleMigrationTaskStats; + missingMacros?: string[]; +} +export const MacrosDataInput = React.memo( + ({ + dataInputStep, + migrationStats, + missingMacros, + onMacrosCreated, + onMissingResourcesFetched, + }) => { + const dataInputStatus = useMemo( + () => getStatus(DataInputStepNumber, dataInputStep), + [dataInputStep] + ); + + return ( + + + + + + + + + + {i18n.MACROS_DATA_INPUT_TITLE} + + + + + {dataInputStatus === 'current' && migrationStats && missingMacros && ( + + + + )} + + + ); + } +); +MacrosDataInput.displayName = 'MacrosDataInput'; + +const END = 10 as const; +type SubStep = 1 | 2 | 3 | typeof END; +export const MacrosDataInputSubSteps = React.memo( + ({ migrationStats, missingMacros, onMacrosCreated, onMissingResourcesFetched }) => { + const [subStep, setSubStep] = useState(missingMacros.length ? 1 : 3); + + // Copy query step + const onCopied = useCallback(() => { + setSubStep(2); + }, []); + const copyStep = useCopyExportQueryStep({ status: getStatus(1, subStep), onCopied }); + + // Upload macros step + const onMacrosCreatedStep = useCallback(() => { + onMacrosCreated(); + setSubStep(3); + }, [onMacrosCreated]); + const uploadStep = useMacrosFileUploadStep({ + status: getStatus(2, subStep), + migrationStats, + missingMacros, + onMacrosCreated: onMacrosCreatedStep, + }); + + // Check missing resources step + const onMissingResourcesFetchedStep = useCallback( + (newMissingResources) => { + onMissingResourcesFetched(newMissingResources); + setSubStep(END); + }, + [onMissingResourcesFetched] + ); + const resourcesStep = useCheckResourcesStep({ + status: getStatus(3, subStep), + migrationStats, + onMissingResourcesFetched: onMissingResourcesFetchedStep, + }); + + const steps = useMemo( + () => [copyStep, uploadStep, resourcesStep], + [copyStep, uploadStep, resourcesStep] + ); + + return ( + + + + ); + } +); +MacrosDataInputSubSteps.displayName = 'MacrosDataInputActive'; diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/macros/sub_steps/check_resources/index.tsx b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/macros/sub_steps/check_resources/index.tsx new file mode 100644 index 0000000000000..d83890e1f260c --- /dev/null +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/macros/sub_steps/check_resources/index.tsx @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useEffect, useMemo } from 'react'; +import { EuiText, type EuiStepProps, type EuiStepStatus } from '@elastic/eui'; +// import { useGetMissingResources } from '../../../../../../logic/use_get_migration_missing_resources'; +import type { RuleMigrationTaskStats } from '../../../../../../../../../common/siem_migrations/model/rule_migration.gen'; +import { useGetMissingResources } from '../../../../../../service/hooks/use_get_missing_resources'; +import type { OnMissingResourcesFetched } from '../../../../types'; +import * as i18n from './translations'; + +export interface CheckResourcesStepProps { + status: EuiStepStatus; + migrationStats: RuleMigrationTaskStats | undefined; + onMissingResourcesFetched: OnMissingResourcesFetched; +} +export const useCheckResourcesStep = ({ + status, + migrationStats, + onMissingResourcesFetched, +}: CheckResourcesStepProps): EuiStepProps => { + const { getMissingResources, isLoading, error } = + useGetMissingResources(onMissingResourcesFetched); + + useEffect(() => { + if (status === 'current' && migrationStats?.id) { + getMissingResources(migrationStats.id); + } + }, [getMissingResources, status, migrationStats?.id]); + + const uploadStepStatus = useMemo(() => { + if (isLoading) { + return 'loading'; + } + if (error) { + return 'danger'; + } + return status; + }, [isLoading, error, status]); + + return { + title: i18n.RULES_DATA_INPUT_CHECK_RESOURCES_TITLE, + status: uploadStepStatus, + children: ( + + {i18n.RULES_DATA_INPUT_CHECK_RESOURCES_DESCRIPTION} + + ), + }; +}; diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/macros/sub_steps/check_resources/translations.ts b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/macros/sub_steps/check_resources/translations.ts new file mode 100644 index 0000000000000..159b4033fafd6 --- /dev/null +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/macros/sub_steps/check_resources/translations.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; + +export const RULES_DATA_INPUT_CHECK_RESOURCES_TITLE = i18n.translate( + 'xpack.securitySolution.siemMigrations.rules.dataInputFlyout.rules.checkResources.title', + { defaultMessage: 'Check for macros and lookups' } +); + +export const RULES_DATA_INPUT_CHECK_RESOURCES_DESCRIPTION = i18n.translate( + 'xpack.securitySolution.siemMigrations.rules.dataInputFlyout.rules.checkResources.description', + { + defaultMessage: `For best translation results, we will automatically review your rules for macros and lookups and ask you to upload them. Once uploaded, we'll be able to deliver a more complete rule translation for all rules using those macros or lookups.`, + } +); diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/macros/sub_steps/copy_export_query/copy_export_query.tsx b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/macros/sub_steps/copy_export_query/copy_export_query.tsx new file mode 100644 index 0000000000000..93f2ce715184c --- /dev/null +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/macros/sub_steps/copy_export_query/copy_export_query.tsx @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useCallback } from 'react'; +import { EuiCodeBlock, EuiSpacer, EuiText } from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n-react'; +import { MACROS_SPLUNK_QUERY } from '../../../../constants'; +import * as i18n from './translations'; + +interface CopyExportQueryProps { + onCopied: () => void; +} +export const CopyExportQuery = React.memo(({ onCopied }) => { + const onClick: React.MouseEventHandler = useCallback( + (ev) => { + // The only button inside the element is the "copy" button. + if ((ev.target as Element).tagName === 'BUTTON') { + onCopied(); + } + }, + [onCopied] + ); + + return ( + <> + {/* The click event is also dispatched when using the keyboard actions (space or enter) for "copy" button. + No need to use keyboard specific events, disabling the a11y lint rule:*/} + {/* eslint-disable-next-line jsx-a11y/click-events-have-key-events */} +
+ {/* onCopy react event is dispatched when the user copies text manually */} + + {MACROS_SPLUNK_QUERY} + +
+ + + {i18n.RULES_DATA_INPUT_COPY_DESCRIPTION_SECTION}, + format: {'JSON'}, + }} + /> + + + ); +}); +CopyExportQuery.displayName = 'CopyExportQuery'; diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/macros/sub_steps/copy_export_query/index.tsx b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/macros/sub_steps/copy_export_query/index.tsx new file mode 100644 index 0000000000000..3d2adcc78857b --- /dev/null +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/macros/sub_steps/copy_export_query/index.tsx @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import type { EuiStepProps, EuiStepStatus } from '@elastic/eui'; +import { CopyExportQuery } from './copy_export_query'; +import * as i18n from './translations'; + +export interface CopyExportQueryStepProps { + status: EuiStepStatus; + onCopied: () => void; +} +export const useCopyExportQueryStep = ({ + status, + onCopied, +}: CopyExportQueryStepProps): EuiStepProps => { + return { + title: i18n.RULES_DATA_INPUT_COPY_TITLE, + status, + children: , + }; +}; diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/macros/sub_steps/copy_export_query/translations.ts b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/macros/sub_steps/copy_export_query/translations.ts new file mode 100644 index 0000000000000..71466a54dd138 --- /dev/null +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/macros/sub_steps/copy_export_query/translations.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; + +export const RULES_DATA_INPUT_COPY_TITLE = i18n.translate( + 'xpack.securitySolution.siemMigrations.rules.dataInputFlyout.macros.copyExportQuery.title', + { defaultMessage: 'Copy macros query' } +); + +export const RULES_DATA_INPUT_COPY_DESCRIPTION_SECTION = i18n.translate( + 'xpack.securitySolution.siemMigrations.rules.dataInputFlyout.macros.copyExportQuery.description.section', + { defaultMessage: 'Search and Reporting' } +); diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/macros/sub_steps/macros_file_upload/index.tsx b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/macros/sub_steps/macros_file_upload/index.tsx new file mode 100644 index 0000000000000..f2353e3f0276a --- /dev/null +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/macros/sub_steps/macros_file_upload/index.tsx @@ -0,0 +1,94 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useCallback, useMemo } from 'react'; +import type { EuiStepProps, EuiStepStatus } from '@elastic/eui'; +import { ResourceIdentifier } from '../../../../../../../../../common/siem_migrations/rules/resources'; +import { useUpsertResources } from '../../../../../../service/hooks/use_upsert_resources'; +import type { + RuleMigrationResourceData, + RuleMigrationTaskStats, +} from '../../../../../../../../../common/siem_migrations/model/rule_migration.gen'; +import type { OnResourcesCreated } from '../../../../types'; +import { MacrosFileUpload } from './macros_file_upload'; +import * as i18n from './translations'; + +export interface RulesFileUploadStepProps { + status: EuiStepStatus; + migrationStats: RuleMigrationTaskStats; + missingMacros: string[]; + onMacrosCreated: OnResourcesCreated; +} +export const useMacrosFileUploadStep = ({ + status, + migrationStats, + missingMacros, + onMacrosCreated, +}: RulesFileUploadStepProps): EuiStepProps => { + const { upsertResources, isLoading, error } = useUpsertResources(onMacrosCreated); + + const upsertMigrationResources = useCallback( + (macrosFromFile: RuleMigrationResourceData[]) => { + const macrosIndexed: Record = Object.fromEntries( + macrosFromFile.map((macro) => [macro.name, macro]) + ); + const resourceIdentifier = new ResourceIdentifier('splunk'); + const macrosToUpsert: RuleMigrationResourceData[] = []; + let missingMacrosIt: string[] = missingMacros; + + while (missingMacrosIt.length > 0) { + const macros: RuleMigrationResourceData[] = []; + missingMacrosIt.forEach((macroName) => { + const macro = macrosIndexed[macroName]; + if (macro) { + macros.push(macro); + } else { + // Macro missing from file + } + }); + macrosToUpsert.push(...macros); + + missingMacrosIt = resourceIdentifier + .fromResources(macros) + .reduce((acc, resource) => { + if (resource.type === 'macro') { + acc.push(resource.name); + } + return acc; + }, []); + } + + if (macrosToUpsert.length === 0) { + return; // No missing macros provided + } + upsertResources(migrationStats.id, macrosToUpsert); + }, + [upsertResources, migrationStats, missingMacros] + ); + + const uploadStepStatus = useMemo(() => { + if (isLoading) { + return 'loading'; + } + if (error) { + return 'danger'; + } + return status; + }, [isLoading, error, status]); + + return { + title: i18n.RULES_DATA_INPUT_FILE_UPLOAD_TITLE, + status: uploadStepStatus, + children: ( + + ), + }; +}; diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/macros/sub_steps/macros_file_upload/macros_file_upload.tsx b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/macros/sub_steps/macros_file_upload/macros_file_upload.tsx new file mode 100644 index 0000000000000..b8d7022d9b454 --- /dev/null +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/macros/sub_steps/macros_file_upload/macros_file_upload.tsx @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useCallback, useMemo } from 'react'; +import { EuiFilePicker, EuiFormRow, EuiText } from '@elastic/eui'; +import { isPlainObject } from 'lodash'; +import type { RuleMigrationResourceData } from '../../../../../../../../../common/siem_migrations/model/rule_migration.gen'; +import { FILE_UPLOAD_ERROR } from '../../../../translations'; +import type { SPLUNK_MACROS_COLUMNS } from '../../../../constants'; +import { useParseFileInput, type SplunkRow } from '../../../common/use_parse_file_input'; +import * as i18n from './translations'; + +type SplunkMacroResult = Partial>; + +export interface MacrosFileUploadProps { + createResources: (resources: RuleMigrationResourceData[]) => void; + apiError?: string; + isLoading?: boolean; +} +export const MacrosFileUpload = React.memo( + ({ createResources, apiError, isLoading }) => { + const onFileParsed = useCallback( + (content: Array>) => { + const rules = content.map(formatMacroRow); + createResources(rules); + }, + [createResources] + ); + + const { parseFile, isParsing, error: fileError } = useParseFileInput(onFileParsed); + + const error = useMemo(() => { + if (apiError) { + return apiError; + } + return fileError; + }, [apiError, fileError]); + + return ( + + {error} + + } + isInvalid={error != null} + fullWidth + > + + + {i18n.RULES_DATA_INPUT_FILE_UPLOAD_PROMPT} + + + } + accept="application/json, application/x-ndjson" + onChange={parseFile} + display="large" + aria-label="Upload logs sample file" + isLoading={isParsing || isLoading} + disabled={isParsing || isLoading} + data-test-subj="macrosFilePicker" + data-loading={isParsing} + /> + + ); + } +); +MacrosFileUpload.displayName = 'MacrosFileUpload'; + +const formatMacroRow = (row: SplunkRow): RuleMigrationResourceData => { + if (!isPlainObject(row.result)) { + throw new Error(FILE_UPLOAD_ERROR.NOT_OBJECT); + } + const macroResource: Partial = { + type: 'macro', + name: row.result.title, + content: row.result.definition, + }; + // resource document format validation delegated to API + return macroResource as RuleMigrationResourceData; +}; diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/macros/sub_steps/macros_file_upload/translations.ts b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/macros/sub_steps/macros_file_upload/translations.ts new file mode 100644 index 0000000000000..25b64787d6dcd --- /dev/null +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/macros/sub_steps/macros_file_upload/translations.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; + +export const RULES_DATA_INPUT_FILE_UPLOAD_TITLE = i18n.translate( + 'xpack.securitySolution.siemMigrations.rules.dataInputFlyout.macros.macrosFileUpload.title', + { defaultMessage: 'Update your macros export' } +); +export const RULES_DATA_INPUT_FILE_UPLOAD_PROMPT = i18n.translate( + 'xpack.securitySolution.siemMigrations.rules.dataInputFlyout.macros.macrosFileUpload.prompt', + { defaultMessage: 'Select or drag and drop the exported JSON file' } +); + +export const RULES_DATA_INPUT_CREATE_MIGRATION_SUCCESS = i18n.translate( + 'xpack.securitySolution.siemMigrations.rules.dataInputFlyout.macros.macrosFileUpload.createSuccess', + { defaultMessage: 'Macros uploaded successfully' } +); +export const RULES_DATA_INPUT_CREATE_MIGRATION_ERROR = i18n.translate( + 'xpack.securitySolution.siemMigrations.rules.dataInputFlyout.macros.macrosFileUpload.createError', + { defaultMessage: 'Failed to upload macros file' } +); diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/macros/translations.ts b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/macros/translations.ts new file mode 100644 index 0000000000000..7061e91311308 --- /dev/null +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/macros/translations.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; + +export const MACROS_DATA_INPUT_TITLE = i18n.translate( + 'xpack.securitySolution.siemMigrations.rules.dataInputFlyout.macros.title', + { defaultMessage: 'Upload identified macros' } +); diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/rules/rules_data_input.tsx b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/rules/rules_data_input.tsx index 2b20dcda0cea7..acc22a030b02f 100644 --- a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/rules/rules_data_input.tsx +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/rules/rules_data_input.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import type { EuiStepProps, EuiStepStatus } from '@elastic/eui'; +import type { EuiStepProps } from '@elastic/eui'; import { EuiFlexGroup, EuiFlexItem, @@ -14,57 +14,31 @@ import { EuiSteps, EuiTitle, } from '@elastic/eui'; -import React, { useMemo, useState } from 'react'; -import { SubStepWrapper } from '../common/sub_step_wrapper'; -import type { OnMigrationCreated } from '../../types'; +import React, { useCallback, useMemo, useState } from 'react'; +import type { RuleMigrationTaskStats } from '../../../../../../../common/siem_migrations/model/rule_migration.gen'; +import { SubStepsWrapper } from '../common/sub_step_wrapper'; +import type { OnMigrationCreated, OnMissingResourcesFetched, DataInputStep } from '../../types'; import { useCopyExportQueryStep } from './sub_steps/copy_export_query'; import { useRulesFileUploadStep } from './sub_steps/rules_file_upload'; import * as i18n from './translations'; import { useCheckResourcesStep } from './sub_steps/check_resources'; +import { getStatus } from '../common/get_status'; -type Step = 1 | 2 | 3 | 4; -const getStatus = (step: Step, currentStep: Step): EuiStepStatus => { - if (step === currentStep) { - return 'current'; - } - if (step < currentStep) { - return 'complete'; - } - return 'incomplete'; -}; +const DataInputStepNumber: DataInputStep = 1; -interface RulesDataInputProps { - selected: boolean; +interface RulesDataInputSubStepsProps { + migrationStats?: RuleMigrationTaskStats; onMigrationCreated: OnMigrationCreated; + onMissingResourcesFetched: OnMissingResourcesFetched; +} +interface RulesDataInputProps extends RulesDataInputSubStepsProps { + dataInputStep: DataInputStep; } - export const RulesDataInput = React.memo( - ({ selected, onMigrationCreated }) => { - const [step, setStep] = useState(1); - - const copyStep = useCopyExportQueryStep({ - status: getStatus(1, step), - onCopied: () => setStep(2), - }); - - const uploadStep = useRulesFileUploadStep({ - status: getStatus(2, step), - onMigrationCreated: (stats) => { - onMigrationCreated(stats); - setStep(3); - }, - }); - - const resourcesStep = useCheckResourcesStep({ - status: getStatus(3, step), - onComplete: () => { - setStep(4); - }, - }); - - const steps = useMemo( - () => [copyStep, uploadStep, resourcesStep], - [copyStep, uploadStep, resourcesStep] + ({ dataInputStep, migrationStats, onMigrationCreated, onMissingResourcesFetched }) => { + const dataInputStatus = useMemo( + () => getStatus(DataInputStepNumber, dataInputStep), + [dataInputStep] ); return ( @@ -73,7 +47,11 @@ export const RulesDataInput = React.memo( - + @@ -82,14 +60,72 @@ export const RulesDataInput = React.memo( - - - - - + {dataInputStatus === 'current' && ( + + + + )} ); } ); RulesDataInput.displayName = 'RulesDataInput'; + +const END = 10 as const; +type SubStep = 1 | 2 | 3 | typeof END; +export const RulesDataInputSubSteps = React.memo( + ({ migrationStats, onMigrationCreated, onMissingResourcesFetched }) => { + const [subStep, setSubStep] = useState(migrationStats ? 3 : 1); + + // Copy query step + const onCopied = useCallback(() => { + setSubStep(2); + }, []); + const copyStep = useCopyExportQueryStep({ status: getStatus(1, subStep), onCopied }); + + // Upload rules step + const onMigrationCreatedStep = useCallback( + (stats) => { + onMigrationCreated(stats); + setSubStep(3); + }, + [onMigrationCreated] + ); + const uploadStep = useRulesFileUploadStep({ + status: getStatus(2, subStep), + migrationStats, + onMigrationCreated: onMigrationCreatedStep, + }); + + // Check missing resources step + const onMissingResourcesFetchedStep = useCallback( + (missingResources) => { + onMissingResourcesFetched(missingResources); + setSubStep(END); + }, + [onMissingResourcesFetched] + ); + const resourcesStep = useCheckResourcesStep({ + status: getStatus(3, subStep), + migrationStats, + onMissingResourcesFetched: onMissingResourcesFetchedStep, + }); + + const steps = useMemo( + () => [copyStep, uploadStep, resourcesStep], + [copyStep, uploadStep, resourcesStep] + ); + + return ( + + + + ); + } +); +RulesDataInputSubSteps.displayName = 'RulesDataInputActive'; diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/rules/sub_steps/check_resources/index.tsx b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/rules/sub_steps/check_resources/index.tsx index 3b081eb203267..02aa109872f4a 100644 --- a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/rules/sub_steps/check_resources/index.tsx +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/rules/sub_steps/check_resources/index.tsx @@ -5,22 +5,45 @@ * 2.0. */ -import React from 'react'; +import React, { useEffect, useMemo } from 'react'; import { EuiText, type EuiStepProps, type EuiStepStatus } from '@elastic/eui'; +import type { RuleMigrationTaskStats } from '../../../../../../../../../common/siem_migrations/model/rule_migration.gen'; +import { useGetMissingResources } from '../../../../../../service/hooks/use_get_missing_resources'; +import type { OnMissingResourcesFetched } from '../../../../types'; import * as i18n from './translations'; export interface CheckResourcesStepProps { status: EuiStepStatus; - onComplete: () => void; + migrationStats: RuleMigrationTaskStats | undefined; + onMissingResourcesFetched: OnMissingResourcesFetched; } export const useCheckResourcesStep = ({ status, - onComplete, + migrationStats, + onMissingResourcesFetched, }: CheckResourcesStepProps): EuiStepProps => { - // onComplete(); // TODO: check the resources + const { getMissingResources, isLoading, error } = + useGetMissingResources(onMissingResourcesFetched); + + useEffect(() => { + if (status === 'current' && migrationStats?.id) { + getMissingResources(migrationStats.id); + } + }, [getMissingResources, status, migrationStats?.id]); + + const uploadStepStatus = useMemo(() => { + if (isLoading) { + return 'loading'; + } + if (error) { + return 'danger'; + } + return status; + }, [isLoading, error, status]); + return { title: i18n.RULES_DATA_INPUT_CHECK_RESOURCES_TITLE, - status, + status: uploadStepStatus, children: ( {i18n.RULES_DATA_INPUT_CHECK_RESOURCES_DESCRIPTION} diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/rules/sub_steps/copy_export_query/translations.ts b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/rules/sub_steps/copy_export_query/translations.ts index d76eb71f2e378..78a0636661604 100644 --- a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/rules/sub_steps/copy_export_query/translations.ts +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/rules/sub_steps/copy_export_query/translations.ts @@ -9,7 +9,7 @@ import { i18n } from '@kbn/i18n'; export const RULES_DATA_INPUT_COPY_TITLE = i18n.translate( 'xpack.securitySolution.siemMigrations.rules.dataInputFlyout.rules.copyExportQuery.title', - { defaultMessage: 'Copy and export query' } + { defaultMessage: 'Copy rules query' } ); export const RULES_DATA_INPUT_COPY_DESCRIPTION_SECTION = i18n.translate( diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/rules/sub_steps/rules_file_upload/index.tsx b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/rules/sub_steps/rules_file_upload/index.tsx index ab7838b28908b..97dfd903e499e 100644 --- a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/rules/sub_steps/rules_file_upload/index.tsx +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/rules/sub_steps/rules_file_upload/index.tsx @@ -7,6 +7,7 @@ import React, { useCallback, useMemo, useState } from 'react'; import type { EuiStepProps, EuiStepStatus } from '@elastic/eui'; +import type { RuleMigrationTaskStats } from '../../../../../../../../../common/siem_migrations/model/rule_migration.gen'; import type { OnMigrationCreated } from '../../../../types'; import { RulesFileUpload } from './rules_file_upload'; import { @@ -17,13 +18,15 @@ import * as i18n from './translations'; export interface RulesFileUploadStepProps { status: EuiStepStatus; + migrationStats?: RuleMigrationTaskStats; onMigrationCreated: OnMigrationCreated; } export const useRulesFileUploadStep = ({ status, + migrationStats, onMigrationCreated, }: RulesFileUploadStepProps): EuiStepProps => { - const [isCreated, setIsCreated] = useState(false); + const [isCreated, setIsCreated] = useState(!!migrationStats); const onSuccess = useCallback( (stats) => { setIsCreated(true); diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/rules/sub_steps/rules_file_upload/parse_rules_file.ts b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/rules/sub_steps/rules_file_upload/parse_rules_file.ts deleted file mode 100644 index 3d5dbb32ccde8..0000000000000 --- a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/rules/sub_steps/rules_file_upload/parse_rules_file.ts +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { isPlainObject } from 'lodash/fp'; -import type { OriginalRule } from '../../../../../../../../../common/siem_migrations/model/rule_migration.gen'; -import type { SPL_RULES_COLUMNS } from '../../../../constants'; -import * as i18n from './translations'; - -type SplunkResult = Partial>; -interface SplunkRow { - result: SplunkResult; -} - -export const parseContent = (fileContent: string): OriginalRule[] => { - const trimmedContent = fileContent.trim(); - let arrayContent: SplunkRow[]; - if (trimmedContent.startsWith('[')) { - arrayContent = parseJSONArray(trimmedContent); - } else { - arrayContent = parseNDJSON(trimmedContent); - } - if (arrayContent.length === 0) { - throw new Error(i18n.RULES_DATA_INPUT_FILE_UPLOAD_ERROR.EMPTY); - } - return arrayContent.map(convertFormat); -}; - -const parseNDJSON = (fileContent: string): SplunkRow[] => { - return fileContent - .split(/\n(?=\{)/) // split at newline followed by '{'. - .filter((entry) => entry.trim() !== '') // Remove empty entries. - .map(parseJSON); // Parse each entry as JSON. -}; - -const parseJSONArray = (fileContent: string): SplunkRow[] => { - const parsedContent = parseJSON(fileContent); - if (!Array.isArray(parsedContent)) { - throw new Error(i18n.RULES_DATA_INPUT_FILE_UPLOAD_ERROR.NOT_ARRAY); - } - return parsedContent; -}; - -const parseJSON = (fileContent: string) => { - try { - return JSON.parse(fileContent); - } catch (error) { - if (error instanceof RangeError) { - throw new Error(i18n.RULES_DATA_INPUT_FILE_UPLOAD_ERROR.TOO_LARGE_TO_PARSE); - } - throw new Error(i18n.RULES_DATA_INPUT_FILE_UPLOAD_ERROR.CAN_NOT_PARSE); - } -}; - -const convertFormat = (row: SplunkRow): OriginalRule => { - if (!isPlainObject(row) || !isPlainObject(row.result)) { - throw new Error(i18n.RULES_DATA_INPUT_FILE_UPLOAD_ERROR.NOT_OBJECT); - } - const originalRule: Partial = { - id: row.result.id, - vendor: 'splunk', - title: row.result.title, - query: row.result.search, - query_language: 'spl', - description: row.result['action.escu.eli5']?.trim() || row.result.description, - }; - - if (row.result['action.correlationsearch.annotations']) { - try { - originalRule.annotations = JSON.parse(row.result['action.correlationsearch.annotations']); - } catch (error) { - delete originalRule.annotations; - } - } - return originalRule as OriginalRule; -}; diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/rules/sub_steps/rules_file_upload/rules_file_upload.tsx b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/rules/sub_steps/rules_file_upload/rules_file_upload.tsx index 0f9787a4ddf68..bec9182420073 100644 --- a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/rules/sub_steps/rules_file_upload/rules_file_upload.tsx +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/rules/sub_steps/rules_file_upload/rules_file_upload.tsx @@ -5,12 +5,17 @@ * 2.0. */ -import React, { useCallback, useMemo, useState } from 'react'; +import React, { useCallback, useMemo } from 'react'; import { EuiFilePicker, EuiFormRow, EuiText } from '@elastic/eui'; +import { isPlainObject } from 'lodash'; import type { OriginalRule } from '../../../../../../../../../common/siem_migrations/model/rule_migration.gen'; import type { CreateMigration } from '../../../../../../service/hooks/use_create_migration'; -import { parseContent } from './parse_rules_file'; import * as i18n from './translations'; +import { FILE_UPLOAD_ERROR } from '../../../../translations'; +import { useParseFileInput, type SplunkRow } from '../../../common/use_parse_file_input'; +import type { SPLUNK_RULES_COLUMNS } from '../../../../constants'; + +type SplunkRulesResult = Partial>; export interface RulesFileUploadProps { createMigration: CreateMigration; @@ -20,65 +25,16 @@ export interface RulesFileUploadProps { } export const RulesFileUpload = React.memo( ({ createMigration, apiError, isLoading, isCreated }) => { - const [isParsing, setIsParsing] = useState(false); - const [fileError, setFileError] = useState(); - - const onChangeFile = useCallback( - (files: FileList | null) => { - if (!files) { - return; - } - - setFileError(undefined); - - const rulesFile = files[0]; - const reader = new FileReader(); - - reader.onloadstart = () => setIsParsing(true); - reader.onloadend = () => setIsParsing(false); - - reader.onload = function (e) { - // We can safely cast to string since we call `readAsText` to load the file. - const fileContent = e.target?.result as string | undefined; - - if (fileContent == null) { - setFileError(i18n.RULES_DATA_INPUT_FILE_UPLOAD_ERROR.CAN_NOT_READ); - return; - } - - if (fileContent === '' && e.loaded > 100000) { - // V8-based browsers can't handle large files and return an empty string - // instead of an error; see https://stackoverflow.com/a/61316641 - setFileError(i18n.RULES_DATA_INPUT_FILE_UPLOAD_ERROR.TOO_LARGE_TO_PARSE); - return; - } - - let data: OriginalRule[]; - try { - data = parseContent(fileContent); - createMigration(data); - } catch (err) { - setFileError(err.message); - } - }; - - const handleReaderError = function () { - const message = reader.error?.message; - if (message) { - setFileError(i18n.RULES_DATA_INPUT_FILE_UPLOAD_ERROR.CAN_NOT_READ_WITH_REASON(message)); - } else { - setFileError(i18n.RULES_DATA_INPUT_FILE_UPLOAD_ERROR.CAN_NOT_READ); - } - }; - - reader.onerror = handleReaderError; - reader.onabort = handleReaderError; - - reader.readAsText(rulesFile); + const onFileParsed = useCallback( + (content: Array>) => { + const rules = content.map(formatRuleRow); + createMigration(rules); }, [createMigration] ); + const { parseFile, isParsing, error: fileError } = useParseFileInput(onFileParsed); + const error = useMemo(() => { if (apiError) { return apiError; @@ -106,10 +62,10 @@ export const RulesFileUpload = React.memo( } - accept="application/json" - onChange={onChangeFile} + accept="application/json, application/x-ndjson" + onChange={parseFile} display="large" - aria-label="Upload logs sample file" + aria-label="Upload rules file" isLoading={isParsing || isLoading} disabled={isLoading || isCreated} data-test-subj="rulesFilePicker" @@ -120,3 +76,27 @@ export const RulesFileUpload = React.memo( } ); RulesFileUpload.displayName = 'RulesFileUpload'; + +const formatRuleRow = (row: SplunkRow): OriginalRule => { + if (!isPlainObject(row.result)) { + throw new Error(FILE_UPLOAD_ERROR.NOT_OBJECT); + } + const originalRule: Partial = { + id: row.result.id, + vendor: 'splunk', + title: row.result.title, + query: row.result.search, + query_language: 'spl', + description: row.result['action.escu.eli5']?.trim() || row.result.description, + }; + + if (row.result['action.correlationsearch.annotations']) { + try { + originalRule.annotations = JSON.parse(row.result['action.correlationsearch.annotations']); + } catch (error) { + delete originalRule.annotations; + } + } + // rule document format validation delegated to API + return originalRule as OriginalRule; +}; diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/rules/sub_steps/rules_file_upload/translations.ts b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/rules/sub_steps/rules_file_upload/translations.ts index 675eed61f4973..b560849ca1cd7 100644 --- a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/rules/sub_steps/rules_file_upload/translations.ts +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/steps/rules/sub_steps/rules_file_upload/translations.ts @@ -9,57 +9,13 @@ import { i18n } from '@kbn/i18n'; export const RULES_DATA_INPUT_FILE_UPLOAD_TITLE = i18n.translate( 'xpack.securitySolution.siemMigrations.rules.dataInputFlyout.rules.rulesFileUpload.title', - { defaultMessage: 'Update your rule export' } + { defaultMessage: 'Update your rules export' } ); export const RULES_DATA_INPUT_FILE_UPLOAD_PROMPT = i18n.translate( 'xpack.securitySolution.siemMigrations.rules.dataInputFlyout.rules.rulesFileUpload.prompt', { defaultMessage: 'Select or drag and drop the exported JSON file' } ); -export const RULES_DATA_INPUT_FILE_UPLOAD_ERROR = { - CAN_NOT_READ: i18n.translate( - 'xpack.securitySolution.siemMigrations.rules.dataInputFlyout.rules.rulesFileUpload.error.canNotRead', - { defaultMessage: 'Failed to read the rules export file' } - ), - CAN_NOT_READ_WITH_REASON: (reason: string) => - i18n.translate( - 'xpack.securitySolution.siemMigrations.rules.dataInputFlyout.rules.rulesFileUpload.error.canNotReadWithReason', - { - defaultMessage: 'An error occurred when reading rules export file: {reason}', - values: { reason }, - } - ), - CAN_NOT_PARSE: i18n.translate( - 'xpack.securitySolution.siemMigrations.rules.dataInputFlyout.rules.rulesFileUpload.error.canNotParse', - { defaultMessage: 'Cannot parse the rules export file as either a JSON file' } - ), - TOO_LARGE_TO_PARSE: i18n.translate( - 'xpack.securitySolution.siemMigrations.rules.dataInputFlyout.rules.rulesFileUpload.error.tooLargeToParse', - { defaultMessage: 'This rules export file is too large to parse' } - ), - NOT_ARRAY: i18n.translate( - 'xpack.securitySolution.siemMigrations.rules.dataInputFlyout.rules.rulesFileUpload.error.notArray', - { defaultMessage: 'The rules export file is not an array' } - ), - EMPTY: i18n.translate( - 'xpack.securitySolution.siemMigrations.rules.dataInputFlyout.rules.rulesFileUpload.error.empty', - { defaultMessage: 'The rules export file is empty' } - ), - NOT_OBJECT: i18n.translate( - 'xpack.securitySolution.siemMigrations.rules.dataInputFlyout.rules.rulesFileUpload.error.notObject', - { defaultMessage: 'The rules export file contains non-object entries' } - ), - WRONG_FORMAT: (formatError: string) => { - return i18n.translate( - 'xpack.securitySolution.siemMigrations.rules.dataInputFlyout.rules.rulesFileUpload.error.wrongFormat', - { - defaultMessage: 'The rules export file has wrong format: {formatError}', - values: { formatError }, - } - ); - }, -}; - export const RULES_DATA_INPUT_CREATE_MIGRATION_SUCCESS = i18n.translate( 'xpack.securitySolution.siemMigrations.rules.dataInputFlyout.rules.rulesFileUpload.createSuccess', { defaultMessage: 'Rules uploaded successfully' } diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/translations.ts b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/translations.ts new file mode 100644 index 0000000000000..1e7988b596e42 --- /dev/null +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/translations.ts @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; + +export const FILE_UPLOAD_ERROR = { + CAN_NOT_READ: i18n.translate( + 'xpack.securitySolution.siemMigrations.rules.dataInputFlyout.fileUploadError.canNotRead', + { defaultMessage: 'Failed to read file' } + ), + CAN_NOT_READ_WITH_REASON: (reason: string) => + i18n.translate( + 'xpack.securitySolution.siemMigrations.rules.dataInputFlyout.fileUploadError.canNotReadWithReason', + { + defaultMessage: 'An error occurred when reading file: {reason}', + values: { reason }, + } + ), + CAN_NOT_PARSE: i18n.translate( + 'xpack.securitySolution.siemMigrations.rules.dataInputFlyout.fileUploadError.canNotParse', + { defaultMessage: 'Cannot parse the file as either a JSON file or NDJSON file' } + ), + TOO_LARGE_TO_PARSE: i18n.translate( + 'xpack.securitySolution.siemMigrations.rules.dataInputFlyout.fileUploadError.tooLargeToParse', + { defaultMessage: 'This file is too large to parse' } + ), + NOT_ARRAY: i18n.translate( + 'xpack.securitySolution.siemMigrations.rules.dataInputFlyout.fileUploadError.notArray', + { defaultMessage: 'The file content is not an array' } + ), + EMPTY: i18n.translate( + 'xpack.securitySolution.siemMigrations.rules.dataInputFlyout.fileUploadError.empty', + { defaultMessage: 'The file is empty' } + ), + NOT_OBJECT: i18n.translate( + 'xpack.securitySolution.siemMigrations.rules.dataInputFlyout.fileUploadError.notObject', + { defaultMessage: 'The file contains non-object entries' } + ), + WRONG_FORMAT: (formatError: string) => { + return i18n.translate( + 'xpack.securitySolution.siemMigrations.rules.dataInputFlyout.fileUploadError.wrongFormat', + { + defaultMessage: 'The file has wrong format: {formatError}', + values: { formatError }, + } + ); + }, +}; diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/types.ts b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/types.ts index 16d8f60043bcb..b293a9394ba54 100644 --- a/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/types.ts +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/components/data_input_flyout/types.ts @@ -5,6 +5,18 @@ * 2.0. */ -import type { RuleMigrationTaskStats } from '../../../../../common/siem_migrations/model/rule_migration.gen'; +import type { + RuleMigrationResourceData, + RuleMigrationTaskStats, +} from '../../../../../common/siem_migrations/model/rule_migration.gen'; export type OnMigrationCreated = (migrationStats: RuleMigrationTaskStats) => void; +export type OnResourcesCreated = () => void; +export type OnMissingResourcesFetched = (missingResources: RuleMigrationResourceData[]) => void; + +export enum DataInputStep { + Rules = 1, + Macros = 2, + Lookups = 3, + End = 10, +} diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/service/hooks/use_create_migration.ts b/x-pack/plugins/security_solution/public/siem_migrations/rules/service/hooks/use_create_migration.ts index 94082cf59d359..18a4ebe47bf8d 100644 --- a/x-pack/plugins/security_solution/public/siem_migrations/rules/service/hooks/use_create_migration.ts +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/service/hooks/use_create_migration.ts @@ -12,10 +12,15 @@ import type { CreateRuleMigrationRequestBody } from '../../../../../common/siem_ import { useKibana } from '../../../../common/lib/kibana/kibana_react'; import { reducer, initialState } from './common/api_request_reducer'; -export const RULES_DATA_INPUT_CREATE_MIGRATION_SUCCESS = i18n.translate( - 'xpack.securitySolution.siemMigrations.rules.service.createRuleSuccess', - { defaultMessage: 'Rules uploaded successfully' } +export const RULES_DATA_INPUT_CREATE_MIGRATION_SUCCESS_TITLE = i18n.translate( + 'xpack.securitySolution.siemMigrations.rules.service.createRuleSuccess.title', + { defaultMessage: 'Rule migration created successfully' } ); +export const RULES_DATA_INPUT_CREATE_MIGRATION_SUCCESS_DESCRIPTION = (rules: number) => + i18n.translate( + 'xpack.securitySolution.siemMigrations.rules.service.createRuleSuccess.description', + { defaultMessage: '{rules} rules uploaded', values: { rules } } + ); export const RULES_DATA_INPUT_CREATE_MIGRATION_ERROR = i18n.translate( 'xpack.securitySolution.siemMigrations.rules.service.createRuleError', { defaultMessage: 'Failed to upload rules file' } @@ -36,7 +41,10 @@ export const useCreateMigration = (onSuccess: OnSuccess) => { const migrationId = await siemMigrations.rules.createRuleMigration(data); const stats = await siemMigrations.rules.getRuleMigrationStats(migrationId); - notifications.toasts.addSuccess(RULES_DATA_INPUT_CREATE_MIGRATION_SUCCESS); + notifications.toasts.addSuccess({ + title: RULES_DATA_INPUT_CREATE_MIGRATION_SUCCESS_TITLE, + text: RULES_DATA_INPUT_CREATE_MIGRATION_SUCCESS_DESCRIPTION(data.length), + }); onSuccess(stats); dispatch({ type: 'success' }); } catch (err) { diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/service/hooks/use_get_missing_resources.ts b/x-pack/plugins/security_solution/public/siem_migrations/rules/service/hooks/use_get_missing_resources.ts new file mode 100644 index 0000000000000..a0679aa1e8bd2 --- /dev/null +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/service/hooks/use_get_missing_resources.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useCallback, useReducer } from 'react'; +import { i18n } from '@kbn/i18n'; +import type { RuleMigrationResourceData } from '../../../../../common/siem_migrations/model/rule_migration.gen'; +import { useKibana } from '../../../../common/lib/kibana/kibana_react'; +import { reducer, initialState } from './common/api_request_reducer'; + +export const RULES_DATA_INPUT_CREATE_MIGRATION_ERROR = i18n.translate( + 'xpack.securitySolution.siemMigrations.rules.service.getMissingResourcesError', + { defaultMessage: 'Failed to fetch missing macros & lookups' } +); + +export type GetMissingResources = (migrationId: string) => void; +export type OnSuccess = (missingResources: RuleMigrationResourceData[]) => void; + +export const useGetMissingResources = (onSuccess: OnSuccess) => { + const { siemMigrations, notifications } = useKibana().services; + const [state, dispatch] = useReducer(reducer, initialState); + + const getMissingResources = useCallback( + (migrationId) => { + (async () => { + try { + dispatch({ type: 'start' }); + const missingResources = await siemMigrations.rules.getMissingResources(migrationId); + + onSuccess(missingResources); + dispatch({ type: 'success' }); + } catch (err) { + const apiError = err.body ?? err; + notifications.toasts.addError(apiError, { + title: RULES_DATA_INPUT_CREATE_MIGRATION_ERROR, + }); + dispatch({ type: 'error', error: apiError }); + } + })(); + }, + [siemMigrations.rules, notifications.toasts, onSuccess] + ); + + return { isLoading: state.loading, error: state.error, getMissingResources }; +}; diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/service/hooks/use_latest_stats.ts b/x-pack/plugins/security_solution/public/siem_migrations/rules/service/hooks/use_latest_stats.ts index 8b692f07eb3cb..88c6b798e2f40 100644 --- a/x-pack/plugins/security_solution/public/siem_migrations/rules/service/hooks/use_latest_stats.ts +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/service/hooks/use_latest_stats.ts @@ -6,7 +6,7 @@ */ import useObservable from 'react-use/lib/useObservable'; -import { useEffect, useMemo } from 'react'; +import { useCallback, useEffect, useMemo } from 'react'; import { useKibana } from '../../../../common/lib/kibana/kibana_react'; export const useLatestStats = () => { @@ -16,8 +16,12 @@ export const useLatestStats = () => { siemMigrations.rules.startPolling(); }, [siemMigrations.rules]); + const refreshStats = useCallback(() => { + siemMigrations.rules.getRuleMigrationsStats(); // this updates latestStats$ internally + }, [siemMigrations.rules]); + const latestStats$ = useMemo(() => siemMigrations.rules.getLatestStats$(), [siemMigrations]); const latestStats = useObservable(latestStats$, null); - return { data: latestStats ?? [], isLoading: latestStats === null }; + return { data: latestStats ?? [], isLoading: latestStats === null, refreshStats }; }; diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/service/hooks/use_upsert_resources.ts b/x-pack/plugins/security_solution/public/siem_migrations/rules/service/hooks/use_upsert_resources.ts new file mode 100644 index 0000000000000..eab3888422bae --- /dev/null +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/service/hooks/use_upsert_resources.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useCallback, useReducer } from 'react'; +import { i18n } from '@kbn/i18n'; +import type { UpsertRuleMigrationResourcesRequestBody } from '../../../../../common/siem_migrations/model/api/rules/rule_migration.gen'; +import { useKibana } from '../../../../common/lib/kibana/kibana_react'; +import { reducer, initialState } from './common/api_request_reducer'; + +export const RULES_DATA_INPUT_UPSERT_MIGRATION_RESOURCES_ERROR = i18n.translate( + 'xpack.securitySolution.siemMigrations.rules.service.upsertRuleMigrationResourcesError', + { defaultMessage: 'Failed to upload rule migration resources' } +); + +export type UpsertResources = ( + migrationId: string, + data: UpsertRuleMigrationResourcesRequestBody +) => void; +export type OnSuccess = () => void; + +export const useUpsertResources = (onSuccess: OnSuccess) => { + const { siemMigrations, notifications } = useKibana().services; + const [state, dispatch] = useReducer(reducer, initialState); + + const upsertResources = useCallback( + (migrationId, data) => { + (async () => { + try { + dispatch({ type: 'start' }); + await siemMigrations.rules.upsertMigrationResources(migrationId, data); + + onSuccess(); + dispatch({ type: 'success' }); + } catch (err) { + const apiError = err.body ?? err; + notifications.toasts.addError(apiError, { + title: RULES_DATA_INPUT_UPSERT_MIGRATION_RESOURCES_ERROR, + }); + dispatch({ type: 'error', error: apiError }); + } + })(); + }, + [siemMigrations.rules, notifications.toasts, onSuccess] + ); + + return { isLoading: state.loading, error: state.error, upsertResources }; +}; diff --git a/x-pack/plugins/security_solution/public/siem_migrations/rules/service/rule_migrations_service.ts b/x-pack/plugins/security_solution/public/siem_migrations/rules/service/rule_migrations_service.ts index c13b0606d771d..75b7887db6525 100644 --- a/x-pack/plugins/security_solution/public/siem_migrations/rules/service/rule_migrations_service.ts +++ b/x-pack/plugins/security_solution/public/siem_migrations/rules/service/rule_migrations_service.ts @@ -13,11 +13,15 @@ import { TRACE_OPTIONS_SESSION_STORAGE_KEY, } from '@kbn/elastic-assistant/impl/assistant_context/constants'; import type { LangSmithOptions } from '../../../../common/siem_migrations/model/common.gen'; -import type { RuleMigrationTaskStats } from '../../../../common/siem_migrations/model/rule_migration.gen'; +import type { + RuleMigrationResourceData, + RuleMigrationTaskStats, +} from '../../../../common/siem_migrations/model/rule_migration.gen'; import type { CreateRuleMigrationRequestBody, GetAllStatsRuleMigrationResponse, GetRuleMigrationStatsResponse, + UpsertRuleMigrationResourcesRequestBody, } from '../../../../common/siem_migrations/model/api/rules/rule_migration.gen'; import { SiemMigrationTaskStatus } from '../../../../common/siem_migrations/constants'; import type { StartPluginsDependencies } from '../../../types'; @@ -29,6 +33,8 @@ import { getRuleMigrationsStatsAll, startRuleMigration, type GetRuleMigrationsStatsAllParams, + getMissingResources, + upsertMigrationResources, } from '../api'; import type { RuleMigrationStats } from '../types'; import { getSuccessToast } from './success_notification'; @@ -99,6 +105,20 @@ export class SiemRulesMigrationsService { return migrationId as string; } + public async upsertMigrationResources( + migrationId: string, + body: UpsertRuleMigrationResourcesRequestBody + ): Promise { + if (body.length === 0) { + throw new Error(i18n.EMPTY_RULES_ERROR); + } + // Batching creation to avoid hitting the max payload size limit of the API + for (let i = 0; i < body.length; i += CREATE_MIGRATION_BODY_BATCH_SIZE) { + const bodyBatch = body.slice(i, i + CREATE_MIGRATION_BODY_BATCH_SIZE); + await upsertMigrationResources({ migrationId, body: bodyBatch }); + } + } + public async startRuleMigration(migrationId: string): Promise { const connectorId = this.connectorIdStorage.get(); if (!connectorId) { @@ -135,6 +155,10 @@ export class SiemRulesMigrationsService { return results; } + public async getMissingResources(migrationId: string): Promise { + return getMissingResources({ migrationId }); + } + private async getRuleMigrationsStatsWithRetry( params: GetRuleMigrationsStatsAllParams = {}, sleepSecs?: number diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/create.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/create.ts index 19669fa75cd3d..1e0a2fc5cb8c5 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/create.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/create.ts @@ -8,6 +8,7 @@ import type { IKibanaResponse, Logger } from '@kbn/core/server'; import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; import { v4 as uuidV4 } from 'uuid'; +import { ResourceIdentifier } from '../../../../../common/siem_migrations/rules/resources'; import { SIEM_RULE_MIGRATION_CREATE_PATH } from '../../../../../common/siem_migrations/constants'; import { CreateRuleMigrationRequestBody, @@ -43,6 +44,11 @@ export const registerSiemRuleMigrationsCreateRoute = ( const originalRules = req.body; const migrationId = req.params.migration_id ?? uuidV4(); try { + const [firstOriginalRule] = originalRules; + if (!firstOriginalRule) { + return res.noContent(); + } + const ctx = await context.resolve(['securitySolution']); const ruleMigrationsClient = ctx.securitySolution.getSiemRuleMigrationsClient(); @@ -53,6 +59,16 @@ export const registerSiemRuleMigrationsCreateRoute = ( await ruleMigrationsClient.data.rules.create(ruleMigrations); + // Create identified resource documents without content to keep track of them + const resourceIdentifier = new ResourceIdentifier(firstOriginalRule.vendor); + const resources = resourceIdentifier + .fromOriginalRules(originalRules) + .map((resource) => ({ ...resource, migration_id: migrationId })); + + if (resources.length > 0) { + await ruleMigrationsClient.data.resources.create(resources); + } + return res.ok({ body: { migration_id: migrationId } }); } catch (err) { logger.error(err); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/index.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/index.ts index a327d4b28a9bd..241e59ac02a27 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/index.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/index.ts @@ -20,6 +20,7 @@ import { registerSiemRuleMigrationsResourceGetRoute } from './resources/get'; import { registerSiemRuleMigrationsRetryRoute } from './retry'; import { registerSiemRuleMigrationsInstallRoute } from './install'; import { registerSiemRuleMigrationsInstallTranslatedRoute } from './install_translated'; +import { registerSiemRuleMigrationsResourceGetMissingRoute } from './resources/missing'; import { registerSiemRuleMigrationsPrebuiltRulesRoute } from './get_prebuilt_rules'; export const registerSiemRuleMigrationsRoutes = ( @@ -41,4 +42,5 @@ export const registerSiemRuleMigrationsRoutes = ( registerSiemRuleMigrationsResourceUpsertRoute(router, logger); registerSiemRuleMigrationsResourceGetRoute(router, logger); + registerSiemRuleMigrationsResourceGetMissingRoute(router, logger); }; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/resources/get.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/resources/get.ts index 7f2cfc8743f07..8d1e1d353e32d 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/resources/get.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/resources/get.ts @@ -39,16 +39,13 @@ export const registerSiemRuleMigrationsResourceGetRoute = ( withLicense( async (context, req, res): Promise> => { const migrationId = req.params.migration_id; - const { type, names } = req.query; + const { type, names, from, size } = req.query; try { const ctx = await context.resolve(['securitySolution']); const ruleMigrationsClient = ctx.securitySolution.getSiemRuleMigrationsClient(); - const resources = await ruleMigrationsClient.data.resources.get( - migrationId, - type, - names - ); + const options = { filters: { type, names }, from, size }; + const resources = await ruleMigrationsClient.data.resources.get(migrationId, options); return res.ok({ body: resources }); } catch (err) { diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/resources/missing.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/resources/missing.ts new file mode 100644 index 0000000000000..0c9ad11f4cce6 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/resources/missing.ts @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IKibanaResponse, Logger } from '@kbn/core/server'; +import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; +import type { RuleMigrationResourceData } from '../../../../../../common/siem_migrations/model/rule_migration.gen'; +import { + GetRuleMigrationResourcesMissingRequestParams, + type GetRuleMigrationResourcesMissingResponse, +} from '../../../../../../common/siem_migrations/model/api/rules/rule_migration.gen'; +import { SIEM_RULE_MIGRATION_RESOURCES_MISSING_PATH } from '../../../../../../common/siem_migrations/constants'; +import type { SecuritySolutionPluginRouter } from '../../../../../types'; +import { withLicense } from '../util/with_license'; + +export const registerSiemRuleMigrationsResourceGetMissingRoute = ( + router: SecuritySolutionPluginRouter, + logger: Logger +) => { + router.versioned + .get({ + path: SIEM_RULE_MIGRATION_RESOURCES_MISSING_PATH, + access: 'internal', + security: { authz: { requiredPrivileges: ['securitySolution'] } }, + }) + .addVersion( + { + version: '1', + validate: { + request: { + params: buildRouteValidationWithZod(GetRuleMigrationResourcesMissingRequestParams), + }, + }, + }, + withLicense( + async ( + context, + req, + res + ): Promise> => { + const migrationId = req.params.migration_id; + try { + const ctx = await context.resolve(['securitySolution']); + const ruleMigrationsClient = ctx.securitySolution.getSiemRuleMigrationsClient(); + + const options = { filters: { hasContent: false } }; + const batches = ruleMigrationsClient.data.resources.searchBatches(migrationId, options); + + const missingResources: RuleMigrationResourceData[] = []; + let results = await batches.next(); + while (results.length) { + missingResources.push(...results.map(({ type, name }) => ({ type, name }))); + results = await batches.next(); + } + + return res.ok({ body: missingResources }); + } catch (err) { + logger.error(err); + return res.badRequest({ body: err.message }); + } + } + ) + ); +}; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/resources/upsert.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/resources/upsert.ts index 645fa09b49dc1..9557c5cfd652f 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/resources/upsert.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/resources/upsert.ts @@ -7,6 +7,7 @@ import type { IKibanaResponse, Logger } from '@kbn/core/server'; import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; +import { ResourceIdentifier } from '../../../../../../common/siem_migrations/rules/resources'; import { UpsertRuleMigrationResourcesRequestBody, UpsertRuleMigrationResourcesRequestParams, @@ -49,13 +50,30 @@ export const registerSiemRuleMigrationsResourceUpsertRoute = ( const ctx = await context.resolve(['securitySolution']); const ruleMigrationsClient = ctx.securitySolution.getSiemRuleMigrationsClient(); + // Check if the migration exists + const { data } = await ruleMigrationsClient.data.rules.get(migrationId, { size: 1 }); + const [rule] = data; + if (!rule) { + return res.notFound({ body: { message: 'Migration not found' } }); + } + + // Upsert identified resource documents with content const ruleMigrations = resources.map((resource) => ({ - migration_id: migrationId, ...resource, + migration_id: migrationId, })); - await ruleMigrationsClient.data.resources.upsert(ruleMigrations); + // Create identified resource documents without content to keep track of them + const resourceIdentifier = new ResourceIdentifier(rule.original_rule.vendor); + const resourcesToCreate = resourceIdentifier + .fromResources(resources) + .map((resource) => ({ + ...resource, + migration_id: migrationId, + })); + await ruleMigrationsClient.data.resources.create(resourcesToCreate); + return res.ok({ body: { acknowledged: true } }); } catch (err) { logger.error(err); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/__mocks__/mocks.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/__mocks__/mocks.ts index d8dc1bb168a72..77ed5e87084e9 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/__mocks__/mocks.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/__mocks__/mocks.ts @@ -11,6 +11,10 @@ import type { RuleMigrationsDataRulesClient } from '../rule_migrations_data_rule export const mockRuleMigrationsDataRulesClient = { create: jest.fn().mockResolvedValue(undefined), get: jest.fn().mockResolvedValue([]), + searchBatches: jest.fn().mockReturnValue({ + next: jest.fn().mockResolvedValue([]), + all: jest.fn().mockResolvedValue([]), + }), takePending: jest.fn().mockResolvedValue([]), saveCompleted: jest.fn().mockResolvedValue(undefined), saveError: jest.fn().mockResolvedValue(undefined), @@ -27,6 +31,10 @@ export const MockRuleMigrationsDataRulesClient = jest export const mockRuleMigrationsDataResourcesClient = { upsert: jest.fn().mockResolvedValue(undefined), get: jest.fn().mockResolvedValue(undefined), + searchBatches: jest.fn().mockReturnValue({ + next: jest.fn().mockResolvedValue([]), + all: jest.fn().mockResolvedValue([]), + }), }; export const MockRuleMigrationsDataResourcesClient = jest .fn() diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_base_client.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_base_client.ts index 14825326eee0e..31931fce0b1d9 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_base_client.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_base_client.ts @@ -5,12 +5,19 @@ * 2.0. */ -import type { SearchHit, SearchResponse } from '@elastic/elasticsearch/lib/api/types'; +import type { + SearchHit, + SearchRequest, + SearchResponse, + Duration, +} from '@elastic/elasticsearch/lib/api/types'; import type { ElasticsearchClient, Logger } from '@kbn/core/server'; import assert from 'assert'; import type { Stored } from '../types'; import type { IndexNameProvider } from './rule_migrations_data_client'; +const DEFAULT_PIT_KEEP_ALIVE: Duration = '30s' as const; + export class RuleMigrationsDataBaseClient { constructor( protected getIndexName: IndexNameProvider, @@ -42,4 +49,48 @@ export class RuleMigrationsDataBaseClient { ? response.hits.total : response.hits.total?.value ?? 0; } + + /** Returns functions to iterate over all the search results in batches */ + protected getSearchBatches( + search: SearchRequest, + keepAlive: Duration = DEFAULT_PIT_KEEP_ALIVE + ) { + const pitPromise = this.getIndexName().then((index) => + this.esClient + .openPointInTime({ index, keep_alive: keepAlive }) + .then(({ id }) => ({ id, keep_alive: keepAlive })) + ); + + let currentBatchSearch: Promise> | undefined; + /* Returns the next batch of search results */ + const next = async (): Promise>> => { + const pit = await pitPromise; + if (!currentBatchSearch) { + currentBatchSearch = this.esClient.search({ ...search, pit }); + } else { + currentBatchSearch = currentBatchSearch.then((previousResponse) => { + if (previousResponse.hits.hits.length === 0) { + return previousResponse; + } + const lastSort = previousResponse.hits.hits[previousResponse.hits.hits.length - 1].sort; + return this.esClient.search({ ...search, pit, search_after: lastSort }); + }); + } + const response = await currentBatchSearch; + return this.processResponseHits(response); + }; + + /** Returns all the search results */ + const all = async (): Promise>> => { + const allResults: Array> = []; + let results = await next(); + while (results.length) { + allResults.push(...results); + results = await next(); + } + return allResults; + }; + + return { next, all }; + } } diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_resources_client.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_resources_client.ts index 888a41aca944c..97e51e9bafdb0 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_resources_client.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_resources_client.ts @@ -6,7 +6,7 @@ */ import { sha256 } from 'js-sha256'; -import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; +import type { QueryDslQueryContainer, Duration } from '@elastic/elasticsearch/lib/api/types'; import type { RuleMigrationResource, RuleMigrationResourceType, @@ -14,12 +14,30 @@ import type { import type { StoredRuleMigrationResource } from '../types'; import { RuleMigrationsDataBaseClient } from './rule_migrations_data_base_client'; -export type CreateRuleMigrationResourceInput = Omit; +export type CreateRuleMigrationResourceInput = Pick< + RuleMigrationResource, + 'migration_id' | 'type' | 'name' | 'metadata' +> & { + content?: string; +}; +export interface RuleMigrationResourceFilters { + type?: RuleMigrationResourceType; + names?: string[]; + hasContent?: boolean; +} +export interface RuleMigrationResourceGetOptions { + filters?: RuleMigrationResourceFilters; + size?: number; + from?: number; +} /* BULK_MAX_SIZE defines the number to break down the bulk operations by. * The 500 number was chosen as a reasonable number to avoid large payloads. It can be adjusted if needed. */ const BULK_MAX_SIZE = 500 as const; +/* DEFAULT_SEARCH_BATCH_SIZE defines the default number of documents to retrieve per search operation + * when retrieving search results in batches. */ +const DEFAULT_SEARCH_BATCH_SIZE = 500 as const; export class RuleMigrationsDataResourcesClient extends RuleMigrationsDataBaseClient { public async upsert(resources: CreateRuleMigrationResourceInput[]): Promise { @@ -52,24 +70,43 @@ export class RuleMigrationsDataResourcesClient extends RuleMigrationsDataBaseCli } } + /** Creates the resources in the index only if they do not exist */ + public async create(resources: CreateRuleMigrationResourceInput[]): Promise { + const index = await this.getIndexName(); + + let resourcesSlice: CreateRuleMigrationResourceInput[]; + const createdAt = new Date().toISOString(); + while ((resourcesSlice = resources.splice(0, BULK_MAX_SIZE)).length > 0) { + await this.esClient + .bulk({ + refresh: 'wait_for', + operations: resourcesSlice.flatMap((resource) => [ + { create: { _id: this.createId(resource), _index: index } }, + { + ...resource, + '@timestamp': createdAt, + updated_by: this.username, + updated_at: createdAt, + }, + ]), + }) + .catch((error) => { + this.logger.error(`Error upsert resources: ${error.message}`); + throw error; + }); + } + } + public async get( migrationId: string, - type?: RuleMigrationResourceType, - names?: string[] + options: RuleMigrationResourceGetOptions = {} ): Promise { + const { filters, size, from } = options; const index = await this.getIndexName(); - - const filter: QueryDslQueryContainer[] = [{ term: { migration_id: migrationId } }]; - if (type) { - filter.push({ term: { type } }); - } - if (names) { - filter.push({ terms: { name: names } }); - } - const query = { bool: { filter } }; + const query = this.getFilterQuery(migrationId, filters); return this.esClient - .search({ index, query }) + .search({ index, query, size, from }) .then(this.processResponseHits.bind(this)) .catch((error) => { this.logger.error(`Error searching resources: ${error.message}`); @@ -77,8 +114,46 @@ export class RuleMigrationsDataResourcesClient extends RuleMigrationsDataBaseCli }); } + /** Returns batching functions to traverse all the migration resources search results */ + searchBatches( + migrationId: string, + options: { scroll?: Duration; size?: number; filters?: RuleMigrationResourceFilters } = {} + ) { + const { size = DEFAULT_SEARCH_BATCH_SIZE, filters = {}, scroll } = options; + const query = this.getFilterQuery(migrationId, filters); + const search = { query, sort: '_doc', scroll, size }; // sort by _doc to ensure consistent order + try { + return this.getSearchBatches(search); + } catch (error) { + this.logger.error(`Error scrolling rule migration resources: ${error.message}`); + throw error; + } + } + private createId(resource: CreateRuleMigrationResourceInput): string { const key = `${resource.migration_id}-${resource.type}-${resource.name}`; return sha256.create().update(key).hex(); } + + private getFilterQuery( + migrationId: string, + filters: RuleMigrationResourceFilters = {} + ): QueryDslQueryContainer { + const filter: QueryDslQueryContainer[] = [{ term: { migration_id: migrationId } }]; + if (filters.type) { + filter.push({ term: { type: filters.type } }); + } + if (filters.names) { + filter.push({ terms: { name: filters.names } }); + } + if (filters.hasContent != null) { + const existContent = { exists: { field: 'content' } }; + if (filters.hasContent) { + filter.push(existContent); + } else { + filter.push({ bool: { must_not: existContent } }); + } + } + return { bool: { filter } }; + } } diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_rules_client.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_rules_client.ts index 1eeb3ced0572a..8cdc776f631b0 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_rules_client.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_rules_client.ts @@ -13,6 +13,7 @@ import type { AggregationsStringTermsAggregate, AggregationsStringTermsBucket, QueryDslQueryContainer, + Duration, } from '@elastic/elasticsearch/lib/api/types'; import type { StoredRuleMigration } from '../types'; import { SiemMigrationStatus } from '../../../../../common/siem_migrations/constants'; @@ -52,9 +53,11 @@ export interface RuleMigrationGetOptions { } /* BULK_MAX_SIZE defines the number to break down the bulk operations by. - * The 500 number was chosen as a reasonable number to avoid large payloads. It can be adjusted if needed. - */ + * The 500 number was chosen as a reasonable number to avoid large payloads. It can be adjusted if needed. */ const BULK_MAX_SIZE = 500 as const; +/* DEFAULT_SEARCH_BATCH_SIZE defines the default number of documents to retrieve per search operation + * when retrieving search results in batches. */ +const DEFAULT_SEARCH_BATCH_SIZE = 500 as const; export class RuleMigrationsDataRulesClient extends RuleMigrationsDataBaseClient { /** Indexes an array of rule migrations to be processed */ @@ -123,7 +126,7 @@ export class RuleMigrationsDataRulesClient extends RuleMigrationsDataBaseClient { filters = {}, sort = {}, from, size }: RuleMigrationGetOptions = {} ): Promise<{ total: number; data: StoredRuleMigration[] }> { const index = await this.getIndexName(); - const query = this.getFilterQuery(migrationId, { ...filters }); + const query = this.getFilterQuery(migrationId, filters); const result = await this.esClient .search({ @@ -143,6 +146,22 @@ export class RuleMigrationsDataRulesClient extends RuleMigrationsDataBaseClient }; } + /** Returns batching functions to traverse all the migration rules search results */ + searchBatches( + migrationId: string, + options: { scroll?: Duration; size?: number; filters?: RuleMigrationFilters } = {} + ) { + const { size = DEFAULT_SEARCH_BATCH_SIZE, filters = {}, scroll } = options; + const query = this.getFilterQuery(migrationId, filters); + const search = { query, sort: '_doc', scroll, size }; // sort by _doc to ensure consistent order + try { + return this.getSearchBatches(search); + } catch (error) { + this.logger.error(`Error scrolling rule migrations: ${error.message}`); + throw error; + } + } + /** * Retrieves `pending` rule migrations with the provided id and updates their status to `processing`. * This operation is not atomic at migration level: diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_migrations_retriever.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_migrations_retriever.ts index 22c884fa4043b..29852558cda48 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_migrations_retriever.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_migrations_retriever.ts @@ -5,19 +5,42 @@ * 2.0. */ +import type { RulesClient } from '@kbn/alerting-plugin/server'; +import type { SavedObjectsClientContract } from '@kbn/core/server'; import type { RuleMigrationsDataClient } from '../../data/rule_migrations_data_client'; import { IntegrationRetriever } from './integration_retriever'; import { PrebuiltRulesRetriever } from './prebuilt_rules_retriever'; import { RuleResourceRetriever } from './rule_resource_retriever'; +interface RuleMigrationsRetrieverDeps { + data: RuleMigrationsDataClient; + rules: RulesClient; + savedObjects: SavedObjectsClientContract; +} + export class RuleMigrationsRetriever { public readonly resources: RuleResourceRetriever; public readonly integrations: IntegrationRetriever; public readonly prebuiltRules: PrebuiltRulesRetriever; - constructor(dataClient: RuleMigrationsDataClient, migrationId: string) { - this.resources = new RuleResourceRetriever(migrationId, dataClient); - this.integrations = new IntegrationRetriever(dataClient); - this.prebuiltRules = new PrebuiltRulesRetriever(dataClient); + constructor(migrationId: string, private readonly clients: RuleMigrationsRetrieverDeps) { + this.resources = new RuleResourceRetriever(migrationId, this.clients.data); + this.integrations = new IntegrationRetriever(this.clients.data); + this.prebuiltRules = new PrebuiltRulesRetriever(this.clients.data); + } + + public async initialize() { + await Promise.all([ + this.resources.initialize(), + // Populates the indices used for RAG searches on prebuilt rules and integrations. + this.clients.data.prebuiltRules.create({ + rulesClient: this.clients.rules, + soClient: this.clients.savedObjects, + }), + // Will use Fleet API client for integration retrieval as an argument once feature is available + this.clients.data.integrations.create(), + ]).catch((error) => { + throw new Error(`Failed to initialize RuleMigrationsRetriever: ${error}`); + }); } } diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_resource_retriever.test.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_resource_retriever.test.ts index 51618d5f3ca13..1e02eec2315e7 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_resource_retriever.test.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_resource_retriever.test.ts @@ -5,176 +5,124 @@ * 2.0. */ -import { MAX_RECURSION_DEPTH, RuleResourceRetriever } from './rule_resource_retriever'; // Adjust path as needed +import { RuleResourceRetriever } from './rule_resource_retriever'; // Adjust path as needed import type { OriginalRule } from '../../../../../../common/siem_migrations/model/rule_migration.gen'; -import { MockRuleMigrationsDataClient } from '../../data/__mocks__/mocks'; - -const mockRuleResourceIdentifier = jest.fn(); -const mockGetRuleResourceIdentifier = jest.fn((_: unknown) => mockRuleResourceIdentifier); -jest.mock('../../../../../../common/siem_migrations/rules/resources', () => ({ - getRuleResourceIdentifier: (params: unknown) => mockGetRuleResourceIdentifier(params), -})); +import { ResourceIdentifier } from '../../../../../../common/siem_migrations/rules/resources'; +import type { RuleMigrationsDataClient } from '../../data/rule_migrations_data_client'; jest.mock('../../data/rule_migrations_data_service'); +jest.mock('../../../../../../common/siem_migrations/rules/resources'); + +const MockResourceIdentifier = ResourceIdentifier as jest.Mock; describe('RuleResourceRetriever', () => { let retriever: RuleResourceRetriever; - const mockRuleMigrationsDataClient = new MockRuleMigrationsDataClient(); - const migrationId = 'test-migration-id'; - const ruleQuery = 'rule-query'; - const originalRule = { query: ruleQuery } as OriginalRule; + let mockDataClient: jest.Mocked; + let mockResourceIdentifier: jest.Mocked; beforeEach(() => { - retriever = new RuleResourceRetriever(migrationId, mockRuleMigrationsDataClient); - mockRuleResourceIdentifier.mockReturnValue({ list: [], macro: [] }); - - mockRuleMigrationsDataClient.resources.get.mockImplementation( - async (_: string, type: string, names: string[]) => - names.map((name) => ({ type, name, content: `${name}-content` })) - ); - - mockRuleResourceIdentifier.mockImplementation((query) => { - if (query === ruleQuery) { - return { list: ['list1', 'list2'], macro: ['macro1'] }; - } - return { list: [], macro: [] }; - }); - - jest.clearAllMocks(); + mockDataClient = { + resources: { searchBatches: jest.fn().mockReturnValue({ next: jest.fn(() => []) }) }, + } as unknown as RuleMigrationsDataClient; + + retriever = new RuleResourceRetriever('mockMigrationId', mockDataClient); + + MockResourceIdentifier.mockImplementation(() => ({ + fromOriginalRule: jest.fn().mockReturnValue([]), + fromResources: jest.fn().mockReturnValue([]), + })); + mockResourceIdentifier = new MockResourceIdentifier( + 'splunk' + ) as jest.Mocked; }); - describe('getResources', () => { - it('should call resource identification', async () => { - await retriever.getResources(originalRule); + it('throws an error if initialize is not called before getResources', async () => { + const originalRule = { vendor: 'splunk' } as unknown as OriginalRule; - expect(mockGetRuleResourceIdentifier).toHaveBeenCalledWith(originalRule); - expect(mockRuleResourceIdentifier).toHaveBeenCalledWith(ruleQuery); - expect(mockRuleResourceIdentifier).toHaveBeenCalledWith('macro1-content'); - }); - - it('should retrieve resources', async () => { - const resources = await retriever.getResources(originalRule); - - expect(mockRuleMigrationsDataClient.resources.get).toHaveBeenCalledWith(migrationId, 'list', [ - 'list1', - 'list2', - ]); - expect(mockRuleMigrationsDataClient.resources.get).toHaveBeenCalledWith( - migrationId, - 'macro', - ['macro1'] - ); - - expect(resources).toEqual({ - list: [ - { type: 'list', name: 'list1', content: 'list1-content' }, - { type: 'list', name: 'list2', content: 'list2-content' }, - ], - macro: [{ type: 'macro', name: 'macro1', content: 'macro1-content' }], - }); - }); + await expect(retriever.getResources(originalRule)).rejects.toThrow( + 'initialize must be called before calling getResources' + ); + }); - it('should retrieve nested resources', async () => { - mockRuleResourceIdentifier.mockImplementation((query) => { - if (query === ruleQuery) { - return { list: ['list1', 'list2'], macro: ['macro1'] }; - } - if (query === 'macro1-content') { - return { list: ['list3'], macro: [] }; - } - return { list: [], macro: [] }; - }); - - const resources = await retriever.getResources(originalRule); - - expect(mockRuleMigrationsDataClient.resources.get).toHaveBeenCalledWith(migrationId, 'list', [ - 'list1', - 'list2', - ]); - expect(mockRuleMigrationsDataClient.resources.get).toHaveBeenCalledWith( - migrationId, - 'macro', - ['macro1'] - ); - expect(mockRuleMigrationsDataClient.resources.get).toHaveBeenCalledWith(migrationId, 'list', [ - 'list3', - ]); - - expect(resources).toEqual({ - list: [ - { type: 'list', name: 'list1', content: 'list1-content' }, - { type: 'list', name: 'list2', content: 'list2-content' }, - { type: 'list', name: 'list3', content: 'list3-content' }, - ], - macro: [{ type: 'macro', name: 'macro1', content: 'macro1-content' }], - }); - }); + it('returns an empty object if no matching resources are found', async () => { + const originalRule = { vendor: 'splunk' } as unknown as OriginalRule; - it('should handle missing macros', async () => { - mockRuleMigrationsDataClient.resources.get.mockImplementation( - async (_: string, type: string, names: string[]) => { - if (type === 'macro') { - return []; - } - return names.map((name) => ({ type, name, content: `${name}-content` })); - } - ); - - const resources = await retriever.getResources(originalRule); - - expect(resources).toEqual({ - list: [ - { type: 'list', name: 'list1', content: 'list1-content' }, - { type: 'list', name: 'list2', content: 'list2-content' }, - ], - }); - }); + // Mock the resource identifier to return no resources + mockResourceIdentifier.fromOriginalRule.mockReturnValue([]); + await retriever.initialize(); // Pretend initialize has been called - it('should handle missing lists', async () => { - mockRuleMigrationsDataClient.resources.get.mockImplementation( - async (_: string, type: string, names: string[]) => { - if (type === 'list') { - return []; - } - return names.map((name) => ({ type, name, content: `${name}-content` })); - } - ); - - const resources = await retriever.getResources(originalRule); - - expect(resources).toEqual({ - macro: [{ type: 'macro', name: 'macro1', content: 'macro1-content' }], - }); - }); + const result = await retriever.getResources(originalRule); + expect(result).toEqual({}); + }); - it('should not include resources with missing content', async () => { - mockRuleMigrationsDataClient.resources.get.mockImplementation( - async (_: string, type: string, names: string[]) => { - return names.map((name) => { - if (name === 'list1') { - return { type, name, content: '' }; - } - return { type, name, content: `${name}-content` }; - }); - } - ); - - const resources = await retriever.getResources(originalRule); - - expect(resources).toEqual({ - list: [{ type: 'list', name: 'list2', content: 'list2-content' }], - macro: [{ type: 'macro', name: 'macro1', content: 'macro1-content' }], - }); + it('returns matching macro and list resources', async () => { + const mockExistingResources = { + macro: { macro1: { name: 'macro1', type: 'macro' } }, + list: { list1: { name: 'list1', type: 'list' } }, + }; + // Inject existing resources manually + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (retriever as any).existingResources = mockExistingResources; + + const mockResourcesIdentified = [ + { name: 'macro1', type: 'macro' as const }, + { name: 'list1', type: 'list' as const }, + ]; + MockResourceIdentifier.mockImplementation(() => ({ + fromOriginalRule: jest.fn().mockReturnValue(mockResourcesIdentified), + fromResources: jest.fn().mockReturnValue([]), + })); + + const originalRule = { vendor: 'splunk' } as unknown as OriginalRule; + + const result = await retriever.getResources(originalRule); + expect(result).toEqual({ + macro: [{ name: 'macro1', type: 'macro' }], + list: [{ name: 'list1', type: 'list' }], }); + }); - it('should stop recursion after reaching MAX_RECURSION_DEPTH', async () => { - mockRuleResourceIdentifier.mockImplementation(() => { - return { list: [], macro: ['infinite-macro'] }; - }); - - const resources = await retriever.getResources(originalRule); - - expect(resources.macro?.length).toEqual(MAX_RECURSION_DEPTH); + it('handles nested resources properly', async () => { + const originalRule = { vendor: 'splunk' } as unknown as OriginalRule; + + const mockExistingResources = { + macro: { + macro1: { name: 'macro1', type: 'macro' }, + macro2: { name: 'macro2', type: 'macro' }, + }, + list: { + list1: { name: 'list1', type: 'list' }, + list2: { name: 'list2', type: 'list' }, + }, + }; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (retriever as any).existingResources = mockExistingResources; + + const mockResourcesIdentifiedFromRule = [ + { name: 'macro1', type: 'macro' as const }, + { name: 'list1', type: 'list' as const }, + ]; + + const mockNestedResources = [ + { name: 'macro2', type: 'macro' as const }, + { name: 'list2', type: 'list' as const }, + ]; + + MockResourceIdentifier.mockImplementation(() => ({ + fromOriginalRule: jest.fn().mockReturnValue(mockResourcesIdentifiedFromRule), + fromResources: jest.fn().mockReturnValue([]).mockReturnValueOnce(mockNestedResources), + })); + + const result = await retriever.getResources(originalRule); + expect(result).toEqual({ + macro: [ + { name: 'macro1', type: 'macro' }, + { name: 'macro2', type: 'macro' }, + ], + list: [ + { name: 'list1', type: 'list' }, + { name: 'list2', type: 'list' }, + ], }); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_resource_retriever.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_resource_retriever.ts index d80646dc27c4d..b89939e199e5a 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_resource_retriever.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_resource_retriever.ts @@ -5,9 +5,7 @@ * 2.0. */ -import { isEmpty } from 'lodash/fp'; -import type { QueryResourceIdentifier } from '../../../../../../common/siem_migrations/rules/resources/types'; -import { getRuleResourceIdentifier } from '../../../../../../common/siem_migrations/rules/resources'; +import { ResourceIdentifier } from '../../../../../../common/siem_migrations/rules/resources'; import type { OriginalRule, RuleMigrationResource, @@ -15,86 +13,91 @@ import type { } from '../../../../../../common/siem_migrations/model/rule_migration.gen'; import type { RuleMigrationsDataClient } from '../../data/rule_migrations_data_client'; +export interface RuleMigrationDefinedResource extends RuleMigrationResource { + content: string; // ensures content exists +} export type RuleMigrationResources = Partial< - Record + Record >; - -/* It's not a common practice to have more than 2-3 nested levels of resources. - * This limit is just to prevent infinite recursion in case something goes wrong. - */ -export const MAX_RECURSION_DEPTH = 30; +interface ExistingResources { + macro: Record; + list: Record; +} export class RuleResourceRetriever { + private existingResources?: ExistingResources; + constructor( private readonly migrationId: string, private readonly dataClient: RuleMigrationsDataClient ) {} - public async getResources(originalRule: OriginalRule): Promise { - const resourceIdentifier = getRuleResourceIdentifier(originalRule); - return this.recursiveRetriever(originalRule.query, resourceIdentifier); + public async initialize(): Promise { + const batches = this.dataClient.resources.searchBatches( + this.migrationId, + { filters: { hasContent: true } } + ); + + const existingRuleResources: ExistingResources = { macro: {}, list: {} }; + let resources; + do { + resources = await batches.next(); + resources.forEach((resource) => { + existingRuleResources[resource.type][resource.name] = resource; + }); + } while (resources.length > 0); + + this.existingResources = existingRuleResources; } - private recursiveRetriever = async ( - query: string, - resourceIdentifier: QueryResourceIdentifier, - it = 0 - ): Promise => { - if (it >= MAX_RECURSION_DEPTH) { - return {}; + public async getResources(originalRule: OriginalRule): Promise { + const existingResources = this.existingResources; + if (!existingResources) { + throw new Error('initialize must be called before calling getResources'); } - const identifiedResources = resourceIdentifier(query); - const resources: RuleMigrationResources = {}; - - const listNames = identifiedResources.list; - if (listNames.length > 0) { - const listsWithContent = await this.dataClient.resources - .get(this.migrationId, 'list', listNames) - .then(withContent); + const resourceIdentifier = new ResourceIdentifier(originalRule.vendor); + const resourcesIdentifiedFromRule = resourceIdentifier.fromOriginalRule(originalRule); - if (listsWithContent.length > 0) { - resources.list = listsWithContent; + const macrosFound = new Map(); + const listsFound = new Map(); + resourcesIdentifiedFromRule.forEach((resource) => { + const existingResource = existingResources[resource.type][resource.name]; + if (existingResource) { + if (resource.type === 'macro') { + macrosFound.set(resource.name, existingResource); + } else if (resource.type === 'list') { + listsFound.set(resource.name, existingResource); + } } - } + }); - const macroNames = identifiedResources.macro; - if (macroNames.length > 0) { - const macrosWithContent = await this.dataClient.resources - .get(this.migrationId, 'macro', macroNames) - .then(withContent); + const resourcesFound = [...macrosFound.values(), ...listsFound.values()]; + if (!resourcesFound.length) { + return {}; + } - if (macrosWithContent.length > 0) { - // retrieve nested resources inside macros - const macrosNestedResources = await Promise.all( - macrosWithContent.map(({ content }) => - this.recursiveRetriever(content, resourceIdentifier, it + 1) - ) - ); + let nestedResourcesFound = resourcesFound; + do { + const nestedResourcesIdentified = resourceIdentifier.fromResources(nestedResourcesFound); - // Process lists inside macros - const macrosNestedLists = macrosNestedResources.flatMap( - (macroNestedResources) => macroNestedResources.list ?? [] - ); - if (macrosNestedLists.length > 0) { - resources.list = (resources.list ?? []).concat(macrosNestedLists); + nestedResourcesFound = []; + nestedResourcesIdentified.forEach((resource) => { + const existingResource = existingResources[resource.type][resource.name]; + if (existingResource) { + nestedResourcesFound.push(existingResource); + if (resource.type === 'macro') { + macrosFound.set(resource.name, existingResource); + } else if (resource.type === 'list') { + listsFound.set(resource.name, existingResource); + } } + }); + } while (nestedResourcesFound.length > 0); - // Process macros inside macros - const macrosNestedMacros = macrosNestedResources.flatMap( - (macroNestedResources) => macroNestedResources.macro ?? [] - ); - - if (macrosNestedMacros.length > 0) { - macrosWithContent.push(...macrosNestedMacros); - } - resources.macro = macrosWithContent; - } - } - return resources; - }; + return { + ...(macrosFound.size > 0 ? { macro: Array.from(macrosFound.values()) } : {}), + ...(listsFound.size > 0 ? { list: Array.from(listsFound.values()) } : {}), + }; + } } - -const withContent = (resources: RuleMigrationResource[]) => { - return resources.filter((resource) => !isEmpty(resource.content)); -}; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_client.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_client.ts index 6dbee5c64ee47..fe3e01fe84925 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_client.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_client.ts @@ -20,9 +20,8 @@ import type { MigrateRuleState } from './agent/types'; import { RuleMigrationsRetriever } from './retrievers'; import type { MigrationAgent, - RuleMigrationTaskPrepareParams, - RuleMigrationTaskRunParams, RuleMigrationTaskStartParams, + RuleMigrationTaskCreateAgentParams, RuleMigrationTaskStartResult, RuleMigrationTaskStopResult, } from './types'; @@ -63,82 +62,40 @@ export class RuleMigrationsTaskClient { return { exists: true, started: false }; } - const abortController = new AbortController(); - - // Retrieve agent from prepare and pass it to run right after without awaiting but using .then - this.prepare({ ...params, abortController }) - .then((agent) => this.run({ ...params, agent, abortController })) - .catch((error) => { - this.logger.error(`Error starting migration ID:${migrationId} with error:${error}`, error); - }); - - return { exists: true, started: true }; - } - - private async prepare({ - migrationId, - connectorId, - inferenceClient, - actionsClient, - rulesClient, - soClient, - abortController, - }: RuleMigrationTaskPrepareParams): Promise { - await Promise.all([ - // Populates the indices used for RAG searches on prebuilt rules and integrations. - await this.data.prebuiltRules.create({ rulesClient, soClient }), - // Will use Fleet API client for integration retrieval as an argument once feature is available - await this.data.integrations.create(), - ]).catch((error) => { - this.logger.error(`Error preparing RAG indices for migration ID:${migrationId}`, error); - throw error; + // run the migration without awaiting it to execute it in the background + this.run(params).catch((error) => { + this.logger.error(`Error executing migration ID:${migrationId}`, error); }); - const ruleMigrationsRetriever = new RuleMigrationsRetriever(this.data, migrationId); - - const actionsClientChat = new ActionsClientChat(connectorId, actionsClient, this.logger); - const model = await actionsClientChat.createModel({ - signal: abortController.signal, - temperature: 0.05, - }); - - const agent = getRuleMigrationAgent({ - connectorId, - model, - inferenceClient, - ruleMigrationsRetriever, - logger: this.logger, - }); - return agent; + return { exists: true, started: true }; } - private async run({ - migrationId, - agent, - invocationConfig, - abortController, - }: RuleMigrationTaskRunParams): Promise { + private async run(params: RuleMigrationTaskStartParams): Promise { + const { migrationId, invocationConfig } = params; if (this.migrationsRunning.has(migrationId)) { // This should never happen, but just in case throw new Error(`Task already running for migration ID:${migrationId} `); } this.logger.info(`Starting migration ID:${migrationId}`); + const abortController = new AbortController(); this.migrationsRunning.set(migrationId, { user: this.currentUser.username, abortController }); - const config: RunnableConfig = { - ...invocationConfig, - // signal: abortController.signal, // not working properly https://github.com/langchain-ai/langgraphjs/issues/319 - }; const abortPromise = abortSignalToPromise(abortController.signal); + const withAbortRace = async (task: Promise) => Promise.race([task, abortPromise.promise]); + + const sleep = async (seconds: number) => { + this.logger.debug(`Sleeping ${seconds}s for migration ID:${migrationId}`); + await withAbortRace(new Promise((resolve) => setTimeout(resolve, seconds * 1000))); + }; try { - const sleep = async (seconds: number) => { - this.logger.debug(`Sleeping ${seconds}s for migration ID:${migrationId}`); - await Promise.race([ - new Promise((resolve) => setTimeout(resolve, seconds * 1000)), - abortPromise.promise, - ]); + this.logger.debug(`Creating agent for migration ID:${migrationId}`); + const agent = await withAbortRace(this.createAgent({ ...params, abortController })); + + const config: RunnableConfig = { + ...invocationConfig, + // signal: abortController.signal, // not working properly https://github.com/langchain-ai/langgraphjs/issues/319 }; let isDone: boolean = false; @@ -154,10 +111,12 @@ export class RuleMigrationsTaskClient { try { const start = Date.now(); - const migrationResult: MigrateRuleState = await Promise.race([ - agent.invoke({ original_rule: ruleMigration.original_rule }, config), - abortPromise.promise, // workaround for the issue with the langGraph signal - ]); + const invocation = agent.invoke( + { original_rule: ruleMigration.original_rule }, + config + ); + // using withAbortRace is a workaround for the issue with the langGraph signal not working properly + const migrationResult = await withAbortRace(invocation); const duration = (Date.now() - start) / 1000; this.logger.debug( @@ -211,6 +170,38 @@ export class RuleMigrationsTaskClient { } } + private async createAgent({ + migrationId, + connectorId, + inferenceClient, + actionsClient, + rulesClient, + soClient, + abortController, + }: RuleMigrationTaskCreateAgentParams): Promise { + const actionsClientChat = new ActionsClientChat(connectorId, actionsClient, this.logger); + const model = await actionsClientChat.createModel({ + signal: abortController.signal, + temperature: 0.05, + }); + + const ruleMigrationsRetriever = new RuleMigrationsRetriever(migrationId, { + data: this.data, + rules: rulesClient, + savedObjects: soClient, + }); + await ruleMigrationsRetriever.initialize(); + + const agent = getRuleMigrationAgent({ + connectorId, + model, + inferenceClient, + ruleMigrationsRetriever, + logger: this.logger, + }); + return agent; + } + /** Updates all the rules in a migration to be re-executed */ public async updateToRetry(migrationId: string): Promise<{ updated: boolean }> { if (this.migrationsRunning.has(migrationId)) { diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/types.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/types.ts index 7ac7e848ba80d..7ddb08f1e47d6 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/types.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/types.ts @@ -30,20 +30,7 @@ export interface RuleMigrationTaskStartParams { soClient: SavedObjectsClientContract; } -export interface RuleMigrationTaskPrepareParams { - migrationId: string; - connectorId: string; - inferenceClient: InferenceClient; - actionsClient: ActionsClient; - rulesClient: RulesClient; - soClient: SavedObjectsClientContract; - abortController: AbortController; -} - -export interface RuleMigrationTaskRunParams { - migrationId: string; - invocationConfig: RunnableConfig; - agent: MigrationAgent; +export interface RuleMigrationTaskCreateAgentParams extends RuleMigrationTaskStartParams { abortController: AbortController; } diff --git a/x-pack/test/api_integration/services/security_solution_api.gen.ts b/x-pack/test/api_integration/services/security_solution_api.gen.ts index a6d0ac86a810c..30903c2f572b2 100644 --- a/x-pack/test/api_integration/services/security_solution_api.gen.ts +++ b/x-pack/test/api_integration/services/security_solution_api.gen.ts @@ -104,6 +104,7 @@ import { GetRuleMigrationResourcesRequestQueryInput, GetRuleMigrationResourcesRequestParamsInput, } from '@kbn/security-solution-plugin/common/siem_migrations/model/api/rules/rule_migration.gen'; +import { GetRuleMigrationResourcesMissingRequestParamsInput } from '@kbn/security-solution-plugin/common/siem_migrations/model/api/rules/rule_migration.gen'; import { GetRuleMigrationStatsRequestParamsInput } from '@kbn/security-solution-plugin/common/siem_migrations/model/api/rules/rule_migration.gen'; import { GetRuleMigrationTranslationStatsRequestParamsInput } from '@kbn/security-solution-plugin/common/siem_migrations/model/api/rules/rule_migration.gen'; import { GetTimelineRequestQueryInput } from '@kbn/security-solution-plugin/common/api/timeline/get_timeline/get_timeline_route.gen'; @@ -998,6 +999,27 @@ finalize it. .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') .query(props.query); }, + /** + * Identifies missing resources from all the rules of an existing SIEM rules migration + */ + getRuleMigrationResourcesMissing( + props: GetRuleMigrationResourcesMissingProps, + kibanaSpace: string = 'default' + ) { + return supertest + .get( + routeWithNamespace( + replaceParams( + '/internal/siem_migrations/rules/{migration_id}/resources/missing', + props.params + ), + kibanaSpace + ) + ) + .set('kbn-xsrf', 'true') + .set(ELASTIC_HTTP_VERSION_HEADER, '1') + .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana'); + }, /** * Retrieves the stats of a SIEM rules migration using the migration id provided */ @@ -1760,6 +1782,9 @@ export interface GetRuleMigrationResourcesProps { query: GetRuleMigrationResourcesRequestQueryInput; params: GetRuleMigrationResourcesRequestParamsInput; } +export interface GetRuleMigrationResourcesMissingProps { + params: GetRuleMigrationResourcesMissingRequestParamsInput; +} export interface GetRuleMigrationStatsProps { params: GetRuleMigrationStatsRequestParamsInput; }