[SecuritySolution][siem migrations] Onboarding UI flyout macros input (…

…elastic#203483) ## Summary From: elastic/security-team#10667 This is the part 2 of the issue - The macros input Implementation of the Onboarding card to create migrations using the flyout. > [!NOTE] > This feature needs `siemMigrationsEnabled` experimental flag enabled to work. Otherwise only the default topic will be available and the topic selector won't be displayed. ### Screenshots <img width="1457" alt="Macros step" src="https://github.com/user-attachments/assets/48ec806e-1fcf-4dbb-998e-cb6a06d9ebaa"> <img width="1457" alt="loading" src="https://github.com/user-attachments/assets/a643de40-3d62-4dbf-a7aa-4f30839bc1b8"> <img width="1457" alt="done" src="https://github.com/user-attachments/assets/b384af03-6bdb-4b10-bd26-18d3b4715677"> #### To do in part 3: - Implement missing steps in the flyout: Lookups ### Test Enable experimental flag Rule file: [rules_test.json](https://github.com/user-attachments/files/18082165/rules_test.json) Macros file: [macros_test.json](https://github.com/user-attachments/files/18082169/macros_test.json) --------- Co-authored-by: Elastic Machine <[email protected]> Co-authored-by: kibanamachine <[email protected]>
e40pud · Dec 12, 2024 · 8c7883f · 8c7883f
1 parent 7218d01
commit 8c7883f
Show file tree

Hide file tree

Showing 61 changed files with 2,102 additions and 766 deletions.
diff --git a/x-pack/plugins/security_solution/common/api/quickstart_client.gen.ts b/x-pack/plugins/security_solution/common/api/quickstart_client.gen.ts
@@ -368,6 +368,8 @@ import type {
   GetRuleMigrationResourcesRequestQueryInput,
   GetRuleMigrationResourcesRequestParamsInput,
   GetRuleMigrationResourcesResponse,
+  GetRuleMigrationResourcesMissingRequestParamsInput,
+  GetRuleMigrationResourcesMissingResponse,
   GetRuleMigrationStatsRequestParamsInput,
   GetRuleMigrationStatsResponse,
   GetRuleMigrationTranslationStatsRequestParamsInput,
@@ -1471,6 +1473,24 @@ finalize it.
       })
       .catch(catchAxiosErrorFormatAndThrow);
   }
+  /**
+   * Identifies missing resources from all the rules of an existing SIEM rules migration
+   */
+  async getRuleMigrationResourcesMissing(props: GetRuleMigrationResourcesMissingProps) {
+    this.log.info(`${new Date().toISOString()} Calling API GetRuleMigrationResourcesMissing`);
+    return this.kbnClient
+      .request<GetRuleMigrationResourcesMissingResponse>({
+        path: replaceParams(
+          '/internal/siem_migrations/rules/{migration_id}/resources/missing',
+          props.params
+        ),
+        headers: {
+          [ELASTIC_HTTP_VERSION_HEADER]: '1',
+        },
+        method: 'GET',
+      })
+      .catch(catchAxiosErrorFormatAndThrow);
+  }
   /**
    * Retrieves the stats of a SIEM rules migration using the migration id provided
    */
@@ -2423,6 +2443,9 @@ export interface GetRuleMigrationResourcesProps {
   query: GetRuleMigrationResourcesRequestQueryInput;
   params: GetRuleMigrationResourcesRequestParamsInput;
 }
+export interface GetRuleMigrationResourcesMissingProps {
+  params: GetRuleMigrationResourcesMissingRequestParamsInput;
+}
 export interface GetRuleMigrationStatsProps {
   params: GetRuleMigrationStatsRequestParamsInput;
 }

diff --git a/x-pack/plugins/security_solution/common/siem_migrations/constants.ts b/x-pack/plugins/security_solution/common/siem_migrations/constants.ts
@@ -27,6 +27,8 @@ export const SIEM_RULE_MIGRATIONS_PREBUILT_RULES_PATH =
   `${SIEM_RULE_MIGRATION_PATH}/prebuilt_rules` as const;
 
 export const SIEM_RULE_MIGRATION_RESOURCES_PATH = `${SIEM_RULE_MIGRATION_PATH}/resources` as const;
+export const SIEM_RULE_MIGRATION_RESOURCES_MISSING_PATH =
+  `${SIEM_RULE_MIGRATION_RESOURCES_PATH}/missing` as const;
 
 export enum SiemMigrationTaskStatus {
   READY = 'ready',

diff --git a/...ck/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.gen.ts b/...ck/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.gen.ts
@@ -103,6 +103,8 @@ export type GetRuleMigrationResourcesRequestQuery = z.infer<
 export const GetRuleMigrationResourcesRequestQuery = z.object({
   type: RuleMigrationResourceType.optional(),
   names: ArrayFromString(z.string()).optional(),
+  from: z.coerce.number().optional(),
+  size: z.coerce.number().optional(),
 });
 export type GetRuleMigrationResourcesRequestQueryInput = z.input<
   typeof GetRuleMigrationResourcesRequestQuery
@@ -121,6 +123,24 @@ export type GetRuleMigrationResourcesRequestParamsInput = z.input<
 export type GetRuleMigrationResourcesResponse = z.infer<typeof GetRuleMigrationResourcesResponse>;
 export const GetRuleMigrationResourcesResponse = z.array(RuleMigrationResource);
 
+export type GetRuleMigrationResourcesMissingRequestParams = z.infer<
+  typeof GetRuleMigrationResourcesMissingRequestParams
+>;
+export const GetRuleMigrationResourcesMissingRequestParams = z.object({
+  migration_id: NonEmptyString,
+});
+export type GetRuleMigrationResourcesMissingRequestParamsInput = z.input<
+  typeof GetRuleMigrationResourcesMissingRequestParams
+>;
+
+/**
+ * The identified resources missing
+ */
+export type GetRuleMigrationResourcesMissingResponse = z.infer<
+  typeof GetRuleMigrationResourcesMissingResponse
+>;
+export const GetRuleMigrationResourcesMissingResponse = z.array(RuleMigrationResourceData);
+
 export type GetRuleMigrationStatsRequestParams = z.infer<typeof GetRuleMigrationStatsRequestParams>;
 export const GetRuleMigrationStatsRequestParams = z.object({
   migration_id: NonEmptyString,

diff --git a/...ugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.schema.yaml b/...ugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.schema.yaml
@@ -473,6 +473,16 @@ paths:
             description: The names of the resource to retrieve
             items:
               type: string
+        - name: from
+          in: query
+          required: false
+          schema:
+            type: number
+        - name: size
+          in: query
+          required: false
+          schema:
+            type: number
       responses:
         200:
           description: Indicates migration resources have been retrieved correctly
@@ -482,3 +492,31 @@ paths:
                 type: array
                 items:
                   $ref: '../../rule_migration.schema.yaml#/components/schemas/RuleMigrationResource'
+
+  /internal/siem_migrations/rules/{migration_id}/resources/missing:
+    get:
+      summary: Gets missing rule migration resources for a migration
+      operationId: GetRuleMigrationResourcesMissing
+      x-codegen-enabled: true
+      x-internal: true
+      description: Identifies missing resources from all the rules of an existing SIEM rules migration
+      tags:
+        - SIEM Rule Migrations
+        - Resources
+      parameters:
+        - name: migration_id
+          in: path
+          required: true
+          schema:
+            description: The migration id to attach the resources
+            $ref: '../../../../../common/api/model/primitives.schema.yaml#/components/schemas/NonEmptyString'
+      responses:
+        200:
+          description: Indicates missing migration resources have been retrieved correctly
+          content:
+            application/json:
+              schema:
+                type: array
+                description: The identified resources missing
+                items:
+                  $ref: '../../rule_migration.schema.yaml#/components/schemas/RuleMigrationResourceData'
diff --git a/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts b/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts
@@ -324,7 +324,7 @@ export const RuleMigrationResourceData = z.object({
   /**
    * The resource content value.
    */
-  content: z.string(),
+  content: z.string().optional(),
   /**
    * The resource arbitrary metadata.
    */

diff --git a/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml b/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml
@@ -291,7 +291,6 @@ components:
       required:
         - type
         - name
-        - content
       properties:
         type:
           $ref: '#/components/schemas/RuleMigrationResourceType'

diff --git a/x-pack/plugins/security_solution/common/siem_migrations/rules/resources/index.ts b/x-pack/plugins/security_solution/common/siem_migrations/rules/resources/index.ts
@@ -5,18 +5,72 @@
  * 2.0.
  */
 
-import type { OriginalRule, OriginalRuleVendor } from '../../model/rule_migration.gen';
-import type { QueryResourceIdentifier, RuleResourceCollection } from './types';
-import { splResourceIdentifier } from './splunk_identifier';
+import type {
+  OriginalRule,
+  OriginalRuleVendor,
+  RuleMigrationResourceData,
+} from '../../model/rule_migration.gen';
+import type { ResourceIdentifiers, RuleResource } from './types';
+import { splResourceIdentifiers } from './splunk';
 
-export const getRuleResourceIdentifier = (rule: OriginalRule): QueryResourceIdentifier => {
-  return ruleResourceIdentifiers[rule.vendor];
+const ruleResourceIdentifiers: Record<OriginalRuleVendor, ResourceIdentifiers> = {
+  splunk: splResourceIdentifiers,
 };
 
-export const identifyRuleResources = (rule: OriginalRule): RuleResourceCollection => {
-  return getRuleResourceIdentifier(rule)(rule.query);
+export const getRuleResourceIdentifier = (vendor: OriginalRuleVendor): ResourceIdentifiers => {
+  return ruleResourceIdentifiers[vendor];
 };
 
-const ruleResourceIdentifiers: Record<OriginalRuleVendor, QueryResourceIdentifier> = {
-  splunk: splResourceIdentifier,
-};
+export class ResourceIdentifier {
+  private identifiers: ResourceIdentifiers;
+
+  constructor(vendor: OriginalRuleVendor) {
+    // The constructor may need query_language as an argument for other vendors
+    this.identifiers = ruleResourceIdentifiers[vendor];
+  }
+
+  public fromOriginalRule(originalRule: OriginalRule): RuleResource[] {
+    return this.identifiers.fromOriginalRule(originalRule);
+  }
+
+  public fromResource(resource: RuleMigrationResourceData): RuleResource[] {
+    return this.identifiers.fromResource(resource);
+  }
+
+  public fromOriginalRules(originalRules: OriginalRule[]): RuleResource[] {
+    const lists = new Set<string>();
+    const macros = new Set<string>();
+    originalRules.forEach((rule) => {
+      const resources = this.identifiers.fromOriginalRule(rule);
+      resources.forEach((resource) => {
+        if (resource.type === 'macro') {
+          macros.add(resource.name);
+        } else if (resource.type === 'list') {
+          lists.add(resource.name);
+        }
+      });
+    });
+    return [
+      ...Array.from(macros).map<RuleResource>((name) => ({ type: 'macro', name })),
+      ...Array.from(lists).map<RuleResource>((name) => ({ type: 'list', name })),
+    ];
+  }
+
+  public fromResources(resources: RuleMigrationResourceData[]): RuleResource[] {
+    const lists = new Set<string>();
+    const macros = new Set<string>();
+    resources.forEach((resource) => {
+      this.identifiers.fromResource(resource).forEach((identifiedResource) => {
+        if (identifiedResource.type === 'macro') {
+          macros.add(identifiedResource.name);
+        } else if (identifiedResource.type === 'list') {
+          lists.add(identifiedResource.name);
+        }
+      });
+    });
+    return [
+      ...Array.from(macros).map<RuleResource>((name) => ({ type: 'macro', name })),
+      ...Array.from(lists).map<RuleResource>((name) => ({ type: 'list', name })),
+    ];
+  }
+}
diff --git a/x-pack/plugins/security_solution/common/siem_migrations/rules/resources/splunk/index.ts b/x-pack/plugins/security_solution/common/siem_migrations/rules/resources/splunk/index.ts
@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { ResourceIdentifiers } from '../types';
+import { splResourceIdentifier } from './splunk_identifier';
+
+export const splResourceIdentifiers: ResourceIdentifiers = {
+  fromOriginalRule: (originalRule) => splResourceIdentifier(originalRule.query),
+  fromResource: (resource) => {
+    if (resource.type === 'macro' && resource.content) {
+      return splResourceIdentifier(resource.content);
+    }
+    return [];
+  },
+};
diff --git a/...security_solution/common/siem_migrations/rules/resources/splunk/splunk_identifier.test.ts b/...security_solution/common/siem_migrations/rules/resources/splunk/splunk_identifier.test.ts
@@ -0,0 +1,135 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { splResourceIdentifier } from './splunk_identifier';
+
+describe('splResourceIdentifier', () => {
+  it('should extract macros correctly', () => {
+    const query =
+      '`macro_zero`, `macro_one(arg1)`, some search command `macro_two(arg1, arg2)` another command `macro_three(arg1, arg2, arg3)`';
+
+    const result = splResourceIdentifier(query);
+    expect(result).toEqual([
+      { type: 'macro', name: 'macro_zero' },
+      { type: 'macro', name: 'macro_one(1)' },
+      { type: 'macro', name: 'macro_two(2)' },
+      { type: 'macro', name: 'macro_three(3)' },
+    ]);
+  });
+
+  it('should extract macros with double quotes parameters correctly', () => {
+    const query = '| `macro_one("90","2")` | `macro_two("20")`';
+
+    const result = splResourceIdentifier(query);
+    expect(result).toEqual([
+      { type: 'macro', name: 'macro_one(2)' },
+      { type: 'macro', name: 'macro_two(1)' },
+    ]);
+  });
+
+  it('should extract macros with single quotes parameters correctly', () => {
+    const query = "| `macro_one('90','2')` | `macro_two('20')`";
+
+    const result = splResourceIdentifier(query);
+    expect(result).toEqual([
+      { type: 'macro', name: 'macro_one(2)' },
+      { type: 'macro', name: 'macro_two(1)' },
+    ]);
+  });
+
+  it('should extract lookup tables correctly', () => {
+    const query =
+      'search ... | lookup my_lookup_table field AS alias OUTPUT new_field | lookup other_lookup_list | lookup third_lookup';
+
+    const result = splResourceIdentifier(query);
+    expect(result).toEqual([
+      { type: 'list', name: 'my_lookup_table' },
+      { type: 'list', name: 'other_lookup_list' },
+      { type: 'list', name: 'third_lookup' },
+    ]);
+  });
+
+  it('should extract both macros and lookup tables correctly', () => {
+    const query =
+      '`macro_one` some search command | lookup my_lookup_table field AS alias OUTPUT new_field | lookup other_lookup_list | lookup third_lookup';
+
+    const result = splResourceIdentifier(query);
+    expect(result).toEqual([
+      { type: 'macro', name: 'macro_one' },
+      { type: 'list', name: 'my_lookup_table' },
+      { type: 'list', name: 'other_lookup_list' },
+      { type: 'list', name: 'third_lookup' },
+    ]);
+  });
+
+  it('should extract lookup correctly when there are modifiers', () => {
+    const query =
+      'lookup my_lookup_1 field AS alias OUTPUT new_field | lookup local=true my_lookup_2 | lookup update=true my_lookup_3 | lookup local=true update=true my_lookup_4 | lookup update=false local=true  my_lookup_5';
+
+    const result = splResourceIdentifier(query);
+    expect(result).toEqual([
+      { type: 'list', name: 'my_lookup_1' },
+      { type: 'list', name: 'my_lookup_2' },
+      { type: 'list', name: 'my_lookup_3' },
+      { type: 'list', name: 'my_lookup_4' },
+      { type: 'list', name: 'my_lookup_5' },
+    ]);
+  });
+
+  it('should return empty arrays if no macros or lookup tables are found', () => {
+    const query = 'search | stats count';
+
+    const result = splResourceIdentifier(query);
+    expect(result).toEqual([]);
+  });
+
+  it('should handle queries with both macros and lookup tables mixed with other commands', () => {
+    const query =
+      'search `macro_one` | `my_lookup_table` field AS alias myfakelookup new_field | lookup real_lookup_list | `third_macro`';
+
+    const result = splResourceIdentifier(query);
+    expect(result).toEqual([
+      { type: 'macro', name: 'macro_one' },
+      { type: 'macro', name: 'my_lookup_table' },
+      { type: 'macro', name: 'third_macro' },
+      { type: 'list', name: 'real_lookup_list' },
+    ]);
+  });
+
+  it('should ignore macros or lookup tables inside string literals with double quotes', () => {
+    const query =
+      '`macro_one` | lookup my_lookup_table | search title="`macro_two` and lookup another_table"';
+
+    const result = splResourceIdentifier(query);
+    expect(result).toEqual([
+      { type: 'macro', name: 'macro_one' },
+      { type: 'list', name: 'my_lookup_table' },
+    ]);
+  });
+
+  it('should ignore macros or lookup tables inside string literals with single quotes', () => {
+    const query =
+      "`macro_one` | lookup my_lookup_table | search title='`macro_two` and lookup another_table'";
+
+    const result = splResourceIdentifier(query);
+    expect(result).toEqual([
+      { type: 'macro', name: 'macro_one' },
+      { type: 'list', name: 'my_lookup_table' },
+    ]);
+  });
+
+  it('should ignore macros or lookup tables inside comments wrapped by ```', () => {
+    const query =
+      '`macro_one` ```this is a comment with `macro_two` and lookup another_table``` | lookup my_lookup_table ```this is another comment```';
+
+    const result = splResourceIdentifier(query);
+    expect(result).toEqual([
+      { type: 'macro', name: 'macro_one' },
+      { type: 'list', name: 'my_lookup_table' },
+    ]);
+  });
+});
-Original file line number
+Diff line change
@@ Expand Up / @@ -291,7 +291,6 @@ components: @@
           required:
             - type
             - name
-            - content
           properties:
             type:
               $ref: '#/components/schemas/RuleMigrationResourceType'
@@ Expand Down @@