From 597fd3e82e549f7a746728af1a577e9fa982b89d Mon Sep 17 00:00:00 2001 From: Pablo Machado Date: Tue, 22 Oct 2024 09:58:20 +0200 Subject: [PATCH] [SecuritySolution] Fix entity-store to support asset criticality delete (#196680) ## Summary Update the entity store API so it does not return the asset criticality field when the value is 'deleted'. ### How to test it * Open kibana with data * Install the entity store * Update asset criticality for a host or user * Wait for the engine to run (I don't know a reliable way to do this) * Refresh the entity analytics dashboard, and it should show empty fields for deleted asset criticality - [ ] Backport it to 8.16 ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --- .../entity_store_data_client.test.ts | 72 +++++++++++++++---- .../entity_store/entity_store_data_client.ts | 18 ++++- .../entity_analytics/entity_store/types.ts | 26 +++++++ .../field_retention_operators.ts | 28 +------- .../entity_store/utils/ingest.ts | 43 +++++++++++ 5 files changed, 146 insertions(+), 41 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/types.ts create mode 100644 x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/utils/ingest.ts diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.test.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.test.ts index 8079e54ac9ba6..858047952801d 100644 --- a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.test.ts +++ b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.test.ts @@ -39,23 +39,25 @@ describe('EntityStoreDataClient', () => { sortOrder: 'asc' as SortOrder, }; + const emptySearchResponse = { + took: 0, + timed_out: false, + _shards: { + total: 0, + successful: 0, + skipped: 0, + failed: 0, + }, + hits: { + total: 0, + hits: [], + }, + }; + describe('search entities', () => { beforeEach(() => { jest.resetAllMocks(); - esClientMock.search.mockResolvedValue({ - took: 0, - timed_out: false, - _shards: { - total: 0, - successful: 0, - skipped: 0, - failed: 0, - }, - hits: { - total: 0, - hits: [], - }, - }); + esClientMock.search.mockResolvedValue(emptySearchResponse); }); it('searches in the entities store indices', async () => { @@ -133,5 +135,47 @@ describe('EntityStoreDataClient', () => { expect(response.inspect).toMatchSnapshot(); }); + + it('returns searched entity record', async () => { + const fakeEntityRecord = { entity_record: true, asset: { criticality: 'low' } }; + + esClientMock.search.mockResolvedValue({ + ...emptySearchResponse, + hits: { + total: 1, + hits: [ + { + _index: '.entities.v1.latest.security_host_default', + _source: fakeEntityRecord, + }, + ], + }, + }); + + const response = await dataClient.searchEntities(defaultSearchParams); + + expect(response.records[0]).toEqual(fakeEntityRecord); + }); + + it("returns empty asset criticality when criticality value is 'deleted'", async () => { + const fakeEntityRecord = { entity_record: true }; + + esClientMock.search.mockResolvedValue({ + ...emptySearchResponse, + hits: { + total: 1, + hits: [ + { + _index: '.entities.v1.latest.security_host_default', + _source: { asset: { criticality: 'deleted' }, ...fakeEntityRecord }, + }, + ], + }, + }); + + const response = await dataClient.searchEntities(defaultSearchParams); + + expect(response.records[0]).toEqual(fakeEntityRecord); + }); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts index 50e500fae40f2..2cb119e6d37fe 100644 --- a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts +++ b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts @@ -53,6 +53,8 @@ import { isPromiseFulfilled, isPromiseRejected, } from './utils'; +import type { EntityRecord } from './types'; +import { CRITICALITY_VALUES } from '../asset_criticality/constants'; interface EntityStoreClientOpts { logger: Logger; @@ -407,7 +409,7 @@ export class EntityStoreDataClient { const sort = sortField ? [{ [sortField]: sortOrder }] : undefined; const query = filterQuery ? JSON.parse(filterQuery) : undefined; - const response = await this.esClient.search({ + const response = await this.esClient.search({ index, query, size: Math.min(perPage, MAX_SEARCH_RESPONSE_SIZE), @@ -419,7 +421,19 @@ export class EntityStoreDataClient { const total = typeof hits.total === 'number' ? hits.total : hits.total?.value ?? 0; - const records = hits.hits.map((hit) => hit._source as Entity); + const records = hits.hits.map((hit) => { + const { asset, ...source } = hit._source as EntityRecord; + + const assetOverwrite: Pick = + asset && asset.criticality !== CRITICALITY_VALUES.DELETED + ? { asset: { criticality: asset.criticality } } + : {}; + + return { + ...source, + ...assetOverwrite, + }; + }); const inspect: InspectQuery = { dsl: [JSON.stringify({ index, body: query }, null, 2)], diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/types.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/types.ts new file mode 100644 index 0000000000000..e5f1e6db36bca --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/types.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { HostEntity, UserEntity } from '../../../../common/api/entity_analytics'; +import type { CriticalityValues } from '../asset_criticality/constants'; + +export interface HostEntityRecord extends Omit { + asset?: { + criticality: CriticalityValues; + }; +} + +export interface UserEntityRecord extends Omit { + asset?: { + criticality: CriticalityValues; + }; +} + +/** + * It represents the data stored in the entity store index. + */ +export type EntityRecord = HostEntityRecord | UserEntityRecord; diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/field_retention_operators.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/field_retention_operators.ts index ced0327194364..6e8f7dc3f6578 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/field_retention_operators.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/field_retention_operators.ts @@ -11,6 +11,7 @@ import { fieldOperatorToIngestProcessor, } from '@kbn/security-solution-plugin/server/lib/entity_analytics/entity_store/field_retention_definition'; import { FtrProviderContext } from '../../../../ftr_provider_context'; +import { applyIngestProcessorToDoc } from '../utils/ingest'; export default ({ getService }: FtrProviderContext) => { const es = getService('es'); const log = getService('log'); @@ -26,31 +27,8 @@ export default ({ getService }: FtrProviderContext) => { docSource: any ): Promise => { const step = fieldOperatorToIngestProcessor(operator, { enrichField: 'historical' }); - const doc = { - _index: 'index', - _id: 'id', - _source: docSource, - }; - - const res = await es.ingest.simulate({ - pipeline: { - description: 'test', - processors: [step], - }, - docs: [doc], - }); - - const firstDoc = res.docs?.[0]; - - // @ts-expect-error error is not in the types - const error = firstDoc?.error; - if (error) { - log.error('Full painless error below: '); - log.error(JSON.stringify(error, null, 2)); - throw new Error('Painless error running pipelie see logs for full detail : ' + error?.type); - } - return firstDoc?.doc?._source; + return applyIngestProcessorToDoc([step], docSource, es, log); }; describe('@ess @serverless @skipInServerlessMKI Entity store - Field Retention Pipeline Steps', () => { @@ -90,7 +68,7 @@ export default ({ getService }: FtrProviderContext) => { expectArraysMatchAnyOrder(resultDoc.test_field, ['foo']); }); - it('should take from history if latest field doesnt have maxLength values', async () => { + it("should take from history if latest field doesn't have maxLength values", async () => { const op: FieldRetentionOperator = { operation: 'collect_values', field: 'test_field', diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/utils/ingest.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/utils/ingest.ts new file mode 100644 index 0000000000000..24f7d759190b5 --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/utils/ingest.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Client } from '@elastic/elasticsearch'; +import { IngestProcessorContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { ToolingLog } from '@kbn/tooling-log'; + +export const applyIngestProcessorToDoc = async ( + steps: IngestProcessorContainer[], + docSource: any, + es: Client, + log: ToolingLog +): Promise => { + const doc = { + _index: 'index', + _id: 'id', + _source: docSource, + }; + + const res = await es.ingest.simulate({ + pipeline: { + description: 'test', + processors: steps, + }, + docs: [doc], + }); + + const firstDoc = res.docs?.[0]; + + // @ts-expect-error error is not in the types + const error = firstDoc?.error; + if (error) { + log.error('Full painless error below: '); + log.error(JSON.stringify(error, null, 2)); + throw new Error('Painless error running pipeline see logs for full detail : ' + error?.type); + } + + return firstDoc?.doc?._source; +};