[SecuritySolution] Fix entity-store to support asset criticality dele…

…te (elastic#196680)

## Summary

Update the entity store API so it does not return the asset criticality
field when the value is 'deleted'.


### How to test it
* Open kibana with data
* Install the entity store
* Update asset criticality for a host or user
* Wait for the engine to run (I don't know a reliable way to do this)
* Refresh the entity analytics dashboard, and it should show empty
fields for deleted asset criticality

- [ ] Backport it to 8.16

### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.test.ts

@@ -39,23 +39,25 @@ describe('EntityStoreDataClient', () => {
     sortOrder: 'asc' as SortOrder,
   };
 
+  const emptySearchResponse = {
+    took: 0,
+    timed_out: false,
+    _shards: {
+      total: 0,
+      successful: 0,
+      skipped: 0,
+      failed: 0,
+    },
+    hits: {
+      total: 0,
+      hits: [],
+    },
+  };
+
   describe('search entities', () => {
     beforeEach(() => {
       jest.resetAllMocks();
-      esClientMock.search.mockResolvedValue({
-        took: 0,
-        timed_out: false,
-        _shards: {
-          total: 0,
-          successful: 0,
-          skipped: 0,
-          failed: 0,
-        },
-        hits: {
-          total: 0,
-          hits: [],
-        },
-      });
+      esClientMock.search.mockResolvedValue(emptySearchResponse);
     });
 
     it('searches in the entities store indices', async () => {
@@ -133,5 +135,47 @@ describe('EntityStoreDataClient', () => {
 
       expect(response.inspect).toMatchSnapshot();
     });
+
+    it('returns searched entity record', async () => {
+      const fakeEntityRecord = { entity_record: true, asset: { criticality: 'low' } };
+
+      esClientMock.search.mockResolvedValue({
+        ...emptySearchResponse,
+        hits: {
+          total: 1,
+          hits: [
+            {
+              _index: '.entities.v1.latest.security_host_default',
+              _source: fakeEntityRecord,
+            },
+          ],
+        },
+      });
+
+      const response = await dataClient.searchEntities(defaultSearchParams);
+
+      expect(response.records[0]).toEqual(fakeEntityRecord);
+    });
+
+    it("returns empty asset criticality when criticality value is 'deleted'", async () => {
+      const fakeEntityRecord = { entity_record: true };
+
+      esClientMock.search.mockResolvedValue({
+        ...emptySearchResponse,
+        hits: {
+          total: 1,
+          hits: [
+            {
+              _index: '.entities.v1.latest.security_host_default',
+              _source: { asset: { criticality: 'deleted' }, ...fakeEntityRecord },
+            },
+          ],
+        },
+      });
+
+      const response = await dataClient.searchEntities(defaultSearchParams);
+
+      expect(response.records[0]).toEqual(fakeEntityRecord);
+    });
   });
 });

x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts

@@ -53,6 +53,8 @@ import {
   isPromiseFulfilled,
   isPromiseRejected,
 } from './utils';
+import type { EntityRecord } from './types';
+import { CRITICALITY_VALUES } from '../asset_criticality/constants';
 
 interface EntityStoreClientOpts {
   logger: Logger;
@@ -407,7 +409,7 @@ export class EntityStoreDataClient {
     const sort = sortField ? [{ [sortField]: sortOrder }] : undefined;
     const query = filterQuery ? JSON.parse(filterQuery) : undefined;
 
-    const response = await this.esClient.search<Entity>({
+    const response = await this.esClient.search<EntityRecord>({
       index,
       query,
       size: Math.min(perPage, MAX_SEARCH_RESPONSE_SIZE),
@@ -419,7 +421,19 @@ export class EntityStoreDataClient {
 
     const total = typeof hits.total === 'number' ? hits.total : hits.total?.value ?? 0;
 
-    const records = hits.hits.map((hit) => hit._source as Entity);
+    const records = hits.hits.map((hit) => {
+      const { asset, ...source } = hit._source as EntityRecord;
+
+      const assetOverwrite: Pick<Entity, 'asset'> =
+        asset && asset.criticality !== CRITICALITY_VALUES.DELETED
+          ? { asset: { criticality: asset.criticality } }
+          : {};
+
+      return {
+        ...source,
+        ...assetOverwrite,
+      };
+    });
 
     const inspect: InspectQuery = {
       dsl: [JSON.stringify({ index, body: query }, null, 2)],

x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/types.ts

@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { HostEntity, UserEntity } from '../../../../common/api/entity_analytics';
+import type { CriticalityValues } from '../asset_criticality/constants';
+
+export interface HostEntityRecord extends Omit<HostEntity, 'asset'> {
+  asset?: {
+    criticality: CriticalityValues;
+  };
+}
+
+export interface UserEntityRecord extends Omit<UserEntity, 'asset'> {
+  asset?: {
+    criticality: CriticalityValues;
+  };
+}
+
+/**
+ * It represents the data stored in the entity store index.
+ */
+export type EntityRecord = HostEntityRecord | UserEntityRecord;

x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/field_retention_operators.ts

@@ -11,6 +11,7 @@ import {
   fieldOperatorToIngestProcessor,
 } from '@kbn/security-solution-plugin/server/lib/entity_analytics/entity_store/field_retention_definition';
 import { FtrProviderContext } from '../../../../ftr_provider_context';
+import { applyIngestProcessorToDoc } from '../utils/ingest';
 export default ({ getService }: FtrProviderContext) => {
   const es = getService('es');
   const log = getService('log');
@@ -26,31 +27,8 @@ export default ({ getService }: FtrProviderContext) => {
     docSource: any
   ): Promise<any> => {
     const step = fieldOperatorToIngestProcessor(operator, { enrichField: 'historical' });
-    const doc = {
-      _index: 'index',
-      _id: 'id',
-      _source: docSource,
-    };
-
-    const res = await es.ingest.simulate({
-      pipeline: {
-        description: 'test',
-        processors: [step],
-      },
-      docs: [doc],
-    });
-
-    const firstDoc = res.docs?.[0];
-
-    // @ts-expect-error error is not in the types
-    const error = firstDoc?.error;
-    if (error) {
-      log.error('Full painless error below: ');
-      log.error(JSON.stringify(error, null, 2));
-      throw new Error('Painless error running pipelie see logs for full detail : ' + error?.type);
-    }
 
-    return firstDoc?.doc?._source;
+    return applyIngestProcessorToDoc([step], docSource, es, log);
   };
 
   describe('@ess @serverless @skipInServerlessMKI Entity store - Field Retention Pipeline Steps', () => {
@@ -90,7 +68,7 @@ export default ({ getService }: FtrProviderContext) => {
         expectArraysMatchAnyOrder(resultDoc.test_field, ['foo']);
       });
 
-      it('should take from history if latest field doesnt have maxLength values', async () => {
+      it("should take from history if latest field doesn't have maxLength values", async () => {
         const op: FieldRetentionOperator = {
           operation: 'collect_values',
           field: 'test_field',

x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/utils/ingest.ts

@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { Client } from '@elastic/elasticsearch';
+import { IngestProcessorContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import { ToolingLog } from '@kbn/tooling-log';
+
+export const applyIngestProcessorToDoc = async (
+  steps: IngestProcessorContainer[],
+  docSource: any,
+  es: Client,
+  log: ToolingLog
+): Promise<any> => {
+  const doc = {
+    _index: 'index',
+    _id: 'id',
+    _source: docSource,
+  };
+
+  const res = await es.ingest.simulate({
+    pipeline: {
+      description: 'test',
+      processors: steps,
+    },
+    docs: [doc],
+  });
+
+  const firstDoc = res.docs?.[0];
+
+  // @ts-expect-error error is not in the types
+  const error = firstDoc?.error;
+  if (error) {
+    log.error('Full painless error below: ');
+    log.error(JSON.stringify(error, null, 2));
+    throw new Error('Painless error running pipeline see logs for full detail : ' + error?.type);
+  }
+
+  return firstDoc?.doc?._source;
+};