[Security Solution][Endpoint] Fix Manifest Manger so that it works wi…

…th large (>10k) (elastic#174411)

## Summary

### Fleet Changes:

- Two new utilities that return  `AsyncIterator`'s:
    - one for working with ElasticSearch `.search()` method
    - one for working with SavedObjects `.find()` method
- NOTE: although the `SavedObjects` client already supports getting back
an `find` interface that returns an `AysncIterable`, I was not
convenient to use in our use cases where we are returning the data from
the SO back to an external consumer (services exposed by Fleet). We need
to be able to first process the data out of the SO before returning it
to the consumer, thus having this utility facilitates that.
- both handle looping through ALL data in a given query (even if >10k)
- new `fetchAllArtifacts()` method in `ArtifactsClient`: Returns an
`AsyncIterator` enabling one to loop through all artifacts (even if
>10k)
- new `fetchAllItemIds()` method in `PackagePolicyService`: return an
`AsyncIterator` enabling one to loop through all item IDs (even if >10k)
- new `fetchAllItems()` method in `PackagePolicyService`: returns an
`AsyncIterator` enabling one to loop through all package policies (even
if >10k)


### Endpoint Changes:

- Retrieval of existing artifacts as well as list of all policies and
policy IDs now use new methods introduced into fleet services (above)
- Added new config property -
`xpack.securitySolution.packagerTaskTimeout` - to enable customer to
adjust the timeout value for how long the artifact packager task can
run. Default has been set to `20m`
- Efficiencies around batch processing of updates to Policies and
artifact creation
- improved logging



### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

x-pack/plugins/fleet/server/mocks/index.ts

@@ -31,6 +31,8 @@ import { packageServiceMock } from '../services/epm/package_service.mock';
 import type { UninstallTokenServiceInterface } from '../services/security/uninstall_token_service';
 import type { MessageSigningServiceInterface } from '../services/security';
 
+import { PackagePolicyMocks } from './package_policy.mocks';
+
 // Export all mocks from artifacts
 export * from '../services/artifacts/mocks';
 
@@ -40,6 +42,8 @@ export * from '../services/files/mocks';
 // export all mocks from fleet actions client
 export * from '../services/actions/mocks';
 
+export * from './package_policy.mocks';
+
 export interface MockedFleetAppContext extends FleetAppContext {
   elasticsearch: ReturnType<typeof elasticsearchServiceMock.createStart>;
   data: ReturnType<typeof dataPluginMock.createStartContract>;
@@ -144,6 +148,22 @@ export const createPackagePolicyServiceMock = (): jest.Mocked<PackagePolicyClien
     getUpgradePackagePolicyInfo: jest.fn(),
     enrichPolicyWithDefaultsFromPackage: jest.fn(),
     findAllForAgentPolicy: jest.fn(),
+    fetchAllItems: jest.fn((..._) => {
+      return {
+        async *[Symbol.asyncIterator]() {
+          yield Promise.resolve([PackagePolicyMocks.generatePackagePolicy({ id: '111' })]);
+          yield Promise.resolve([PackagePolicyMocks.generatePackagePolicy({ id: '222' })]);
+        },
+      };
+    }),
+    fetchAllItemIds: jest.fn((..._) => {
+      return {
+        async *[Symbol.asyncIterator]() {
+          yield Promise.resolve(['111']);
+          yield Promise.resolve(['222']);
+        },
+      };
+    }),
   };
 };
 

x-pack/plugins/fleet/server/mocks/package_policy.mocks.ts

@@ -0,0 +1,109 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { SavedObjectsFindResponse } from '@kbn/core-saved-objects-api-server';
+
+import type { SavedObjectsFindResult } from '@kbn/core-saved-objects-api-server';
+
+import { mapPackagePolicySavedObjectToPackagePolicy } from '../services/package_policies';
+
+import type { PackagePolicy } from '../../common';
+import { PACKAGE_POLICY_SAVED_OBJECT_TYPE } from '../../common';
+
+import type { PackagePolicySOAttributes } from '../types';
+
+const generatePackagePolicySOAttributesMock = (
+  overrides: Partial<PackagePolicySOAttributes> = {}
+): PackagePolicySOAttributes => {
+  return {
+    name: `Package Policy 1`,
+    description: 'Policy for things',
+    created_at: '2024-01-24T15:21:13.389Z',
+    created_by: 'elastic',
+    updated_at: '2024-01-25T15:21:13.389Z',
+    updated_by: 'user-a',
+    policy_id: '444-555-666',
+    enabled: true,
+    inputs: [],
+    namespace: 'default',
+    package: {
+      name: 'endpoint',
+      title: 'Elastic Endpoint',
+      version: '1.0.0',
+    },
+    revision: 1,
+    is_managed: false,
+    secret_references: [],
+    vars: {},
+    elasticsearch: {
+      privileges: {
+        cluster: [],
+      },
+    },
+    agents: 2,
+
+    ...overrides,
+  };
+};
+
+const generatePackagePolicyMock = (overrides: Partial<PackagePolicy> = {}) => {
+  return {
+    ...mapPackagePolicySavedObjectToPackagePolicy(generatePackagePolicySavedObjectMock()),
+    ...overrides,
+  };
+};
+
+const generatePackagePolicySavedObjectMock = (
+  soAttributes: PackagePolicySOAttributes = generatePackagePolicySOAttributesMock()
+): SavedObjectsFindResult<PackagePolicySOAttributes> => {
+  return {
+    score: 1,
+    id: 'so-123',
+    type: PACKAGE_POLICY_SAVED_OBJECT_TYPE,
+    version: 'abc',
+    created_at: soAttributes.created_at,
+    updated_at: soAttributes.updated_at,
+    attributes: soAttributes,
+    references: [],
+    sort: ['created_at'],
+  };
+};
+
+const generatePackagePolicySavedObjectFindResponseMock = (
+  soResults?: PackagePolicySOAttributes[]
+): SavedObjectsFindResponse<PackagePolicySOAttributes> => {
+  const soList = soResults ?? [
+    generatePackagePolicySOAttributesMock(),
+    generatePackagePolicySOAttributesMock(),
+  ];
+
+  return {
+    saved_objects: soList.map((soAttributes) => {
+      return {
+        score: 1,
+        id: 'so-123',
+        type: PACKAGE_POLICY_SAVED_OBJECT_TYPE,
+        version: 'abc',
+        created_at: soAttributes.created_at,
+        updated_at: soAttributes.updated_at,
+        attributes: soAttributes,
+        references: [],
+        sort: ['created_at'],
+      };
+    }),
+    total: soList.length,
+    per_page: 10,
+    page: 1,
+    pit_id: 'pit-id-1',
+  };
+};
+
+export const PackagePolicyMocks = Object.freeze({
+  generatePackagePolicySOAttributes: generatePackagePolicySOAttributesMock,
+  generatePackagePolicySavedObjectFindResponse: generatePackagePolicySavedObjectFindResponseMock,
+  generatePackagePolicy: generatePackagePolicyMock,
+});

x-pack/plugins/fleet/server/services/artifacts/artifacts.test.ts

@@ -11,6 +11,8 @@ import { errors } from '@elastic/elasticsearch';
 
 import type { TransportResult } from '@elastic/elasticsearch';
 
+import { set } from '@kbn/safer-lodash-set';
+
 import { FLEET_SERVER_ARTIFACTS_INDEX } from '../../../common';
 
 import { ArtifactsElasticsearchError } from '../../errors';
@@ -33,12 +35,14 @@ import {
   createArtifact,
   deleteArtifact,
   encodeArtifactContent,
+  fetchAllArtifacts,
   generateArtifactContentHash,
   getArtifact,
   listArtifacts,
 } from './artifacts';
 
 import type { NewArtifact } from './types';
+import type { FetchAllArtifactsOptions } from './types';
 
 describe('When using the artifacts services', () => {
   let esClientMock: ReturnType<typeof elasticsearchServiceMock.createInternalClient>;
@@ -324,8 +328,28 @@ describe('When using the artifacts services', () => {
         newArtifact,
       ]);
 
-      expect(responseErrors).toEqual([new Error('error')]);
-      expect(artifacts).toBeUndefined();
+      expect(responseErrors).toEqual([
+        new Error(
+          'Create of artifact id [undefined] returned: result [undefined], status [400], reason [{"reason":"error"}]'
+        ),
+      ]);
+      expect(artifacts).toEqual([
+        {
+          body: 'eJyrVkrNKynKTC1WsoqOrQUAJxkFKQ==',
+          compressionAlgorithm: 'zlib',
+          created: expect.any(String),
+          decodedSha256: 'd801aa1fb',
+          decodedSize: 14,
+          encodedSha256: 'd29238d40',
+          encodedSize: 22,
+          encryptionAlgorithm: 'none',
+          id: 'endpoint:trustlist-v1-d801aa1fb',
+          identifier: 'trustlist-v1',
+          packageName: 'endpoint',
+          relative_url: '/api/fleet/artifacts/trustlist-v1/d801aa1fb',
+          type: 'trustlist',
+        },
+      ]);
     });
   });
 
@@ -488,4 +512,113 @@ describe('When using the artifacts services', () => {
       });
     });
   });
+
+  describe('and calling `fetchAll()`', () => {
+    beforeEach(() => {
+      esClientMock.search
+        .mockResolvedValueOnce(generateArtifactEsSearchResultHitsMock())
+        .mockResolvedValueOnce(generateArtifactEsSearchResultHitsMock())
+        .mockResolvedValueOnce(set(generateArtifactEsSearchResultHitsMock(), 'hits.hits', []));
+    });
+
+    it('should return an iterator', async () => {
+      expect(fetchAllArtifacts(esClientMock)).toEqual({
+        [Symbol.asyncIterator]: expect.any(Function),
+      });
+    });
+
+    it('should provide artifacts on each iteration', async () => {
+      for await (const artifacts of fetchAllArtifacts(esClientMock)) {
+        expect(artifacts[0]).toEqual({
+          body: expect.anything(),
+          compressionAlgorithm: expect.anything(),
+          created: expect.anything(),
+          decodedSha256: expect.anything(),
+          decodedSize: expect.anything(),
+          encodedSha256: expect.anything(),
+          encodedSize: expect.anything(),
+          encryptionAlgorithm: expect.anything(),
+          id: expect.anything(),
+          identifier: expect.anything(),
+          packageName: expect.anything(),
+          relative_url: expect.anything(),
+          type: expect.anything(),
+        });
+      }
+
+      expect(esClientMock.search).toHaveBeenCalledTimes(3);
+    });
+
+    it('should use defaults if no `options` were provided', async () => {
+      for await (const artifacts of fetchAllArtifacts(esClientMock)) {
+        expect(artifacts.length).toBeGreaterThan(0);
+      }
+
+      expect(esClientMock.search).toHaveBeenLastCalledWith(
+        expect.objectContaining({
+          q: '',
+          size: 1000,
+          sort: [{ created: { order: 'asc' } }],
+          _source_excludes: undefined,
+        })
+      );
+    });
+
+    it('should use custom options when provided', async () => {
+      const options: FetchAllArtifactsOptions = {
+        kuery: 'foo: something',
+        sortOrder: 'desc',
+        perPage: 500,
+        sortField: 'someField',
+        includeArtifactBody: false,
+      };
+
+      for await (const artifacts of fetchAllArtifacts(esClientMock, options)) {
+        expect(artifacts.length).toBeGreaterThan(0);
+      }
+
+      expect(esClientMock.search).toHaveBeenCalledWith(
+        expect.objectContaining({
+          q: options.kuery,
+          size: options.perPage,
+          sort: [{ [options.sortField!]: { order: options.sortOrder } }],
+          _source_excludes: 'body',
+        })
+      );
+    });
+
+    it('should set `done` to true if loop `break`s out', async () => {
+      const iterator = fetchAllArtifacts(esClientMock);
+
+      for await (const _ of iterator) {
+        break;
+      }
+
+      await expect(iterator[Symbol.asyncIterator]().next()).resolves.toEqual({
+        done: true,
+        value: expect.any(Array),
+      });
+
+      expect(esClientMock.search).toHaveBeenCalledTimes(1);
+    });
+
+    it('should handle throwing in loop by setting `done` to `true`', async () => {
+      const iterator = fetchAllArtifacts(esClientMock);
+
+      try {
+        for await (const _ of iterator) {
+          throw new Error('test');
+        }
+      } catch (e) {
+        expect(e); // just to silence eslint
+      }
+
+      await expect(iterator[Symbol.asyncIterator]().next()).resolves.toEqual({
+        done: true,
+        value: expect.any(Array),
+      });
+
+      expect(esClientMock.search).toHaveBeenCalledTimes(1);
+    });
+  });
 });