[Logs+] API to create a basic integration (#160777)

## Summary

Closes #159991

Fields that have been utilised to fulfil `basic` and `agent` fields can
be easily amended if these are incorrect.

Multiple datasets are supported, and these can contain more than one
type.

## Testing

A curl command similar to the following should allow you to hit the API
(check the credentials etc):

```
curl -XPOST -u 'elastic:changeme' -H 'kbn-xsrf: something' -d '{
    "integrationName": "web_custom_nginx",
    "datasets": [{"name": "access", "type": "logs"}, {"name": "error", "type": "metrics"}, {"name": "warning", "type":"logs"}]
}' 'http://localhost:5601/<BASE_PATH>/api/fleet/epm/custom_integrations'
```

## History / context

- [Prototype
learnings](#158552 (comment))
- [Prototype PR](#160003)

## Results / expectations

API response (with installed assets):

![Screenshot 2023-07-05 at 16 56
33](https://github.com/elastic/kibana/assets/471693/fc4a0bab-7057-430a-8c03-18dd4ee17ab7)

We see the custom integration in "installed integrations" (albeit with a
verification warning):

![Screenshot 2023-07-05 at 16 57
14](https://github.com/elastic/kibana/assets/471693/0c9177d2-2871-490f-9b5c-f338e96484c4)

We see the custom integration in Discover with the logs explorer
profile:

![Screenshot 2023-07-05 at 16 58
20](https://github.com/elastic/kibana/assets/471693/30c556f2-9fcd-416e-8047-5976fc11ffa2)

The assets are installed correctly:

![Screenshot 2023-07-05 at 16 59
06](https://github.com/elastic/kibana/assets/471693/abb82632-f619-4fc3-be93-dc6ce97abedd)

![Screenshot 2023-07-05 at 16 59
20](https://github.com/elastic/kibana/assets/471693/ca1c1da5-1e4b-422c-9edb-0f56e0ed3f98)

![Screenshot 2023-07-05 at 16 59
36](https://github.com/elastic/kibana/assets/471693/8bd60d7e-aebc-4833-b423-eba3336fb42c)

x-pack/plugins/fleet/common/constants/epm.ts

@@ -30,6 +30,9 @@ export const PACKAGE_TEMPLATE_SUFFIX = '@package';
 export const USER_SETTINGS_TEMPLATE_SUFFIX = '@custom';
 
 export const DATASET_VAR_NAME = 'data_stream.dataset';
+
+export const CUSTOM_INTEGRATION_PACKAGE_SPEC_VERSION = '2.9.0';
+
 /*
  Package rules:
 |               | autoUpdatePackages |

x-pack/plugins/fleet/common/constants/routes.ts

@@ -34,6 +34,7 @@ export const EPM_API_ROUTES = {
   DATA_STREAMS_PATTERN: `${EPM_API_ROOT}/data_streams`,
   INSTALL_FROM_REGISTRY_PATTERN: EPM_PACKAGES_ONE,
   INSTALL_BY_UPLOAD_PATTERN: EPM_PACKAGES_MANY,
+  CUSTOM_INTEGRATIONS_PATTERN: `${EPM_API_ROOT}/custom_integrations`,
   DELETE_PATTERN: EPM_PACKAGES_ONE,
   FILEPATH_PATTERN: `${EPM_PACKAGES_ONE}/{filePath*}`,
   CATEGORIES_PATTERN: `${EPM_API_ROOT}/categories`,

x-pack/plugins/fleet/common/types/models/epm.ts

@@ -34,7 +34,7 @@ export interface DefaultPackagesInstallationError {
 }
 
 export type InstallType = 'reinstall' | 'reupdate' | 'rollback' | 'update' | 'install' | 'unknown';
-export type InstallSource = 'registry' | 'upload' | 'bundled';
+export type InstallSource = 'registry' | 'upload' | 'bundled' | 'custom';
 
 export type EpmPackageInstallStatus = 'installed' | 'installing' | 'install_failed';
 

x-pack/plugins/fleet/server/routes/epm/handlers.ts

@@ -52,6 +52,7 @@ import type {
   UpdatePackageRequestSchema,
   GetLimitedPackagesRequestSchema,
   GetBulkAssetsRequestSchema,
+  CreateCustomIntegrationRequestSchema,
 } from '../../types';
 import {
   bulkInstallPackages,
@@ -403,6 +404,45 @@ export const installPackageFromRegistryHandler: FleetRequestHandler<
     return await defaultFleetErrorHandler({ error: res.error, response });
   }
 };
+export const createCustomIntegrationHandler: FleetRequestHandler<
+  undefined,
+  undefined,
+  TypeOf<typeof CreateCustomIntegrationRequestSchema.body>
+> = async (context, request, response) => {
+  const coreContext = await context.core;
+  const fleetContext = await context.fleet;
+  const savedObjectsClient = fleetContext.internalSoClient;
+  const esClient = coreContext.elasticsearch.client.asInternalUser;
+  const user = (await appContextService.getSecurity()?.authc.getCurrentUser(request)) || undefined;
+  const kibanaVersion = appContextService.getKibanaVersion();
+  const authorizationHeader = HTTPAuthorizationHeader.parseFromRequest(request, user?.username);
+  const spaceId = fleetContext.spaceId;
+  const { integrationName, force, datasets } = request.body;
+
+  const res = await installPackage({
+    installSource: 'custom',
+    savedObjectsClient,
+    pkgName: integrationName,
+    datasets,
+    esClient,
+    spaceId,
+    force,
+    authorizationHeader,
+    kibanaVersion,
+  });
+
+  if (!res.error) {
+    const body: InstallPackageResponse = {
+      items: res.assets || [],
+      _meta: {
+        install_source: res.installSource,
+      },
+    };
+    return response.ok({ body });
+  } else {
+    return await defaultFleetErrorHandler({ error: res.error, response });
+  }
+};
 
 const bulkInstallServiceResponseToHttpEntry = (
   result: BulkInstallResponse

x-pack/plugins/fleet/server/routes/epm/index.ts

@@ -43,6 +43,7 @@ import {
   UpdatePackageRequestSchemaDeprecated,
   ReauthorizeTransformRequestSchema,
   GetDataStreamsRequestSchema,
+  CreateCustomIntegrationRequestSchema,
 } from '../../types';
 
 import {
@@ -62,6 +63,7 @@ import {
   getVerificationKeyIdHandler,
   reauthorizeTransformsHandler,
   getDataStreamsHandler,
+  createCustomIntegrationHandler,
 } from './handlers';
 
 const MAX_FILE_SIZE_BYTES = 104857600; // 100MB
@@ -196,6 +198,17 @@ export const registerRoutes = (router: FleetAuthzRouter) => {
     installPackageByUploadHandler
   );
 
+  router.post(
+    {
+      path: EPM_API_ROUTES.CUSTOM_INTEGRATIONS_PATTERN,
+      validate: CreateCustomIntegrationRequestSchema,
+      fleetAuthz: {
+        integrations: { installPackages: true },
+      },
+    },
+    createCustomIntegrationHandler
+  );
+
   router.delete(
     {
       path: EPM_API_ROUTES.DELETE_PATTERN,

x-pack/plugins/fleet/server/services/epm/packages/custom_integrations/assets/cache.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { setArchiveEntry, setArchiveFilelist } from '../../../archive';
+
+interface Assets {
+  path: string;
+  content: Buffer;
+}
+export const cacheAssets = (assets: Assets[], name: string, version: string) => {
+  const paths = assets.map((asset) => asset.path);
+
+  setArchiveFilelist({ name, version }, paths);
+
+  assets.forEach((asset) => {
+    setArchiveEntry(asset.path, asset.content);
+  });
+
+  return paths;
+};