[EDR Workflows] Initialize agent with latest fleet supported version (e…

…lastic#189174)

This PR introduces a call to `/agents/available_versions` to fetch the
latest available agent version in Serverless environment. This version
is then used to create agents throughout our tests.

x-pack/plugins/security_solution/common/endpoint/data_generators/endpoint_rule_alert_generator.ts

@@ -36,7 +36,7 @@ export class EndpointRuleAlertGenerator extends BaseDataGenerator {
   generate(overrides: DeepPartial<EndpointRuleAlert> = {}): EndpointRuleAlert {
     const endpointMetadataGenerator = new EndpointMetadataGenerator();
     const endpointMetadata = endpointMetadataGenerator.generate({
-      agent: { version: kibanaPackageJson.version },
+      agent: { version: overrides?.agent?.version ?? kibanaPackageJson.version },
       host: { hostname: overrides?.host?.hostname },
       Endpoint: { state: { isolation: overrides?.Endpoint?.state?.isolation } },
     });
@@ -50,7 +50,7 @@ export class EndpointRuleAlertGenerator extends BaseDataGenerator {
         agent: {
           id: endpointAgentId,
           type: 'endpoint',
-          version: kibanaPackageJson.version,
+          version: endpointMetadata.agent.version,
         },
         elastic: endpointMetadata.elastic,
         host: endpointMetadata.host,

x-pack/plugins/security_solution/common/endpoint/data_loaders/index_endpoint_rule_alerts.ts

@@ -15,6 +15,9 @@ import type {
   MappingTypeMapping,
   Name,
 } from '@elastic/elasticsearch/lib/api/types';
+import type { KbnClient } from '@kbn/test';
+import { isServerlessKibanaFlavor } from '../utils/kibana_status';
+import { fetchFleetLatestAvailableAgentVersion } from '../utils/fetch_fleet_version';
 import { createToolingLogger, wrapErrorIfNeeded } from './utils';
 import { DEFAULT_ALERTS_INDEX } from '../../constants';
 import { EndpointRuleAlertGenerator } from '../data_generators/endpoint_rule_alert_generator';
@@ -26,6 +29,7 @@ export interface IndexEndpointRuleAlertsOptions {
   endpointIsolated?: boolean;
   count?: number;
   log?: ToolingLog;
+  kbnClient?: KbnClient;
 }
 
 export interface IndexedEndpointRuleAlerts {
@@ -41,6 +45,7 @@ export interface DeletedIndexedEndpointRuleAlerts {
  * Loads alerts for Endpoint directly into the internal index that the Endpoint Rule would have
  * written them to for a given endpoint
  * @param esClient
+ * @param kbnClient
  * @param endpointAgentId
  * @param endpointHostname
  * @param endpointIsolated
@@ -49,6 +54,7 @@ export interface DeletedIndexedEndpointRuleAlerts {
  */
 export const indexEndpointRuleAlerts = async ({
   esClient,
+  kbnClient,
   endpointAgentId,
   endpointHostname,
   endpointIsolated,
@@ -59,12 +65,20 @@ export const indexEndpointRuleAlerts = async ({
 
   await ensureEndpointRuleAlertsIndexExists(esClient);
 
+  let version = kibanaPackageJson.version;
+  if (kbnClient) {
+    const isServerless = await isServerlessKibanaFlavor(kbnClient);
+    if (isServerless) {
+      version = await fetchFleetLatestAvailableAgentVersion(kbnClient);
+    }
+  }
+
   const alertsGenerator = new EndpointRuleAlertGenerator();
   const indexedAlerts: estypes.IndexResponse[] = [];
 
   for (let n = 0; n < count; n++) {
     const alert = alertsGenerator.generate({
-      agent: { id: endpointAgentId },
+      agent: { id: endpointAgentId, version },
       host: { hostname: endpointHostname },
       ...(endpointIsolated ? { Endpoint: { state: { isolation: endpointIsolated } } } : {}),
     });

x-pack/plugins/security_solution/common/endpoint/data_loaders/index_fleet_server.ts

@@ -23,6 +23,7 @@ import {
   packagePolicyRouteService,
 } from '@kbn/fleet-plugin/common';
 import type { ToolingLog } from '@kbn/tooling-log';
+import { fetchFleetLatestAvailableAgentVersion } from '../utils/fetch_fleet_version';
 import { indexFleetServerAgent } from './index_fleet_agent';
 import { catchAxiosErrorFormatAndThrow } from '../format_axios_error';
 import { usageTracker } from './usage_tracker';
@@ -47,6 +48,12 @@ export const enableFleetServerIfNecessary = usageTracker.track(
     log: ToolingLog = createToolingLogger(),
     version: string = kibanaPackageJson.version
   ) => {
+    let agentVersion = version;
+
+    if (isServerless) {
+      agentVersion = await fetchFleetLatestAvailableAgentVersion(kbnClient);
+    }
+
     const agentPolicy = await getOrCreateFleetServerAgentPolicy(kbnClient, log);
 
     if (!isServerless && !(await hasFleetServerAgent(esClient, agentPolicy.id))) {
@@ -56,7 +63,7 @@ export const enableFleetServerIfNecessary = usageTracker.track(
 
       const indexedAgent = await indexFleetServerAgent(esClient, log, {
         policy_id: agentPolicy.id,
-        agent: { version },
+        agent: { version: agentVersion },
         last_checkin_status: 'online',
         last_checkin: lastCheckin.toISOString(),
       });

x-pack/plugins/security_solution/common/endpoint/utils/fetch_fleet_version.ts

@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/**
+ * Fetches the latest version of the Elastic Agent available for download
+ * @param kbnClient
+ */
+import type { KbnClient } from '@kbn/test';
+import { AGENT_API_ROUTES } from '@kbn/fleet-plugin/common';
+import type { GetAvailableVersionsResponse } from '@kbn/fleet-plugin/common/types';
+import { catchAxiosErrorFormatAndThrow } from '../format_axios_error';
+
+export const fetchFleetLatestAvailableAgentVersion = async (
+  kbnClient: KbnClient
+): Promise<string> => {
+  return kbnClient
+    .request<GetAvailableVersionsResponse>({
+      method: 'GET',
+      path: AGENT_API_ROUTES.AVAILABLE_VERSIONS_PATTERN,
+      headers: {
+        'elastic-api-version': '2023-10-31',
+      },
+    })
+    .then((response) => response.data.items[0])
+    .catch(catchAxiosErrorFormatAndThrow);
+};

x-pack/plugins/security_solution/common/endpoint/utils/kibana_status.ts

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { KbnClient } from '@kbn/test';
+import type { Client } from '@elastic/elasticsearch';
+import type { StatusResponse } from '@kbn/core-status-common-internal';
+import { catchAxiosErrorFormatAndThrow } from '../format_axios_error';
+
+export const fetchKibanaStatus = async (kbnClient: KbnClient): Promise<StatusResponse> => {
+  return (await kbnClient.status.get().catch(catchAxiosErrorFormatAndThrow)) as StatusResponse;
+};
+/**
+ * Checks to see if Kibana/ES is running in serverless mode
+ * @param client
+ */
+export const isServerlessKibanaFlavor = async (client: KbnClient | Client): Promise<boolean> => {
+  if (client instanceof KbnClient) {
+    const kbnStatus = await fetchKibanaStatus(client);
+
+    // If we don't have status for plugins, then error
+    // the Status API will always return something (its an open API), but if auth was successful,
+    // it will also return more data.
+    if (!kbnStatus?.status?.plugins) {
+      throw new Error(
+        `Unable to retrieve Kibana plugins status (likely an auth issue with the username being used for kibana)`
+      );
+    }
+
+    return kbnStatus.status.plugins?.serverless?.level === 'available';
+  } else {
+    return (await client.info()).version.build_flavor === 'serverless';
+  }
+};

x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/scan.cy.ts

@@ -56,8 +56,8 @@ describe(
             policy = indexedPolicy.integrationPolicies[0];
 
             return enableAllPolicyProtections(policy.id).then(() => {
-              // Create and enroll a new Endpoint host
-              return createEndpointHost(policy.policy_ids[0]).then((host) => {
+              // At this point 8.14.2 is GA and this functionality is not available until 8.15.0
+              return createEndpointHost(policy.policy_ids[0], '8.15.0').then((host) => {
                 createdHost = host as CreateAndEnrollEndpointHostResponse;
               });
             });

x-pack/plugins/security_solution/public/management/cypress/support/create_and_enroll_endpoint_host_ci.ts

@@ -10,13 +10,14 @@ import type { Client } from '@elastic/elasticsearch';
 import type { ToolingLog } from '@kbn/tooling-log';
 import type { KbnClient } from '@kbn/test/src/kbn_client';
 import { kibanaPackageJson } from '@kbn/repo-info';
+import { isServerlessKibanaFlavor } from '../../../../common/endpoint/utils/kibana_status';
+import { fetchFleetLatestAvailableAgentVersion } from '../../../../common/endpoint/utils/fetch_fleet_version';
 import { isFleetServerRunning } from '../../../../scripts/endpoint/common/fleet_server/fleet_server_services';
 import type { HostVm } from '../../../../scripts/endpoint/common/types';
 import type { BaseVmCreateOptions } from '../../../../scripts/endpoint/common/vm_services';
 import { createVm } from '../../../../scripts/endpoint/common/vm_services';
 import {
   fetchAgentPolicyEnrollmentKey,
-  fetchFleetAvailableVersions,
   fetchFleetServerUrl,
   getAgentDownloadUrl,
   getAgentFileName,
@@ -38,12 +39,12 @@ export interface CreateAndEnrollEndpointHostCIOptions
   agentPolicyId: string;
   /** version of the Agent to install. Defaults to stack version */
   version?: string;
+  /** skip all checks and use provided version */
+  forceVersion?: boolean;
   /** The name for the host. Will also be the name of the VM */
   hostname?: string;
   /** If `version` should be exact, or if this is `true`, then the closest version will be used. Defaults to `false` */
   useClosestVersionMatch?: boolean;
-  /** If the environment is MKI */
-  isMkiEnvironment?: boolean;
 }
 
 export interface CreateAndEnrollEndpointHostCIResponse {
@@ -66,14 +67,16 @@ export const createAndEnrollEndpointHostCI = async ({
   hostname,
   version = kibanaPackageJson.version,
   useClosestVersionMatch = true,
-  isMkiEnvironment = false,
+  forceVersion = false,
 }: CreateAndEnrollEndpointHostCIOptions): Promise<CreateAndEnrollEndpointHostCIResponse> => {
-  let agentVersion = version;
   const vmName = hostname ?? `test-host-${Math.random().toString().substring(2, 6)}`;
+  let agentVersion = version;
 
-  if (isMkiEnvironment) {
-    // MKI env provides own fleet server. We must be sure that currently deployed FS is compatible with agent version we want to deploy.
-    agentVersion = await fetchFleetAvailableVersions(kbnClient);
+  if (!forceVersion) {
+    const isServerless = await isServerlessKibanaFlavor(kbnClient);
+    if (isServerless) {
+      agentVersion = await fetchFleetLatestAvailableAgentVersion(kbnClient);
+    }
   }
 
   const fileNameNoExtension = getAgentFileName(agentVersion);

x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts

@@ -246,11 +246,12 @@ export const dataLoaders = (
     },
 
     indexEndpointRuleAlerts: async (options: { endpointAgentId: string; count?: number }) => {
-      const { esClient, log } = await stackServicesPromise;
+      const { esClient, log, kbnClient } = await stackServicesPromise;
       return (
         await indexEndpointRuleAlerts({
           ...options,
           esClient,
+          kbnClient,
           log,
         })
       ).alerts;
@@ -326,8 +327,6 @@ export const dataLoadersForRealEndpoints = (
   config: Cypress.PluginConfigOptions
 ): void => {
   const stackServicesPromise = setupStackServicesUsingCypressConfig(config);
-  const isServerless = Boolean(config.env.IS_SERVERLESS);
-  const isCloudServerless = Boolean(config.env.CLOUD_SERVERLESS);
 
   on('task', {
     createSentinelOneHost: async () => {
@@ -415,7 +414,6 @@ ${s1Info.status}
       options: Omit<CreateAndEnrollEndpointHostCIOptions, 'log' | 'kbnClient'>
     ): Promise<CreateAndEnrollEndpointHostCIResponse> => {
       const { kbnClient, log, esClient } = await stackServicesPromise;
-      const isMkiEnvironment = isServerless && isCloudServerless;
       let retryAttempt = 0;
       const attemptCreateEndpointHost =
         async (): Promise<CreateAndEnrollEndpointHostCIResponse> => {
@@ -424,7 +422,6 @@ ${s1Info.status}
             const newHost = process.env.CI
               ? await createAndEnrollEndpointHostCI({
                   useClosestVersionMatch: true,
-                  isMkiEnvironment,
                   ...options,
                   log,
                   kbnClient,

x-pack/plugins/security_solution/public/management/cypress/support/plugin_handlers/endpoint_data_loader.ts

@@ -10,6 +10,7 @@ import type { KbnClient } from '@kbn/test';
 import pRetry from 'p-retry';
 import { kibanaPackageJson } from '@kbn/repo-info';
 import type { ToolingLog } from '@kbn/tooling-log';
+import { fetchFleetLatestAvailableAgentVersion } from '../../../../../common/endpoint/utils/fetch_fleet_version';
 import { dump } from '../../../../../scripts/endpoint/common/utils';
 import { STARTED_TRANSFORM_STATES } from '../../../../../common/constants';
 import {
@@ -77,8 +78,18 @@ export const cyLoadEndpointDataHandler = async (
     isServerless = false,
   } = options;
 
+  let agentVersion = version;
+
+  if (isServerless) {
+    agentVersion = await fetchFleetLatestAvailableAgentVersion(kbnClient);
+  }
+
   const DocGenerator = EndpointDocGenerator.custom({
-    CustomMetadataGenerator: EndpointMetadataGenerator.custom({ version, os, isolation }),
+    CustomMetadataGenerator: EndpointMetadataGenerator.custom({
+      version: agentVersion,
+      os,
+      isolation,
+    }),
   });
 
   if (waitUntilTransformed) {
@@ -192,6 +203,7 @@ const startTransform = async (
  * the united metadata index
  *
  * @param esClient
+ * @param log
  * @param location
  * @param ids
  */

x-pack/plugins/security_solution/public/management/cypress/tasks/create_endpoint_host.ts

@@ -10,12 +10,14 @@ import type { CreateAndEnrollEndpointHostResponse } from '../../../../scripts/en
 // only used in "real" endpoint tests not in mocked ones
 export const createEndpointHost = (
   agentPolicyId: string,
+  version?: string,
   timeout?: number
 ): Cypress.Chainable<CreateAndEnrollEndpointHostResponse> => {
   return cy.task(
     'createEndpointHost',
     {
       agentPolicyId,
+      ...(version ? { version, forceVersion: true } : {}),
     },
     { timeout: timeout ?? 30 * 60 * 1000 }
   );

x-pack/plugins/security_solution/scripts/endpoint/agent_emulator/services/endpoint_loader.ts

@@ -11,6 +11,8 @@ import pMap from 'p-map';
 import type { CreatePackagePolicyResponse } from '@kbn/fleet-plugin/common';
 import type { ToolingLog } from '@kbn/tooling-log';
 import { kibanaPackageJson } from '@kbn/repo-info';
+import { isServerlessKibanaFlavor } from '../../../../common/endpoint/utils/kibana_status';
+import { fetchFleetLatestAvailableAgentVersion } from '../../../../common/endpoint/utils/fetch_fleet_version';
 import { indexAlerts } from '../../../../common/endpoint/data_loaders/index_alerts';
 import { EndpointDocGenerator } from '../../../../common/endpoint/generate_data';
 import { fetchEndpointMetadataList } from '../../common/endpoint_metadata_services';
@@ -21,16 +23,9 @@ import { METADATA_DATASTREAM } from '../../../../common/endpoint/constants';
 import { EndpointMetadataGenerator } from '../../../../common/endpoint/data_generators/endpoint_metadata_generator';
 import { getEndpointPackageInfo } from '../../../../common/endpoint/utils/package';
 import { ENDPOINT_ALERTS_INDEX, ENDPOINT_EVENTS_INDEX } from '../../common/constants';
-import { isServerlessKibanaFlavor } from '../../common/stack_services';
 
 let WAS_FLEET_SETUP_DONE = false;
 
-const CurrentKibanaVersionDocGenerator = EndpointDocGenerator.custom({
-  CustomMetadataGenerator: EndpointMetadataGenerator.custom({
-    version: kibanaPackageJson.version,
-  }),
-});
-
 export const loadEndpointsIfNoneExist = async (
   esClient: Client,
   kbnClient: KbnClient,
@@ -84,14 +79,20 @@ export const loadEndpoints = async ({
   log,
   onProgress,
   count = 2,
-  DocGeneratorClass = CurrentKibanaVersionDocGenerator,
+  DocGeneratorClass,
 }: LoadEndpointsOptions): Promise<void> => {
   if (log) {
     log.verbose(`loadEndpoints(): Loading ${count} endpoints...`);
   }
 
+  const isServerless = await isServerlessKibanaFlavor(kbnClient);
+  let version = kibanaPackageJson.version;
+
+  if (isServerless) {
+    version = await fetchFleetLatestAvailableAgentVersion(kbnClient);
+  }
+
   if (!WAS_FLEET_SETUP_DONE) {
-    const isServerless = await isServerlessKibanaFlavor(kbnClient);
     await setupFleetForEndpoint(kbnClient);
     await enableFleetServerIfNecessary(esClient, isServerless, kbnClient, log);
     // eslint-disable-next-line require-atomic-updates
@@ -120,10 +121,16 @@ export const loadEndpoints = async ({
     }
   };
 
+  const CurrentKibanaVersionDocGenerator = EndpointDocGenerator.custom({
+    CustomMetadataGenerator: EndpointMetadataGenerator.custom({
+      version,
+    }),
+  });
+
   await pMap(
     Array.from({ length: count }),
     async () => {
-      const endpointGenerator = new DocGeneratorClass();
+      const endpointGenerator = new (DocGeneratorClass ?? CurrentKibanaVersionDocGenerator)();
 
       await indexEndpointHostDocs({
         numDocs: 1,