[Fleet] Prevent concurrent runs of Fleet setup (elastic#183636)

Closes elastic/ingest-dev#3346

- [x] Unit and integration tests are created or updated
- [x] Turn down info logging

The linked issue seems to be caused by multiple kibana instances running
Fleet setup at the same time, trying to create the preconfigured cloud
policy concurrently, and in case of failures, the agent policy is left
with a revision with no inputs, this way preventing fleet-server to
start properly.

See the concurrent errors in the logs:
https://platform-logging.kb.us-west2.gcp.elastic-cloud.com/app/r/s/tUpMP

This fix introduces a `fleet-setup-lock` SO type, which is used to
create a document as a lock by Fleet setup, and is deleted when the
setup is completed. Concurrent calls to Fleet setup will return early if
this doc exists.

To verify:
Run the test `./run_fleet_setup_parallel.sh` from local kibana, and
verify the generated logs that only one of them ran Fleet setup.

---------

Co-authored-by: kibanamachine <[email protected]>

packages/kbn-check-mappings-update-cli/current_fields.json

@@ -457,6 +457,11 @@
     "proxy_headers",
     "url"
   ],
+  "fleet-setup-lock": [
+    "started_at",
+    "status",
+    "uuid"
+  ],
   "fleet-uninstall-tokens": [
     "policy_id",
     "token_plain"

packages/kbn-check-mappings-update-cli/current_mappings.json

@@ -1546,6 +1546,19 @@
       }
     }
   },
+  "fleet-setup-lock": {
+    "properties": {
+      "started_at": {
+        "type": "date"
+      },
+      "status": {
+        "type": "keyword"
+      },
+      "uuid": {
+        "type": "text"
+      }
+    }
+  },
   "fleet-uninstall-tokens": {
     "dynamic": false,
     "properties": {

run_fleet_setup_parallel.sh

@@ -0,0 +1,7 @@
+node scripts/jest_integration.js x-pack/plugins/fleet/server/integration_tests/es.test.ts &
+
+sleep 5 
+node scripts/jest_integration.js x-pack/plugins/fleet/server/integration_tests/fleet_setup.test.ts &
+node scripts/jest_integration.js x-pack/plugins/fleet/server/integration_tests/fleet_setup.test.ts &
+node scripts/jest_integration.js x-pack/plugins/fleet/server/integration_tests/fleet_setup.test.ts &
+exit 0

src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts

@@ -101,6 +101,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "fleet-message-signing-keys": "93421f43fed2526b59092a4e3c65d64bc2266c0f",
         "fleet-preconfiguration-deletion-record": "c52ea1e13c919afe8a5e8e3adbb7080980ecc08e",
         "fleet-proxy": "6cb688f0d2dd856400c1dbc998b28704ff70363d",
+        "fleet-setup-lock": "0dc784792c79b5af5a6e6b5dcac06b0dbaa90bde",
         "fleet-uninstall-tokens": "ed8aa37e3cdd69e4360709e64944bb81cae0c025",
         "graph-workspace": "5cc6bb1455b078fd848c37324672163f09b5e376",
         "guided-onboarding-guide-state": "d338972ed887ac480c09a1a7fbf582d6a3827c91",

src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts

@@ -69,6 +69,7 @@ const previouslyRegisteredTypes = [
   'fleet-preconfiguration-deletion-record',
   'fleet-proxy',
   'fleet-uninstall-tokens',
+  'fleet-setup-lock',
   'graph-workspace',
   'guided-setup-state',
   'guided-onboarding-guide-state',

src/core/server/integration_tests/saved_objects/migrations/group5/dot_kibana_split.test.ts

@@ -221,6 +221,7 @@ describe('split .kibana index into multiple system indices', () => {
             "fleet-message-signing-keys",
             "fleet-preconfiguration-deletion-record",
             "fleet-proxy",
+            "fleet-setup-lock",
             "fleet-uninstall-tokens",
             "graph-workspace",
             "guided-onboarding-guide-state",

x-pack/plugins/fleet/common/constants/index.ts

@@ -6,7 +6,7 @@
  */
 
 export { INTEGRATIONS_PLUGIN_ID, PLUGIN_ID } from './plugin';
-export { INGEST_SAVED_OBJECT_INDEX } from './saved_objects';
+export { INGEST_SAVED_OBJECT_INDEX, FLEET_SETUP_LOCK_TYPE } from './saved_objects';
 export * from './routes';
 export * from './agent';
 export * from './agent_policy';

x-pack/plugins/fleet/common/constants/saved_objects.ts

@@ -6,3 +6,5 @@
  */
 
 export const INGEST_SAVED_OBJECT_INDEX = '.kibana_ingest';
+
+export const FLEET_SETUP_LOCK_TYPE = 'fleet-setup-lock';

x-pack/plugins/fleet/common/types/models/agent_policy.ts

@@ -62,6 +62,7 @@ export interface AgentPolicy extends Omit<NewAgentPolicy, 'id'> {
   agents?: number;
   unprivileged_agents?: number;
   is_protected: boolean;
+  version?: string;
 }
 
 export interface FullAgentPolicyInputStream {

x-pack/plugins/fleet/common/types/models/fleet_setup_lock.ts

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export interface FleetSetupLock {
+  status: string;
+  uuid: string;
+  started_at: string;
+}

x-pack/plugins/fleet/common/types/models/index.ts

@@ -20,3 +20,4 @@ export * from './fleet_server_policy_config';
 export * from './fleet_proxy';
 export * from './secret';
 export * from './setup_technology';
+export * from './fleet_setup_lock';

x-pack/plugins/fleet/server/constants/index.ts

@@ -90,6 +90,7 @@ export {
   OUTPUT_SECRETS_MINIMUM_FLEET_SERVER_VERSION,
   // outputs
   OUTPUT_HEALTH_DATA_STREAM,
+  FLEET_SETUP_LOCK_TYPE,
   type PrivilegeMapObject,
 } from '../../common/constants';
 

x-pack/plugins/fleet/server/integration_tests/es.test.ts

@@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import type { TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server';
+import { createTestServers } from '@kbn/core-test-helpers-kbn-server';
+
+/**
+ * Verifies that multiple Kibana instances running in parallel will not create duplicate preconfiguration objects.
+ */
+describe.skip('Fleet setup preconfiguration with multiple instances Kibana', () => {
+  let esServer: TestElasticsearchUtils;
+
+  const startServers = async () => {
+    const { startES } = createTestServers({
+      adjustTimeout: (t) => jest.setTimeout(t),
+      settings: {
+        es: {
+          license: 'trial',
+        },
+      },
+    });
+
+    esServer = await startES();
+  };
+
+  const stopServers = async () => {
+    if (esServer) {
+      await esServer.stop();
+    }
+
+    await new Promise((res) => setTimeout(res, 10000));
+  };
+
+  beforeEach(async () => {
+    await startServers();
+  });
+
+  afterEach(async () => {
+    await stopServers();
+  });
+
+  describe('startES', () => {
+    it('start es', async () => {
+      await new Promise((resolve) => setTimeout(resolve, 60000));
+    });
+  });
+});