From c8ff7ea27ab02548c25c3c295bbd988e3adfbba4 Mon Sep 17 00:00:00 2001 From: Jill Guyonnet Date: Fri, 20 Sep 2024 14:28:23 +0100 Subject: [PATCH] =?UTF-8?q?[Fleet]=20Add=20@custom=20component=20tem?= =?UTF-8?q?plate=20to=20integrations=20index=20te=E2=80=A6=20(#192731)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Relates https://github.com/elastic/kibana/issues/190730 This PR adds a `@custom `component template to integrations index template's `composed_of` array. ### Testing Integration install/update should be tested. From my manual testing, Fleet seems to behave normally and the new component template is added to the `composed_of` array. For example, the output of `GET /_index_template/logs-system.auth` is: ```json { "index_templates": [ { "name": "logs-system.auth", "index_template": { "index_patterns": [ "logs-system.auth-*" ], "template": { "settings": {}, "mappings": { "_meta": { "package": { "name": "system" }, "managed_by": "fleet", "managed": true } } }, "composed_of": [ "logs@mappings", "logs@settings", "logs-system.auth@package", "logs@custom", "logs-system.auth@custom", "ecs@mappings", ".fleet_globals-1", ".fleet_agent_id_verification-1" ], "priority": 200, "_meta": { "package": { "name": "system" }, "managed_by": "fleet", "managed": true }, "data_stream": { "hidden": false, "allow_custom_routing": false, "failure_store": false }, "ignore_missing_component_templates": [ "logs@custom", "logs-system.auth@custom" ] } } ] } ``` ### Screenshot Screenshot 2024-09-19 at 16 44 57 Screenshot 2024-09-19 at 17 04 45 ### Checklist - [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [ ] [Flaky Test Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was used on any tests changed --- .../fleet/server/constants/fleet_es_assets.ts | 2 +- .../services/epm/elasticsearch/template/install.ts | 14 ++++++++++++++ .../steps/step_save_system_object.test.ts | 4 ++-- .../apis/epm/__snapshots__/bulk_get_assets.snap | 12 ++++++++++++ .../apis/epm/install_by_upload.ts | 6 +++--- .../apis/epm/install_overrides.ts | 1 + .../apis/epm/install_remove_assets.ts | 8 ++++++++ .../apis/epm/update_assets.ts | 9 +++++++++ .../package_policy/input_package_create_upgrade.ts | 1 + .../apis/package_policy/update.ts | 1 + .../apis/package_policy/upgrade.ts | 2 ++ 11 files changed, 54 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts b/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts index 83c283591a9bd..55e6493c77891 100644 --- a/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts +++ b/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts @@ -11,7 +11,7 @@ import { getESAssetMetadata } from '../services/epm/elasticsearch/meta'; const meta = getESAssetMetadata(); -export const FLEET_INSTALL_FORMAT_VERSION = '1.2.0'; +export const FLEET_INSTALL_FORMAT_VERSION = '1.3.0'; export const FLEET_AGENT_POLICIES_SCHEMA_VERSION = '1.1.1'; diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts index 2ee8477e04f42..2a17768ac1f9c 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts @@ -319,6 +319,7 @@ export function buildComponentTemplates(params: { experimentalDataStreamFeature?: ExperimentalDataStreamFeature; lifecycle?: IndexTemplate['template']['lifecycle']; fieldCount?: number; + type?: string; }) { const { templateName, @@ -330,6 +331,7 @@ export function buildComponentTemplates(params: { experimentalDataStreamFeature, lifecycle, fieldCount, + type, } = params; const packageTemplateName = `${templateName}${PACKAGE_TEMPLATE_SUFFIX}`; const userSettingsTemplateName = `${templateName}${USER_SETTINGS_TEMPLATE_SUFFIX}`; @@ -417,6 +419,17 @@ export function buildComponentTemplates(params: { _meta, }; + // Stub custom template + if (type) { + const customTemplateName = `${type}${USER_SETTINGS_TEMPLATE_SUFFIX}`; + templatesMap[customTemplateName] = { + template: { + settings: {}, + }, + _meta, + }; + } + // return empty/stub template templatesMap[userSettingsTemplateName] = { template: { @@ -580,6 +593,7 @@ export function prepareTemplate({ experimentalDataStreamFeature, lifecycle: lifecyle, fieldCount: countFields(validFields), + type: dataStream.type, }); const template = getTemplate({ diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_system_object.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_system_object.test.ts index e91826c99793c..aecdd0b2552c4 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_system_object.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_system_object.test.ts @@ -91,7 +91,7 @@ describe('updateLatestExecutedState', () => { 'epm-packages', 'test-integration', { - install_format_schema_version: '1.2.0', + install_format_schema_version: '1.3.0', install_status: 'installed', install_version: '1.0.0', latest_install_failed_attempts: [], @@ -157,7 +157,7 @@ describe('updateLatestExecutedState', () => { 'epm-packages', 'test-integration', { - install_format_schema_version: '1.2.0', + install_format_schema_version: '1.3.0', install_status: 'installed', install_version: '1.0.0', latest_install_failed_attempts: [], diff --git a/x-pack/test/fleet_api_integration/apis/epm/__snapshots__/bulk_get_assets.snap b/x-pack/test/fleet_api_integration/apis/epm/__snapshots__/bulk_get_assets.snap index 5fd219958c319..37c7882fff203 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/__snapshots__/bulk_get_assets.snap +++ b/x-pack/test/fleet_api_integration/apis/epm/__snapshots__/bulk_get_assets.snap @@ -62,6 +62,12 @@ Array [ "id": "logs-all_assets.test_logs@package", "type": "component_template", }, + Object { + "appLink": "/app/management/data/index_management/component_templates/logs@custom", + "attributes": Object {}, + "id": "logs@custom", + "type": "component_template", + }, Object { "appLink": "/app/management/data/index_management/component_templates/logs-all_assets.test_logs@custom", "attributes": Object {}, @@ -80,6 +86,12 @@ Array [ "id": "metrics-all_assets.test_metrics@package", "type": "component_template", }, + Object { + "appLink": "/app/management/data/index_management/component_templates/metrics@custom", + "attributes": Object {}, + "id": "metrics@custom", + "type": "component_template", + }, Object { "appLink": "/app/management/data/index_management/component_templates/metrics-all_assets.test_metrics@custom", "attributes": Object {}, diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_by_upload.ts b/x-pack/test/fleet_api_integration/apis/epm/install_by_upload.ts index 30b2eacce0fb7..331cae3058bf1 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/install_by_upload.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/install_by_upload.ts @@ -97,7 +97,7 @@ export default function (providerContext: FtrProviderContext) { it('should install a tar archive correctly', async function () { const res = await uploadPackage(); - expect(res.body.items.length).to.be(30); + expect(res.body.items.length).to.be(32); }); it('should upgrade when uploading a newer zip archive', async () => { @@ -111,7 +111,7 @@ export default function (providerContext: FtrProviderContext) { .type('application/zip') .send(buf) .expect(200); - expect(res.body.items.length).to.be(30); + expect(res.body.items.length).to.be(32); expect(res.body.items.some((item: any) => item.id.includes(testPkgNewVersion))); await deletePackage(testPkgName, testPkgNewVersion); @@ -182,7 +182,7 @@ export default function (providerContext: FtrProviderContext) { .type('application/zip') .send(buf) .expect(200); - expect(res.body.items.length).to.be(30); + expect(res.body.items.length).to.be(32); }); it('should throw an error if the archive is zip but content type is gzip', async function () { diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_overrides.ts b/x-pack/test/fleet_api_integration/apis/epm/install_overrides.ts index 36f98fb3434b3..8b85502bdf5ad 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/install_overrides.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/install_overrides.ts @@ -61,6 +61,7 @@ export default function (providerContext: FtrProviderContext) { `logs@mappings`, `logs@settings`, `${templateName}@package`, + 'logs@custom', `${templateName}@custom`, `ecs@mappings`, '.fleet_globals-1', diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts b/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts index 24564014633f2..fc8225e9df02d 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts @@ -599,6 +599,10 @@ const expectAssetsInstalled = ({ id: 'logs-all_assets.test_logs@package', type: 'component_template', }, + { + id: 'logs@custom', + type: 'component_template', + }, { id: 'logs-all_assets.test_logs@custom', type: 'component_template', @@ -607,6 +611,10 @@ const expectAssetsInstalled = ({ id: 'metrics-all_assets.test_metrics@package', type: 'component_template', }, + { + id: 'metrics@custom', + type: 'component_template', + }, { id: 'metrics-all_assets.test_metrics@custom', type: 'component_template', diff --git a/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts b/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts index 69aedb72947bd..17d54786245af 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts @@ -417,6 +417,10 @@ export default function (providerContext: FtrProviderContext) { id: 'logs-all_assets.test_logs@package', type: 'component_template', }, + { + id: 'logs@custom', + type: 'component_template', + }, { id: 'logs-all_assets.test_logs@custom', type: 'component_template', @@ -441,6 +445,11 @@ export default function (providerContext: FtrProviderContext) { id: 'metrics-all_assets.test_metrics@package', type: 'component_template', }, + + { + id: 'metrics@custom', + type: 'component_template', + }, { id: 'metrics-all_assets.test_metrics@custom', type: 'component_template', diff --git a/x-pack/test/fleet_api_integration/apis/package_policy/input_package_create_upgrade.ts b/x-pack/test/fleet_api_integration/apis/package_policy/input_package_create_upgrade.ts index 77aeadcae93a7..481f4e09c68d9 100644 --- a/x-pack/test/fleet_api_integration/apis/package_policy/input_package_create_upgrade.ts +++ b/x-pack/test/fleet_api_integration/apis/package_policy/input_package_create_upgrade.ts @@ -207,6 +207,7 @@ export default function (providerContext: FtrProviderContext) { { id: 'logs-dataset1', type: 'index_template' }, { id: 'logs-dataset1@package', type: 'component_template' }, { id: 'logs-dataset1@custom', type: 'component_template' }, + { id: 'logs@custom', type: 'component_template' }, ]); // now check the package component template was created correctly diff --git a/x-pack/test/fleet_api_integration/apis/package_policy/update.ts b/x-pack/test/fleet_api_integration/apis/package_policy/update.ts index 971ed42b578df..273f051dfcec6 100644 --- a/x-pack/test/fleet_api_integration/apis/package_policy/update.ts +++ b/x-pack/test/fleet_api_integration/apis/package_policy/update.ts @@ -948,6 +948,7 @@ export default function (providerContext: FtrProviderContext) { { id: 'logs-somedataset', type: 'index_template' }, { id: 'logs-somedataset@package', type: 'component_template' }, { id: 'logs-somedataset@custom', type: 'component_template' }, + { id: 'logs@custom', type: 'component_template' }, ]); const dataset3PkgComponentTemplate = await getComponentTemplate('logs-somedataset@package'); diff --git a/x-pack/test/fleet_api_integration/apis/package_policy/upgrade.ts b/x-pack/test/fleet_api_integration/apis/package_policy/upgrade.ts index 6982878c7d111..021eebcdcc0c1 100644 --- a/x-pack/test/fleet_api_integration/apis/package_policy/upgrade.ts +++ b/x-pack/test/fleet_api_integration/apis/package_policy/upgrade.ts @@ -1317,6 +1317,8 @@ export default function (providerContext: FtrProviderContext) { for (let i = 0; i < POLICY_COUNT; i++) { await createPackagePolicy(i.toString()); } + + expectedAssets.push({ id: 'logs@custom', type: 'component_template' }); }); afterEach(async function () {