From 2fee796ddce2cc66338ad7af6b78aa19a557c3df Mon Sep 17 00:00:00 2001
From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
Date: Thu, 9 Jan 2025 03:29:41 +1100
Subject: [PATCH] [8.x] Fix generation of dynamic mapping for object with
specific subfield (#204104) (#204493)
# Backport
This will backport the following commits from `main` to `8.x`:
- [Fix generation of dynamic mapping for object with specific subfield
(#204104)](https://github.com/elastic/kibana/pull/204104)
<!--- Backport version: 9.4.3 -->
### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)
<!--BACKPORT [{"author":{"name":"Jaime Soriano
Pastor","email":"jaime.soriano@elastic.co"},"sourceCommit":{"committedDate":"2024-12-16T20:58:30Z","message":"Fix
generation of dynamic mapping for object with specific subfield
(#204104)\n\nFix generation of dynamic mapping for objects that have
more specific\r\nsubfields in separate definitions.\r\n\r\nThis can be
reproduced for example with:\r\n```\r\n- name: labels\r\n type:
object\r\n object_type: keyword\r\n object_type_mapping_type: '*'\r\n-
name: labels.count\r\n type: long\r\n```\r\n\r\nFleet expands and
deduplicates field definitions before generating the\r\nmappings, so the
definitions above are converted to something like
the\r\nfollowing:\r\n```\r\n- name: labels\r\n type: group\r\n
object_type: keyword\r\n object_type_mapping_type: '*'\r\n fields:\r\n -
name: count\r\n type: long\r\n```\r\n\r\nUsually fields of type `group`
don't have an `object_type`, so this was\r\nbeing ignored, the dynamic
mapping was not being generated.\r\n\r\nThis issue was not reproduced if
the object field name includes a\r\nwildcard, like in `labels.*`,
because then the expansion and\r\ndeduplication resolves to something
like this:\r\n```\r\n- name: labels\r\n type: group\r\n fields:\r\n -
name: '*'\r\n type: object\r\n object_type: keyword\r\n
object_type_mapping_type: '*'\r\n - name: count\r\n type:
long\r\n```","sha":"e3877e053405bae71b5576648cb7c637c4a23f9a","branchLabelMapping":{"^v9.0.0$":"main","^v8.18.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:fix","Team:Fleet","v9.0.0","backport:prev-major"],"title":"Fix
generation of dynamic mapping for object with specific
subfield","number":204104,"url":"https://github.com/elastic/kibana/pull/204104","mergeCommit":{"message":"Fix
generation of dynamic mapping for object with specific subfield
(#204104)\n\nFix generation of dynamic mapping for objects that have
more specific\r\nsubfields in separate definitions.\r\n\r\nThis can be
reproduced for example with:\r\n```\r\n- name: labels\r\n type:
object\r\n object_type: keyword\r\n object_type_mapping_type: '*'\r\n-
name: labels.count\r\n type: long\r\n```\r\n\r\nFleet expands and
deduplicates field definitions before generating the\r\nmappings, so the
definitions above are converted to something like
the\r\nfollowing:\r\n```\r\n- name: labels\r\n type: group\r\n
object_type: keyword\r\n object_type_mapping_type: '*'\r\n fields:\r\n -
name: count\r\n type: long\r\n```\r\n\r\nUsually fields of type `group`
don't have an `object_type`, so this was\r\nbeing ignored, the dynamic
mapping was not being generated.\r\n\r\nThis issue was not reproduced if
the object field name includes a\r\nwildcard, like in `labels.*`,
because then the expansion and\r\ndeduplication resolves to something
like this:\r\n```\r\n- name: labels\r\n type: group\r\n fields:\r\n -
name: '*'\r\n type: object\r\n object_type: keyword\r\n
object_type_mapping_type: '*'\r\n - name: count\r\n type:
long\r\n```","sha":"e3877e053405bae71b5576648cb7c637c4a23f9a"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/204104","number":204104,"mergeCommit":{"message":"Fix
generation of dynamic mapping for object with specific subfield
(#204104)\n\nFix generation of dynamic mapping for objects that have
more specific\r\nsubfields in separate definitions.\r\n\r\nThis can be
reproduced for example with:\r\n```\r\n- name: labels\r\n type:
object\r\n object_type: keyword\r\n object_type_mapping_type: '*'\r\n-
name: labels.count\r\n type: long\r\n```\r\n\r\nFleet expands and
deduplicates field definitions before generating the\r\nmappings, so the
definitions above are converted to something like
the\r\nfollowing:\r\n```\r\n- name: labels\r\n type: group\r\n
object_type: keyword\r\n object_type_mapping_type: '*'\r\n fields:\r\n -
name: count\r\n type: long\r\n```\r\n\r\nUsually fields of type `group`
don't have an `object_type`, so this was\r\nbeing ignored, the dynamic
mapping was not being generated.\r\n\r\nThis issue was not reproduced if
the object field name includes a\r\nwildcard, like in `labels.*`,
because then the expansion and\r\ndeduplication resolves to something
like this:\r\n```\r\n- name: labels\r\n type: group\r\n fields:\r\n -
name: '*'\r\n type: object\r\n object_type: keyword\r\n
object_type_mapping_type: '*'\r\n - name: count\r\n type:
long\r\n```","sha":"e3877e053405bae71b5576648cb7c637c4a23f9a"}}]}]
BACKPORT-->
Co-authored-by: Jaime Soriano Pastor <jaime.soriano@elastic.co>
---
.../elasticsearch/template/template.test.ts | 78 ++++++
.../epm/elasticsearch/template/template.ts | 242 +++++++++---------
2 files changed, 203 insertions(+), 117 deletions(-)
diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/template/template.test.ts
index 6b3d0c77ad74d..15f7e1f526085 100644
--- a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/template/template.test.ts
+++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/template/template.test.ts
@@ -786,6 +786,84 @@ describe('EPM template', () => {
expect(mappings).toEqual(objectFieldWithPropertyReversedMapping);
});
+ it('tests processing object field with more specific properties without wildcard', () => {
+ const objectFieldWithPropertyReversedLiteralYml = `
+- name: labels
+ type: object
+ object_type: keyword
+ object_type_mapping_type: '*'
+- name: labels.count
+ type: long
+`;
+ const objectFieldWithPropertyReversedMapping = {
+ dynamic_templates: [
+ {
+ labels: {
+ path_match: 'labels.*',
+ match_mapping_type: '*',
+ mapping: {
+ type: 'keyword',
+ },
+ },
+ },
+ ],
+ properties: {
+ labels: {
+ dynamic: true,
+ type: 'object',
+ properties: {
+ count: {
+ type: 'long',
+ },
+ },
+ },
+ },
+ };
+ const fields: Field[] = safeLoad(objectFieldWithPropertyReversedLiteralYml);
+ const processedFields = processFields(fields);
+ const mappings = generateMappings(processedFields);
+ expect(mappings).toEqual(objectFieldWithPropertyReversedMapping);
+ });
+
+ it('tests processing object field with more specific properties with wildcard', () => {
+ const objectFieldWithPropertyReversedLiteralYml = `
+- name: labels.*
+ type: object
+ object_type: keyword
+ object_type_mapping_type: '*'
+- name: labels.count
+ type: long
+`;
+ const objectFieldWithPropertyReversedMapping = {
+ dynamic_templates: [
+ {
+ 'labels.*': {
+ path_match: 'labels.*',
+ match_mapping_type: '*',
+ mapping: {
+ type: 'keyword',
+ },
+ },
+ },
+ ],
+ properties: {
+ labels: {
+ dynamic: true,
+ type: 'object',
+ properties: {
+ count: {
+ type: 'long',
+ },
+ },
+ },
+ },
+ };
+ const fields: Field[] = safeLoad(objectFieldWithPropertyReversedLiteralYml);
+ const processedFields = processFields(fields);
+ const mappings = generateMappings(processedFields);
+ expect(mappings).toEqual(objectFieldWithPropertyReversedMapping);
+ });
+
it('tests processing object field with subobjects set to false (case B)', () => {
const objectFieldWithPropertyReversedLiteralYml = `
- name: b.labels.*
diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/template/template.ts
index e48e6251a092a..89608d76ebf92 100644
--- a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/template/template.ts
+++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/template/template.ts
@@ -299,6 +299,124 @@ function _generateMappings(
}
}
+ function addObjectAsDynamicMapping(field: Field) {
+ const path = ctx.groupFieldName ? `${ctx.groupFieldName}.${field.name}` : field.name;
+ const pathMatch = path.includes('*') ? path : `${path}.*`;
+
+ let dynProperties: Properties = getDefaultProperties(field);
+ let matchingType: string | undefined;
+ switch (field.object_type) {
+ case 'histogram':
+ dynProperties = histogram(field);
+ matchingType = field.object_type_mapping_type ?? '*';
+ break;
+ case 'ip':
+ case 'keyword':
+ case 'match_only_text':
+ case 'text':
+ case 'wildcard':
+ dynProperties.type = field.object_type;
+ matchingType = field.object_type_mapping_type ?? 'string';
+ break;
+ case 'scaled_float':
+ dynProperties = scaledFloat(field);
+ matchingType = field.object_type_mapping_type ?? '*';
+ break;
+ case 'aggregate_metric_double':
+ dynProperties.type = field.object_type;
+ dynProperties.metrics = field.metrics;
+ dynProperties.default_metric = field.default_metric;
+ matchingType = field.object_type_mapping_type ?? '*';
+ break;
+ case 'double':
+ case 'float':
+ case 'half_float':
+ dynProperties.type = field.object_type;
+ if (isIndexModeTimeSeries) {
+ dynProperties.time_series_metric = field.metric_type;
+ }
+ matchingType = field.object_type_mapping_type ?? 'double';
+ break;
+ case 'byte':
+ case 'long':
+ case 'short':
+ case 'unsigned_long':
+ dynProperties.type = field.object_type;
+ if (isIndexModeTimeSeries) {
+ dynProperties.time_series_metric = field.metric_type;
+ }
+ matchingType = field.object_type_mapping_type ?? 'long';
+ break;
+ case 'integer':
+ // Map integers as long, as in other cases.
+ dynProperties.type = 'long';
+ if (isIndexModeTimeSeries) {
+ dynProperties.time_series_metric = field.metric_type;
+ }
+ matchingType = field.object_type_mapping_type ?? 'long';
+ break;
+ case 'boolean':
+ dynProperties.type = field.object_type;
+ if (isIndexModeTimeSeries) {
+ dynProperties.time_series_metric = field.metric_type;
+ }
+ matchingType = field.object_type_mapping_type ?? field.object_type;
+ break;
+ case 'group':
+ if (!field?.fields) {
+ break;
+ }
+ const subFields = field.fields.map((subField) => ({
+ ...subField,
+ type: 'object',
+ object_type: subField.object_type ?? subField.type,
+ }));
+ const mappings = _generateMappings(
+ subFields,
+ {
+ ...ctx,
+ groupFieldName: ctx.groupFieldName ? `${ctx.groupFieldName}.${field.name}` : field.name,
+ },
+ isIndexModeTimeSeries
+ );
+ if (mappings.hasDynamicTemplateMappings) {
+ hasDynamicTemplateMappings = true;
+ }
+ break;
+ case 'flattened':
+ dynProperties.type = field.object_type;
+ matchingType = field.object_type_mapping_type ?? 'object';
+ break;
+ default:
+ throw new PackageInvalidArchiveError(
+ `No dynamic mapping generated for field ${path} of type ${field.object_type}`
+ );
+ }
+
+ if (field.dimension && isIndexModeTimeSeries) {
+ dynProperties.time_series_dimension = field.dimension;
+ }
+
+ // When a wildcard field specifies the subobjects setting,
+ // the parent intermediate object should set the subobjects
+ // setting.
+ //
+ // For example, if a wildcard field `foo.*` has subobjects,
+ // we should set subobjects on the intermediate object `foo`.
+ //
+ if (field.subobjects !== undefined && path.includes('*')) {
+ subobjects = field.subobjects;
+ }
+
+ if (dynProperties && matchingType) {
+ addDynamicMappingWithIntermediateObjects(path, pathMatch, matchingType, dynProperties);
+
+ // Add the parent object as static property, this is needed for
+ // index templates not using `"dynamic": true`.
+ addParentObjectAsStaticProperty(field);
+ }
+ }
+
// TODO: this can happen when the fields property in fields.yml is present but empty
// Maybe validation should be moved to fields/field.ts
if (fields) {
@@ -360,123 +478,7 @@ function _generateMappings(
}
if (type === 'object' && field.object_type) {
- const path = ctx.groupFieldName ? `${ctx.groupFieldName}.${field.name}` : field.name;
- const pathMatch = path.includes('*') ? path : `${path}.*`;
-
- let dynProperties: Properties = getDefaultProperties(field);
- let matchingType: string | undefined;
- switch (field.object_type) {
- case 'histogram':
- dynProperties = histogram(field);
- matchingType = field.object_type_mapping_type ?? '*';
- break;
- case 'ip':
- case 'keyword':
- case 'match_only_text':
- case 'text':
- case 'wildcard':
- dynProperties.type = field.object_type;
- matchingType = field.object_type_mapping_type ?? 'string';
- break;
- case 'scaled_float':
- dynProperties = scaledFloat(field);
- matchingType = field.object_type_mapping_type ?? '*';
- break;
- case 'aggregate_metric_double':
- dynProperties.type = field.object_type;
- dynProperties.metrics = field.metrics;
- dynProperties.default_metric = field.default_metric;
- matchingType = field.object_type_mapping_type ?? '*';
- break;
- case 'double':
- case 'float':
- case 'half_float':
- dynProperties.type = field.object_type;
- if (isIndexModeTimeSeries) {
- dynProperties.time_series_metric = field.metric_type;
- }
- matchingType = field.object_type_mapping_type ?? 'double';
- break;
- case 'byte':
- case 'long':
- case 'short':
- case 'unsigned_long':
- dynProperties.type = field.object_type;
- if (isIndexModeTimeSeries) {
- dynProperties.time_series_metric = field.metric_type;
- }
- matchingType = field.object_type_mapping_type ?? 'long';
- break;
- case 'integer':
- // Map integers as long, as in other cases.
- dynProperties.type = 'long';
- if (isIndexModeTimeSeries) {
- dynProperties.time_series_metric = field.metric_type;
- }
- matchingType = field.object_type_mapping_type ?? 'long';
- break;
- case 'boolean':
- dynProperties.type = field.object_type;
- if (isIndexModeTimeSeries) {
- dynProperties.time_series_metric = field.metric_type;
- }
- matchingType = field.object_type_mapping_type ?? field.object_type;
- break;
- case 'group':
- if (!field?.fields) {
- break;
- }
- const subFields = field.fields.map((subField) => ({
- ...subField,
- type: 'object',
- object_type: subField.object_type ?? subField.type,
- }));
- const mappings = _generateMappings(
- subFields,
- {
- ...ctx,
- groupFieldName: ctx.groupFieldName
- ? `${ctx.groupFieldName}.${field.name}`
- : field.name,
- },
- isIndexModeTimeSeries
- );
- if (mappings.hasDynamicTemplateMappings) {
- hasDynamicTemplateMappings = true;
- }
- break;
- case 'flattened':
- dynProperties.type = field.object_type;
- matchingType = field.object_type_mapping_type ?? 'object';
- break;
- default:
- throw new PackageInvalidArchiveError(
- `No dynamic mapping generated for field ${path} of type ${field.object_type}`
- );
- }
-
- if (field.dimension && isIndexModeTimeSeries) {
- dynProperties.time_series_dimension = field.dimension;
- }
-
- // When a wildcard field specifies the subobjects setting,
- // the parent intermediate object should set the subobjects
- // setting.
- //
- // For example, if a wildcard field `foo.*` has subobjects,
- // we should set subobjects on the intermediate object `foo`.
- //
- if (field.subobjects !== undefined && path.includes('*')) {
- subobjects = field.subobjects;
- }
-
- if (dynProperties && matchingType) {
- addDynamicMappingWithIntermediateObjects(path, pathMatch, matchingType, dynProperties);
-
- // Add the parent object as static property, this is needed for
- // index templates not using `"dynamic": true`.
- addParentObjectAsStaticProperty(field);
- }
+ addObjectAsDynamicMapping(field);
} else {
let fieldProps = getDefaultProperties(field);
@@ -492,6 +494,12 @@ function _generateMappings(
},
isIndexModeTimeSeries
);
+ if (field.object_type) {
+ // A group can have an object_type if it has been merged with an object during deduplication,
+ // generate also the dynamic mapping for the object.
+ addObjectAsDynamicMapping(field);
+ mappings.hasDynamicTemplateMappings = true;
+ }
if (mappings.hasNonDynamicTemplateMappings) {
fieldProps = {
properties: