Skip to content

Commit

Permalink
Fix generation of dynamic mapping for object with specific subfield (#…
Browse files Browse the repository at this point in the history 
…204104)

Fix generation of dynamic mapping for objects that have more specific
subfields in separate definitions.

This can be reproduced for example with:
```
- name: labels
  type: object
  object_type: keyword
  object_type_mapping_type: '*'
- name: labels.count
  type: long
```

Fleet expands and deduplicates field definitions before generating the
mappings, so the definitions above are converted to something like the
following:
```
- name: labels
  type: group
  object_type: keyword
  object_type_mapping_type: '*'
  fields:
  - name: count
    type: long
```

Usually fields of type `group` don't have an `object_type`, so this was
being ignored, the dynamic mapping was not being generated.

This issue was not reproduced if the object field name includes a
wildcard, like in `labels.*`, because then the expansion and
deduplication resolves to something like this:
```
- name: labels
  type: group
  fields:
  - name: '*'
    type: object
    object_type: keyword
    object_type_mapping_type: '*'
  - name: count
    type: long
```
  • Loading branch information
@jsoriano
jsoriano authored Dec 16, 2024
1 parent 05f2cba commit e3877e0
Show file tree
Hide file tree
Showing 2 changed files with 203 additions and 117 deletions.
78 changes: 78 additions & 0 deletions x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -816,6 +816,84 @@ describe('EPM template', () => {
expect(mappings).toEqual(objectFieldWithPropertyReversedMapping);
});

it('tests processing object field with more specific properties without wildcard', () => {
const objectFieldWithPropertyReversedLiteralYml = `
- name: labels
type: object
object_type: keyword
object_type_mapping_type: '*'
- name: labels.count
type: long
`;
const objectFieldWithPropertyReversedMapping = {
dynamic_templates: [
{
labels: {
path_match: 'labels.*',
match_mapping_type: '*',
mapping: {
type: 'keyword',
},
},
},
],
properties: {
labels: {
dynamic: true,
type: 'object',
properties: {
count: {
type: 'long',
},
},
},
},
};
const fields: Field[] = load(objectFieldWithPropertyReversedLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(mappings).toEqual(objectFieldWithPropertyReversedMapping);
});

it('tests processing object field with more specific properties with wildcard', () => {
const objectFieldWithPropertyReversedLiteralYml = `
- name: labels.*
type: object
object_type: keyword
object_type_mapping_type: '*'
- name: labels.count
type: long
`;
const objectFieldWithPropertyReversedMapping = {
dynamic_templates: [
{
'labels.*': {
path_match: 'labels.*',
match_mapping_type: '*',
mapping: {
type: 'keyword',
},
},
},
],
properties: {
labels: {
dynamic: true,
type: 'object',
properties: {
count: {
type: 'long',
},
},
},
},
};
const fields: Field[] = load(objectFieldWithPropertyReversedLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(mappings).toEqual(objectFieldWithPropertyReversedMapping);
});

it('tests processing object field with subobjects set to false (case B)', () => {
const objectFieldWithPropertyReversedLiteralYml = `
- name: b.labels.*
Expand Down
242 changes: 125 additions & 117 deletions x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -310,6 +310,124 @@ function _generateMappings(
}
}

function addObjectAsDynamicMapping(field: Field) {
const path = ctx.groupFieldName ? `${ctx.groupFieldName}.${field.name}` : field.name;
const pathMatch = path.includes('*') ? path : `${path}.*`;

let dynProperties: Properties = getDefaultProperties(field);
let matchingType: string | undefined;
switch (field.object_type) {
case 'histogram':
dynProperties = histogram(field);
matchingType = field.object_type_mapping_type ?? '*';
break;
case 'ip':
case 'keyword':
case 'match_only_text':
case 'text':
case 'wildcard':
dynProperties.type = field.object_type;
matchingType = field.object_type_mapping_type ?? 'string';
break;
case 'scaled_float':
dynProperties = scaledFloat(field);
matchingType = field.object_type_mapping_type ?? '*';
break;
case 'aggregate_metric_double':
dynProperties.type = field.object_type;
dynProperties.metrics = field.metrics;
dynProperties.default_metric = field.default_metric;
matchingType = field.object_type_mapping_type ?? '*';
break;
case 'double':
case 'float':
case 'half_float':
dynProperties.type = field.object_type;
if (isIndexModeTimeSeries) {
dynProperties.time_series_metric = field.metric_type;
}
matchingType = field.object_type_mapping_type ?? 'double';
break;
case 'byte':
case 'long':
case 'short':
case 'unsigned_long':
dynProperties.type = field.object_type;
if (isIndexModeTimeSeries) {
dynProperties.time_series_metric = field.metric_type;
}
matchingType = field.object_type_mapping_type ?? 'long';
break;
case 'integer':
// Map integers as long, as in other cases.
dynProperties.type = 'long';
if (isIndexModeTimeSeries) {
dynProperties.time_series_metric = field.metric_type;
}
matchingType = field.object_type_mapping_type ?? 'long';
break;
case 'boolean':
dynProperties.type = field.object_type;
if (isIndexModeTimeSeries) {
dynProperties.time_series_metric = field.metric_type;
}
matchingType = field.object_type_mapping_type ?? field.object_type;
break;
case 'group':
if (!field?.fields) {
break;
}
const subFields = field.fields.map((subField) => ({
...subField,
type: 'object',
object_type: subField.object_type ?? subField.type,
}));
const mappings = _generateMappings(
subFields,
{
...ctx,
groupFieldName: ctx.groupFieldName ? `${ctx.groupFieldName}.${field.name}` : field.name,
},
isIndexModeTimeSeries
);
if (mappings.hasDynamicTemplateMappings) {
hasDynamicTemplateMappings = true;
}
break;
case 'flattened':
dynProperties.type = field.object_type;
matchingType = field.object_type_mapping_type ?? 'object';
break;
default:
throw new PackageInvalidArchiveError(
`No dynamic mapping generated for field ${path} of type ${field.object_type}`
);
}

if (field.dimension && isIndexModeTimeSeries) {
dynProperties.time_series_dimension = field.dimension;
}

// When a wildcard field specifies the subobjects setting,
// the parent intermediate object should set the subobjects
// setting.
//
// For example, if a wildcard field `foo.*` has subobjects,
// we should set subobjects on the intermediate object `foo`.
//
if (field.subobjects !== undefined && path.includes('*')) {
subobjects = field.subobjects;
}

if (dynProperties && matchingType) {
addDynamicMappingWithIntermediateObjects(path, pathMatch, matchingType, dynProperties);

// Add the parent object as static property, this is needed for
// index templates not using `"dynamic": true`.
addParentObjectAsStaticProperty(field);
}
}

// TODO: this can happen when the fields property in fields.yml is present but empty
// Maybe validation should be moved to fields/field.ts
if (fields) {
Expand Down Expand Up @@ -371,123 +489,7 @@ function _generateMappings(
}

if (type === 'object' && field.object_type) {
const path = ctx.groupFieldName ? `${ctx.groupFieldName}.${field.name}` : field.name;
const pathMatch = path.includes('*') ? path : `${path}.*`;

let dynProperties: Properties = getDefaultProperties(field);
let matchingType: string | undefined;
switch (field.object_type) {
case 'histogram':
dynProperties = histogram(field);
matchingType = field.object_type_mapping_type ?? '*';
break;
case 'ip':
case 'keyword':
case 'match_only_text':
case 'text':
case 'wildcard':
dynProperties.type = field.object_type;
matchingType = field.object_type_mapping_type ?? 'string';
break;
case 'scaled_float':
dynProperties = scaledFloat(field);
matchingType = field.object_type_mapping_type ?? '*';
break;
case 'aggregate_metric_double':
dynProperties.type = field.object_type;
dynProperties.metrics = field.metrics;
dynProperties.default_metric = field.default_metric;
matchingType = field.object_type_mapping_type ?? '*';
break;
case 'double':
case 'float':
case 'half_float':
dynProperties.type = field.object_type;
if (isIndexModeTimeSeries) {
dynProperties.time_series_metric = field.metric_type;
}
matchingType = field.object_type_mapping_type ?? 'double';
break;
case 'byte':
case 'long':
case 'short':
case 'unsigned_long':
dynProperties.type = field.object_type;
if (isIndexModeTimeSeries) {
dynProperties.time_series_metric = field.metric_type;
}
matchingType = field.object_type_mapping_type ?? 'long';
break;
case 'integer':
// Map integers as long, as in other cases.
dynProperties.type = 'long';
if (isIndexModeTimeSeries) {
dynProperties.time_series_metric = field.metric_type;
}
matchingType = field.object_type_mapping_type ?? 'long';
break;
case 'boolean':
dynProperties.type = field.object_type;
if (isIndexModeTimeSeries) {
dynProperties.time_series_metric = field.metric_type;
}
matchingType = field.object_type_mapping_type ?? field.object_type;
break;
case 'group':
if (!field?.fields) {
break;
}
const subFields = field.fields.map((subField) => ({
...subField,
type: 'object',
object_type: subField.object_type ?? subField.type,
}));
const mappings = _generateMappings(
subFields,
{
...ctx,
groupFieldName: ctx.groupFieldName
? `${ctx.groupFieldName}.${field.name}`
: field.name,
},
isIndexModeTimeSeries
);
if (mappings.hasDynamicTemplateMappings) {
hasDynamicTemplateMappings = true;
}
break;
case 'flattened':
dynProperties.type = field.object_type;
matchingType = field.object_type_mapping_type ?? 'object';
break;
default:
throw new PackageInvalidArchiveError(
`No dynamic mapping generated for field ${path} of type ${field.object_type}`
);
}

if (field.dimension && isIndexModeTimeSeries) {
dynProperties.time_series_dimension = field.dimension;
}

// When a wildcard field specifies the subobjects setting,
// the parent intermediate object should set the subobjects
// setting.
//
// For example, if a wildcard field `foo.*` has subobjects,
// we should set subobjects on the intermediate object `foo`.
//
if (field.subobjects !== undefined && path.includes('*')) {
subobjects = field.subobjects;
}

if (dynProperties && matchingType) {
addDynamicMappingWithIntermediateObjects(path, pathMatch, matchingType, dynProperties);

// Add the parent object as static property, this is needed for
// index templates not using `"dynamic": true`.
addParentObjectAsStaticProperty(field);
}
addObjectAsDynamicMapping(field);
} else {
let fieldProps = getDefaultProperties(field);

Expand All @@ -503,6 +505,12 @@ function _generateMappings(
},
isIndexModeTimeSeries
);
if (field.object_type) {
// A group can have an object_type if it has been merged with an object during deduplication,
// generate also the dynamic mapping for the object.
addObjectAsDynamicMapping(field);
mappings.hasDynamicTemplateMappings = true;
}
if (mappings.hasNonDynamicTemplateMappings) {
fieldProps = {
properties:
Expand Down

0 comments on commit e3877e0

Please sign in to comment.