Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix generation of dynamic mapping for object with specific subfield #204104

Merged
merged 4 commits into from
Dec 16, 2024
Merged

Fix generation of dynamic mapping for object with specific subfield #204104

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3ff9e2a
Fix generation of dynamic mapping for object with specific subfield
jsoriano Dec 12, 2024
3aa3782
Linting
jsoriano Dec 13, 2024
eb6b005
Merge branch 'main' into dynamic-mapping-non-empty-object
jsoriano Dec 16, 2024
82fddd4
Merge branch 'main' into dynamic-mapping-non-empty-object
jsoriano Dec 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 78 additions & 0 deletions x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -816,6 +816,84 @@ describe('EPM template', () => {
expect(mappings).toEqual(objectFieldWithPropertyReversedMapping);
});

it('tests processing object field with more specific properties without wildcard', () => {
const objectFieldWithPropertyReversedLiteralYml = `
- name: labels
type: object
object_type: keyword
object_type_mapping_type: '*'
- name: labels.count
type: long
`;
const objectFieldWithPropertyReversedMapping = {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should there be a test with group type?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The group type cannot have object_type. Only internally in fleet after the deduplication.

dynamic_templates: [
{
labels: {
path_match: 'labels.*',
match_mapping_type: '*',
mapping: {
type: 'keyword',
},
},
},
],
properties: {
labels: {
dynamic: true,
type: 'object',
properties: {
count: {
type: 'long',
},
},
},
},
};
const fields: Field[] = load(objectFieldWithPropertyReversedLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(mappings).toEqual(objectFieldWithPropertyReversedMapping);
});

it('tests processing object field with more specific properties with wildcard', () => {
const objectFieldWithPropertyReversedLiteralYml = `
- name: labels.*
type: object
object_type: keyword
object_type_mapping_type: '*'
- name: labels.count
type: long
`;
const objectFieldWithPropertyReversedMapping = {
dynamic_templates: [
{
'labels.*': {
path_match: 'labels.*',
match_mapping_type: '*',
mapping: {
type: 'keyword',
},
},
},
],
properties: {
labels: {
dynamic: true,
type: 'object',
properties: {
count: {
type: 'long',
},
},
},
},
};
const fields: Field[] = load(objectFieldWithPropertyReversedLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(mappings).toEqual(objectFieldWithPropertyReversedMapping);
});

it('tests processing object field with subobjects set to false (case B)', () => {
const objectFieldWithPropertyReversedLiteralYml = `
- name: b.labels.*
Expand Down
242 changes: 125 additions & 117 deletions x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -310,6 +310,124 @@ function _generateMappings(
}
}

function addObjectAsDynamicMapping(field: Field) {
const path = ctx.groupFieldName ? `${ctx.groupFieldName}.${field.name}` : field.name;
const pathMatch = path.includes('*') ? path : `${path}.*`;

let dynProperties: Properties = getDefaultProperties(field);
let matchingType: string | undefined;
switch (field.object_type) {
case 'histogram':
dynProperties = histogram(field);
matchingType = field.object_type_mapping_type ?? '*';
break;
case 'ip':
case 'keyword':
case 'match_only_text':
case 'text':
case 'wildcard':
dynProperties.type = field.object_type;
matchingType = field.object_type_mapping_type ?? 'string';
break;
case 'scaled_float':
dynProperties = scaledFloat(field);
matchingType = field.object_type_mapping_type ?? '*';
break;
case 'aggregate_metric_double':
dynProperties.type = field.object_type;
dynProperties.metrics = field.metrics;
dynProperties.default_metric = field.default_metric;
matchingType = field.object_type_mapping_type ?? '*';
break;
case 'double':
case 'float':
case 'half_float':
dynProperties.type = field.object_type;
if (isIndexModeTimeSeries) {
dynProperties.time_series_metric = field.metric_type;
}
matchingType = field.object_type_mapping_type ?? 'double';
break;
case 'byte':
case 'long':
case 'short':
case 'unsigned_long':
dynProperties.type = field.object_type;
if (isIndexModeTimeSeries) {
dynProperties.time_series_metric = field.metric_type;
}
matchingType = field.object_type_mapping_type ?? 'long';
break;
case 'integer':
// Map integers as long, as in other cases.
dynProperties.type = 'long';
if (isIndexModeTimeSeries) {
dynProperties.time_series_metric = field.metric_type;
}
matchingType = field.object_type_mapping_type ?? 'long';
break;
case 'boolean':
dynProperties.type = field.object_type;
if (isIndexModeTimeSeries) {
dynProperties.time_series_metric = field.metric_type;
}
matchingType = field.object_type_mapping_type ?? field.object_type;
break;
case 'group':
if (!field?.fields) {
break;
}
const subFields = field.fields.map((subField) => ({
...subField,
type: 'object',
object_type: subField.object_type ?? subField.type,
}));
const mappings = _generateMappings(
subFields,
{
...ctx,
groupFieldName: ctx.groupFieldName ? `${ctx.groupFieldName}.${field.name}` : field.name,
},
isIndexModeTimeSeries
);
if (mappings.hasDynamicTemplateMappings) {
hasDynamicTemplateMappings = true;
}
break;
case 'flattened':
dynProperties.type = field.object_type;
matchingType = field.object_type_mapping_type ?? 'object';
break;
default:
throw new PackageInvalidArchiveError(
`No dynamic mapping generated for field ${path} of type ${field.object_type}`
);
}

if (field.dimension && isIndexModeTimeSeries) {
dynProperties.time_series_dimension = field.dimension;
}

// When a wildcard field specifies the subobjects setting,
// the parent intermediate object should set the subobjects
// setting.
//
// For example, if a wildcard field `foo.*` has subobjects,
// we should set subobjects on the intermediate object `foo`.
//
if (field.subobjects !== undefined && path.includes('*')) {
subobjects = field.subobjects;
}

if (dynProperties && matchingType) {
addDynamicMappingWithIntermediateObjects(path, pathMatch, matchingType, dynProperties);

// Add the parent object as static property, this is needed for
// index templates not using `"dynamic": true`.
addParentObjectAsStaticProperty(field);
}
}

// TODO: this can happen when the fields property in fields.yml is present but empty
// Maybe validation should be moved to fields/field.ts
if (fields) {
Expand Down Expand Up @@ -371,123 +489,7 @@ function _generateMappings(
}

if (type === 'object' && field.object_type) {
const path = ctx.groupFieldName ? `${ctx.groupFieldName}.${field.name}` : field.name;
const pathMatch = path.includes('*') ? path : `${path}.*`;

let dynProperties: Properties = getDefaultProperties(field);
let matchingType: string | undefined;
switch (field.object_type) {
case 'histogram':
dynProperties = histogram(field);
matchingType = field.object_type_mapping_type ?? '*';
break;
case 'ip':
case 'keyword':
case 'match_only_text':
case 'text':
case 'wildcard':
dynProperties.type = field.object_type;
matchingType = field.object_type_mapping_type ?? 'string';
break;
case 'scaled_float':
dynProperties = scaledFloat(field);
matchingType = field.object_type_mapping_type ?? '*';
break;
case 'aggregate_metric_double':
dynProperties.type = field.object_type;
dynProperties.metrics = field.metrics;
dynProperties.default_metric = field.default_metric;
matchingType = field.object_type_mapping_type ?? '*';
break;
case 'double':
case 'float':
case 'half_float':
dynProperties.type = field.object_type;
if (isIndexModeTimeSeries) {
dynProperties.time_series_metric = field.metric_type;
}
matchingType = field.object_type_mapping_type ?? 'double';
break;
case 'byte':
case 'long':
case 'short':
case 'unsigned_long':
dynProperties.type = field.object_type;
if (isIndexModeTimeSeries) {
dynProperties.time_series_metric = field.metric_type;
}
matchingType = field.object_type_mapping_type ?? 'long';
break;
case 'integer':
// Map integers as long, as in other cases.
dynProperties.type = 'long';
if (isIndexModeTimeSeries) {
dynProperties.time_series_metric = field.metric_type;
}
matchingType = field.object_type_mapping_type ?? 'long';
break;
case 'boolean':
dynProperties.type = field.object_type;
if (isIndexModeTimeSeries) {
dynProperties.time_series_metric = field.metric_type;
}
matchingType = field.object_type_mapping_type ?? field.object_type;
break;
case 'group':
if (!field?.fields) {
break;
}
const subFields = field.fields.map((subField) => ({
...subField,
type: 'object',
object_type: subField.object_type ?? subField.type,
}));
const mappings = _generateMappings(
subFields,
{
...ctx,
groupFieldName: ctx.groupFieldName
? `${ctx.groupFieldName}.${field.name}`
: field.name,
},
isIndexModeTimeSeries
);
if (mappings.hasDynamicTemplateMappings) {
hasDynamicTemplateMappings = true;
}
break;
case 'flattened':
dynProperties.type = field.object_type;
matchingType = field.object_type_mapping_type ?? 'object';
break;
default:
throw new PackageInvalidArchiveError(
`No dynamic mapping generated for field ${path} of type ${field.object_type}`
);
}

if (field.dimension && isIndexModeTimeSeries) {
dynProperties.time_series_dimension = field.dimension;
}

// When a wildcard field specifies the subobjects setting,
// the parent intermediate object should set the subobjects
// setting.
//
// For example, if a wildcard field `foo.*` has subobjects,
// we should set subobjects on the intermediate object `foo`.
//
if (field.subobjects !== undefined && path.includes('*')) {
subobjects = field.subobjects;
}

if (dynProperties && matchingType) {
addDynamicMappingWithIntermediateObjects(path, pathMatch, matchingType, dynProperties);

// Add the parent object as static property, this is needed for
// index templates not using `"dynamic": true`.
addParentObjectAsStaticProperty(field);
}
addObjectAsDynamicMapping(field);
} else {
let fieldProps = getDefaultProperties(field);

Expand All @@ -503,6 +505,12 @@ function _generateMappings(
},
isIndexModeTimeSeries
);
if (field.object_type) {
// A group can have an object_type if it has been merged with an object during deduplication,
// generate also the dynamic mapping for the object.
addObjectAsDynamicMapping(field);
mappings.hasDynamicTemplateMappings = true;
}
if (mappings.hasNonDynamicTemplateMappings) {
fieldProps = {
properties:
Expand Down
Loading