From e8d32afdf428c9a533167931c6a5e40095a7ced4 Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Tue, 24 Dec 2024 11:41:44 -0800 Subject: [PATCH 1/2] Extract synthetic source logic from DocumentParser (#116049) --- .../LogsDbDocumentParsingBenchmark.java | 400 ++++++++++++++++++ .../DataStreamGetWriteIndexTests.java | 2 +- .../CustomSyntheticSourceFieldLookup.java | 88 ++++ .../index/mapper/DocumentMapper.java | 4 +- .../index/mapper/DocumentParser.java | 284 +++++++++---- .../index/mapper/DocumentParserContext.java | 25 +- .../index/mapper/DocumentParserListener.java | 221 ++++++++++ .../index/mapper/MapperService.java | 1 + .../index/mapper/MappingLookup.java | 31 +- ...SyntheticSourceDocumentParserListener.java | 398 +++++++++++++++++ ...CustomSyntheticSourceFieldLookupTests.java | 263 ++++++++++++ .../index/mapper/DocumentMapperTests.java | 1 + .../mapper/DocumentParserListenerTests.java | 253 +++++++++++ .../index/mapper/DocumentParserTests.java | 1 + .../mapper/DynamicFieldsBuilderTests.java | 12 +- .../FieldAliasMapperValidationTests.java | 2 +- .../mapper/FieldNamesFieldTypeTests.java | 3 - .../mapper/IgnoredSourceFieldMapperTests.java | 4 +- .../index/mapper/MappingLookupTests.java | 12 +- .../index/mapper/MappingParserTests.java | 2 +- ...eticSourceDocumentParserListenerTests.java | 385 +++++++++++++++++ .../query/SearchExecutionContextTests.java | 10 +- .../indices/IndicesRequestCacheTests.java | 8 +- .../search/SearchServiceTests.java | 3 +- .../AbstractSuggestionBuilderTestCase.java | 2 +- .../metadata/DataStreamTestHelper.java | 2 +- .../mapper/TestDocumentParserContext.java | 1 + .../aggregations/AggregatorTestCase.java | 5 +- .../DocumentSubsetBitsetCacheTests.java | 2 +- ...ityIndexReaderWrapperIntegrationTests.java | 2 +- .../mapper/WildcardFieldMapperTests.java | 2 +- 31 files changed, 2299 insertions(+), 130 deletions(-) create mode 100644 benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/LogsDbDocumentParsingBenchmark.java create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/CustomSyntheticSourceFieldLookup.java create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/DocumentParserListener.java create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/SyntheticSourceDocumentParserListener.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/CustomSyntheticSourceFieldLookupTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/DocumentParserListenerTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/SyntheticSourceDocumentParserListenerTests.java diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/LogsDbDocumentParsingBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/LogsDbDocumentParsingBenchmark.java new file mode 100644 index 0000000000000..8924f84fdc908 --- /dev/null +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/LogsDbDocumentParsingBenchmark.java @@ -0,0 +1,400 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.benchmark.index.mapper; + +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.logging.LogConfigurator; +import org.elasticsearch.index.mapper.LuceneDocument; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.SourceToParse; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentType; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + +import java.io.IOException; +import java.util.List; +import java.util.Random; +import java.util.concurrent.TimeUnit; + +@Fork(value = 1) +@Warmup(iterations = 5) +@Measurement(iterations = 5) +@BenchmarkMode(Mode.Throughput) +@OutputTimeUnit(TimeUnit.SECONDS) +@State(Scope.Benchmark) +public class LogsDbDocumentParsingBenchmark { + private Random random; + private MapperService mapperServiceEnabled; + private MapperService mapperServiceEnabledWithStoreArrays; + private MapperService mapperServiceDisabled; + private SourceToParse[] documents; + + static { + LogConfigurator.configureESLogging(); // doc values implementations need logging + } + + private static String SAMPLE_LOGS_MAPPING_ENABLED = """ + { + "_source": { + "mode": "synthetic" + }, + "properties": { + "kafka": { + "properties": { + "log": { + "properties": { + "component": { + "ignore_above": 1024, + "type": "keyword" + }, + "trace": { + "properties": { + "message": { + "type": "text" + }, + "class": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "thread": { + "ignore_above": 1024, + "type": "keyword" + }, + "class": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "host": { + "properties": { + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "properties": { + "build": { + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "codename": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword", + "fields": { + "text": { + "type": "text" + } + } + }, + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "ip" + }, + "containerized": { + "type": "boolean" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "architecture": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + """; + + private static String SAMPLE_LOGS_MAPPING_ENABLED_WITH_STORE_ARRAYS = """ + { + "_source": { + "mode": "synthetic" + }, + "properties": { + "kafka": { + "properties": { + "log": { + "properties": { + "component": { + "ignore_above": 1024, + "type": "keyword" + }, + "trace": { + "synthetic_source_keep": "arrays", + "properties": { + "message": { + "type": "text" + }, + "class": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "thread": { + "ignore_above": 1024, + "type": "keyword" + }, + "class": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "host": { + "properties": { + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "properties": { + "build": { + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "codename": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword", + "fields": { + "text": { + "type": "text" + } + } + }, + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "ip" + }, + "containerized": { + "type": "boolean" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "architecture": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + """; + + private static String SAMPLE_LOGS_MAPPING_DISABLED = """ + { + "_source": { + "mode": "synthetic" + }, + "enabled": false + } + """; + + @Setup + public void setUp() throws IOException { + this.random = new Random(); + this.mapperServiceEnabled = MapperServiceFactory.create(SAMPLE_LOGS_MAPPING_ENABLED); + this.mapperServiceEnabledWithStoreArrays = MapperServiceFactory.create(SAMPLE_LOGS_MAPPING_ENABLED_WITH_STORE_ARRAYS); + this.mapperServiceDisabled = MapperServiceFactory.create(SAMPLE_LOGS_MAPPING_DISABLED); + this.documents = generateRandomDocuments(10_000); + } + + @Benchmark + public List benchmarkEnabledObject() { + return mapperServiceEnabled.documentMapper().parse(randomFrom(documents)).docs(); + } + + @Benchmark + public List benchmarkEnabledObjectWithStoreArrays() { + return mapperServiceEnabledWithStoreArrays.documentMapper().parse(randomFrom(documents)).docs(); + } + + @Benchmark + public List benchmarkDisabledObject() { + return mapperServiceDisabled.documentMapper().parse(randomFrom(documents)).docs(); + } + + @SafeVarargs + @SuppressWarnings("varargs") + private T randomFrom(T... items) { + return items[random.nextInt(items.length)]; + } + + private SourceToParse[] generateRandomDocuments(int count) throws IOException { + var docs = new SourceToParse[count]; + for (int i = 0; i < count; i++) { + docs[i] = generateRandomDocument(); + } + return docs; + } + + private SourceToParse generateRandomDocument() throws IOException { + var builder = XContentBuilder.builder(XContentType.JSON.xContent()); + + builder.startObject(); + + builder.startObject("kafka"); + { + builder.startObject("log"); + { + builder.field("component", randomString(10)); + builder.startArray("trace"); + { + builder.startObject(); + { + builder.field("message", randomString(50)); + builder.field("class", randomString(10)); + } + builder.endObject(); + builder.startObject(); + { + builder.field("message", randomString(50)); + builder.field("class", randomString(10)); + } + builder.endObject(); + } + builder.endArray(); + builder.field("thread", randomString(10)); + builder.field("class", randomString(10)); + + } + builder.endObject(); + } + builder.endObject(); + + builder.startObject("host"); + { + builder.field("hostname", randomString(10)); + builder.startObject("os"); + { + builder.field("name", randomString(10)); + } + builder.endObject(); + + builder.field("domain", randomString(10)); + builder.field("ip", randomIp()); + builder.field("name", randomString(10)); + } + + builder.endObject(); + + builder.endObject(); + + return new SourceToParse(UUIDs.randomBase64UUID(), BytesReference.bytes(builder), XContentType.JSON); + } + + private String randomIp() { + return "" + random.nextInt(255) + '.' + random.nextInt(255) + '.' + random.nextInt(255) + '.' + random.nextInt(255); + } + + private String randomString(int maxLength) { + var length = random.nextInt(maxLength); + var builder = new StringBuilder(length); + for (int i = 0; i < length; i++) { + builder.append((byte) (32 + random.nextInt(94))); + } + return builder.toString(); + } +} diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java index 3d08be1f24a42..2c0d6d47e0233 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java @@ -241,7 +241,7 @@ public void setup() throws Exception { new MetadataFieldMapper[] { dtfm }, Collections.emptyMap() ); - MappingLookup mappingLookup = MappingLookup.fromMappers(mapping, List.of(dtfm, dateFieldMapper), List.of()); + MappingLookup mappingLookup = MappingLookup.fromMappers(mapping, List.of(dtfm, dateFieldMapper), List.of(), null); indicesService = DataStreamTestHelper.mockIndicesServices(mappingLookup); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/CustomSyntheticSourceFieldLookup.java b/server/src/main/java/org/elasticsearch/index/mapper/CustomSyntheticSourceFieldLookup.java new file mode 100644 index 0000000000000..dbbee8a9035d4 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/CustomSyntheticSourceFieldLookup.java @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.IndexSettings; + +import java.util.HashMap; +import java.util.Map; + +/** + * Contains lookup information needed to perform custom synthetic source logic. + * For example fields that use fallback synthetic source implementation or fields that preserve array ordering + * in synthetic source; + */ +public class CustomSyntheticSourceFieldLookup { + private final Map fieldsWithCustomSyntheticSourceHandling; + + public CustomSyntheticSourceFieldLookup(Mapping mapping, @Nullable IndexSettings indexSettings, boolean isSourceSynthetic) { + var fields = new HashMap(); + if (isSourceSynthetic && indexSettings != null) { + populateFields(fields, mapping.getRoot(), indexSettings.sourceKeepMode()); + } + this.fieldsWithCustomSyntheticSourceHandling = Map.copyOf(fields); + } + + private void populateFields(Map fields, ObjectMapper currentLevel, Mapper.SourceKeepMode defaultSourceKeepMode) { + if (currentLevel.isEnabled() == false) { + fields.put(currentLevel.fullPath(), Reason.DISABLED_OBJECT); + return; + } + if (sourceKeepMode(currentLevel, defaultSourceKeepMode) == Mapper.SourceKeepMode.ALL) { + fields.put(currentLevel.fullPath(), Reason.SOURCE_KEEP_ALL); + return; + } + if (currentLevel.isNested() == false && sourceKeepMode(currentLevel, defaultSourceKeepMode) == Mapper.SourceKeepMode.ARRAYS) { + fields.put(currentLevel.fullPath(), Reason.SOURCE_KEEP_ARRAYS); + } + + for (Mapper child : currentLevel) { + if (child instanceof ObjectMapper objectMapper) { + populateFields(fields, objectMapper, defaultSourceKeepMode); + } else if (child instanceof FieldMapper fieldMapper) { + // The order here is important. + // If fallback logic is used, it should be always correctly marked as FALLBACK_SYNTHETIC_SOURCE. + // This allows us to apply an optimization for SOURCE_KEEP_ARRAYS and don't store arrays that have one element. + // If this order is changed and a field that both has SOURCE_KEEP_ARRAYS and FALLBACK_SYNTHETIC_SOURCE + // is marked as SOURCE_KEEP_ARRAYS we would lose data for this field by applying such an optimization. + if (fieldMapper.syntheticSourceMode() == FieldMapper.SyntheticSourceMode.FALLBACK) { + fields.put(fieldMapper.fullPath(), Reason.FALLBACK_SYNTHETIC_SOURCE); + } else if (sourceKeepMode(fieldMapper, defaultSourceKeepMode) == Mapper.SourceKeepMode.ALL) { + fields.put(fieldMapper.fullPath(), Reason.SOURCE_KEEP_ALL); + } else if (sourceKeepMode(fieldMapper, defaultSourceKeepMode) == Mapper.SourceKeepMode.ARRAYS) { + fields.put(fieldMapper.fullPath(), Reason.SOURCE_KEEP_ARRAYS); + } + } + } + } + + private Mapper.SourceKeepMode sourceKeepMode(ObjectMapper mapper, Mapper.SourceKeepMode defaultSourceKeepMode) { + return mapper.sourceKeepMode().orElse(defaultSourceKeepMode); + } + + private Mapper.SourceKeepMode sourceKeepMode(FieldMapper mapper, Mapper.SourceKeepMode defaultSourceKeepMode) { + return mapper.sourceKeepMode().orElse(defaultSourceKeepMode); + } + + public Map getFieldsWithCustomSyntheticSourceHandling() { + return fieldsWithCustomSyntheticSourceHandling; + } + + /** + * Specifies why this field needs custom handling. + */ + public enum Reason { + SOURCE_KEEP_ARRAYS, + SOURCE_KEEP_ALL, + FALLBACK_SYNTHETIC_SOURCE, + DISABLED_OBJECT + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java index 03e6c343c7ab9..5abb7b5a1b728 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java @@ -49,6 +49,7 @@ public static DocumentMapper createEmpty(MapperService mapperService) { mapping, mapping.toCompressedXContent(), IndexVersion.current(), + mapperService.getIndexSettings(), mapperService.getMapperMetrics(), mapperService.index().getName() ); @@ -59,12 +60,13 @@ public static DocumentMapper createEmpty(MapperService mapperService) { Mapping mapping, CompressedXContent source, IndexVersion version, + IndexSettings indexSettings, MapperMetrics mapperMetrics, String indexName ) { this.documentParser = documentParser; this.type = mapping.getRoot().fullPath(); - this.mappingLookup = MappingLookup.fromMapping(mapping); + this.mappingLookup = MappingLookup.fromMapping(mapping, indexSettings); this.mappingSource = source; this.mapperMetrics = mapperMetrics; this.indexVersion = version; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 9ddb6f0d496a0..0fc14d4dbeeb9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -27,6 +27,7 @@ import org.elasticsearch.plugins.internal.XContentMeteringParserDecorator; import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.lookup.Source; +import org.elasticsearch.xcontent.FilterXContentParserWrapper; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentLocation; import org.elasticsearch.xcontent.XContentParseException; @@ -43,6 +44,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.function.BiFunction; import java.util.function.Consumer; import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.MAX_DIMS_COUNT; @@ -60,10 +62,22 @@ public final class DocumentParser { private final XContentParserConfiguration parserConfiguration; private final MappingParserContext mappingParserContext; + private final BiFunction listenersFactory; DocumentParser(XContentParserConfiguration parserConfiguration, MappingParserContext mappingParserContext) { this.mappingParserContext = mappingParserContext; this.parserConfiguration = parserConfiguration; + this.listenersFactory = this::createDefaultListeners; + } + + DocumentParser( + XContentParserConfiguration parserConfiguration, + MappingParserContext mappingParserContext, + BiFunction listenersFactory + ) { + this.mappingParserContext = mappingParserContext; + this.parserConfiguration = parserConfiguration; + this.listenersFactory = listenersFactory; } /** @@ -81,13 +95,11 @@ public ParsedDocument parseDocument(SourceToParse source, MappingLookup mappingL final RootDocumentParserContext context; final XContentType xContentType = source.getXContentType(); + Listeners listeners = listenersFactory.apply(mappingLookup, xContentType); + XContentMeteringParserDecorator meteringParserDecorator = source.getMeteringParserDecorator(); - try ( - XContentParser parser = meteringParserDecorator.decorate( - XContentHelper.createParser(parserConfiguration, source.source(), xContentType) - ) - ) { - context = new RootDocumentParserContext(mappingLookup, mappingParserContext, source, parser); + try (XContentParser parser = meteringParserDecorator.decorate(createParser(source, xContentType, listeners))) { + context = new RootDocumentParserContext(mappingLookup, mappingParserContext, source, listeners, parser); validateStart(context.parser()); MetadataFieldMapper[] metadataFieldsMappers = mappingLookup.getMapping().getSortedMetadataMappers(); internalParseDocument(metadataFieldsMappers, context); @@ -121,6 +133,152 @@ public String documentDescription() { }; } + private Listeners createDefaultListeners(MappingLookup mappingLookup, XContentType xContentType) { + if (mappingLookup.isSourceSynthetic() && mappingParserContext.getIndexSettings().getSkipIgnoredSourceWrite() == false) { + return new Listeners.Single(new SyntheticSourceDocumentParserListener(mappingLookup, xContentType)); + } + + return Listeners.NOOP; + } + + private XContentParser createParser(SourceToParse sourceToParse, XContentType xContentType, Listeners listeners) throws IOException { + XContentParser plainParser = XContentHelper.createParser(parserConfiguration, sourceToParse.source(), xContentType); + + if (listeners.isNoop()) { + return plainParser; + } + + return new ListenerAwareXContentParser(plainParser, listeners); + } + + static class ListenerAwareXContentParser extends FilterXContentParserWrapper { + private final Listeners listeners; + + ListenerAwareXContentParser(XContentParser parser, Listeners listeners) { + super(parser); + this.listeners = listeners; + } + + @Override + public Token nextToken() throws IOException { + var token = delegate().nextToken(); + + if (listeners.anyActive()) { + var listenerToken = DocumentParserListener.Token.current(delegate()); + listeners.publish(listenerToken); + } + + return token; + } + + @Override + public void skipChildren() throws IOException { + // We can not use "native" implementation because some listeners may want to see + // skipped parts. + Token token = currentToken(); + if (token != Token.START_OBJECT && token != Token.START_ARRAY) { + return; + } + + int depth = 0; + while (token != null) { + if (token == Token.START_OBJECT || token == Token.START_ARRAY) { + depth += 1; + } + if (token == Token.END_OBJECT || token == Token.END_ARRAY) { + depth -= 1; + if (depth == 0) { + return; + } + } + + token = nextToken(); + } + } + } + + /** + * Encapsulates listeners that are subscribed to this document parser. This allows to generalize logic without knowing + * how many listeners are present (and if they are present at all). + */ + public interface Listeners { + void publish(DocumentParserListener.Event event, DocumentParserContext context) throws IOException; + + void publish(DocumentParserListener.Token token) throws IOException; + + DocumentParserListener.Output finish(); + + boolean isNoop(); + + boolean anyActive(); + + /** + * No listeners are present. + */ + Listeners NOOP = new Listeners() { + @Override + public void publish(DocumentParserListener.Event event, DocumentParserContext context) {} + + @Override + public void publish(DocumentParserListener.Token token) { + + } + + @Override + public DocumentParserListener.Output finish() { + return DocumentParserListener.Output.empty(); + } + + @Override + public boolean isNoop() { + return true; + } + + @Override + public boolean anyActive() { + return false; + } + }; + + /** + * One or more listeners are present. + */ + class Single implements Listeners { + private final DocumentParserListener listener; + + public Single(DocumentParserListener listener) { + this.listener = listener; + } + + @Override + public void publish(DocumentParserListener.Event event, DocumentParserContext context) throws IOException { + listener.consume(event); + } + + @Override + public void publish(DocumentParserListener.Token token) throws IOException { + if (listener.isActive()) { + listener.consume(token); + } + } + + @Override + public DocumentParserListener.Output finish() { + return listener.finish(); + } + + @Override + public boolean isNoop() { + return false; + } + + @Override + public boolean anyActive() { + return listener.isActive(); + } + } + } + private void internalParseDocument(MetadataFieldMapper[] metadataFieldsMappers, DocumentParserContext context) { try { final boolean emptyDoc = isEmptyDoc(context.root(), context.parser()); @@ -129,26 +287,19 @@ private void internalParseDocument(MetadataFieldMapper[] metadataFieldsMappers, metadataMapper.preParse(context); } + context.publishEvent(new DocumentParserListener.Event.DocumentStart(context.root(), context.doc())); + if (context.root().isEnabled() == false) { // entire type is disabled - if (context.canAddIgnoredField()) { - context.addIgnoredField( - new IgnoredSourceFieldMapper.NameValue( - MapperService.SINGLE_MAPPING_NAME, - 0, - context.encodeFlattenedToken(), - context.doc() - ) - ); - } else { - context.parser().skipChildren(); - } + context.parser().skipChildren(); } else if (emptyDoc == false) { parseObjectOrNested(context); } executeIndexTimeScripts(context); + context.finishListeners(); + for (MetadataFieldMapper metadataMapper : metadataFieldsMappers) { metadataMapper.postParse(context); } @@ -274,22 +425,12 @@ static Mapping createDynamicUpdate(DocumentParserContext context) { } static void parseObjectOrNested(DocumentParserContext context) throws IOException { + XContentParser parser = context.parser(); String currentFieldName = parser.currentName(); if (context.parent().isEnabled() == false) { // entire type is disabled - if (context.canAddIgnoredField()) { - context.addIgnoredField( - new IgnoredSourceFieldMapper.NameValue( - context.parent().fullPath(), - context.parent().fullPath().lastIndexOf(context.parent().leafName()), - context.encodeFlattenedToken(), - context.doc() - ) - ); - } else { - parser.skipChildren(); - } + parser.skipChildren(); return; } XContentParser.Token token = parser.currentToken(); @@ -302,22 +443,6 @@ static void parseObjectOrNested(DocumentParserContext context) throws IOExceptio throwOnConcreteValue(context.parent(), currentFieldName, context); } - var sourceKeepMode = getSourceKeepMode(context, context.parent().sourceKeepMode()); - if (context.canAddIgnoredField() - && (sourceKeepMode == Mapper.SourceKeepMode.ALL - || (sourceKeepMode == Mapper.SourceKeepMode.ARRAYS && context.inArrayScope()))) { - context = context.addIgnoredFieldFromContext( - new IgnoredSourceFieldMapper.NameValue( - context.parent().fullPath(), - context.parent().fullPath().lastIndexOf(context.parent().leafName()), - null, - context.doc() - ) - ); - token = context.parser().currentToken(); - parser = context.parser(); - } - if (context.parent().isNested()) { // Handle a nested object that doesn't contain an array. Arrays are handled in #parseNonDynamicArray. context = context.createNestedContext((NestedObjectMapper) context.parent()); @@ -441,6 +566,9 @@ private static void addFields(IndexVersion indexCreatedVersion, LuceneDocument n static void parseObjectOrField(DocumentParserContext context, Mapper mapper) throws IOException { if (mapper instanceof ObjectMapper objectMapper) { + context.publishEvent( + new DocumentParserListener.Event.ObjectStart(objectMapper, context.inArrayScope(), context.parent(), context.doc()) + ); parseObjectOrNested(context.createChildContext(objectMapper)); } else if (mapper instanceof FieldMapper fieldMapper) { if (shouldFlattenObject(context, fieldMapper)) { @@ -450,12 +578,19 @@ static void parseObjectOrField(DocumentParserContext context, Mapper mapper) thr parseObjectOrNested(context.createFlattenContext(currentFieldName)); context.path().add(currentFieldName); } else { - var sourceKeepMode = getSourceKeepMode(context, fieldMapper.sourceKeepMode()); + context.publishEvent( + new DocumentParserListener.Event.LeafValue( + fieldMapper, + context.inArrayScope(), + context.parent(), + context.doc(), + context.parser() + ) + ); + if (context.canAddIgnoredField() - && (fieldMapper.syntheticSourceMode() == FieldMapper.SyntheticSourceMode.FALLBACK - || sourceKeepMode == Mapper.SourceKeepMode.ALL - || (sourceKeepMode == Mapper.SourceKeepMode.ARRAYS && context.inArrayScope()) - || (context.isWithinCopyTo() == false && context.isCopyToDestinationField(mapper.fullPath())))) { + && context.isWithinCopyTo() == false + && context.isCopyToDestinationField(mapper.fullPath())) { context = context.addIgnoredFieldFromContext( IgnoredSourceFieldMapper.NameValue.fromContext(context, fieldMapper.fullPath(), null) ); @@ -685,40 +820,17 @@ private static void parseNonDynamicArray( ) throws IOException { String fullPath = context.path().pathAsText(arrayFieldName); + if (mapper instanceof ObjectMapper objectMapper) { + context.publishEvent(new DocumentParserListener.Event.ObjectArrayStart(objectMapper, context.parent(), context.doc())); + } else if (mapper instanceof FieldMapper fieldMapper) { + context.publishEvent(new DocumentParserListener.Event.LeafArrayStart(fieldMapper, context.parent(), context.doc())); + } + // Check if we need to record the array source. This only applies to synthetic source. - boolean canRemoveSingleLeafElement = false; if (context.canAddIgnoredField()) { - Mapper.SourceKeepMode mode = Mapper.SourceKeepMode.NONE; - boolean objectWithFallbackSyntheticSource = false; - if (mapper instanceof ObjectMapper objectMapper) { - mode = getSourceKeepMode(context, objectMapper.sourceKeepMode()); - objectWithFallbackSyntheticSource = mode == Mapper.SourceKeepMode.ALL - || (mode == Mapper.SourceKeepMode.ARRAYS && objectMapper instanceof NestedObjectMapper == false); - } - boolean fieldWithFallbackSyntheticSource = false; - boolean fieldWithStoredArraySource = false; - if (mapper instanceof FieldMapper fieldMapper) { - mode = getSourceKeepMode(context, fieldMapper.sourceKeepMode()); - fieldWithFallbackSyntheticSource = fieldMapper.syntheticSourceMode() == FieldMapper.SyntheticSourceMode.FALLBACK; - fieldWithStoredArraySource = mode != Mapper.SourceKeepMode.NONE; - } boolean copyToFieldHasValuesInDocument = context.isWithinCopyTo() == false && context.isCopyToDestinationField(fullPath); - - canRemoveSingleLeafElement = mapper instanceof FieldMapper - && mode == Mapper.SourceKeepMode.ARRAYS - && fieldWithFallbackSyntheticSource == false - && copyToFieldHasValuesInDocument == false; - - if (objectWithFallbackSyntheticSource - || fieldWithFallbackSyntheticSource - || fieldWithStoredArraySource - || copyToFieldHasValuesInDocument) { + if (copyToFieldHasValuesInDocument) { context = context.addIgnoredFieldFromContext(IgnoredSourceFieldMapper.NameValue.fromContext(context, fullPath, null)); - } else if (mapper instanceof ObjectMapper objectMapper && (objectMapper.isEnabled() == false)) { - // No need to call #addIgnoredFieldFromContext as both singleton and array instances of this object - // get tracked through ignored source. - context.addIgnoredField(IgnoredSourceFieldMapper.NameValue.fromContext(context, fullPath, context.encodeFlattenedToken())); - return; } } @@ -729,28 +841,20 @@ private static void parseNonDynamicArray( XContentParser parser = context.parser(); XContentParser.Token token; - int elements = 0; while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { if (token == XContentParser.Token.START_OBJECT) { - elements = Integer.MAX_VALUE; parseObject(context, lastFieldName); } else if (token == XContentParser.Token.START_ARRAY) { - elements = Integer.MAX_VALUE; parseArray(context, lastFieldName); } else if (token == XContentParser.Token.VALUE_NULL) { - elements++; parseNullValue(context, lastFieldName); } else if (token == null) { throwEOFOnParseArray(arrayFieldName, context); } else { assert token.isValue(); - elements++; parseValue(context, lastFieldName); } } - if (elements <= 1 && canRemoveSingleLeafElement) { - context.removeLastIgnoredField(fullPath); - } postProcessDynamicArrayMapping(context, lastFieldName); } @@ -1049,12 +1153,14 @@ private static class RootDocumentParserContext extends DocumentParserContext { MappingLookup mappingLookup, MappingParserContext mappingParserContext, SourceToParse source, + Listeners listeners, XContentParser parser ) throws IOException { super( mappingLookup, mappingParserContext, source, + listeners, mappingLookup.getMapping().getRoot(), ObjectMapper.Dynamic.getRootDynamic(mappingLookup) ); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index 51e4e9f4c1b5e..625f3a3d19af7 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -117,6 +117,7 @@ private enum Scope { private final MappingLookup mappingLookup; private final MappingParserContext mappingParserContext; private final SourceToParse sourceToParse; + private final DocumentParser.Listeners listeners; private final Set ignoredFields; private final List ignoredFieldValues; @@ -149,6 +150,7 @@ private DocumentParserContext( MappingLookup mappingLookup, MappingParserContext mappingParserContext, SourceToParse sourceToParse, + DocumentParser.Listeners listeners, Set ignoreFields, List ignoredFieldValues, Scope currentScope, @@ -169,6 +171,7 @@ private DocumentParserContext( this.mappingLookup = mappingLookup; this.mappingParserContext = mappingParserContext; this.sourceToParse = sourceToParse; + this.listeners = listeners; this.ignoredFields = ignoreFields; this.ignoredFieldValues = ignoredFieldValues; this.currentScope = currentScope; @@ -192,6 +195,7 @@ private DocumentParserContext(ObjectMapper parent, ObjectMapper.Dynamic dynamic, in.mappingLookup, in.mappingParserContext, in.sourceToParse, + in.listeners, in.ignoredFields, in.ignoredFieldValues, in.currentScope, @@ -215,6 +219,7 @@ protected DocumentParserContext( MappingLookup mappingLookup, MappingParserContext mappingParserContext, SourceToParse source, + DocumentParser.Listeners listeners, ObjectMapper parent, ObjectMapper.Dynamic dynamic ) { @@ -222,6 +227,7 @@ protected DocumentParserContext( mappingLookup, mappingParserContext, source, + listeners, new HashSet<>(), new ArrayList<>(), Scope.SINGLETON, @@ -301,12 +307,6 @@ public final void addIgnoredField(IgnoredSourceFieldMapper.NameValue values) { } } - final void removeLastIgnoredField(String name) { - if (ignoredFieldValues.isEmpty() == false && ignoredFieldValues.getLast().name().equals(name)) { - ignoredFieldValues.removeLast(); - } - } - /** * Return the collection of values for fields that have been ignored so far. */ @@ -470,6 +470,15 @@ public Set getCopyToFields() { return copyToFields; } + public void publishEvent(DocumentParserListener.Event event) throws IOException { + listeners.publish(event, this); + } + + public void finishListeners() { + var output = listeners.finish(); + ignoredFieldValues.addAll(output.ignoredSourceValues()); + } + /** * Add a new mapper dynamically created while parsing. * @@ -659,7 +668,7 @@ public final DocumentParserContext createChildContext(ObjectMapper parent) { /** * Return a new context that will be used within a nested document. */ - public final DocumentParserContext createNestedContext(NestedObjectMapper nestedMapper) { + public final DocumentParserContext createNestedContext(NestedObjectMapper nestedMapper) throws IOException { if (isWithinCopyTo()) { // nested context will already have been set up for copy_to fields return this; @@ -688,7 +697,7 @@ public final DocumentParserContext createNestedContext(NestedObjectMapper nested /** * Return a new context that has the provided document as the current document. */ - public final DocumentParserContext switchDoc(final LuceneDocument document) { + public final DocumentParserContext switchDoc(final LuceneDocument document) throws IOException { DocumentParserContext cloned = new Wrapper(this.parent, this) { @Override public LuceneDocument doc() { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserListener.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserListener.java new file mode 100644 index 0000000000000..7ea902235da1c --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserListener.java @@ -0,0 +1,221 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.List; + +/** + * Component that listens to events produced by {@link DocumentParser} in order to implement some parsing related logic. + * It allows to keep such logic separate from actual document parsing workflow which is by itself complex. + */ +public interface DocumentParserListener { + /** + * Specifies if this listener is currently actively consuming tokens. + * This is used to avoid doing unnecessary work. + * @return + */ + boolean isActive(); + + /** + * Sends a {@link Token} to this listener. + * This is only called when {@link #isActive()} returns true since it involves a somewhat costly operation of creating a token instance + * and tokens are low level meaning this is called very frequently. + * @param token + * @throws IOException + */ + void consume(Token token) throws IOException; + + /** + * Sends an {@link Event} to this listener. Unlike tokens events are always sent to a listener. + * The logic here is that based on the event listener can decide to change the return value of {@link #isActive()}. + * @param event + * @throws IOException + */ + void consume(Event event) throws IOException; + + Output finish(); + + /** + * A lower level notification passed from the parser to a listener. + * This token is closely related to {@link org.elasticsearch.xcontent.XContentParser.Token} and is used for use cases like + * preserving the exact structure of the parsed document. + */ + sealed interface Token permits Token.FieldName, Token.StartObject, Token.EndObject, Token.StartArray, Token.EndArray, + Token.StringAsCharArrayValue, Token.NullValue, Token.ValueToken { + + record FieldName(String name) implements Token {} + + record StartObject() implements Token {} + + record EndObject() implements Token {} + + record StartArray() implements Token {} + + record EndArray() implements Token {} + + record NullValue() implements Token {} + + final class StringAsCharArrayValue implements Token { + private final XContentParser parser; + + public StringAsCharArrayValue(XContentParser parser) { + this.parser = parser; + } + + char[] buffer() throws IOException { + return parser.textCharacters(); + } + + int length() throws IOException { + return parser.textLength(); + } + + int offset() throws IOException { + return parser.textOffset(); + } + } + + non-sealed interface ValueToken extends Token { + T value() throws IOException; + } + + Token START_OBJECT = new StartObject(); + Token END_OBJECT = new EndObject(); + Token START_ARRAY = new StartArray(); + Token END_ARRAY = new EndArray(); + + static Token current(XContentParser parser) throws IOException { + return switch (parser.currentToken()) { + case START_OBJECT -> Token.START_OBJECT; + case END_OBJECT -> Token.END_OBJECT; + case START_ARRAY -> Token.START_ARRAY; + case END_ARRAY -> Token.END_ARRAY; + case FIELD_NAME -> new FieldName(parser.currentName()); + case VALUE_STRING -> { + if (parser.hasTextCharacters()) { + yield new StringAsCharArrayValue(parser); + } else { + yield (ValueToken) parser::text; + } + } + case VALUE_NUMBER -> switch (parser.numberType()) { + case INT -> (ValueToken) parser::intValue; + case BIG_INTEGER -> (ValueToken) () -> (BigInteger) parser.numberValue(); + case LONG -> (ValueToken) parser::longValue; + case FLOAT -> (ValueToken) parser::floatValue; + case DOUBLE -> (ValueToken) parser::doubleValue; + case BIG_DECIMAL -> { + // See @XContentGenerator#copyCurrentEvent + assert false : "missing xcontent number handling for type [" + parser.numberType() + "]"; + yield null; + } + }; + case VALUE_BOOLEAN -> (ValueToken) parser::booleanValue; + case VALUE_EMBEDDED_OBJECT -> (ValueToken) parser::binaryValue; + case VALUE_NULL -> new NullValue(); + case null -> null; + }; + } + } + + /** + * High level notification passed from the parser to a listener. + * Events represent meaningful logical operations during parsing and contain relevant context for the operation + * like a mapper being used. + * A listener can use events and/or tokens depending on the use case. For example, it can wait for a specific event and then switch + * to consuming tokens instead. + */ + sealed interface Event permits Event.DocumentStart, Event.ObjectStart, Event.ObjectArrayStart, Event.LeafArrayStart, Event.LeafValue { + record DocumentStart(RootObjectMapper rootObjectMapper, LuceneDocument document) implements Event {} + + record ObjectStart(ObjectMapper objectMapper, boolean insideObjectArray, ObjectMapper parentMapper, LuceneDocument document) + implements + Event {} + + record ObjectArrayStart(ObjectMapper objectMapper, ObjectMapper parentMapper, LuceneDocument document) implements Event {} + + final class LeafValue implements Event { + private final FieldMapper fieldMapper; + private final boolean insideObjectArray; + private final ObjectMapper parentMapper; + private final LuceneDocument document; + private final XContentParser parser; + private final boolean isObjectOrArray; + private final boolean isArray; + + public LeafValue( + FieldMapper fieldMapper, + boolean insideObjectArray, + ObjectMapper parentMapper, + LuceneDocument document, + XContentParser parser + ) { + this.fieldMapper = fieldMapper; + this.insideObjectArray = insideObjectArray; + this.parentMapper = parentMapper; + this.document = document; + this.parser = parser; + this.isObjectOrArray = parser.currentToken().isValue() == false && parser.currentToken() != XContentParser.Token.VALUE_NULL; + this.isArray = parser.currentToken() == XContentParser.Token.START_ARRAY; + } + + public FieldMapper fieldMapper() { + return fieldMapper; + } + + public boolean insideObjectArray() { + return insideObjectArray; + } + + public ObjectMapper parentMapper() { + return parentMapper; + } + + public LuceneDocument document() { + return document; + } + + /** + * @return whether a value is an object or an array vs a single value like a long. + */ + boolean isContainer() { + return isObjectOrArray; + } + + boolean isArray() { + return isArray; + } + + BytesRef encodeValue() throws IOException { + assert isContainer() == false : "Objects should not be handled with direct encoding"; + + return XContentDataHelper.encodeToken(parser); + } + } + + record LeafArrayStart(FieldMapper fieldMapper, ObjectMapper parentMapper, LuceneDocument document) implements Event {} + } + + record Output(List ignoredSourceValues) { + static Output empty() { + return new Output(new ArrayList<>()); + } + + void merge(Output part) { + this.ignoredSourceValues.addAll(part.ignoredSourceValues); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 1673b1719d8bf..c20a243bf6696 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -590,6 +590,7 @@ private DocumentMapper newDocumentMapper(Mapping mapping, MergeReason reason, Co mapping, mappingSource, indexVersionCreated, + indexSettings, mapperMetrics, index().getName() ); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java index ed02e5fc29617..80dfa37a0ee03 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java @@ -43,7 +43,7 @@ private CacheKey() {} * A lookup representing an empty mapping. It can be used to look up fields, although it won't hold any, but it does not * hold a valid {@link DocumentParser}, {@link IndexSettings} or {@link IndexAnalyzers}. */ - public static final MappingLookup EMPTY = fromMappers(Mapping.EMPTY, List.of(), List.of()); + public static final MappingLookup EMPTY = fromMappers(Mapping.EMPTY, List.of(), List.of(), null); private final CacheKey cacheKey = new CacheKey(); @@ -59,14 +59,16 @@ private CacheKey() {} private final List indexTimeScriptMappers; private final Mapping mapping; private final int totalFieldsCount; + private final CustomSyntheticSourceFieldLookup customSyntheticSourceFieldLookup; /** * Creates a new {@link MappingLookup} instance by parsing the provided mapping and extracting its field definitions. * * @param mapping the mapping source + * @param indexSettings index settings * @return the newly created lookup instance */ - public static MappingLookup fromMapping(Mapping mapping) { + public static MappingLookup fromMapping(Mapping mapping, IndexSettings indexSettings) { List newObjectMappers = new ArrayList<>(); List newFieldMappers = new ArrayList<>(); List newFieldAliasMappers = new ArrayList<>(); @@ -79,7 +81,7 @@ public static MappingLookup fromMapping(Mapping mapping) { for (Mapper child : mapping.getRoot()) { collect(child, newObjectMappers, newFieldMappers, newFieldAliasMappers, newPassThroughMappers); } - return new MappingLookup(mapping, newFieldMappers, newObjectMappers, newFieldAliasMappers, newPassThroughMappers); + return new MappingLookup(mapping, newFieldMappers, newObjectMappers, newFieldAliasMappers, newPassThroughMappers, indexSettings); } private static void collect( @@ -120,6 +122,7 @@ private static void collect( * @param objectMappers the object mappers * @param aliasMappers the field alias mappers * @param passThroughMappers the pass-through mappers + * @param indexSettings index settings * @return the newly created lookup instance */ public static MappingLookup fromMappers( @@ -127,13 +130,19 @@ public static MappingLookup fromMappers( Collection mappers, Collection objectMappers, Collection aliasMappers, - Collection passThroughMappers + Collection passThroughMappers, + @Nullable IndexSettings indexSettings ) { - return new MappingLookup(mapping, mappers, objectMappers, aliasMappers, passThroughMappers); + return new MappingLookup(mapping, mappers, objectMappers, aliasMappers, passThroughMappers, indexSettings); } - public static MappingLookup fromMappers(Mapping mapping, Collection mappers, Collection objectMappers) { - return new MappingLookup(mapping, mappers, objectMappers, List.of(), List.of()); + public static MappingLookup fromMappers( + Mapping mapping, + Collection mappers, + Collection objectMappers, + @Nullable IndexSettings indexSettings + ) { + return new MappingLookup(mapping, mappers, objectMappers, List.of(), List.of(), indexSettings); } private MappingLookup( @@ -141,7 +150,8 @@ private MappingLookup( Collection mappers, Collection objectMappers, Collection aliasMappers, - Collection passThroughMappers + Collection passThroughMappers, + @Nullable IndexSettings indexSettings ) { this.totalFieldsCount = mapping.getRoot().getTotalFieldsCount(); this.mapping = mapping; @@ -207,6 +217,7 @@ private MappingLookup( this.runtimeFieldMappersCount = runtimeFields.size(); this.indexAnalyzersMap = Map.copyOf(indexAnalyzersMap); this.indexTimeScriptMappers = List.copyOf(indexTimeScriptMappers); + this.customSyntheticSourceFieldLookup = new CustomSyntheticSourceFieldLookup(mapping, indexSettings, isSourceSynthetic()); runtimeFields.stream().flatMap(RuntimeField::asMappedFieldTypes).map(MappedFieldType::name).forEach(this::validateDoesNotShadow); assert assertMapperNamesInterned(this.fieldMappers, this.objectMappers); @@ -543,4 +554,8 @@ public void validateDoesNotShadow(String name) { throw new MapperParsingException("Field [" + name + "] attempted to shadow a time_series_metric"); } } + + public CustomSyntheticSourceFieldLookup getCustomSyntheticSourceFieldLookup() { + return customSyntheticSourceFieldLookup; + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SyntheticSourceDocumentParserListener.java b/server/src/main/java/org/elasticsearch/index/mapper/SyntheticSourceDocumentParserListener.java new file mode 100644 index 0000000000000..eabb117635570 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/SyntheticSourceDocumentParserListener.java @@ -0,0 +1,398 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentType; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Listens for document parsing events and stores an additional copy of source data when it is needed for synthetic _source. + *
+ * Note that synthetic source logic for dynamic fields and fields involved in copy_to logic is still handled in {@link DocumentParser}. + */ +class SyntheticSourceDocumentParserListener implements DocumentParserListener { + private final CustomSyntheticSourceFieldLookup customSyntheticSourceFieldLookup; + private final XContentType xContentType; + + private final Map>> ignoredSourceValues; + + private State state; + + SyntheticSourceDocumentParserListener(MappingLookup mappingLookup, XContentType xContentType) { + this.customSyntheticSourceFieldLookup = mappingLookup.getCustomSyntheticSourceFieldLookup(); + this.xContentType = xContentType; + + this.ignoredSourceValues = new HashMap<>(); + this.state = new Tracking(); + } + + @Override + public boolean isActive() { + return state instanceof Storing; + } + + @Override + public void consume(Token token) throws IOException { + if (token == null) { + return; + } + + this.state = state.consume(token); + } + + @Override + public void consume(Event event) throws IOException { + if (event == null) { + return; + } + + this.state = state.consume(event); + } + + @Override + public Output finish() { + var values = new ArrayList(); + + for (var fieldToValueMap : ignoredSourceValues.values()) { + for (var fieldValues : fieldToValueMap.values()) { + long singleElementArrays = 0; + long stashedValuesForSourceKeepArrays = 0; + + for (var fieldValue : fieldValues) { + if (fieldValue instanceof StoredValue.Array arr) { + // Arrays are stored to preserve the order of elements. + // If there is a single element it does not matter and we can drop such data. + if (arr.length == 1 && arr.reason() == StoreReason.LEAF_STORED_ARRAY) { + singleElementArrays += 1; + } + } + if (fieldValue instanceof StoredValue.Singleton singleton) { + // Stash values are values of fields that are inside object arrays and have synthetic_source_keep: "arrays". + // With current logic either all field values should be in ignored source + // or none of them. + // With object arrays the same field can be parsed multiple times (one time for every object array entry) + // and it is possible that one of the value is an array. + // Due to the rule above we need to proactively store all values of such fields because we may later discover + // that there is an array and we need to "switch" to ignored source usage. + // However if we stored all values but the array is not there, the field will be correctly constructed + // using regular logic and therefore we can drop this and save some space. + if (singleton.reason() == StoreReason.LEAF_VALUE_STASH_FOR_STORED_ARRAYS) { + stashedValuesForSourceKeepArrays += 1; + } + } + } + + // Only if all values match one of the optimization criteria we skip them, otherwise add all of them to resulting list. + if (singleElementArrays != fieldValues.size() && stashedValuesForSourceKeepArrays != fieldValues.size()) { + for (var storedValue : fieldValues) { + values.add(storedValue.nameValue()); + } + } + } + } + + return new Output(values); + } + + sealed interface StoredValue permits StoredValue.Array, StoredValue.Singleton { + IgnoredSourceFieldMapper.NameValue nameValue(); + + /** + * An array of values is stored f.e. due to synthetic_source_keep: "arrays". + */ + record Array(IgnoredSourceFieldMapper.NameValue nameValue, StoreReason reason, long length) implements StoredValue {} + + /** + * A single value. + */ + record Singleton(IgnoredSourceFieldMapper.NameValue nameValue, StoreReason reason) implements StoredValue {} + + } + + /** + * Reason for storing this value. + */ + enum StoreReason { + /** + * Leaf array that is stored due to "synthetic_source_keep": "arrays". + */ + LEAF_STORED_ARRAY, + + /** + * "Stashed" value needed to only in case there are mixed arrays and single values + * for this field. + * Can be dropped in some cases. + */ + LEAF_VALUE_STASH_FOR_STORED_ARRAYS, + + /** + * There is currently no need to distinguish other reasons. + */ + OTHER + } + + private void addIgnoredSourceValue(StoredValue storedValue, String fullPath, LuceneDocument luceneDocument) { + var values = ignoredSourceValues.computeIfAbsent(luceneDocument, ld -> new HashMap<>()) + .computeIfAbsent(fullPath, p -> new ArrayList<>()); + + values.add(storedValue); + } + + interface State { + State consume(Token token) throws IOException; + + State consume(Event event) throws IOException; + } + + class Storing implements State { + private final State returnState; + private final String fullPath; + private final ObjectMapper parentMapper; + private final StoreReason reason; + private final LuceneDocument document; + + private final XContentBuilder builder; + // Current object/array depth, needed to understand when the top-most object/arrays ends vs a nested one. + private int depth; + // If we are storing an array this is the length of the array. + private int length; + + Storing( + State returnState, + Token startingToken, + String fullPath, + ObjectMapper parentMapper, + StoreReason reason, + LuceneDocument document + ) throws IOException { + this.returnState = returnState; + this.fullPath = fullPath; + this.parentMapper = parentMapper; + this.reason = reason; + this.document = document; + + this.builder = XContentBuilder.builder(xContentType.xContent()); + + this.depth = 0; + this.length = 0; + + consume(startingToken); + } + + public State consume(Token token) throws IOException { + switch (token) { + case Token.StartObject startObject -> { + builder.startObject(); + if (depth == 1) { + length += 1; + } + depth += 1; + } + case Token.EndObject endObject -> { + builder.endObject(); + + if (processEndObjectOrArray(endObject)) { + return returnState; + } + } + case Token.StartArray startArray -> { + builder.startArray(); + depth += 1; + } + case Token.EndArray endArray -> { + builder.endArray(); + + if (processEndObjectOrArray(endArray)) { + return returnState; + } + } + case Token.FieldName fieldName -> builder.field(fieldName.name()); + case Token.StringAsCharArrayValue stringAsCharArrayValue -> { + if (depth == 1) { + length += 1; + } + builder.generator() + .writeString(stringAsCharArrayValue.buffer(), stringAsCharArrayValue.offset(), stringAsCharArrayValue.length()); + } + case Token.ValueToken valueToken -> { + if (depth == 1) { + length += 1; + } + builder.value(valueToken.value()); + } + case Token.NullValue nullValue -> { + if (depth == 1) { + length += 1; + } + builder.nullValue(); + } + case null -> { + } + } + + return this; + } + + public State consume(Event event) { + // We are currently storing something so events are not relevant. + return this; + } + + private boolean processEndObjectOrArray(Token token) throws IOException { + assert token instanceof Token.EndObject || token instanceof Token.EndArray + : "Unexpected token when storing ignored source value"; + + depth -= 1; + if (depth == 0) { + var parentOffset = parentMapper.isRoot() ? 0 : parentMapper.fullPath().length() + 1; + var nameValue = new IgnoredSourceFieldMapper.NameValue( + fullPath, + parentOffset, + XContentDataHelper.encodeXContentBuilder(builder), + document + ); + var storedValue = token instanceof Token.EndObject + ? new StoredValue.Singleton(nameValue, reason) + : new StoredValue.Array(nameValue, reason, length); + + addIgnoredSourceValue(storedValue, fullPath, document); + + return true; + } + + return false; + } + } + + class Tracking implements State { + public State consume(Token token) throws IOException { + return this; + } + + public State consume(Event event) throws IOException { + switch (event) { + case Event.DocumentStart documentStart -> { + if (documentStart.rootObjectMapper().isEnabled() == false) { + return new Storing( + this, + Token.START_OBJECT, + documentStart.rootObjectMapper().fullPath(), + documentStart.rootObjectMapper(), + StoreReason.OTHER, + documentStart.document() + ); + } + } + case Event.ObjectStart objectStart -> { + var reason = customSyntheticSourceFieldLookup.getFieldsWithCustomSyntheticSourceHandling() + .get(objectStart.objectMapper().fullPath()); + if (reason == null) { + return this; + } + if (reason == CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ARRAYS && objectStart.insideObjectArray() == false) { + return this; + } + + return new Storing( + this, + Token.START_OBJECT, + objectStart.objectMapper().fullPath(), + objectStart.parentMapper(), + StoreReason.OTHER, + objectStart.document() + ); + } + case Event.ObjectArrayStart objectArrayStart -> { + var reason = customSyntheticSourceFieldLookup.getFieldsWithCustomSyntheticSourceHandling() + .get(objectArrayStart.objectMapper().fullPath()); + if (reason == null) { + return this; + } + + return new Storing( + this, + Token.START_ARRAY, + objectArrayStart.objectMapper().fullPath(), + objectArrayStart.parentMapper(), + StoreReason.OTHER, + objectArrayStart.document() + ); + } + case Event.LeafValue leafValue -> { + var reason = customSyntheticSourceFieldLookup.getFieldsWithCustomSyntheticSourceHandling() + .get(leafValue.fieldMapper().fullPath()); + if (reason == null) { + return this; + } + if (reason == CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ARRAYS && leafValue.insideObjectArray() == false) { + return this; + } + + var storeReason = reason == CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ARRAYS + ? StoreReason.LEAF_VALUE_STASH_FOR_STORED_ARRAYS + : StoreReason.OTHER; + + if (leafValue.isContainer()) { + return new Storing( + this, + leafValue.isArray() ? Token.START_ARRAY : Token.START_OBJECT, + leafValue.fieldMapper().fullPath(), + leafValue.parentMapper(), + storeReason, + leafValue.document() + ); + } + + var parentMapper = leafValue.parentMapper(); + var parentOffset = parentMapper.isRoot() ? 0 : parentMapper.fullPath().length() + 1; + + var nameValue = new IgnoredSourceFieldMapper.NameValue( + leafValue.fieldMapper().fullPath(), + parentOffset, + leafValue.encodeValue(), + leafValue.document() + ); + addIgnoredSourceValue( + new StoredValue.Singleton(nameValue, storeReason), + leafValue.fieldMapper().fullPath(), + leafValue.document() + ); + } + case Event.LeafArrayStart leafArrayStart -> { + var reason = customSyntheticSourceFieldLookup.getFieldsWithCustomSyntheticSourceHandling() + .get(leafArrayStart.fieldMapper().fullPath()); + if (reason == null) { + return this; + } + + var storeReason = reason == CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ARRAYS + ? StoreReason.LEAF_STORED_ARRAY + : StoreReason.OTHER; + return new Storing( + this, + Token.START_ARRAY, + leafArrayStart.fieldMapper().fullPath(), + leafArrayStart.parentMapper(), + storeReason, + leafArrayStart.document() + ); + } + } + + return this; + } + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CustomSyntheticSourceFieldLookupTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CustomSyntheticSourceFieldLookupTests.java new file mode 100644 index 0000000000000..c3d6bbf285d77 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/CustomSyntheticSourceFieldLookupTests.java @@ -0,0 +1,263 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexVersion; + +import java.io.IOException; +import java.util.Map; + +public class CustomSyntheticSourceFieldLookupTests extends MapperServiceTestCase { + private static String MAPPING = """ + { + "_doc": { + "properties": { + "keep_all": { + "type": "keyword", + "synthetic_source_keep": "all" + }, + "keep_arrays": { + "type": "keyword", + "synthetic_source_keep": "arrays" + }, + "fallback_impl": { + "type": "long", + "doc_values": "false" + }, + "object_keep_all": { + "properties": {}, + "synthetic_source_keep": "all" + }, + "object_keep_arrays": { + "properties": {}, + "synthetic_source_keep": "arrays" + }, + "object_disabled": { + "properties": {}, + "enabled": "false" + }, + "nested_keep_all": { + "type": "nested", + "properties": {}, + "synthetic_source_keep": "all" + }, + "nested_disabled": { + "type": "nested", + "properties": {}, + "enabled": "false" + }, + "just_field": { + "type": "boolean" + }, + "just_object": { + "properties": {} + }, + "nested_obj": { + "properties": { + "keep_all": { + "type": "keyword", + "synthetic_source_keep": "all" + }, + "keep_arrays": { + "type": "keyword", + "synthetic_source_keep": "arrays" + }, + "fallback_impl": { + "type": "long", + "doc_values": "false" + }, + "object_keep_all": { + "properties": {}, + "synthetic_source_keep": "all" + }, + "object_keep_arrays": { + "properties": {}, + "synthetic_source_keep": "arrays" + }, + "object_disabled": { + "properties": {}, + "enabled": "false" + }, + "nested_keep_all": { + "type": "nested", + "properties": {}, + "synthetic_source_keep": "all" + }, + "nested_disabled": { + "type": "nested", + "properties": {}, + "enabled": "false" + }, + "just_field": { + "type": "boolean" + }, + "just_object": { + "properties": {} + } + } + }, + "nested_nested": { + "properties": { + "keep_all": { + "type": "keyword", + "synthetic_source_keep": "all" + }, + "keep_arrays": { + "type": "keyword", + "synthetic_source_keep": "arrays" + }, + "fallback_impl": { + "type": "long", + "doc_values": "false" + }, + "object_keep_all": { + "properties": {}, + "synthetic_source_keep": "all" + }, + "object_keep_arrays": { + "properties": {}, + "synthetic_source_keep": "arrays" + }, + "object_disabled": { + "properties": {}, + "enabled": "false" + }, + "nested_keep_all": { + "type": "nested", + "properties": {}, + "synthetic_source_keep": "all" + }, + "nested_disabled": { + "type": "nested", + "properties": {}, + "enabled": "false" + }, + "just_field": { + "type": "boolean" + }, + "just_object": { + "properties": {} + } + } + } + } + } + } + """; + + public void testIsNoopWhenSourceIsNotSynthetic() throws IOException { + var mapping = createMapperService(MAPPING).mappingLookup().getMapping(); + var indexSettings = indexSettings(Mapper.SourceKeepMode.NONE); + var sut = new CustomSyntheticSourceFieldLookup(mapping, indexSettings, false); + + assertEquals(sut.getFieldsWithCustomSyntheticSourceHandling(), Map.of()); + } + + public void testDetectsLeafWithKeepAll() throws IOException { + var mapping = createMapperService(MAPPING).mappingLookup().getMapping(); + var indexSettings = indexSettings(Mapper.SourceKeepMode.NONE); + var sut = new CustomSyntheticSourceFieldLookup(mapping, indexSettings, true); + + var fields = sut.getFieldsWithCustomSyntheticSourceHandling(); + assertEquals(CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ALL, fields.get("keep_all")); + assertEquals(CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ALL, fields.get("nested_obj.keep_all")); + assertEquals(CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ALL, fields.get("nested_nested.keep_all")); + } + + public void testDetectsLeafWithKeepArrays() throws IOException { + var mapping = createMapperService(MAPPING).mappingLookup().getMapping(); + var indexSettings = indexSettings(Mapper.SourceKeepMode.NONE); + var sut = new CustomSyntheticSourceFieldLookup(mapping, indexSettings, true); + + var fields = sut.getFieldsWithCustomSyntheticSourceHandling(); + assertEquals(CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ARRAYS, fields.get("keep_arrays")); + assertEquals(CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ARRAYS, fields.get("nested_obj.keep_arrays")); + assertEquals(CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ARRAYS, fields.get("nested_nested.keep_arrays")); + } + + public void testDetectsLeafWithFallback() throws IOException { + var mapping = createMapperService(MAPPING).mappingLookup().getMapping(); + var indexSettings = indexSettings(Mapper.SourceKeepMode.NONE); + var sut = new CustomSyntheticSourceFieldLookup(mapping, indexSettings, true); + + var fields = sut.getFieldsWithCustomSyntheticSourceHandling(); + assertEquals(CustomSyntheticSourceFieldLookup.Reason.FALLBACK_SYNTHETIC_SOURCE, fields.get("fallback_impl")); + assertEquals(CustomSyntheticSourceFieldLookup.Reason.FALLBACK_SYNTHETIC_SOURCE, fields.get("nested_obj.fallback_impl")); + assertEquals(CustomSyntheticSourceFieldLookup.Reason.FALLBACK_SYNTHETIC_SOURCE, fields.get("nested_nested.fallback_impl")); + } + + public void testDetectsObjectWithKeepAll() throws IOException { + var mapping = createMapperService(MAPPING).mappingLookup().getMapping(); + var indexSettings = indexSettings(Mapper.SourceKeepMode.NONE); + var sut = new CustomSyntheticSourceFieldLookup(mapping, indexSettings, true); + + var fields = sut.getFieldsWithCustomSyntheticSourceHandling(); + + assertEquals(CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ALL, fields.get("object_keep_all")); + assertEquals(CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ALL, fields.get("nested_obj.object_keep_all")); + assertEquals(CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ALL, fields.get("nested_nested.object_keep_all")); + + assertEquals(CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ALL, fields.get("nested_keep_all")); + assertEquals(CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ALL, fields.get("nested_obj.nested_keep_all")); + assertEquals(CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ALL, fields.get("nested_nested.nested_keep_all")); + } + + public void testDetectsObjectWithKeepArrays() throws IOException { + var mapping = createMapperService(MAPPING).mappingLookup().getMapping(); + var indexSettings = indexSettings(Mapper.SourceKeepMode.NONE); + var sut = new CustomSyntheticSourceFieldLookup(mapping, indexSettings, true); + + var fields = sut.getFieldsWithCustomSyntheticSourceHandling(); + assertEquals(CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ARRAYS, fields.get("object_keep_arrays")); + assertEquals(CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ARRAYS, fields.get("nested_obj.object_keep_arrays")); + assertEquals(CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ARRAYS, fields.get("nested_nested.object_keep_arrays")); + } + + public void testDetectsDisabledObject() throws IOException { + var mapping = createMapperService(MAPPING).mappingLookup().getMapping(); + var indexSettings = indexSettings(Mapper.SourceKeepMode.NONE); + var sut = new CustomSyntheticSourceFieldLookup(mapping, indexSettings, true); + + var fields = sut.getFieldsWithCustomSyntheticSourceHandling(); + + assertEquals(CustomSyntheticSourceFieldLookup.Reason.DISABLED_OBJECT, fields.get("object_disabled")); + assertEquals(CustomSyntheticSourceFieldLookup.Reason.DISABLED_OBJECT, fields.get("nested_obj.object_disabled")); + assertEquals(CustomSyntheticSourceFieldLookup.Reason.DISABLED_OBJECT, fields.get("nested_nested.object_disabled")); + + assertEquals(CustomSyntheticSourceFieldLookup.Reason.DISABLED_OBJECT, fields.get("nested_disabled")); + assertEquals(CustomSyntheticSourceFieldLookup.Reason.DISABLED_OBJECT, fields.get("nested_obj.nested_disabled")); + assertEquals(CustomSyntheticSourceFieldLookup.Reason.DISABLED_OBJECT, fields.get("nested_nested.nested_disabled")); + } + + public void testAppliesIndexLevelSourceKeepMode() throws IOException { + var mapping = createMapperService(MAPPING).mappingLookup().getMapping(); + var indexSettings = indexSettings(Mapper.SourceKeepMode.ARRAYS); + var sut = new CustomSyntheticSourceFieldLookup(mapping, indexSettings, true); + + var fields = sut.getFieldsWithCustomSyntheticSourceHandling(); + + assertEquals(CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ARRAYS, fields.get("just_field")); + assertEquals(CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ARRAYS, fields.get("nested_obj.just_field")); + assertEquals(CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ARRAYS, fields.get("nested_nested.just_field")); + + assertEquals(CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ARRAYS, fields.get("just_object")); + assertEquals(CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ARRAYS, fields.get("nested_obj.just_object")); + assertEquals(CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ARRAYS, fields.get("nested_nested.just_object")); + } + + private static IndexSettings indexSettings(Mapper.SourceKeepMode sourceKeepMode) { + return createIndexSettings( + IndexVersion.current(), + Settings.builder().put(Mapper.SYNTHETIC_SOURCE_KEEP_INDEX_SETTING.getKey(), sourceKeepMode).build() + ); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java index b2ba3d60d2174..9e617f638e755 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java @@ -81,6 +81,7 @@ public void testAddFields() throws Exception { merged, merged.toCompressedXContent(), IndexVersion.current(), + null, MapperMetrics.NOOP, "myIndex" ); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserListenerTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserListenerTests.java new file mode 100644 index 0000000000000..a1dafacfb7b23 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserListenerTests.java @@ -0,0 +1,253 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.XContentType; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class DocumentParserListenerTests extends MapperServiceTestCase { + private static class MemorizingDocumentParserListener implements DocumentParserListener { + private final List events = new ArrayList<>(); + private final List tokens = new ArrayList<>(); + + @Override + public boolean isActive() { + return true; + } + + @Override + public void consume(Token token) throws IOException { + // Tokens contains information tied to current parser state so we need to "materialize" them. + if (token instanceof Token.StringAsCharArrayValue charArray) { + var string = String.copyValueOf(charArray.buffer(), charArray.offset(), charArray.length()); + tokens.add((Token.ValueToken) () -> string); + } else if (token instanceof Token.ValueToken v) { + var value = v.value(); + tokens.add((Token.ValueToken) () -> value); + } else { + tokens.add(token); + } + } + + @Override + public void consume(Event event) throws IOException { + events.add(event); + } + + @Override + public Output finish() { + return new Output(List.of()); + } + + public List getTokens() { + return tokens; + } + + public List getEvents() { + return events; + } + } + + public void testEventFlow() throws IOException { + var mapping = XContentBuilder.builder(XContentType.JSON.xContent()).startObject().startObject("_doc").startObject("properties"); + { + mapping.startObject("leaf").field("type", "keyword").endObject(); + mapping.startObject("leaf_array").field("type", "keyword").endObject(); + + mapping.startObject("object").startObject("properties"); + { + mapping.startObject("leaf").field("type", "keyword").endObject(); + mapping.startObject("leaf_array").field("type", "keyword").endObject(); + } + mapping.endObject().endObject(); + + mapping.startObject("object_array").startObject("properties"); + { + mapping.startObject("leaf").field("type", "keyword").endObject(); + mapping.startObject("leaf_array").field("type", "keyword").endObject(); + } + mapping.endObject().endObject(); + } + mapping.endObject().endObject().endObject(); + var mappingService = createSytheticSourceMapperService(mapping); + + XContentType xContentType = randomFrom(XContentType.values()); + + var listener = new MemorizingDocumentParserListener(); + var documentParser = new DocumentParser( + XContentParserConfiguration.EMPTY, + mappingService.parserContext(), + (ml, xct) -> new DocumentParser.Listeners.Single(listener) + ); + + var source = XContentBuilder.builder(xContentType.xContent()); + source.startObject(); + { + source.field("leaf", "leaf"); + source.array("leaf_array", "one", "two"); + source.startObject("object"); + { + source.field("leaf", "leaf"); + source.array("leaf_array", "one", "two"); + } + source.endObject(); + source.startArray("object_array"); + { + source.startObject(); + { + source.field("leaf", "leaf"); + source.array("leaf_array", "one", "two"); + } + source.endObject(); + } + source.endArray(); + } + source.endObject(); + + documentParser.parseDocument(new SourceToParse("id1", BytesReference.bytes(source), xContentType), mappingService.mappingLookup()); + var events = listener.getEvents(); + + assertEquals("_doc", ((DocumentParserListener.Event.DocumentStart) events.get(0)).rootObjectMapper().fullPath()); + + assertLeafEvents(events, 1, "", "_doc", false); + + var objectStart = (DocumentParserListener.Event.ObjectStart) events.get(5); + assertEquals("object", objectStart.objectMapper().fullPath()); + assertEquals("_doc", objectStart.parentMapper().fullPath()); + assertFalse(objectStart.insideObjectArray()); + assertLeafEvents(events, 6, "object.", "object", false); + + var objectArrayStart = (DocumentParserListener.Event.ObjectArrayStart) events.get(10); + assertEquals("object_array", objectArrayStart.objectMapper().fullPath()); + assertEquals("_doc", objectArrayStart.parentMapper().fullPath()); + + var objectInArrayStart = (DocumentParserListener.Event.ObjectStart) events.get(11); + assertEquals("object_array", objectInArrayStart.objectMapper().fullPath()); + assertEquals("_doc", objectInArrayStart.parentMapper().fullPath()); + assertTrue(objectInArrayStart.insideObjectArray()); + assertLeafEvents(events, 12, "object_array.", "object_array", true); + } + + public void testTokenFlow() throws IOException { + var mapping = XContentBuilder.builder(XContentType.JSON.xContent()) + .startObject() + .startObject("_doc") + .field("enabled", false) + .endObject() + .endObject(); + var mappingService = createSytheticSourceMapperService(mapping); + + XContentType xContentType = randomFrom(XContentType.values()); + + var listener = new MemorizingDocumentParserListener(); + var documentParser = new DocumentParser( + XContentParserConfiguration.EMPTY, + mappingService.parserContext(), + (ml, xct) -> new DocumentParser.Listeners.Single(listener) + ); + + var source = XContentBuilder.builder(xContentType.xContent()); + source.startObject(); + { + source.field("leaf", "leaf"); + source.array("leaf_array", "one", "two"); + source.startObject("object"); + { + source.field("leaf", "leaf"); + source.array("leaf_array", "one", "two"); + } + source.endObject(); + source.startArray("object_array"); + { + source.startObject(); + { + source.field("leaf", "leaf"); + source.array("leaf_array", "one", "two"); + } + source.endObject(); + } + source.endArray(); + } + source.endObject(); + + documentParser.parseDocument(new SourceToParse("id1", BytesReference.bytes(source), xContentType), mappingService.mappingLookup()); + var tokens = listener.getTokens(); + assertTrue(tokens.get(0) instanceof DocumentParserListener.Token.StartObject); + { + assertLeafTokens(tokens, 1); + assertEquals("object", ((DocumentParserListener.Token.FieldName) tokens.get(8)).name()); + assertTrue(tokens.get(9) instanceof DocumentParserListener.Token.StartObject); + { + assertLeafTokens(tokens, 10); + } + assertTrue(tokens.get(17) instanceof DocumentParserListener.Token.EndObject); + assertEquals("object_array", ((DocumentParserListener.Token.FieldName) tokens.get(18)).name()); + assertTrue(tokens.get(19) instanceof DocumentParserListener.Token.StartArray); + { + assertTrue(tokens.get(20) instanceof DocumentParserListener.Token.StartObject); + { + assertLeafTokens(tokens, 21); + } + assertTrue(tokens.get(28) instanceof DocumentParserListener.Token.EndObject); + } + assertTrue(tokens.get(29) instanceof DocumentParserListener.Token.EndArray); + } + assertTrue(tokens.get(30) instanceof DocumentParserListener.Token.EndObject); + } + + private void assertLeafEvents( + List events, + int start, + String prefix, + String parent, + boolean inObjectArray + ) { + var leafValue = (DocumentParserListener.Event.LeafValue) events.get(start); + assertEquals(prefix + "leaf", leafValue.fieldMapper().fullPath()); + assertEquals(parent, leafValue.parentMapper().fullPath()); + assertFalse(leafValue.isArray()); + assertFalse(leafValue.isContainer()); + assertEquals(inObjectArray, leafValue.insideObjectArray()); + + var leafArray = (DocumentParserListener.Event.LeafArrayStart) events.get(start + 1); + assertEquals(prefix + "leaf_array", leafArray.fieldMapper().fullPath()); + assertEquals(parent, leafArray.parentMapper().fullPath()); + + var arrayValue1 = (DocumentParserListener.Event.LeafValue) events.get(start + 2); + assertEquals(prefix + "leaf_array", arrayValue1.fieldMapper().fullPath()); + assertEquals(parent, arrayValue1.parentMapper().fullPath()); + assertFalse(arrayValue1.isArray()); + assertFalse(arrayValue1.isContainer()); + assertEquals(inObjectArray, leafValue.insideObjectArray()); + + var arrayValue2 = (DocumentParserListener.Event.LeafValue) events.get(start + 3); + assertEquals(prefix + "leaf_array", arrayValue2.fieldMapper().fullPath()); + assertEquals(parent, arrayValue2.parentMapper().fullPath()); + assertFalse(arrayValue2.isArray()); + assertFalse(arrayValue2.isContainer()); + assertEquals(inObjectArray, leafValue.insideObjectArray()); + } + + private void assertLeafTokens(List tokens, int start) throws IOException { + assertEquals("leaf", ((DocumentParserListener.Token.FieldName) tokens.get(start)).name()); + assertEquals("leaf", ((DocumentParserListener.Token.ValueToken) tokens.get(start + 1)).value()); + assertEquals("leaf_array", ((DocumentParserListener.Token.FieldName) tokens.get(start + 2)).name()); + assertTrue(tokens.get(start + 3) instanceof DocumentParserListener.Token.StartArray); + assertEquals("one", ((DocumentParserListener.Token.ValueToken) tokens.get(start + 4)).value()); + assertEquals("two", ((DocumentParserListener.Token.ValueToken) tokens.get(start + 5)).value()); + assertTrue(tokens.get(start + 6) instanceof DocumentParserListener.Token.EndArray); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java index d128b25038a59..3699f97e243af 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java @@ -2688,6 +2688,7 @@ same name need to be part of the same mappings (hence the same document). If th newMapping, newMapping.toCompressedXContent(), IndexVersion.current(), + mapperService.getIndexSettings(), MapperMetrics.NOOP, "myIndex" ); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java index d4d0e67ff4141..1f020520e7e35 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java @@ -9,8 +9,11 @@ package org.elasticsearch.index.mapper; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentType; @@ -75,7 +78,14 @@ public void testCreateDynamicStringFieldAsKeywordForDimension() throws IOExcepti ).build(MapperBuilderContext.root(false, false)); Mapping mapping = new Mapping(root, new MetadataFieldMapper[] { sourceMapper }, Map.of()); - DocumentParserContext ctx = new TestDocumentParserContext(MappingLookup.fromMapping(mapping), sourceToParse) { + IndexMetadata indexMetadata = IndexMetadata.builder("index") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) + .numberOfShards(1) + .numberOfReplicas(0) + .build(); + IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); + + DocumentParserContext ctx = new TestDocumentParserContext(MappingLookup.fromMapping(mapping, indexSettings), sourceToParse) { @Override public XContentParser parser() { return parser; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java index e385177b87147..dbfc1f114fffb 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java @@ -204,6 +204,6 @@ private static MappingLookup createMappingLookup( new MetadataFieldMapper[0], Collections.emptyMap() ); - return MappingLookup.fromMappers(mapping, fieldMappers, objectMappers, fieldAliasMappers, emptyList()); + return MappingLookup.fromMappers(mapping, fieldMappers, objectMappers, fieldAliasMappers, emptyList(), null); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java index 4a45824342c74..67b62530f3443 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java @@ -22,8 +22,6 @@ import java.util.List; import java.util.stream.Stream; -import static java.util.Collections.emptyList; - public class FieldNamesFieldTypeTests extends ESTestCase { public void testTermQuery() { @@ -36,7 +34,6 @@ public void testTermQuery() { settings ); List mappers = Stream.of(fieldNamesFieldType, fieldType).map(MockFieldMapper::new).toList(); - MappingLookup mappingLookup = MappingLookup.fromMappers(Mapping.EMPTY, mappers, emptyList()); SearchExecutionContext searchExecutionContext = SearchExecutionContextHelper.createSimple(indexSettings, null, null); Query termQuery = fieldNamesFieldType.termQuery("field_name", searchExecutionContext); assertEquals(new TermQuery(new Term(FieldNamesFieldMapper.CONTENT_TYPE, "field_name")), termQuery); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java index 14902aa419b9f..5c9765eefb98d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java @@ -848,7 +848,7 @@ public void testIndexStoredArraySourceRootObjectArrayWithBypass() throws IOExcep b.field("bool_value", true); }); assertEquals(""" - {"bool_value":true,"path":{"int_value":[20,10]}}""", syntheticSource); + {"bool_value":true,"path":{"int_value":[10,20]}}""", syntheticSource); } public void testIndexStoredArraySourceNestedValueArray() throws IOException { @@ -912,7 +912,7 @@ public void testIndexStoredArraySourceNestedValueArrayDisabled() throws IOExcept b.endObject(); }); assertEquals(""" - {"path":{"bool_value":true,"int_value":[10,20,30],"obj":{"foo":[2,1]}}}""", syntheticSource); + {"path":{"bool_value":true,"int_value":[10,20,30],"obj":{"foo":[1,2]}}}""", syntheticSource); } public void testFieldStoredArraySourceNestedValueArray() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java index fd44e68df19a8..71edce3d1549a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java @@ -13,8 +13,12 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.mapper.TimeSeriesParams.MetricType; @@ -49,7 +53,13 @@ private static MappingLookup createMappingLookup( new MetadataFieldMapper[0], Collections.emptyMap() ); - return MappingLookup.fromMappers(mapping, fieldMappers, objectMappers); + IndexMetadata indexMetadata = IndexMetadata.builder("index") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) + .numberOfShards(1) + .numberOfReplicas(0) + .build(); + IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); + return MappingLookup.fromMappers(mapping, fieldMappers, objectMappers, indexSettings); } public void testOnlyRuntimeField() { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java index b87ab09c530d6..289ec24f7d3b9 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java @@ -107,7 +107,7 @@ public void testFieldNameWithDeepDots() throws Exception { b.endObject(); }); Mapping mapping = createMappingParser(Settings.EMPTY).parse("_doc", new CompressedXContent(BytesReference.bytes(builder))); - MappingLookup mappingLookup = MappingLookup.fromMapping(mapping); + MappingLookup mappingLookup = MappingLookup.fromMapping(mapping, null); assertNotNull(mappingLookup.getMapper("foo.bar")); assertNotNull(mappingLookup.getMapper("foo.baz.deep.field")); assertNotNull(mappingLookup.objectMappers().get("foo")); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SyntheticSourceDocumentParserListenerTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SyntheticSourceDocumentParserListenerTests.java new file mode 100644 index 0000000000000..b5496e1001bf4 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/SyntheticSourceDocumentParserListenerTests.java @@ -0,0 +1,385 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentType; + +import java.io.IOException; +import java.math.BigInteger; + +public class SyntheticSourceDocumentParserListenerTests extends MapperServiceTestCase { + public void testStoreLeafValue() throws IOException { + XContentType xContentType = randomFrom(XContentType.values()); + + var mapping = fieldMapping(b -> b.field("type", "long").field("synthetic_source_keep", "all")); + var mappingLookup = createSytheticSourceMapperService(mapping).mappingLookup(); + var sut = new SyntheticSourceDocumentParserListener(mappingLookup, xContentType); + + var doc = new LuceneDocument(); + + var value = XContentBuilder.builder(xContentType.xContent()).value(1234L); + var parser = createParser(value); + parser.nextToken(); + + sut.consume( + new DocumentParserListener.Event.LeafValue( + (FieldMapper) mappingLookup.getMapper("field"), + false, + mappingLookup.getMapping().getRoot(), + doc, + parser + ) + ); + + var output = sut.finish(); + + assertEquals(1, output.ignoredSourceValues().size()); + var valueToStore = output.ignoredSourceValues().get(0); + assertEquals("field", valueToStore.name()); + var decoded = XContentBuilder.builder(xContentType.xContent()); + XContentDataHelper.decodeAndWrite(decoded, valueToStore.value()); + assertEquals(BytesReference.bytes(value), BytesReference.bytes(decoded)); + } + + public void testStoreLeafArray() throws IOException { + XContentType xContentType = randomFrom(XContentType.values()); + + var mapping = fieldMapping(b -> b.field("type", "long").field("synthetic_source_keep", "all")); + var mappingLookup = createSytheticSourceMapperService(mapping).mappingLookup(); + var sut = new SyntheticSourceDocumentParserListener(mappingLookup, xContentType); + + var values = randomList(0, 10, ESTestCase::randomLong); + + var doc = new LuceneDocument(); + + sut.consume( + new DocumentParserListener.Event.LeafArrayStart( + (FieldMapper) mappingLookup.getMapper("field"), + mappingLookup.getMapping().getRoot(), + doc + ) + ); + for (long l : values) { + sut.consume((DocumentParserListener.Token.ValueToken) () -> l); + } + sut.consume(DocumentParserListener.Token.END_ARRAY); + + var output = sut.finish(); + + assertEquals(1, output.ignoredSourceValues().size()); + var valueToStore = output.ignoredSourceValues().get(0); + assertEquals("field", valueToStore.name()); + + var decoded = XContentBuilder.builder(xContentType.xContent()); + XContentDataHelper.decodeAndWrite(decoded, valueToStore.value()); + + var parser = createParser(decoded); + assertEquals(XContentParser.Token.START_ARRAY, parser.nextToken()); + for (long l : values) { + parser.nextToken(); + assertEquals(XContentParser.Token.VALUE_NUMBER, parser.currentToken()); + assertEquals(l, parser.longValue()); + } + assertEquals(XContentParser.Token.END_ARRAY, parser.nextToken()); + } + + public void testStoreObject() throws IOException { + XContentType xContentType = randomFrom(XContentType.values()); + + var mapping = fieldMapping(b -> b.field("type", "object").field("synthetic_source_keep", "all")); + var mappingLookup = createSytheticSourceMapperService(mapping).mappingLookup(); + var sut = new SyntheticSourceDocumentParserListener(mappingLookup, xContentType); + + var names = randomList(0, 10, () -> randomAlphaOfLength(10)); + var values = randomList(names.size(), names.size(), () -> randomAlphaOfLength(10)); + + var doc = new LuceneDocument(); + + sut.consume( + new DocumentParserListener.Event.ObjectStart( + mappingLookup.objectMappers().get("field"), + false, + mappingLookup.getMapping().getRoot(), + doc + ) + ); + for (int i = 0; i < names.size(); i++) { + sut.consume(new DocumentParserListener.Token.FieldName(names.get(i))); + var value = values.get(i); + sut.consume((DocumentParserListener.Token.ValueToken) () -> value); + } + sut.consume(DocumentParserListener.Token.END_OBJECT); + + var output = sut.finish(); + + assertEquals(1, output.ignoredSourceValues().size()); + var valueToStore = output.ignoredSourceValues().get(0); + assertEquals("field", valueToStore.name()); + + var decoded = XContentBuilder.builder(xContentType.xContent()); + XContentDataHelper.decodeAndWrite(decoded, valueToStore.value()); + + var parser = createParser(decoded); + assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken()); + for (int i = 0; i < names.size(); i++) { + parser.nextToken(); + assertEquals(XContentParser.Token.FIELD_NAME, parser.currentToken()); + assertEquals(names.get(i), parser.currentName()); + + assertEquals(XContentParser.Token.VALUE_STRING, parser.nextToken()); + assertEquals(values.get(i), parser.text()); + } + assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken()); + } + + public void testStoreObjectArray() throws IOException { + XContentType xContentType = randomFrom(XContentType.values()); + + var mapping = fieldMapping(b -> b.field("type", "object").field("synthetic_source_keep", "all")); + var mappingLookup = createSytheticSourceMapperService(mapping).mappingLookup(); + var sut = new SyntheticSourceDocumentParserListener(mappingLookup, xContentType); + + var names = randomList(0, 10, () -> randomAlphaOfLength(10)); + var values = randomList(names.size(), names.size(), () -> randomAlphaOfLength(10)); + + var doc = new LuceneDocument(); + + sut.consume( + new DocumentParserListener.Event.ObjectArrayStart( + mappingLookup.objectMappers().get("field"), + mappingLookup.getMapping().getRoot(), + doc + ) + ); + for (int i = 0; i < names.size(); i++) { + sut.consume(DocumentParserListener.Token.START_OBJECT); + + sut.consume(new DocumentParserListener.Token.FieldName(names.get(i))); + var value = values.get(i); + sut.consume((DocumentParserListener.Token.ValueToken) () -> value); + + sut.consume(DocumentParserListener.Token.END_OBJECT); + } + sut.consume(DocumentParserListener.Token.END_ARRAY); + + var output = sut.finish(); + + assertEquals(1, output.ignoredSourceValues().size()); + var valueToStore = output.ignoredSourceValues().get(0); + assertEquals("field", valueToStore.name()); + + var decoded = XContentBuilder.builder(xContentType.xContent()); + XContentDataHelper.decodeAndWrite(decoded, valueToStore.value()); + + var parser = createParser(decoded); + assertEquals(XContentParser.Token.START_ARRAY, parser.nextToken()); + for (int i = 0; i < names.size(); i++) { + assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken()); + assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken()); + assertEquals(names.get(i), parser.currentName()); + + assertEquals(XContentParser.Token.VALUE_STRING, parser.nextToken()); + assertEquals(values.get(i), parser.text()); + + assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken()); + } + assertEquals(XContentParser.Token.END_ARRAY, parser.nextToken()); + } + + public void testStashedLeafValue() throws IOException { + XContentType xContentType = randomFrom(XContentType.values()); + + var mapping = fieldMapping(b -> b.field("type", "boolean").field("synthetic_source_keep", "arrays")); + var mappingLookup = createSytheticSourceMapperService(mapping).mappingLookup(); + var sut = new SyntheticSourceDocumentParserListener(mappingLookup, xContentType); + + var doc = new LuceneDocument(); + + var value = XContentBuilder.builder(xContentType.xContent()).value(false); + var parser = createParser(value); + parser.nextToken(); + + sut.consume( + new DocumentParserListener.Event.LeafValue( + (FieldMapper) mappingLookup.getMapper("field"), + true, + mappingLookup.getMapping().getRoot(), + doc, + parser + ) + ); + + sut.consume( + new DocumentParserListener.Event.LeafValue( + (FieldMapper) mappingLookup.getMapper("field"), + true, + mappingLookup.getMapping().getRoot(), + doc, + parser + ) + ); + + var output = sut.finish(); + + // Single values are optimized away because there are no arrays mixed in and regular synthetic source logic is sufficient + assertEquals(0, output.ignoredSourceValues().size()); + } + + public void testStashedMixedValues() throws IOException { + XContentType xContentType = randomFrom(XContentType.values()); + + var mapping = fieldMapping(b -> b.field("type", "boolean").field("synthetic_source_keep", "arrays")); + var mappingLookup = createSytheticSourceMapperService(mapping).mappingLookup(); + var sut = new SyntheticSourceDocumentParserListener(mappingLookup, xContentType); + + var doc = new LuceneDocument(); + + var value = XContentBuilder.builder(xContentType.xContent()).value(false); + var parser = createParser(value); + parser.nextToken(); + + sut.consume( + new DocumentParserListener.Event.LeafValue( + (FieldMapper) mappingLookup.getMapper("field"), + true, + mappingLookup.getMapping().getRoot(), + doc, + parser + ) + ); + + sut.consume( + new DocumentParserListener.Event.LeafValue( + (FieldMapper) mappingLookup.getMapper("field"), + true, + mappingLookup.getMapping().getRoot(), + doc, + parser + ) + ); + + sut.consume( + new DocumentParserListener.Event.LeafArrayStart( + (FieldMapper) mappingLookup.getMapper("field"), + mappingLookup.getMapping().getRoot(), + doc + ) + ); + sut.consume((DocumentParserListener.Token.ValueToken) () -> true); + sut.consume((DocumentParserListener.Token.ValueToken) () -> true); + sut.consume(DocumentParserListener.Token.END_ARRAY); + + var output = sut.finish(); + + // Both arrays and individual values are stored. + assertEquals(3, output.ignoredSourceValues().size()); + } + + public void testStashedObjectValue() throws IOException { + XContentType xContentType = randomFrom(XContentType.values()); + + var mapping = fieldMapping(b -> b.field("type", "object").field("synthetic_source_keep", "arrays")); + var mappingLookup = createSytheticSourceMapperService(mapping).mappingLookup(); + var sut = new SyntheticSourceDocumentParserListener(mappingLookup, xContentType); + + var doc = new LuceneDocument(); + + var value = XContentBuilder.builder(xContentType.xContent()).value(1234L); + var parser = createParser(value); + parser.nextToken(); + + sut.consume( + new DocumentParserListener.Event.ObjectStart( + mappingLookup.objectMappers().get("field"), + true, + mappingLookup.getMapping().getRoot(), + doc + ) + ); + sut.consume(new DocumentParserListener.Token.FieldName("hello")); + sut.consume((DocumentParserListener.Token.ValueToken) () -> BigInteger.valueOf(13)); + sut.consume(DocumentParserListener.Token.END_OBJECT); + + var output = sut.finish(); + + // Single value optimization does not work for objects because it is possible that one of the fields + // of this object needs to be stored in ignored source. + // Because we stored the entire object we didn't store individual fields separately. + // Optimizing this away would lead to missing data from synthetic source in some cases. + // We could do both, but we don't do it now. + assertEquals(1, output.ignoredSourceValues().size()); + } + + public void testSingleElementArray() throws IOException { + XContentType xContentType = randomFrom(XContentType.values()); + + var mapping = fieldMapping(b -> b.field("type", "boolean").field("synthetic_source_keep", "arrays")); + var mappingLookup = createSytheticSourceMapperService(mapping).mappingLookup(); + var sut = new SyntheticSourceDocumentParserListener(mappingLookup, xContentType); + + var doc = new LuceneDocument(); + + sut.consume( + new DocumentParserListener.Event.LeafArrayStart( + (FieldMapper) mappingLookup.getMapper("field"), + mappingLookup.getMapping().getRoot(), + doc + ) + ); + sut.consume((DocumentParserListener.Token.ValueToken) () -> true); + sut.consume(DocumentParserListener.Token.END_ARRAY); + + var output = sut.finish(); + + // Since there is only one value in the array, order does not matter, + // and we can drop ignored source value and use standard synthetic source logic. + assertEquals(0, output.ignoredSourceValues().size()); + } + + public void testMultipleSingleElementArrays() throws IOException { + XContentType xContentType = randomFrom(XContentType.values()); + + var mapping = fieldMapping(b -> b.field("type", "boolean").field("synthetic_source_keep", "arrays")); + var mappingLookup = createSytheticSourceMapperService(mapping).mappingLookup(); + var sut = new SyntheticSourceDocumentParserListener(mappingLookup, xContentType); + + var doc = new LuceneDocument(); + + sut.consume( + new DocumentParserListener.Event.LeafArrayStart( + (FieldMapper) mappingLookup.getMapper("field"), + mappingLookup.getMapping().getRoot(), + doc + ) + ); + sut.consume((DocumentParserListener.Token.ValueToken) () -> true); + sut.consume(DocumentParserListener.Token.END_ARRAY); + + sut.consume( + new DocumentParserListener.Event.LeafArrayStart( + (FieldMapper) mappingLookup.getMapper("field"), + mappingLookup.getMapping().getRoot(), + doc + ) + ); + sut.consume((DocumentParserListener.Token.ValueToken) () -> false); + sut.consume(DocumentParserListener.Token.END_ARRAY); + + var output = sut.finish(); + + // Since there is only one value in the array, order does not matter, + // and we can drop ignored source value. + assertEquals(0, output.ignoredSourceValues().size()); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java index dc70c44a89128..df2c9466f3b7d 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java @@ -297,7 +297,7 @@ private static MappingLookup createMappingLookup(List concreteF new MetadataFieldMapper[0], Collections.emptyMap() ); - return MappingLookup.fromMappers(mapping, mappers, Collections.emptyList()); + return MappingLookup.fromMappers(mapping, mappers, Collections.emptyList(), null); } public void testSearchRequestRuntimeFields() { @@ -389,7 +389,13 @@ public void testSyntheticSourceSearchLookup() throws IOException { new KeywordFieldMapper.Builder("cat", IndexVersion.current()).ignoreAbove(100) ).build(MapperBuilderContext.root(true, false)); Mapping mapping = new Mapping(root, new MetadataFieldMapper[] { sourceMapper }, Map.of()); - MappingLookup lookup = MappingLookup.fromMapping(mapping); + IndexMetadata indexMetadata = IndexMetadata.builder("index") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) + .numberOfShards(1) + .numberOfReplicas(0) + .build(); + IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); + MappingLookup lookup = MappingLookup.fromMapping(mapping, indexSettings); SearchExecutionContext sec = createSearchExecutionContext("index", "", lookup, Map.of()); assertTrue(sec.isSourceSynthetic()); diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesRequestCacheTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesRequestCacheTests.java index 773c660caa1c6..8afedbd63f14e 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesRequestCacheTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesRequestCacheTests.java @@ -204,7 +204,7 @@ public void testCacheDifferentReaders() throws Exception { public void testCacheDifferentMapping() throws Exception { IndicesRequestCache cache = new IndicesRequestCache(Settings.EMPTY); MappingLookup.CacheKey mappingKey1 = MappingLookup.EMPTY.cacheKey(); - MappingLookup.CacheKey mappingKey2 = MappingLookup.fromMappers(Mapping.EMPTY, emptyList(), emptyList()).cacheKey(); + MappingLookup.CacheKey mappingKey2 = MappingLookup.fromMappers(Mapping.EMPTY, emptyList(), emptyList(), null).cacheKey(); AtomicBoolean indexShard = new AtomicBoolean(true); ShardRequestCache requestCacheStats = new ShardRequestCache(); Directory dir = newDirectory(); @@ -364,13 +364,13 @@ public void testClearAllEntityIdentity() throws Exception { writer.updateDocument(new Term("id", "0"), newDoc(0, "bar")); DirectoryReader secondReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1)); - MappingLookup.CacheKey secondMappingKey = MappingLookup.fromMappers(Mapping.EMPTY, emptyList(), emptyList()).cacheKey(); + MappingLookup.CacheKey secondMappingKey = MappingLookup.fromMappers(Mapping.EMPTY, emptyList(), emptyList(), null).cacheKey(); TestEntity secondEntity = new TestEntity(requestCacheStats, indexShard); Loader secondLoader = new Loader(secondReader, 0); writer.updateDocument(new Term("id", "0"), newDoc(0, "baz")); DirectoryReader thirdReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1)); - MappingLookup.CacheKey thirdMappingKey = MappingLookup.fromMappers(Mapping.EMPTY, emptyList(), emptyList()).cacheKey(); + MappingLookup.CacheKey thirdMappingKey = MappingLookup.fromMappers(Mapping.EMPTY, emptyList(), emptyList(), null).cacheKey(); AtomicBoolean differentIdentity = new AtomicBoolean(true); TestEntity thirdEntity = new TestEntity(requestCacheStats, differentIdentity); Loader thirdLoader = new Loader(thirdReader, 0); @@ -506,7 +506,7 @@ public void testKeyEqualsAndHashCode() throws IOException { AtomicBoolean trueBoolean = new AtomicBoolean(true); AtomicBoolean falseBoolean = new AtomicBoolean(false); MappingLookup.CacheKey mKey1 = MappingLookup.EMPTY.cacheKey(); - MappingLookup.CacheKey mKey2 = MappingLookup.fromMappers(Mapping.EMPTY, emptyList(), emptyList()).cacheKey(); + MappingLookup.CacheKey mKey2 = MappingLookup.fromMappers(Mapping.EMPTY, emptyList(), emptyList(), null).cacheKey(); Directory dir = newDirectory(); IndexWriterConfig config = newIndexWriterConfig(); IndexWriter writer = new IndexWriter(dir, config); diff --git a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java index d041121b8a96b..926ac534164f3 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java @@ -227,7 +227,8 @@ private SearchExecutionContext createSearchExecutionContext( MappingLookup mappingLookup = MappingLookup.fromMappers( mapping, Collections.singletonList(keywordFieldMapper), - Collections.emptyList() + Collections.emptyList(), + indexSettings ); return new SearchExecutionContext( 0, diff --git a/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java b/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java index 3e4ed0ebac1ba..d78c697e19c81 100644 --- a/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java @@ -169,7 +169,7 @@ public void testBuild() throws IOException { invocation -> new TestTemplateService.MockTemplateScript.Factory(((Script) invocation.getArguments()[0]).getIdOrCode()) ); List mappers = Collections.singletonList(new MockFieldMapper(fieldType)); - MappingLookup lookup = MappingLookup.fromMappers(Mapping.EMPTY, mappers, emptyList()); + MappingLookup lookup = MappingLookup.fromMappers(Mapping.EMPTY, mappers, emptyList(), idxSettings); SearchExecutionContext mockContext = new SearchExecutionContext( 0, 0, diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java b/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java index c3ce32d4ce333..fc0ac3286c6e5 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java @@ -685,7 +685,7 @@ public static MetadataRolloverService getMetadataRolloverService( new MetadataFieldMapper[] { dtfm }, Collections.emptyMap() ); - mappingLookup = MappingLookup.fromMappers(mapping, List.of(dtfm, dateFieldMapper), List.of()); + mappingLookup = MappingLookup.fromMappers(mapping, List.of(dtfm, dateFieldMapper), List.of(), null); } IndicesService indicesService = mockIndicesServices(mappingLookup); diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java index 49fe9d30239ae..02ae0853909fc 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java @@ -70,6 +70,7 @@ private TestDocumentParserContext(MappingLookup mappingLookup, SourceToParse sou } ), source, + DocumentParser.Listeners.NOOP, mappingLookup.getMapping().getRoot(), ObjectMapper.Dynamic.getRootDynamic(mappingLookup) ); diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index f057d35d6e7a9..4fe41692e1500 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -357,7 +357,8 @@ private AggregationContext createAggregationContext( Arrays.stream(fieldTypes) .map(ft -> new FieldAliasMapper(ft.name() + "-alias", ft.name() + "-alias", ft.name())) .collect(toList()), - List.of() + List.of(), + indexSettings ); BiFunction> fieldDataBuilder = (fieldType, context) -> fieldType .fielddataBuilder( @@ -465,7 +466,7 @@ private SubSearchContext buildSubSearchContext( * of stuff. */ SearchExecutionContext subContext = spy(searchExecutionContext); - MappingLookup disableNestedLookup = MappingLookup.fromMappers(Mapping.EMPTY, Set.of(), Set.of()); + MappingLookup disableNestedLookup = MappingLookup.fromMappers(Mapping.EMPTY, Set.of(), Set.of(), indexSettings); doReturn(new NestedDocuments(disableNestedLookup, bitsetFilterCache::getBitSetProducer, indexSettings.getIndexVersionCreated())) .when(subContext) .getNestedDocuments(); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java index 5369c95ad6fa7..ceb375a97d1df 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java @@ -630,7 +630,7 @@ private void runTestOnIndices(int numberIndices, CheckedConsumer concreteFields) { List mappers = concreteFields.stream().map(MockFieldMapper::new).collect(Collectors.toList()); - return MappingLookup.fromMappers(Mapping.EMPTY, mappers, emptyList()); + return MappingLookup.fromMappers(Mapping.EMPTY, mappers, emptyList(), null); } } diff --git a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java index 0b31e96ece84a..108efaa0f7691 100644 --- a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java +++ b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java @@ -1093,7 +1093,7 @@ protected final SearchExecutionContext createMockContext() { IndexFieldData.Builder builder = fieldType.fielddataBuilder(fdc); return builder.build(new IndexFieldDataCache.None(), null); }; - MappingLookup lookup = MappingLookup.fromMapping(Mapping.EMPTY); + MappingLookup lookup = MappingLookup.fromMapping(Mapping.EMPTY, null); return new SearchExecutionContext( 0, 0, From ad1938de1becbaa38eecf401c2b84c5374d6208b Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Wed, 25 Dec 2024 02:48:06 +0400 Subject: [PATCH 2/2] Add missing timeouts to rest-api-spec SLM APIs (#118958) --- docs/changelog/118958.yaml | 5 +++++ .../rest-api-spec/api/slm.delete_lifecycle.json | 11 ++++++++++- .../rest-api-spec/api/slm.execute_lifecycle.json | 11 ++++++++++- .../rest-api-spec/api/slm.execute_retention.json | 11 ++++++++++- .../rest-api-spec/api/slm.get_lifecycle.json | 11 ++++++++++- .../resources/rest-api-spec/api/slm.get_stats.json | 11 ++++++++++- .../resources/rest-api-spec/api/slm.get_status.json | 11 ++++++++++- .../rest-api-spec/api/slm.put_lifecycle.json | 11 ++++++++++- 8 files changed, 75 insertions(+), 7 deletions(-) create mode 100644 docs/changelog/118958.yaml diff --git a/docs/changelog/118958.yaml b/docs/changelog/118958.yaml new file mode 100644 index 0000000000000..fb0fd6388ab61 --- /dev/null +++ b/docs/changelog/118958.yaml @@ -0,0 +1,5 @@ +pr: 118958 +summary: Add missing timeouts to rest-api-spec SLM APIs +area: ILM+SLM +type: bug +issues: [] diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/slm.delete_lifecycle.json b/rest-api-spec/src/main/resources/rest-api-spec/api/slm.delete_lifecycle.json index 12202a7a2a7b1..1d66312f053c7 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/slm.delete_lifecycle.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/slm.delete_lifecycle.json @@ -25,6 +25,15 @@ } ] }, - "params":{} + "params":{ + "master_timeout":{ + "type":"time", + "description":"Explicit operation timeout for connection to master node" + }, + "timeout":{ + "type":"time", + "description":"Explicit operation timeout" + } + } } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/slm.execute_lifecycle.json b/rest-api-spec/src/main/resources/rest-api-spec/api/slm.execute_lifecycle.json index 1395a3d3275ae..71f1727a8638b 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/slm.execute_lifecycle.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/slm.execute_lifecycle.json @@ -25,6 +25,15 @@ } ] }, - "params":{} + "params":{ + "master_timeout":{ + "type":"time", + "description":"Explicit operation timeout for connection to master node" + }, + "timeout":{ + "type":"time", + "description":"Explicit operation timeout" + } + } } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/slm.execute_retention.json b/rest-api-spec/src/main/resources/rest-api-spec/api/slm.execute_retention.json index f6ce3e75cc379..4166122d5bf1d 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/slm.execute_retention.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/slm.execute_retention.json @@ -19,6 +19,15 @@ } ] }, - "params":{} + "params":{ + "master_timeout":{ + "type":"time", + "description":"Explicit operation timeout for connection to master node" + }, + "timeout":{ + "type":"time", + "description":"Explicit operation timeout" + } + } } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/slm.get_lifecycle.json b/rest-api-spec/src/main/resources/rest-api-spec/api/slm.get_lifecycle.json index 94d0772a405da..406fee6015522 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/slm.get_lifecycle.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/slm.get_lifecycle.json @@ -31,6 +31,15 @@ } ] }, - "params":{} + "params":{ + "master_timeout":{ + "type":"time", + "description":"Explicit operation timeout for connection to master node" + }, + "timeout":{ + "type":"time", + "description":"Explicit operation timeout" + } + } } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/slm.get_stats.json b/rest-api-spec/src/main/resources/rest-api-spec/api/slm.get_stats.json index aa693ad31711c..05281ff46cb8d 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/slm.get_stats.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/slm.get_stats.json @@ -19,6 +19,15 @@ } ] }, - "params":{} + "params":{ + "master_timeout":{ + "type":"time", + "description":"Explicit operation timeout for connection to master node" + }, + "timeout":{ + "type":"time", + "description":"Explicit operation timeout" + } + } } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/slm.get_status.json b/rest-api-spec/src/main/resources/rest-api-spec/api/slm.get_status.json index 92ba1b4c321e6..404f92f55921f 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/slm.get_status.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/slm.get_status.json @@ -19,6 +19,15 @@ } ] }, - "params":{} + "params":{ + "master_timeout":{ + "type":"time", + "description":"Explicit operation timeout for connection to master node" + }, + "timeout":{ + "type":"time", + "description":"Explicit operation timeout" + } + } } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/slm.put_lifecycle.json b/rest-api-spec/src/main/resources/rest-api-spec/api/slm.put_lifecycle.json index 7e7babb987c79..621ed870ffdbe 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/slm.put_lifecycle.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/slm.put_lifecycle.json @@ -26,7 +26,16 @@ } ] }, - "params":{}, + "params":{ + "master_timeout":{ + "type":"time", + "description":"Explicit operation timeout for connection to master node" + }, + "timeout":{ + "type":"time", + "description":"Explicit operation timeout" + } + }, "body":{ "description":"The snapshot lifecycle policy definition to register" }