Skip to content

Commit

Permalink
handling edge cases
Browse files Browse the repository at this point in the history 
Signed-off-by: bharath-techie <[email protected]>
  • Loading branch information
@bharath-techie
bharath-techie committed Dec 20, 2024
1 parent b1f8fba commit 995f912
Show file tree
Hide file tree
Showing 5 changed files with 152 additions and 5 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- Added a precaution to handle extreme date values during sorting to prevent `arithmetic_exception: long overflow` ([#16812](https://github.com/opensearch-project/OpenSearch/pull/16812)).
- Add search replica stats to segment replication stats API ([#16678](https://github.com/opensearch-project/OpenSearch/pull/16678))
- Introduce a setting to disable download of full cluster state from remote on term mismatch([#16798](https://github.com/opensearch-project/OpenSearch/pull/16798/))
- Changes to support IP field in star tree indexing([#16641](https://github.com/opensearch-project/OpenSearch/pull/16641/))
- Support object fields in star-tree index([#16728](https://github.com/opensearch-project/OpenSearch/pull/16728/))

### Dependencies
- Bump `com.google.cloud:google-cloud-core-http` from 2.23.0 to 2.47.0 ([#16504](https://github.com/opensearch-project/OpenSearch/pull/16504))
Expand Down
132 changes: 131 additions & 1 deletion server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,98 @@ private static XContentBuilder createNestedTestMapping() {
}
}

private static XContentBuilder createNestedTestMappingForArray() {
try {
return jsonBuilder().startObject()
.startObject("composite")
.startObject("startree-1")
.field("type", "star_tree")
.startObject("config")
.startObject("date_dimension")
.field("name", "timestamp")
.endObject()
.startArray("ordered_dimensions")
.startObject()
.field("name", "status")
.endObject()
.startObject()
.field("name", "nested.nested1.keyword_dv")
.endObject()
.endArray()
.startArray("metrics")
.startObject()
.field("name", "nested3.numeric_dv")
.endObject()
.endArray()
.endObject()
.endObject()
.endObject()
.startObject("properties")
.startObject("timestamp")
.field("type", "date")
.endObject()
.startObject("status")
.field("type", "integer")
.endObject()
.startObject("nested3")
.startObject("properties")
.startObject("numeric_dv")
.field("type", "integer")
.field("doc_values", true)
.endObject()
.endObject()
.endObject()
.startObject("numeric")
.field("type", "integer")
.field("doc_values", false)
.endObject()
.startObject("nested")
.startObject("properties")
.startObject("nested1")
.startObject("properties")
.startObject("status")
.field("type", "integer")
.field("doc_values", true)
.endObject()
.startObject("keyword_dv")
.field("type", "keyword")
.field("doc_values", true)
.endObject()
.endObject()
.endObject()
.endObject()
.endObject()
.startObject("nested-not-startree")
.startObject("properties")
.startObject("nested1")
.startObject("properties")
.startObject("status")
.field("type", "integer")
.field("doc_values", true)
.endObject()
.startObject("keyword_dv")
.field("type", "keyword")
.field("doc_values", true)
.endObject()
.endObject()
.endObject()
.endObject()
.endObject()
.startObject("keyword")
.field("type", "keyword")
.field("doc_values", false)
.endObject()
.startObject("ip")
.field("type", "ip")
.field("doc_values", false)
.endObject()
.endObject()
.endObject();
} catch (IOException e) {
throw new IllegalStateException(e);
}
}

private static XContentBuilder createDateTestMapping(boolean duplicate) {
try {
return jsonBuilder().startObject()
Expand Down Expand Up @@ -693,6 +785,7 @@ public void testCompositeIndexWithArraysInCompositeField() throws IOException {
}

public void testCompositeIndexWithArraysInNestedCompositeField() throws IOException {
// here nested.nested1.status is part of the composite field but "nested" field itself is an array
prepareCreate(TEST_INDEX).setSettings(settings).setMapping(createNestedTestMapping()).get();
// Attempt to index a document with an array field
XContentBuilder doc = jsonBuilder().startObject()
Expand Down Expand Up @@ -726,7 +819,7 @@ public void testCompositeIndexWithArraysInNestedCompositeField() throws IOExcept

public void testCompositeIndexWithArraysInChildNestedCompositeField() throws IOException {
prepareCreate(TEST_INDEX).setSettings(settings).setMapping(createNestedTestMapping()).get();
// Attempt to index a document with an array field
// here nested.nested1.status is part of the composite field but "nested.nested1" field is an array
XContentBuilder doc = jsonBuilder().startObject()
.field("timestamp", "2023-06-01T12:00:00Z")
.startObject("nested")
Expand Down Expand Up @@ -754,6 +847,43 @@ public void testCompositeIndexWithArraysInChildNestedCompositeField() throws IOE
);
}

public void testCompositeIndexWithArraysInNestedCompositeFieldSameNameAsNormalField() throws IOException {
prepareCreate(TEST_INDEX).setSettings(settings).setMapping(createNestedTestMappingForArray()).get();
// here status is part of the composite field but "nested.nested1.status" field is an array which is not
// part of composite field
XContentBuilder doc = jsonBuilder().startObject()
.field("timestamp", "2023-06-01T12:00:00Z")
.startObject("nested")
.startObject("nested1")
.startArray("status")
.value(10)
.value(20)
.value(30)
.endArray()
.endObject()
.endObject()
.field("status", "200")
.endObject();
// Index the document and refresh
// Index the document and refresh
IndexResponse indexResponse = client().prepareIndex(TEST_INDEX).setSource(doc).get();

assertEquals(RestStatus.CREATED, indexResponse.status());

client().admin().indices().prepareRefresh(TEST_INDEX).get();
// Verify the document was indexed
SearchResponse searchResponse = client().prepareSearch(TEST_INDEX).setQuery(QueryBuilders.matchAllQuery()).get();

assertEquals(1, searchResponse.getHits().getTotalHits().value);

// Verify the values in the indexed document
SearchHit hit = searchResponse.getHits().getAt(0);
assertEquals("2023-06-01T12:00:00Z", hit.getSourceAsMap().get("timestamp"));

int values = Integer.parseInt((String) hit.getSourceAsMap().get("status"));
assertEquals(200, values);
}

public void testCompositeIndexWithNestedArraysInNonCompositeField() throws IOException {
prepareCreate(TEST_INDEX).setSettings(settings).setMapping(createNestedTestMapping()).get();
// Attempt to index a document with an array field
Expand Down
2 changes: 1 addition & 1 deletion server/src/main/java/org/opensearch/common/util/FeatureFlags.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ public class FeatureFlags {
* aggregations.
*/
public static final String STAR_TREE_INDEX = "opensearch.experimental.feature.composite_index.star_tree.enabled";
public static final Setting<Boolean> STAR_TREE_INDEX_SETTING = Setting.boolSetting(STAR_TREE_INDEX, false, Property.NodeScope);
public static final Setting<Boolean> STAR_TREE_INDEX_SETTING = Setting.boolSetting(STAR_TREE_INDEX, true, Property.NodeScope);

/**
* Gates the functionality of application based configuration templates.
Expand Down
14 changes: 12 additions & 2 deletions server/src/main/java/org/opensearch/index/mapper/DocumentParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -661,10 +661,20 @@ private static void parseNonDynamicArray(ParseContext context, ObjectMapper mapp
throws IOException {
XContentParser parser = context.parser();
XContentParser.Token token;
String path = context.path().pathAsText(arrayFieldName);
boolean isNested = path.contains(".") || context.mapperService().isCompositeIndexFieldNestedField("nested");
// block array values for composite index fields
// Examples : Assume original index has 2 fields - status , nested.nested1.status
// case 1 : if status is part of composite index and nested.nested1.status is not part of composite index,
// then nested.nested1.status/nested.nested1/nested array should not be blocked
// case 2 : if nested.nested1.status is part of composite index and status is not part of composite index,
// then arrays in nested/nested.nested1 and nested.nested1.status fields should be blocked
// but arrays in status should not be blocked
if (context.indexSettings().isCompositeIndex()
&& (context.mapperService().isFieldPartOfCompositeIndex(arrayFieldName)
|| context.mapperService().isFieldPartOfCompositeIndex(context.path().pathAsText(arrayFieldName)))) {
&& ((isNested == false
&& (context.mapperService().isFieldPartOfCompositeIndex(arrayFieldName)
|| context.mapperService().isCompositeIndexFieldNestedField(arrayFieldName)))
|| (isNested && context.mapperService().isCompositeIndexFieldNestedField(path)))) {
throw new MapperParsingException(
String.format(
Locale.ROOT,
Expand Down
7 changes: 6 additions & 1 deletion server/src/main/java/org/opensearch/index/mapper/MapperService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -707,7 +707,12 @@ private Set<CompositeMappedFieldType> getCompositeFieldTypesFromMapper() {
}

public boolean isFieldPartOfCompositeIndex(String field) {
return fieldsPartOfCompositeMappings.contains(field) || nestedFieldsPartOfCompositeMappings.contains(field);
return fieldsPartOfCompositeMappings.contains(field);
}

public boolean isCompositeIndexFieldNestedField(String field) {
return nestedFieldsPartOfCompositeMappings.contains(field);

}

public ObjectMapper getObjectMapper(String name) {
Expand Down

0 comments on commit 995f912

Please sign in to comment.