Changes to support IP field in star tree indexing (#16641)

* Changes to support IP --------- Signed-off-by: bharath-techie <[email protected]>
opensearch-project · Dec 19, 2024 · e6d71d2 · e6d71d2
1 parent b5f651f
commit e6d71d2
Show file tree

Hide file tree

Showing 13 changed files with 204 additions and 43 deletions.
diff --git a/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java b/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java
@@ -56,7 +56,7 @@ public class StarTreeMapperIT extends OpenSearchIntegTestCase {
         .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(512, ByteSizeUnit.MB))
         .build();
 
-    private static XContentBuilder createMinimalTestMapping(boolean invalidDim, boolean invalidMetric, boolean ipdim) {
+    private static XContentBuilder createMinimalTestMapping(boolean invalidDim, boolean invalidMetric, boolean wildcard) {
         try {
             return jsonBuilder().startObject()
                 .startObject("composite")
@@ -68,7 +68,7 @@ private static XContentBuilder createMinimalTestMapping(boolean invalidDim, bool
                 .endObject()
                 .startArray("ordered_dimensions")
                 .startObject()
-                .field("name", getDim(invalidDim, ipdim))
+                .field("name", getDim(invalidDim, wildcard))
                 .endObject()
                 .startObject()
                 .field("name", "keyword_dv")
@@ -102,8 +102,16 @@ private static XContentBuilder createMinimalTestMapping(boolean invalidDim, bool
                 .field("type", "keyword")
                 .field("doc_values", false)
                 .endObject()
+                .startObject("ip_no_dv")
+                .field("type", "ip")
+                .field("doc_values", false)
+                .endObject()
                 .startObject("ip")
                 .field("type", "ip")
+                .field("doc_values", true)
+                .endObject()
+                .startObject("wildcard")
+                .field("type", "wildcard")
                 .field("doc_values", false)
                 .endObject()
                 .endObject()
@@ -362,11 +370,11 @@ private XContentBuilder getMappingWithDuplicateFields(boolean isDuplicateDim, bo
         return mapping;
     }
 
-    private static String getDim(boolean hasDocValues, boolean isKeyword) {
+    private static String getDim(boolean hasDocValues, boolean isWildCard) {
         if (hasDocValues) {
-            return random().nextBoolean() ? "numeric" : "keyword";
-        } else if (isKeyword) {
-            return "ip";
+            return random().nextBoolean() ? "numeric" : random().nextBoolean() ? "keyword" : "ip_no_dv";
+        } else if (isWildCard) {
+            return "wildcard";
         }
         return "numeric_dv";
     }
@@ -748,7 +756,7 @@ public void testUnsupportedDim() {
             () -> prepareCreate(TEST_INDEX).setSettings(settings).setMapping(createMinimalTestMapping(false, false, true)).get()
         );
         assertEquals(
-            "Failed to parse mapping [_doc]: unsupported field type associated with dimension [ip] as part of star tree field [startree-1]",
+            "Failed to parse mapping [_doc]: unsupported field type associated with dimension [wildcard] as part of star tree field [startree-1]",
             ex.getMessage()
         );
     }

diff --git a/...n/java/org/opensearch/index/codec/composite/composite912/Composite912DocValuesWriter.java b/...n/java/org/opensearch/index/codec/composite/composite912/Composite912DocValuesWriter.java
@@ -33,9 +33,11 @@
 import org.opensearch.index.compositeindex.datacube.startree.builder.StarTreesBuilder;
 import org.opensearch.index.compositeindex.datacube.startree.index.CompositeIndexValues;
 import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues;
+import org.opensearch.index.fielddata.IndexNumericFieldData;
+import org.opensearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData;
 import org.opensearch.index.mapper.CompositeMappedFieldType;
 import org.opensearch.index.mapper.DocCountFieldMapper;
-import org.opensearch.index.mapper.KeywordFieldMapper;
+import org.opensearch.index.mapper.MappedFieldType;
 import org.opensearch.index.mapper.MapperService;
 
 import java.io.IOException;
@@ -44,6 +46,7 @@
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -262,22 +265,38 @@ public SortedSetDocValues getSortedSet(FieldInfo field) {
                         return DocValues.emptySortedSet();
                     }
                 });
-            }
-            // TODO : change this logic to evaluate for sortedNumericField specifically
-            else {
+            } else if (isSortedNumericField(compositeField)) {
                 fieldProducerMap.put(compositeField, new EmptyDocValuesProducer() {
                     @Override
                     public SortedNumericDocValues getSortedNumeric(FieldInfo field) {
                         return DocValues.emptySortedNumeric();
                     }
                 });
+            } else {
+                throw new IllegalStateException(
+                    String.format(Locale.ROOT, "Unsupported DocValues field associated with the composite field : %s", compositeField)
+                );
             }
         }
         compositeFieldSet.remove(compositeField);
     }
 
     private boolean isSortedSetField(String field) {
-        return mapperService.fieldType(field) instanceof KeywordFieldMapper.KeywordFieldType;
+        MappedFieldType ft = mapperService.fieldType(field);
+        assert ft.isAggregatable();
+        return ft.fielddataBuilder(
+            "",
+            () -> { throw new UnsupportedOperationException("SearchLookup not available"); }
+        ) instanceof SortedSetOrdinalsIndexFieldData.Builder;
+    }
+
+    private boolean isSortedNumericField(String field) {
+        MappedFieldType ft = mapperService.fieldType(field);
+        assert ft.isAggregatable();
+        return ft.fielddataBuilder(
+            "",
+            () -> { throw new UnsupportedOperationException("SearchLookup not available"); }
+        ) instanceof IndexNumericFieldData.Builder;
     }
 
     @Override
@@ -370,5 +389,4 @@ private static SegmentWriteState getSegmentWriteState(SegmentWriteState segmentW
             segmentWriteState.segmentSuffix
         );
     }
-
 }
diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionFactory.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionFactory.java
@@ -24,7 +24,8 @@
 import java.util.stream.Collectors;
 
 import static org.opensearch.index.compositeindex.datacube.DateDimension.CALENDAR_INTERVALS;
-import static org.opensearch.index.compositeindex.datacube.KeywordDimension.KEYWORD;
+import static org.opensearch.index.compositeindex.datacube.IpDimension.IP;
+import static org.opensearch.index.compositeindex.datacube.OrdinalDimension.ORDINAL;
 
 /**
  * Dimension factory class mainly used to parse and create dimension from the mappings
@@ -44,8 +45,10 @@ public static Dimension parseAndCreateDimension(
                 return parseAndCreateDateDimension(name, dimensionMap, c);
             case NumericDimension.NUMERIC:
                 return new NumericDimension(name);
-            case KEYWORD:
-                return new KeywordDimension(name);
+            case ORDINAL:
+                return new OrdinalDimension(name);
+            case IP:
+                return new IpDimension(name);
             default:
                 throw new IllegalArgumentException(
                     String.format(Locale.ROOT, "unsupported field type associated with dimension [%s] as part of star tree field", name)
@@ -69,8 +72,10 @@ public static Dimension parseAndCreateDimension(
                 return parseAndCreateDateDimension(name, dimensionMap, c);
             case NUMERIC:
                 return new NumericDimension(name);
-            case KEYWORD:
-                return new KeywordDimension(name);
+            case ORDINAL:
+                return new OrdinalDimension(name);
+            case IP:
+                return new IpDimension(name);
             default:
                 throw new IllegalArgumentException(
                     String.format(Locale.ROOT, "unsupported field type associated with star tree dimension [%s]", name)

diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionType.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionType.java
@@ -30,8 +30,14 @@ public enum DimensionType {
     DATE,
 
     /**
-     * Represents a keyword dimension type.
-     * This is used for dimensions that contain keyword ordinals.
+     * Represents dimension types which uses ordinals.
+     * This is used for dimensions that contain sortedSet ordinals.
      */
-    KEYWORD
+    ORDINAL,
+
+    /**
+     * Represents an IP dimension type.
+     * This is used for dimensions that contain IP ordinals.
+     */
+    IP
 }
diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/IpDimension.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/IpDimension.java
@@ -0,0 +1,82 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.compositeindex.datacube;
+
+import org.apache.lucene.index.DocValuesType;
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.index.mapper.CompositeDataCubeFieldType;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+import java.util.function.Consumer;
+
+/**
+ * Composite index keyword dimension class
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public class IpDimension implements Dimension {
+    public static final String IP = "ip";
+    private final String field;
+
+    public IpDimension(String field) {
+        this.field = field;
+    }
+
+    @Override
+    public String getField() {
+        return field;
+    }
+
+    @Override
+    public int getNumSubDimensions() {
+        return 1;
+    }
+
+    @Override
+    public void setDimensionValues(Long value, Consumer<Long> dimSetter) {
+        // This will set the keyword dimension value's ordinal
+        dimSetter.accept(value);
+    }
+
+    @Override
+    public List<String> getSubDimensionNames() {
+        return List.of(field);
+    }
+
+    @Override
+    public DocValuesType getDocValuesType() {
+        return DocValuesType.SORTED_SET;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(CompositeDataCubeFieldType.NAME, field);
+        builder.field(CompositeDataCubeFieldType.TYPE, IP);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        IpDimension dimension = (IpDimension) o;
+        return Objects.equals(field, dimension.getField());
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(field);
+    }
+}
diff --git a/...ositeindex/datacube/KeywordDimension.java → ...ositeindex/datacube/OrdinalDimension.java b/...ositeindex/datacube/KeywordDimension.java → ...ositeindex/datacube/OrdinalDimension.java
@@ -24,11 +24,11 @@
  * @opensearch.experimental
  */
 @ExperimentalApi
-public class KeywordDimension implements Dimension {
-    public static final String KEYWORD = "keyword";
+public class OrdinalDimension implements Dimension {
+    public static final String ORDINAL = "ordinal";
     private final String field;
 
-    public KeywordDimension(String field) {
+    public OrdinalDimension(String field) {
         this.field = field;
     }
 
@@ -62,7 +62,7 @@ public DocValuesType getDocValuesType() {
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
         builder.field(CompositeDataCubeFieldType.NAME, field);
-        builder.field(CompositeDataCubeFieldType.TYPE, KEYWORD);
+        builder.field(CompositeDataCubeFieldType.TYPE, ORDINAL);
         builder.endObject();
         return builder;
     }
@@ -71,7 +71,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     public boolean equals(Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
-        KeywordDimension dimension = (KeywordDimension) o;
+        OrdinalDimension dimension = (OrdinalDimension) o;
         return Objects.equals(field, dimension.getField());
     }
 

diff --git a/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java
@@ -52,6 +52,7 @@
 import org.opensearch.common.logging.DeprecationLogger;
 import org.opensearch.common.network.InetAddresses;
 import org.opensearch.common.network.NetworkAddress;
+import org.opensearch.index.compositeindex.datacube.DimensionType;
 import org.opensearch.index.fielddata.IndexFieldData;
 import org.opensearch.index.fielddata.ScriptDocValues;
 import org.opensearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData;
@@ -68,6 +69,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.function.BiFunction;
 import java.util.function.Supplier;
 
@@ -161,6 +163,11 @@ public IpFieldMapper build(BuilderContext context) {
             );
         }
 
+        @Override
+        public Optional<DimensionType> getSupportedDataCubeDimensionType() {
+            return Optional.of(DimensionType.IP);
+        }
+
     }
 
     public static final TypeParser PARSER = new TypeParser((n, c) -> {

diff --git a/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java
@@ -259,7 +259,7 @@ public KeywordFieldMapper build(BuilderContext context) {
 
         @Override
         public Optional<DimensionType> getSupportedDataCubeDimensionType() {
-            return Optional.of(DimensionType.KEYWORD);
+            return Optional.of(DimensionType.ORDINAL);
         }
     }