Use FieldDefCreatorContext for field creation (#778)

Yelp · Nov 4, 2024 · 5fab5ab · 5fab5ab
1 parent d502e1e
commit 5fab5ab
Show file tree

Hide file tree

Showing 32 changed files with 310 additions and 98 deletions.
diff --git a/src/main/java/com/yelp/nrtsearch/server/field/AtomFieldDef.java b/src/main/java/com/yelp/nrtsearch/server/field/AtomFieldDef.java
@@ -32,8 +32,9 @@
 public class AtomFieldDef extends TextBaseFieldDef implements Sortable {
   private static final Analyzer keywordAnalyzer = new KeywordAnalyzer();
 
-  public AtomFieldDef(String name, Field requestField) {
-    super(name, requestField);
+  public AtomFieldDef(
+      String name, Field requestField, FieldDefCreator.FieldDefCreatorContext context) {
+    super(name, requestField, context);
   }
 
   @Override

diff --git a/src/main/java/com/yelp/nrtsearch/server/field/BooleanFieldDef.java b/src/main/java/com/yelp/nrtsearch/server/field/BooleanFieldDef.java
@@ -37,8 +37,9 @@
 
 /** Field class for 'BOOLEAN' field type. */
 public class BooleanFieldDef extends IndexableFieldDef<Boolean> implements TermQueryable {
-  protected BooleanFieldDef(String name, Field requestField) {
-    super(name, requestField, Boolean.class);
+  protected BooleanFieldDef(
+      String name, Field requestField, FieldDefCreator.FieldDefCreatorContext context) {
+    super(name, requestField, context, Boolean.class);
   }
 
   @Override

diff --git a/src/main/java/com/yelp/nrtsearch/server/field/ContextSuggestFieldDef.java b/src/main/java/com/yelp/nrtsearch/server/field/ContextSuggestFieldDef.java
@@ -36,9 +36,11 @@ public class ContextSuggestFieldDef extends IndexableFieldDef<Void> {
   /**
    * @param name name of field
    * @param requestField field definition from grpc request
+   * @param context creation context
    */
-  protected ContextSuggestFieldDef(String name, Field requestField) {
-    super(name, requestField, Void.class);
+  protected ContextSuggestFieldDef(
+      String name, Field requestField, FieldDefCreator.FieldDefCreatorContext context) {
+    super(name, requestField, context, Void.class);
     this.indexAnalyzer = this.parseIndexAnalyzer(requestField);
     this.searchAnalyzer = this.parseSearchAnalyzer(requestField);
   }

diff --git a/src/main/java/com/yelp/nrtsearch/server/field/DateTimeFieldDef.java b/src/main/java/com/yelp/nrtsearch/server/field/DateTimeFieldDef.java
@@ -77,8 +77,9 @@ private static DateTimeFormatter createDateTimeFormatter(String dateTimeFormat)
     }
   }
 
-  public DateTimeFieldDef(String name, Field requestField) {
-    super(name, requestField, Instant.class);
+  public DateTimeFieldDef(
+      String name, Field requestField, FieldDefCreator.FieldDefCreatorContext context) {
+    super(name, requestField, context, Instant.class);
     dateTimeFormat = requestField.getDateTimeFormat();
     dateTimeFormatter = createDateTimeFormatter(dateTimeFormat);
   }

diff --git a/src/main/java/com/yelp/nrtsearch/server/field/DoubleFieldDef.java b/src/main/java/com/yelp/nrtsearch/server/field/DoubleFieldDef.java
@@ -37,8 +37,9 @@
 /** Field class for 'DOUBLE' field type. */
 public class DoubleFieldDef extends NumberFieldDef<Double> {
 
-  public DoubleFieldDef(String name, Field requestField) {
-    super(name, requestField, DOUBLE_PARSER, Double.class);
+  public DoubleFieldDef(
+      String name, Field requestField, FieldDefCreator.FieldDefCreatorContext context) {
+    super(name, requestField, DOUBLE_PARSER, context, Double.class);
   }
 
   @Override

diff --git a/src/main/java/com/yelp/nrtsearch/server/field/FieldDefCreator.java b/src/main/java/com/yelp/nrtsearch/server/field/FieldDefCreator.java
@@ -20,6 +20,7 @@
 import com.yelp.nrtsearch.server.grpc.FieldType;
 import com.yelp.nrtsearch.server.plugins.FieldTypePlugin;
 import com.yelp.nrtsearch.server.plugins.Plugin;
+import com.yelp.nrtsearch.server.state.GlobalState;
 import com.yelp.nrtsearch.server.utils.StructValueTransformer;
 import java.util.HashMap;
 import java.util.Map;
@@ -34,6 +35,13 @@ public class FieldDefCreator {
 
   private final Map<String, FieldDefProvider<? extends FieldDef>> fieldDefMap = new HashMap<>();
 
+  /**
+   * Addition context for field definition creation.
+   *
+   * @param config service configuration
+   */
+  public record FieldDefCreatorContext(NrtsearchConfig config) {}
+
   public FieldDefCreator(NrtsearchConfig configuration) {
     register("ATOM", AtomFieldDef::new);
     register("TEXT", TextFieldDef::new);
@@ -52,12 +60,12 @@ public FieldDefCreator(NrtsearchConfig configuration) {
     // completely registered.
     register(
         "VIRTUAL",
-        (name, field) -> {
+        (name, field, context) -> {
           throw new UnsupportedOperationException("Virtual fields should be created directly");
         });
     register(
         "RUNTIME",
-        (name, field) -> {
+        (name, field, context) -> {
           throw new UnsupportedOperationException("Runtime fields should be created directly");
         });
     register("VECTOR", VectorFieldDef::createField);
@@ -72,7 +80,7 @@ public FieldDefCreator(NrtsearchConfig configuration) {
    * @param field grpc request field definition
    * @return field definition
    */
-  public FieldDef createFieldDef(String name, Field field) {
+  public FieldDef createFieldDef(String name, Field field, FieldDefCreatorContext context) {
     String type;
     if (field.getType().equals(FieldType.CUSTOM)) {
       type =
@@ -86,7 +94,17 @@ public FieldDef createFieldDef(String name, Field field) {
     if (provider == null) {
       throw new IllegalArgumentException("Invalid field type: " + type);
     }
-    return provider.get(name, field);
+    return provider.get(name, field, context);
+  }
+
+  /**
+   * Create a new {@link FieldDefCreatorContext} instance.
+   *
+   * @param globalState global state
+   * @return new context instance
+   */
+  public static FieldDefCreatorContext createContext(GlobalState globalState) {
+    return new FieldDefCreatorContext(globalState.getConfiguration());
   }
 
   private void register(Map<String, FieldDefProvider<? extends FieldDef>> fieldDefs) {

diff --git a/src/main/java/com/yelp/nrtsearch/server/field/FieldDefProvider.java b/src/main/java/com/yelp/nrtsearch/server/field/FieldDefProvider.java
@@ -30,7 +30,8 @@ public interface FieldDefProvider<T extends FieldDef> {
    *
    * @param name field name
    * @param requestField field request definition
+   * @param context creation context
    * @return concrete {@link FieldDef} instance for field
    */
-  T get(String name, Field requestField);
+  T get(String name, Field requestField, FieldDefCreator.FieldDefCreatorContext context);
 }
diff --git a/src/main/java/com/yelp/nrtsearch/server/field/FloatFieldDef.java b/src/main/java/com/yelp/nrtsearch/server/field/FloatFieldDef.java
@@ -37,8 +37,9 @@
 /** Field class for 'FLOAT' field type. */
 public class FloatFieldDef extends NumberFieldDef<Float> {
 
-  public FloatFieldDef(String name, Field requestField) {
-    super(name, requestField, FLOAT_PARSER, Float.class);
+  public FloatFieldDef(
+      String name, Field requestField, FieldDefCreator.FieldDefCreatorContext context) {
+    super(name, requestField, FLOAT_PARSER, context, Float.class);
   }
 
   @Override

diff --git a/src/main/java/com/yelp/nrtsearch/server/field/IdFieldDef.java b/src/main/java/com/yelp/nrtsearch/server/field/IdFieldDef.java
@@ -38,8 +38,9 @@
 /** Field class for defining '_ID' fields which are used to update documents */
 public class IdFieldDef extends IndexableFieldDef<String> implements TermQueryable {
 
-  protected IdFieldDef(String name, Field requestField) {
-    super(name, requestField, String.class);
+  protected IdFieldDef(
+      String name, Field requestField, FieldDefCreator.FieldDefCreatorContext context) {
+    super(name, requestField, context, String.class);
   }
 
   /**

diff --git a/src/main/java/com/yelp/nrtsearch/server/field/IndexableFieldDef.java b/src/main/java/com/yelp/nrtsearch/server/field/IndexableFieldDef.java
@@ -73,10 +73,14 @@ public enum FacetValueType {
    *
    * @param name name of field
    * @param requestField field definition from grpc request
+   * @param context creation context
    * @param docValuesObjectClass class of doc values object
    */
   protected IndexableFieldDef(
-      String name, Field requestField, Class<? super T> docValuesObjectClass) {
+      String name,
+      Field requestField,
+      FieldDefCreator.FieldDefCreatorContext context,
+      Class<? super T> docValuesObjectClass) {
     super(name);
 
     validateRequest(requestField);
@@ -116,7 +120,7 @@ protected IndexableFieldDef(
       for (Field field : requestField.getChildFieldsList()) {
         checkChildName(field.getName());
         String childName = getName() + IndexState.CHILD_FIELD_SEPARATOR + field.getName();
-        FieldDef fieldDef = FieldDefCreator.getInstance().createFieldDef(childName, field);
+        FieldDef fieldDef = FieldDefCreator.getInstance().createFieldDef(childName, field, context);
         if (!(fieldDef instanceof IndexableFieldDef)) {
           throw new IllegalArgumentException("Child field is not indexable: " + childName);
         }

diff --git a/src/main/java/com/yelp/nrtsearch/server/field/IntFieldDef.java b/src/main/java/com/yelp/nrtsearch/server/field/IntFieldDef.java
@@ -36,8 +36,9 @@
 /** Field class for 'INT' field type. */
 public class IntFieldDef extends NumberFieldDef<Integer> {
 
-  public IntFieldDef(String name, Field requestField) {
-    super(name, requestField, INT_PARSER, Integer.class);
+  public IntFieldDef(
+      String name, Field requestField, FieldDefCreator.FieldDefCreatorContext context) {
+    super(name, requestField, INT_PARSER, context, Integer.class);
   }
 
   @Override

diff --git a/src/main/java/com/yelp/nrtsearch/server/field/LatLonFieldDef.java b/src/main/java/com/yelp/nrtsearch/server/field/LatLonFieldDef.java
@@ -47,8 +47,9 @@
 
 /** Field class for 'LAT_LON' field type. */
 public class LatLonFieldDef extends IndexableFieldDef<GeoPoint> implements Sortable, GeoQueryable {
-  public LatLonFieldDef(String name, Field requestField) {
-    super(name, requestField, GeoPoint.class);
+  public LatLonFieldDef(
+      String name, Field requestField, FieldDefCreator.FieldDefCreatorContext context) {
+    super(name, requestField, context, GeoPoint.class);
   }
 
   @Override

diff --git a/src/main/java/com/yelp/nrtsearch/server/field/LongFieldDef.java b/src/main/java/com/yelp/nrtsearch/server/field/LongFieldDef.java
@@ -36,8 +36,9 @@
 /** Field class for 'LONG' field type. */
 public class LongFieldDef extends NumberFieldDef<Long> {
 
-  public LongFieldDef(String name, Field requestField) {
-    super(name, requestField, LONG_PARSER, Long.class);
+  public LongFieldDef(
+      String name, Field requestField, FieldDefCreator.FieldDefCreatorContext context) {
+    super(name, requestField, LONG_PARSER, context, Long.class);
   }
 
   @Override

diff --git a/src/main/java/com/yelp/nrtsearch/server/field/NumberFieldDef.java b/src/main/java/com/yelp/nrtsearch/server/field/NumberFieldDef.java
@@ -69,8 +69,9 @@ protected NumberFieldDef(
       String name,
       Field requestField,
       Function<String, Number> fieldParser,
+      FieldDefCreator.FieldDefCreatorContext context,
       Class<T> docValuesClass) {
-    super(name, requestField, docValuesClass);
+    super(name, requestField, context, docValuesClass);
     this.fieldParser = fieldParser;
   }
 

diff --git a/src/main/java/com/yelp/nrtsearch/server/field/ObjectFieldDef.java b/src/main/java/com/yelp/nrtsearch/server/field/ObjectFieldDef.java
@@ -47,8 +47,9 @@ public class ObjectFieldDef extends IndexableFieldDef<Struct> {
   private final Gson gson;
   private final boolean isNestedDoc;
 
-  protected ObjectFieldDef(String name, Field requestField) {
-    super(name, requestField, Struct.class);
+  protected ObjectFieldDef(
+      String name, Field requestField, FieldDefCreator.FieldDefCreatorContext context) {
+    super(name, requestField, context, Struct.class);
     this.isNestedDoc = requestField.getNestedDoc();
     gson = new GsonBuilder().serializeNulls().create();
   }

diff --git a/src/main/java/com/yelp/nrtsearch/server/field/PolygonfieldDef.java b/src/main/java/com/yelp/nrtsearch/server/field/PolygonfieldDef.java
@@ -38,8 +38,9 @@
 
 public class PolygonfieldDef extends IndexableFieldDef<Struct> implements PolygonQueryable {
 
-  protected PolygonfieldDef(String name, Field requestField) {
-    super(name, requestField, Struct.class);
+  protected PolygonfieldDef(
+      String name, Field requestField, FieldDefCreator.FieldDefCreatorContext context) {
+    super(name, requestField, context, Struct.class);
   }
 
   @Override

diff --git a/src/main/java/com/yelp/nrtsearch/server/field/TextBaseFieldDef.java b/src/main/java/com/yelp/nrtsearch/server/field/TextBaseFieldDef.java
@@ -65,15 +65,18 @@ public abstract class TextBaseFieldDef extends IndexableFieldDef<String>
   private final int ignoreAbove;
 
   /**
-   * Field constructor. Uses {@link IndexableFieldDef#IndexableFieldDef(String, Field, Class)} to do
-   * common initialization, then sets up analyzers. Analyzers are parsed through calls to the
-   * protected methods {@link #parseIndexAnalyzer(Field)} and {@link #parseSearchAnalyzer(Field)}.
+   * Field constructor. Uses {@link IndexableFieldDef#IndexableFieldDef(String, Field,
+   * FieldDefCreator.FieldDefCreatorContext, Class)} to do common initialization, then sets up
+   * analyzers. Analyzers are parsed through calls to the protected methods {@link
+   * #parseIndexAnalyzer(Field)} and {@link #parseSearchAnalyzer(Field)}.
    *
    * @param name field name
    * @param requestField field definition from grpc request
+   * @param context creation context
    */
-  protected TextBaseFieldDef(String name, Field requestField) {
-    super(name, requestField, String.class);
+  protected TextBaseFieldDef(
+      String name, Field requestField, FieldDefCreator.FieldDefCreatorContext context) {
+    super(name, requestField, context, String.class);
     indexAnalyzer = parseIndexAnalyzer(requestField);
     searchAnalyzer = parseSearchAnalyzer(requestField);
     eagerFieldGlobalOrdinals = requestField.getEagerFieldGlobalOrdinals();

diff --git a/src/main/java/com/yelp/nrtsearch/server/field/TextFieldDef.java b/src/main/java/com/yelp/nrtsearch/server/field/TextFieldDef.java
@@ -21,8 +21,9 @@
 
 /** Field class for 'TEXT' field type. */
 public class TextFieldDef extends TextBaseFieldDef {
-  public TextFieldDef(String name, Field requestField) {
-    super(name, requestField);
+  public TextFieldDef(
+      String name, Field requestField, FieldDefCreator.FieldDefCreatorContext context) {
+    super(name, requestField, context);
   }
 
   @Override

diff --git a/src/main/java/com/yelp/nrtsearch/server/field/VectorFieldDef.java b/src/main/java/com/yelp/nrtsearch/server/field/VectorFieldDef.java
@@ -208,12 +208,14 @@ private static KnnVectorsFormat getHnswScalarQuantizedVectorsFormat(
    *
    * @param name name of field
    * @param field field definition from grpc request
+   * @param context creation context
    * @return new VectorFieldDef
    */
-  static VectorFieldDef<?> createField(String name, Field field) {
+  static VectorFieldDef<?> createField(
+      String name, Field field, FieldDefCreator.FieldDefCreatorContext context) {
     return switch (field.getVectorElementType()) {
-      case VECTOR_ELEMENT_FLOAT -> new FloatVectorFieldDef(name, field);
-      case VECTOR_ELEMENT_BYTE -> new ByteVectorFieldDef(name, field);
+      case VECTOR_ELEMENT_FLOAT -> new FloatVectorFieldDef(name, field, context);
+      case VECTOR_ELEMENT_BYTE -> new ByteVectorFieldDef(name, field, context);
       default ->
           throw new IllegalArgumentException("Invalid field type: " + field.getVectorElementType());
     };
@@ -222,10 +224,15 @@ static VectorFieldDef<?> createField(String name, Field field) {
   /**
    * @param name name of field
    * @param requestField field definition from grpc request
+   * @param context creation context
    * @param docValuesClass class of doc values object
    */
-  protected VectorFieldDef(String name, Field requestField, Class<T> docValuesClass) {
-    super(name, requestField, docValuesClass);
+  protected VectorFieldDef(
+      String name,
+      Field requestField,
+      FieldDefCreator.FieldDefCreatorContext context,
+      Class<T> docValuesClass) {
+    super(name, requestField, context, docValuesClass);
     this.vectorDimensions = requestField.getVectorDimensions();
     if (isSearchable()) {
       VectorSearchType vectorSearchType = getSearchType(requestField.getVectorIndexingOptions());
@@ -322,8 +329,9 @@ public Query getKnnQuery(KnnQuery knnQuery, Query filterQuery) {
 
   /** Field class for 'FLOAT' vector field type. */
   public static class FloatVectorFieldDef extends VectorFieldDef<FloatVectorType> {
-    public FloatVectorFieldDef(String name, Field requestField) {
-      super(name, requestField, FloatVectorType.class);
+    public FloatVectorFieldDef(
+        String name, Field requestField, FieldDefCreator.FieldDefCreatorContext context) {
+      super(name, requestField, context, FloatVectorType.class);
     }
 
     @Override
@@ -462,8 +470,9 @@ public void validateVectorForSearch(float[] vector) {
 
   /** Field class for 'BYTE' vector field type. */
   public static class ByteVectorFieldDef extends VectorFieldDef<ByteVectorType> {
-    public ByteVectorFieldDef(String name, Field requestField) {
-      super(name, requestField, ByteVectorType.class);
+    public ByteVectorFieldDef(
+        String name, Field requestField, FieldDefCreator.FieldDefCreatorContext context) {
+      super(name, requestField, context, ByteVectorType.class);
     }
 
     @Override

diff --git a/src/main/java/com/yelp/nrtsearch/server/index/BackendStateManager.java b/src/main/java/com/yelp/nrtsearch/server/index/BackendStateManager.java
@@ -16,6 +16,7 @@
 package com.yelp.nrtsearch.server.index;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.yelp.nrtsearch.server.field.FieldDefCreator;
 import com.yelp.nrtsearch.server.grpc.Field;
 import com.yelp.nrtsearch.server.grpc.IndexLiveSettings;
 import com.yelp.nrtsearch.server.grpc.IndexSettings;
@@ -87,7 +88,10 @@ public synchronized void load() throws IOException {
     stateInfo = fixIndexName(stateInfo, indexName);
     UpdatedFieldInfo updatedFieldInfo =
         FieldUpdateUtils.updateFields(
-            new FieldAndFacetState(), Collections.emptyMap(), stateInfo.getFieldsMap().values());
+            new FieldAndFacetState(),
+            Collections.emptyMap(),
+            stateInfo.getFieldsMap().values(),
+            FieldDefCreator.createContext(globalState));
     currentState =
         createIndexState(stateInfo, updatedFieldInfo.fieldAndFacetState, liveSettingsOverrides);
   }
@@ -203,7 +207,8 @@ public synchronized String updateFields(List<Field> fields) throws IOException {
         FieldUpdateUtils.updateFields(
             currentState.getFieldAndFacetState(),
             currentState.getIndexStateInfo().getFieldsMap(),
-            fields);
+            fields,
+            FieldDefCreator.createContext(globalState));
     IndexStateInfo updatedStateInfo =
         replaceFields(currentState.getIndexStateInfo(), updatedFieldInfo.fields);
     ImmutableIndexState updatedIndexState =