Skip to content

Commit

Permalink
variant predefine
Browse files Browse the repository at this point in the history
  • Loading branch information
eldenmoon committed Sep 6, 2024
1 parent 648c622 commit b826b83
Show file tree
Hide file tree
Showing 16 changed files with 415 additions and 21 deletions.
16 changes: 16 additions & 0 deletions be/src/runtime/types.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
#include <utility>

#include "olap/olap_define.h"
#include "runtime/define_primitive_type.h"
#include "runtime/primitive_type.h"

namespace doris {
Expand Down Expand Up @@ -108,6 +109,21 @@ TypeDescriptor::TypeDescriptor(const std::vector<TTypeNode>& types, int* idx)
contains_nulls.push_back(node.contains_nulls[1]);
break;
}
case TTypeNodeType::VARIANT: {
// complex variant type
DCHECK(!node.__isset.scalar_type);
DCHECK_LT(*idx, types.size() - 1);
DCHECK(!node.__isset.contains_nulls);
type = TYPE_VARIANT;
contains_nulls.reserve(node.struct_fields.size());
for (size_t i = 0; i < node.struct_fields.size(); i++) {
++(*idx);
children.push_back(TypeDescriptor(types, idx));
field_names.push_back(node.struct_fields[i].name);
contains_nulls.push_back(node.struct_fields[i].contains_null);
}
break;
}
default:
DCHECK(false) << node.type;
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package org.apache.doris.catalog;


import org.apache.doris.thrift.TTypeDesc;
import org.apache.doris.thrift.TTypeNode;
import org.apache.doris.thrift.TTypeNodeType;

import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.gson.annotations.SerializedName;

import java.util.ArrayList;
import java.util.HashMap;

public class ComplexVariantType extends Type {

@SerializedName(value = "fieldMap")
private final HashMap<String, StructField> fieldMap = Maps.newHashMap();

@SerializedName(value = "fields")
private final ArrayList<StructField> predefinedFields;

public ComplexVariantType() {
this.predefinedFields = Lists.newArrayList();
}

public ComplexVariantType(ArrayList<StructField> fields) {
Preconditions.checkNotNull(fields);
this.predefinedFields = fields;
for (int i = 0; i < this.predefinedFields.size(); ++i) {
this.predefinedFields.get(i).setPosition(i);
fieldMap.put(this.predefinedFields.get(i).getName(), this.predefinedFields.get(i));
}
}

public ArrayList<StructField> getPredefinedFields() {
return predefinedFields;
}

@Override
protected String toSql(int depth) {
if (depth >= MAX_NESTING_DEPTH) {
return "variant<...>";
}
ArrayList<String> fieldsSql = Lists.newArrayList();
for (StructField f : predefinedFields) {
fieldsSql.add(f.toSql(depth + 1));
}
return String.format("variant<%s>", Joiner.on(",").join(fieldsSql));
}

@Override
protected String prettyPrint(int lpad) {
return null;
}

@Override
public void toThrift(TTypeDesc container) {
// not use ScalarType's toThrift for compatibility, because VariantType is not extends ScalarType previously
TTypeNode node = new TTypeNode();
container.types.add(node);
node.setType(TTypeNodeType.VARIANT);
// predefined fields
node.setStructFields(new ArrayList<>());
for (StructField field : predefinedFields) {
field.toThrift(container, node);
}
}

@Override
public PrimitiveType getPrimitiveType() {
return PrimitiveType.VARIANT;
}


@Override
public boolean matchesType(Type t) {
return t.isVariantType() || t.isStringType();
}

@Override
public boolean supportSubType(Type subType) {
for (Type supportedType : Type.getComplexVariantSubTypes()) {
if (subType.getPrimitiveType() == supportedType.getPrimitiveType()) {
return true;
}
}
return false;
}

@Override
public boolean equals(Object other) {
if (!(other instanceof ComplexVariantType)) {
return false;
}
ComplexVariantType otherVariantType = (ComplexVariantType) other;
return otherVariantType.getPredefinedFields().equals(predefinedFields);
}

@Override
public int getSlotSize() {
return PrimitiveType.VARIANT.getSlotSize();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -42,13 +42,17 @@ public class StructField {

public static final String DEFAULT_FIELD_NAME = "col";

public StructField(String name, Type type, String comment, boolean containsNull) {
this.name = name.toLowerCase();
public StructField(String name, Type type, String comment, boolean containsNull, boolean nameCaseSensitive) {
this.name = (nameCaseSensitive ? name : name.toLowerCase());
this.type = type;
this.comment = comment;
this.containsNull = containsNull;
}

public StructField(String name, Type type, String comment, boolean containsNull) {
this(name, type, comment, containsNull, false);
}

public StructField(String name, Type type) {
this(name, type, null, true);
}
Expand Down
42 changes: 41 additions & 1 deletion fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,8 @@ public abstract class Type {
private static final ArrayList<Type> arraySubTypes;
private static final ArrayList<Type> mapSubTypes;
private static final ArrayList<Type> structSubTypes;

private static final ArrayList<Type> complexVariantSubTypes;
private static final ArrayList<ScalarType> trivialTypes;

static {
Expand Down Expand Up @@ -170,6 +172,8 @@ public abstract class Type {
typeMap.put("MAP", Type.MAP);
typeMap.put("OBJECT", Type.UNSUPPORTED);
typeMap.put("ARRAY", Type.ARRAY);
typeMap.put("IPV4", Type.IPV4);
typeMap.put("IPV6", Type.IPV6);
typeMap.put("QUANTILE_STATE", Type.QUANTILE_STATE);
}

Expand Down Expand Up @@ -306,6 +310,27 @@ public abstract class Type {
structSubTypes.add(ARRAY);
structSubTypes.add(MAP);
structSubTypes.add(STRUCT);

complexVariantSubTypes = Lists.newArrayList();
complexVariantSubTypes.add(BOOLEAN);
complexVariantSubTypes.addAll(integerTypes);
complexVariantSubTypes.add(FLOAT);
complexVariantSubTypes.add(DOUBLE);
complexVariantSubTypes.add(DECIMAL32); // same DEFAULT_DECIMALV3
complexVariantSubTypes.add(DECIMAL64);
complexVariantSubTypes.add(DECIMAL128);
complexVariantSubTypes.add(DECIMAL256);
complexVariantSubTypes.add(DATE);
complexVariantSubTypes.add(DATETIME);
complexVariantSubTypes.add(DATEV2);
complexVariantSubTypes.add(DATETIMEV2);
complexVariantSubTypes.add(IPV4);
complexVariantSubTypes.add(IPV6);
complexVariantSubTypes.add(CHAR);
complexVariantSubTypes.add(VARCHAR);
complexVariantSubTypes.add(STRING);
complexVariantSubTypes.add(ARRAY);
complexVariantSubTypes.add(NULL);
}

public static final Set<Class> DATE_SUPPORTED_JAVA_TYPE = Sets.newHashSet(LocalDate.class, java.util.Date.class,
Expand Down Expand Up @@ -373,6 +398,10 @@ public static ArrayList<Type> getStructSubTypes() {
return structSubTypes;
}

public static ArrayList<Type> getComplexVariantSubTypes() {
return complexVariantSubTypes;
}

/**
* Return true if this is complex type and support subType
*/
Expand Down Expand Up @@ -554,7 +583,7 @@ public boolean isJsonbType() {
}

public boolean isVariantType() {
return isScalarType(PrimitiveType.VARIANT);
return isScalarType(PrimitiveType.VARIANT) || isComplexVariant();
}

// only metric types have the following constraint:
Expand Down Expand Up @@ -686,6 +715,10 @@ public boolean isStructType() {
return this instanceof StructType;
}

public boolean isComplexVariant() {
return this instanceof ComplexVariantType;
}

public boolean isAnyType() {
return this instanceof AnyType;
}
Expand Down Expand Up @@ -979,6 +1012,13 @@ private boolean exceedsMaxNestingDepth(int d) {
} else if (isMapType()) {
MapType mapType = (MapType) this;
return mapType.getValueType().exceedsMaxNestingDepth(d + 1);
} else if (isComplexVariant()) {
ComplexVariantType complexVariantType = (ComplexVariantType) this;
for (StructField f : complexVariantType.getPredefinedFields()) {
if (f.getType().exceedsMaxNestingDepth(d + 1)) {
return true;
}
}
} else {
Preconditions.checkState(isScalarType() || isAggStateType());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1658,6 +1658,7 @@ dataType
: complex=ARRAY LT dataType GT #complexDataType
| complex=MAP LT dataType COMMA dataType GT #complexDataType
| complex=STRUCT LT complexColTypeList GT #complexDataType
| VARIANT LT variantSubColTypeList GT #variantPredefinedFields
| AGG_STATE LT functionNameIdentifier
LEFT_PAREN dataTypes+=dataTypeWithNullable
(COMMA dataTypes+=dataTypeWithNullable)* RIGHT_PAREN GT #aggStateDataType
Expand Down Expand Up @@ -1708,6 +1709,13 @@ complexColType
: identifier COLON dataType commentSpec?
;

variantSubColTypeList
: variantSubColType (COMMA variantSubColType)*
;
variantSubColType
: qualifiedName COLON dataType commentSpec?
;

commentSpec
: COMMENT STRING_LITERAL
;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -324,7 +324,7 @@ public void analyze() throws AnalysisException {
FunctionName fnName = new FunctionName(getFnName(type));
Function searchDesc = new Function(fnName, Arrays.asList(getActualArgTypes(collectChildReturnTypes())),
Type.INVALID, false);
if (type.isScalarType()) {
if (type.isScalarType() || type.isComplexVariant()) {
fn = Env.getCurrentEnv().getFunction(searchDesc, Function.CompareMode.IS_IDENTICAL);
} else {
createComplexTypeCastFunction();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ public class InvertedIndexUtil {
public static String INVERTED_INDEX_PARSER_COARSE_GRANULARITY = "coarse_grained";

public static String INVERTED_INDEX_PARSER_CHAR_FILTER_TYPE = "char_filter_type";
public static String INVERTED_INDEX_SUB_COLUMN_PATH = "sub_column_path";
public static String INVERTED_INDEX_PARSER_CHAR_FILTER_PATTERN = "char_filter_pattern";
public static String INVERTED_INDEX_PARSER_CHAR_FILTER_REPLACEMENT = "char_filter_replacement";

Expand Down Expand Up @@ -157,7 +158,8 @@ public static void checkInvertedIndexProperties(Map<String, String> properties)
INVERTED_INDEX_PARSER_CHAR_FILTER_REPLACEMENT,
INVERTED_INDEX_PARSER_IGNORE_ABOVE_KEY,
INVERTED_INDEX_PARSER_LOWERCASE_KEY,
INVERTED_INDEX_PARSER_STOPWORDS_KEY
INVERTED_INDEX_PARSER_STOPWORDS_KEY,
INVERTED_INDEX_SUB_COLUMN_PATH
));

for (String key : properties.keySet()) {
Expand Down
9 changes: 5 additions & 4 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
Original file line number Diff line number Diff line change
Expand Up @@ -330,8 +330,9 @@ public void createChildrenColumn(Type type, Column column) {
v.setIsAllowNull(((MapType) type).getIsValueContainsNull());
column.addChildrenColumn(k);
column.addChildrenColumn(v);
} else if (type.isStructType()) {
ArrayList<StructField> fields = ((StructType) type).getFields();
} else if (type.isStructType() || type.isComplexVariant()) {
ArrayList<StructField> fields = type.isStructType()
? ((StructType) type).getFields() : ((ComplexVariantType) type).getPredefinedFields();
for (StructField field : fields) {
Column c = new Column(field.getName(), field.getType());
c.setIsAllowNull(field.getContainsNull());
Expand Down Expand Up @@ -657,7 +658,7 @@ private void toChildrenThrift(Column column, TColumn tColumn) {
tColumn.setChildrenColumn(new ArrayList<>());
setChildrenTColumn(k, tColumn);
setChildrenTColumn(v, tColumn);
} else if (column.type.isStructType()) {
} else if (column.type.isStructType() || column.type.isComplexVariant()) {
List<Column> childrenColumns = column.getChildren();
tColumn.setChildrenColumn(new ArrayList<>());
for (Column children : childrenColumns) {
Expand Down Expand Up @@ -802,7 +803,7 @@ public OlapFile.ColumnPB toPb(Set<String> bfColumns, List<Index> indexes) throws
builder.addChildrenColumns(k.toPb(Sets.newHashSet(), Lists.newArrayList()));
Column v = this.getChildren().get(1);
builder.addChildrenColumns(v.toPb(Sets.newHashSet(), Lists.newArrayList()));
} else if (this.type.isStructType()) {
} else if (this.type.isStructType() || this.type.isComplexVariant()) {
List<Column> childrenColumns = this.getChildren();
for (Column c : childrenColumns) {
builder.addChildrenColumns(c.toPb(Sets.newHashSet(), Lists.newArrayList()));
Expand Down
Loading

0 comments on commit b826b83

Please sign in to comment.