From b826b837672d9d1b4e888d6dba1154553d1dd0de Mon Sep 17 00:00:00 2001 From: lihangyu <15605149486@163.com> Date: Tue, 20 Aug 2024 22:28:15 +0800 Subject: [PATCH] variant predefine --- be/src/runtime/types.cpp | 16 +++ .../doris/catalog/ComplexVariantType.java | 123 ++++++++++++++++ .../org/apache/doris/catalog/StructField.java | 8 +- .../java/org/apache/doris/catalog/Type.java | 42 +++++- .../org/apache/doris/nereids/DorisParser.g4 | 8 ++ .../org/apache/doris/analysis/CastExpr.java | 2 +- .../doris/analysis/InvertedIndexUtil.java | 4 +- .../java/org/apache/doris/catalog/Column.java | 9 +- .../nereids/parser/LogicalPlanBuilder.java | 33 ++++- .../plans/commands/info/ColumnDefinition.java | 12 ++ .../plans/commands/info/CreateTableInfo.java | 8 +- .../nereids/types/ComplexVariantType.java | 133 ++++++++++++++++++ .../apache/doris/nereids/types/DataType.java | 7 +- .../doris/nereids/types/StructField.java | 19 ++- .../apache/doris/persist/gson/GsonUtils.java | 4 +- .../org/apache/doris/qe/SessionVariable.java | 8 ++ 16 files changed, 415 insertions(+), 21 deletions(-) create mode 100644 fe/fe-common/src/main/java/org/apache/doris/catalog/ComplexVariantType.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/types/ComplexVariantType.java diff --git a/be/src/runtime/types.cpp b/be/src/runtime/types.cpp index 14ba4b2cebdece..a18d73c0e9cc56 100644 --- a/be/src/runtime/types.cpp +++ b/be/src/runtime/types.cpp @@ -28,6 +28,7 @@ #include #include "olap/olap_define.h" +#include "runtime/define_primitive_type.h" #include "runtime/primitive_type.h" namespace doris { @@ -108,6 +109,21 @@ TypeDescriptor::TypeDescriptor(const std::vector& types, int* idx) contains_nulls.push_back(node.contains_nulls[1]); break; } + case TTypeNodeType::VARIANT: { + // complex variant type + DCHECK(!node.__isset.scalar_type); + DCHECK_LT(*idx, types.size() - 1); + DCHECK(!node.__isset.contains_nulls); + type = TYPE_VARIANT; + contains_nulls.reserve(node.struct_fields.size()); + for (size_t i = 0; i < node.struct_fields.size(); i++) { + ++(*idx); + children.push_back(TypeDescriptor(types, idx)); + field_names.push_back(node.struct_fields[i].name); + contains_nulls.push_back(node.struct_fields[i].contains_null); + } + break; + } default: DCHECK(false) << node.type; } diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/ComplexVariantType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/ComplexVariantType.java new file mode 100644 index 00000000000000..421719a8515a77 --- /dev/null +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/ComplexVariantType.java @@ -0,0 +1,123 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.catalog; + + +import org.apache.doris.thrift.TTypeDesc; +import org.apache.doris.thrift.TTypeNode; +import org.apache.doris.thrift.TTypeNodeType; + +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.gson.annotations.SerializedName; + +import java.util.ArrayList; +import java.util.HashMap; + +public class ComplexVariantType extends Type { + + @SerializedName(value = "fieldMap") + private final HashMap fieldMap = Maps.newHashMap(); + + @SerializedName(value = "fields") + private final ArrayList predefinedFields; + + public ComplexVariantType() { + this.predefinedFields = Lists.newArrayList(); + } + + public ComplexVariantType(ArrayList fields) { + Preconditions.checkNotNull(fields); + this.predefinedFields = fields; + for (int i = 0; i < this.predefinedFields.size(); ++i) { + this.predefinedFields.get(i).setPosition(i); + fieldMap.put(this.predefinedFields.get(i).getName(), this.predefinedFields.get(i)); + } + } + + public ArrayList getPredefinedFields() { + return predefinedFields; + } + + @Override + protected String toSql(int depth) { + if (depth >= MAX_NESTING_DEPTH) { + return "variant<...>"; + } + ArrayList fieldsSql = Lists.newArrayList(); + for (StructField f : predefinedFields) { + fieldsSql.add(f.toSql(depth + 1)); + } + return String.format("variant<%s>", Joiner.on(",").join(fieldsSql)); + } + + @Override + protected String prettyPrint(int lpad) { + return null; + } + + @Override + public void toThrift(TTypeDesc container) { + // not use ScalarType's toThrift for compatibility, because VariantType is not extends ScalarType previously + TTypeNode node = new TTypeNode(); + container.types.add(node); + node.setType(TTypeNodeType.VARIANT); + // predefined fields + node.setStructFields(new ArrayList<>()); + for (StructField field : predefinedFields) { + field.toThrift(container, node); + } + } + + @Override + public PrimitiveType getPrimitiveType() { + return PrimitiveType.VARIANT; + } + + + @Override + public boolean matchesType(Type t) { + return t.isVariantType() || t.isStringType(); + } + + @Override + public boolean supportSubType(Type subType) { + for (Type supportedType : Type.getComplexVariantSubTypes()) { + if (subType.getPrimitiveType() == supportedType.getPrimitiveType()) { + return true; + } + } + return false; + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof ComplexVariantType)) { + return false; + } + ComplexVariantType otherVariantType = (ComplexVariantType) other; + return otherVariantType.getPredefinedFields().equals(predefinedFields); + } + + @Override + public int getSlotSize() { + return PrimitiveType.VARIANT.getSlotSize(); + } +} diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/StructField.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/StructField.java index ecbfd30ca23538..c08c93135a8d3c 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/StructField.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/StructField.java @@ -42,13 +42,17 @@ public class StructField { public static final String DEFAULT_FIELD_NAME = "col"; - public StructField(String name, Type type, String comment, boolean containsNull) { - this.name = name.toLowerCase(); + public StructField(String name, Type type, String comment, boolean containsNull, boolean nameCaseSensitive) { + this.name = (nameCaseSensitive ? name : name.toLowerCase()); this.type = type; this.comment = comment; this.containsNull = containsNull; } + public StructField(String name, Type type, String comment, boolean containsNull) { + this(name, type, comment, containsNull, false); + } + public StructField(String name, Type type) { this(name, type, null, true); } diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java index 1dcd062261b2b8..2c4f62e136a4fc 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java @@ -135,6 +135,8 @@ public abstract class Type { private static final ArrayList arraySubTypes; private static final ArrayList mapSubTypes; private static final ArrayList structSubTypes; + + private static final ArrayList complexVariantSubTypes; private static final ArrayList trivialTypes; static { @@ -170,6 +172,8 @@ public abstract class Type { typeMap.put("MAP", Type.MAP); typeMap.put("OBJECT", Type.UNSUPPORTED); typeMap.put("ARRAY", Type.ARRAY); + typeMap.put("IPV4", Type.IPV4); + typeMap.put("IPV6", Type.IPV6); typeMap.put("QUANTILE_STATE", Type.QUANTILE_STATE); } @@ -306,6 +310,27 @@ public abstract class Type { structSubTypes.add(ARRAY); structSubTypes.add(MAP); structSubTypes.add(STRUCT); + + complexVariantSubTypes = Lists.newArrayList(); + complexVariantSubTypes.add(BOOLEAN); + complexVariantSubTypes.addAll(integerTypes); + complexVariantSubTypes.add(FLOAT); + complexVariantSubTypes.add(DOUBLE); + complexVariantSubTypes.add(DECIMAL32); // same DEFAULT_DECIMALV3 + complexVariantSubTypes.add(DECIMAL64); + complexVariantSubTypes.add(DECIMAL128); + complexVariantSubTypes.add(DECIMAL256); + complexVariantSubTypes.add(DATE); + complexVariantSubTypes.add(DATETIME); + complexVariantSubTypes.add(DATEV2); + complexVariantSubTypes.add(DATETIMEV2); + complexVariantSubTypes.add(IPV4); + complexVariantSubTypes.add(IPV6); + complexVariantSubTypes.add(CHAR); + complexVariantSubTypes.add(VARCHAR); + complexVariantSubTypes.add(STRING); + complexVariantSubTypes.add(ARRAY); + complexVariantSubTypes.add(NULL); } public static final Set DATE_SUPPORTED_JAVA_TYPE = Sets.newHashSet(LocalDate.class, java.util.Date.class, @@ -373,6 +398,10 @@ public static ArrayList getStructSubTypes() { return structSubTypes; } + public static ArrayList getComplexVariantSubTypes() { + return complexVariantSubTypes; + } + /** * Return true if this is complex type and support subType */ @@ -554,7 +583,7 @@ public boolean isJsonbType() { } public boolean isVariantType() { - return isScalarType(PrimitiveType.VARIANT); + return isScalarType(PrimitiveType.VARIANT) || isComplexVariant(); } // only metric types have the following constraint: @@ -686,6 +715,10 @@ public boolean isStructType() { return this instanceof StructType; } + public boolean isComplexVariant() { + return this instanceof ComplexVariantType; + } + public boolean isAnyType() { return this instanceof AnyType; } @@ -979,6 +1012,13 @@ private boolean exceedsMaxNestingDepth(int d) { } else if (isMapType()) { MapType mapType = (MapType) this; return mapType.getValueType().exceedsMaxNestingDepth(d + 1); + } else if (isComplexVariant()) { + ComplexVariantType complexVariantType = (ComplexVariantType) this; + for (StructField f : complexVariantType.getPredefinedFields()) { + if (f.getType().exceedsMaxNestingDepth(d + 1)) { + return true; + } + } } else { Preconditions.checkState(isScalarType() || isAggStateType()); } diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index 087af9d717de13..b5fc3b2df1017d 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -1658,6 +1658,7 @@ dataType : complex=ARRAY LT dataType GT #complexDataType | complex=MAP LT dataType COMMA dataType GT #complexDataType | complex=STRUCT LT complexColTypeList GT #complexDataType + | VARIANT LT variantSubColTypeList GT #variantPredefinedFields | AGG_STATE LT functionNameIdentifier LEFT_PAREN dataTypes+=dataTypeWithNullable (COMMA dataTypes+=dataTypeWithNullable)* RIGHT_PAREN GT #aggStateDataType @@ -1708,6 +1709,13 @@ complexColType : identifier COLON dataType commentSpec? ; +variantSubColTypeList + : variantSubColType (COMMA variantSubColType)* + ; +variantSubColType + : qualifiedName COLON dataType commentSpec? + ; + commentSpec : COMMENT STRING_LITERAL ; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java index de257991ca6ba4..7efe8c150dab4a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java @@ -324,7 +324,7 @@ public void analyze() throws AnalysisException { FunctionName fnName = new FunctionName(getFnName(type)); Function searchDesc = new Function(fnName, Arrays.asList(getActualArgTypes(collectChildReturnTypes())), Type.INVALID, false); - if (type.isScalarType()) { + if (type.isScalarType() || type.isComplexVariant()) { fn = Env.getCurrentEnv().getFunction(searchDesc, Function.CompareMode.IS_IDENTICAL); } else { createComplexTypeCastFunction(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/InvertedIndexUtil.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/InvertedIndexUtil.java index abba2762d5664d..c0cccd7ce53c8c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/InvertedIndexUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/InvertedIndexUtil.java @@ -41,6 +41,7 @@ public class InvertedIndexUtil { public static String INVERTED_INDEX_PARSER_COARSE_GRANULARITY = "coarse_grained"; public static String INVERTED_INDEX_PARSER_CHAR_FILTER_TYPE = "char_filter_type"; + public static String INVERTED_INDEX_SUB_COLUMN_PATH = "sub_column_path"; public static String INVERTED_INDEX_PARSER_CHAR_FILTER_PATTERN = "char_filter_pattern"; public static String INVERTED_INDEX_PARSER_CHAR_FILTER_REPLACEMENT = "char_filter_replacement"; @@ -157,7 +158,8 @@ public static void checkInvertedIndexProperties(Map properties) INVERTED_INDEX_PARSER_CHAR_FILTER_REPLACEMENT, INVERTED_INDEX_PARSER_IGNORE_ABOVE_KEY, INVERTED_INDEX_PARSER_LOWERCASE_KEY, - INVERTED_INDEX_PARSER_STOPWORDS_KEY + INVERTED_INDEX_PARSER_STOPWORDS_KEY, + INVERTED_INDEX_SUB_COLUMN_PATH )); for (String key : properties.keySet()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java index fc69c31f0e98f3..44f989356e6c73 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java @@ -330,8 +330,9 @@ public void createChildrenColumn(Type type, Column column) { v.setIsAllowNull(((MapType) type).getIsValueContainsNull()); column.addChildrenColumn(k); column.addChildrenColumn(v); - } else if (type.isStructType()) { - ArrayList fields = ((StructType) type).getFields(); + } else if (type.isStructType() || type.isComplexVariant()) { + ArrayList fields = type.isStructType() + ? ((StructType) type).getFields() : ((ComplexVariantType) type).getPredefinedFields(); for (StructField field : fields) { Column c = new Column(field.getName(), field.getType()); c.setIsAllowNull(field.getContainsNull()); @@ -657,7 +658,7 @@ private void toChildrenThrift(Column column, TColumn tColumn) { tColumn.setChildrenColumn(new ArrayList<>()); setChildrenTColumn(k, tColumn); setChildrenTColumn(v, tColumn); - } else if (column.type.isStructType()) { + } else if (column.type.isStructType() || column.type.isComplexVariant()) { List childrenColumns = column.getChildren(); tColumn.setChildrenColumn(new ArrayList<>()); for (Column children : childrenColumns) { @@ -802,7 +803,7 @@ public OlapFile.ColumnPB toPb(Set bfColumns, List indexes) throws builder.addChildrenColumns(k.toPb(Sets.newHashSet(), Lists.newArrayList())); Column v = this.getChildren().get(1); builder.addChildrenColumns(v.toPb(Sets.newHashSet(), Lists.newArrayList())); - } else if (this.type.isStructType()) { + } else if (this.type.isStructType() || this.type.isComplexVariant()) { List childrenColumns = this.getChildren(); for (Column c : childrenColumns) { builder.addChildrenColumns(c.toPb(Sets.newHashSet(), Lists.newArrayList())); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 324ab808226930..2420145e53e272 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -205,6 +205,9 @@ import org.apache.doris.nereids.DorisParser.UpdateContext; import org.apache.doris.nereids.DorisParser.UserIdentifyContext; import org.apache.doris.nereids.DorisParser.UserVariableContext; +import org.apache.doris.nereids.DorisParser.VariantPredefinedFieldsContext; +import org.apache.doris.nereids.DorisParser.VariantSubColTypeContext; +import org.apache.doris.nereids.DorisParser.VariantSubColTypeListContext; import org.apache.doris.nereids.DorisParser.WhereClauseContext; import org.apache.doris.nereids.DorisParser.WindowFrameContext; import org.apache.doris.nereids.DorisParser.WindowSpecContext; @@ -463,6 +466,7 @@ import org.apache.doris.nereids.types.ArrayType; import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.nereids.types.BooleanType; +import org.apache.doris.nereids.types.ComplexVariantType; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.LargeIntType; import org.apache.doris.nereids.types.MapType; @@ -2748,6 +2752,8 @@ public ColumnDefinition visitColumnDef(ColumnDefContext ctx) { ? visitPrimitiveDataType(((PrimitiveDataTypeContext) ctx.type)) : ctx.type instanceof ComplexDataTypeContext ? visitComplexDataType((ComplexDataTypeContext) ctx.type) + : ctx.type instanceof VariantPredefinedFieldsContext + ? visitVariantPredefinedFields((VariantPredefinedFieldsContext) ctx.type) : visitAggStateDataType((AggStateDataTypeContext) ctx.type); colType = colType.conversion(); boolean isKey = ctx.KEY() != null; @@ -3582,6 +3588,30 @@ public DataType visitPrimitiveDataType(PrimitiveDataTypeContext ctx) { }); } + @Override + public DataType visitVariantPredefinedFields(VariantPredefinedFieldsContext ctx) { + return new ComplexVariantType(visitVariantSubColTypeList(ctx.variantSubColTypeList())); + } + + @Override + public List visitVariantSubColTypeList(VariantSubColTypeListContext ctx) { + return ctx.variantSubColType().stream().map( + this::visitVariantSubColType).collect(ImmutableList.toImmutableList()); + } + + @Override + public StructField visitVariantSubColType(VariantSubColTypeContext ctx) { + String comment; + if (ctx.commentSpec() != null) { + comment = ctx.commentSpec().STRING_LITERAL().getText(); + comment = LogicalPlanBuilderAssistant.escapeBackSlash(comment.substring(1, comment.length() - 1)); + } else { + comment = ""; + } + return new StructField(ctx.qualifiedName().getText(), + typedVisit(ctx.dataType()), true, comment, true /*name case-sensitive*/); + } + @Override public DataType visitComplexDataType(ComplexDataTypeContext ctx) { return ParserUtils.withOrigin(ctx, () -> { @@ -3612,7 +3642,8 @@ public StructField visitComplexColType(ComplexColTypeContext ctx) { } else { comment = ""; } - return new StructField(ctx.identifier().getText(), typedVisit(ctx.dataType()), true, comment); + return new StructField(ctx.identifier().getText(), + typedVisit(ctx.dataType()), true, comment, false /*name case-insensitive*/); } private String parseConstant(ConstantContext context) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java index 0b2694cc311b4c..a867b41c823472 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java @@ -466,6 +466,18 @@ private void validateDataType(Type catalogType) { } } } + } else if (catalogType.isComplexVariant()) { + ArrayList predefinedFields = + ((org.apache.doris.catalog.ComplexVariantType) catalogType).getPredefinedFields(); + Set fieldNames = new HashSet<>(); + for (org.apache.doris.catalog.StructField field : predefinedFields) { + Type fieldType = field.getType(); + validateNestedType(catalogType, fieldType); + if (!fieldNames.add(field.getName())) { + throw new AnalysisException("Duplicate field name " + field.getName() + + " in struct " + catalogType.toSql()); + } + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java index 730c6f115a3da6..77817ebf468d90 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java @@ -610,10 +610,10 @@ public void validate(ConnectContext ctx) { if (distinct.size() != indexes.size()) { throw new AnalysisException("index name must be unique."); } - if (distinctCol.size() != indexes.size()) { - throw new AnalysisException( - "same index columns have multiple same type index is not allowed."); - } + // if (distinctCol.size() != indexes.size()) { + // throw new AnalysisException( + // "same index columns have multiple same type index is not allowed."); + // } } generatedColumnCheck(ctx); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/ComplexVariantType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/ComplexVariantType.java new file mode 100644 index 00000000000000..4f17e4424f1450 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/ComplexVariantType.java @@ -0,0 +1,133 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.types; + +import org.apache.doris.catalog.Type; +import org.apache.doris.nereids.analyzer.ComplexDataType; +import org.apache.doris.nereids.annotation.Developing; +import org.apache.doris.nereids.exceptions.AnalysisException; + +import com.google.common.base.Suppliers; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +/** + * + * ComplexVariant type in Nereids which could predefine some fields of nested columns. + * Example: VARIANT <`a.b`:INT, a.c:DATETIMEV2> + * + */ +@Developing +public class ComplexVariantType extends DataType implements ComplexDataType { + // for test compatibility + + public static final List TEST_PREDEFINE_LIST = new ArrayList() { + { + add(new StructField("PREDEFINE_COL1", StringType.INSTANCE, true, "", true)); + add(new StructField("PREDEFINE_COL2", IntegerType.INSTANCE, true, "", true)); + add(new StructField("PREDEFINE_COL3", DecimalV3Type.SYSTEM_DEFAULT, true, "", true)); + add(new StructField("PREDEFINE_COL4", DateTimeV2Type.SYSTEM_DEFAULT, true, "", true)); + } + }; + public static final ComplexVariantType TEST_INSTANCE = new ComplexVariantType(TEST_PREDEFINE_LIST); + // predefined fields + private final List predefinedFields; + private final Supplier> pathToFields; + + // No predefined fields + public ComplexVariantType() { + predefinedFields = Lists.newArrayList(); + pathToFields = Suppliers.memoize(Maps::newTreeMap); + } + + /** + * Contains predefined fields like struct + */ + public ComplexVariantType(List fields) { + this.predefinedFields = ImmutableList.copyOf(Objects.requireNonNull(fields, "fields should not be null")); + this.pathToFields = Suppliers.memoize(() -> this.predefinedFields.stream().collect(ImmutableMap.toImmutableMap( + StructField::getName, f -> f, (f1, f2) -> { + throw new AnalysisException("The name of the struct field cannot be repeated." + + " same name fields are " + f1 + " and " + f2); + }))); + } + + public Map getPathToFields() { + return pathToFields.get(); + } + + public List getPredefinedFields() { + return predefinedFields; + } + + @Override + public boolean acceptsType(DataType other) { + return other instanceof VariantType || other instanceof ComplexVariantType; + } + + @Override + public Type toCatalogDataType() { + return predefinedFields.isEmpty() ? Type.VARIANT + : new org.apache.doris.catalog.ComplexVariantType(predefinedFields.stream() + .map(StructField::toCatalogDataType) + .collect(Collectors.toCollection(ArrayList::new))); + } + + @Override + public String toSql() { + if (predefinedFields.isEmpty()) { + return "VARIANT"; + } + return "VARIANT<" + predefinedFields.stream().map(StructField::toSql).collect(Collectors.joining(",")) + ">"; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + return super.equals(o); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode()); + } + + @Override + public int width() { + return 24; + } + + @Override + public String toString() { + return toSql(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java index 0c83098bf8f1f2..1c78d1aeb32008 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java @@ -33,6 +33,7 @@ import org.apache.doris.nereids.types.coercion.IntegralType; import org.apache.doris.nereids.types.coercion.NumericType; import org.apache.doris.nereids.types.coercion.PrimitiveType; +import org.apache.doris.qe.ConnectContext; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -303,6 +304,10 @@ public static DataType convertPrimitiveFromStrings(List types) { dataType = IPv6Type.INSTANCE; break; case "variant": + if (ConnectContext.get().getSessionVariable().useVariantAsComplexVariant) { + dataType = ComplexVariantType.TEST_INSTANCE; + break; + } dataType = VariantType.INSTANCE; break; default: @@ -617,7 +622,7 @@ public boolean isMapType() { } public boolean isVariantType() { - return this instanceof VariantType; + return this instanceof VariantType || this instanceof ComplexVariantType; } public boolean isStructType() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructField.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructField.java index e095f25aa66bc8..7567d9295a0be9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructField.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructField.java @@ -32,19 +32,28 @@ public class StructField { private final boolean nullable; private final String comment; + private final boolean nameCaseSensitive; + /** * StructField Constructor * @param name The name of this field * @param dataType The data type of this field * @param nullable Indicates if values of this field can be `null` values + * @param nameCaseSensitive Indicates if name is case-sensitive */ - public StructField(String name, DataType dataType, boolean nullable, String comment) { - this.name = Objects.requireNonNull(name, "name should not be null").toLowerCase(Locale.ROOT); + public StructField(String name, DataType dataType, boolean nullable, String comment, boolean nameCaseSensitive) { + this.nameCaseSensitive = nameCaseSensitive; + this.name = nameCaseSensitive ? Objects.requireNonNull(name, "name should not be null") + : Objects.requireNonNull(name, "name should not be null").toLowerCase(Locale.ROOT); this.dataType = Objects.requireNonNull(dataType, "dataType should not be null"); this.nullable = nullable; this.comment = Objects.requireNonNull(comment, "comment should not be null"); } + public StructField(String name, DataType dataType, boolean nullable, String comment) { + this(name, dataType, nullable, comment, false); + } + public String getName() { return name; } @@ -69,16 +78,16 @@ public StructField conversion() { } public StructField withDataType(DataType dataType) { - return new StructField(name, dataType, nullable, comment); + return new StructField(name, dataType, nullable, comment, nameCaseSensitive); } public StructField withDataTypeAndNullable(DataType dataType, boolean nullable) { - return new StructField(name, dataType, nullable, comment); + return new StructField(name, dataType, nullable, comment, nameCaseSensitive); } public org.apache.doris.catalog.StructField toCatalogDataType() { return new org.apache.doris.catalog.StructField( - name, dataType.toCatalogDataType(), comment, nullable); + name, dataType.toCatalogDataType(), comment, nullable, nameCaseSensitive); } public String toSql() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java index f29dbaceab2313..29760c6d189b96 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java @@ -68,6 +68,7 @@ import org.apache.doris.catalog.AnyType; import org.apache.doris.catalog.ArrayType; import org.apache.doris.catalog.BrokerTable; +import org.apache.doris.catalog.ComplexVariantType; import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.DistributionInfo; import org.apache.doris.catalog.Env; @@ -287,7 +288,8 @@ public class GsonUtils { .registerSubtype(AnyType.class, AnyType.class.getSimpleName()) .registerSubtype(MultiRowType.class, MultiRowType.class.getSimpleName()) .registerSubtype(TemplateType.class, TemplateType.class.getSimpleName()) - .registerSubtype(VariantType.class, VariantType.class.getSimpleName()); + .registerSubtype(VariantType.class, VariantType.class.getSimpleName()) + .registerSubtype(ComplexVariantType.class, ComplexVariantType.class.getSimpleName()); // runtime adapter for class "Expr" private static final RuntimeTypeAdapterFactory exprAdapterFactory diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 8db24006293249..d9aa75f035d9d3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -616,6 +616,9 @@ public class SessionVariable implements Serializable, Writable { public static final String DESCRIBE_EXTEND_VARIANT_COLUMN = "describe_extend_variant_column"; + // test only flag + public static final String USE_VARIANT_AS_COMPLEX_VARIANT = "use_variant_as_complex_variant"; + public static final String FORCE_JNI_SCANNER = "force_jni_scanner"; public static final String ENABLE_COUNT_PUSH_DOWN_FOR_EXTERNAL_TABLE = "enable_count_push_down_for_external_table"; @@ -1158,6 +1161,9 @@ public int getBeNumberForTest() { @VariableMgr.VarAttr(name = DESCRIBE_EXTEND_VARIANT_COLUMN, needForward = true) public boolean enableDescribeExtendVariantColumn = false; + @VariableMgr.VarAttr(name = USE_VARIANT_AS_COMPLEX_VARIANT, needForward = true, fuzzy = true) + public boolean useVariantAsComplexVariant = false; + @VariableMgr.VarAttr(name = PROFILLING) public boolean profiling = false; @@ -2205,12 +2211,14 @@ public void initFuzzyModeVariables() { this.enableDeleteSubPredicateV2 = false; this.topnOptLimitThreshold = 0; this.enableSyncRuntimeFilterSize = true; + this.useVariantAsComplexVariant = true; } else { this.rewriteOrToInPredicateThreshold = 2; this.enableFunctionPushdown = true; this.enableDeleteSubPredicateV2 = true; this.topnOptLimitThreshold = 1024; this.enableSyncRuntimeFilterSize = false; + this.useVariantAsComplexVariant = false; } /*