From 928136c7dea535dc3b416af9c58c2e31b8f40e71 Mon Sep 17 00:00:00 2001 From: jacktengg <18241664+jacktengg@users.noreply.github.com> Date: Thu, 19 Oct 2023 00:34:48 +0800 Subject: [PATCH] fix --- .../apache/doris/catalog/PrimitiveType.java | 38 +++++++++++++++++++ .../org/apache/doris/catalog/ScalarType.java | 37 ++++++++---------- .../java/org/apache/doris/catalog/Type.java | 1 + .../doris/nereids/types/DecimalV3Type.java | 11 +----- .../org/apache/doris/qe/ConnectProcessor.java | 2 +- .../org/apache/doris/qe/SessionVariable.java | 4 +- .../decimalv3/test_decimalv3_overflow.groovy | 1 - 7 files changed, 59 insertions(+), 35 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java index e5a5726e52e8c70..4c9fae123eae1a7 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java @@ -125,6 +125,7 @@ public static ImmutableSetMultimap getImplicitCast builder.put(NULL_TYPE, DECIMAL32); builder.put(NULL_TYPE, DECIMAL64); builder.put(NULL_TYPE, DECIMAL128); + builder.put(NULL_TYPE, DECIMAL256); builder.put(NULL_TYPE, CHAR); builder.put(NULL_TYPE, VARCHAR); builder.put(NULL_TYPE, STRING); @@ -150,6 +151,7 @@ public static ImmutableSetMultimap getImplicitCast builder.put(BOOLEAN, DECIMAL32); builder.put(BOOLEAN, DECIMAL64); builder.put(BOOLEAN, DECIMAL128); + builder.put(BOOLEAN, DECIMAL256); builder.put(BOOLEAN, VARCHAR); builder.put(BOOLEAN, STRING); // Tinyint @@ -169,6 +171,7 @@ public static ImmutableSetMultimap getImplicitCast builder.put(TINYINT, DECIMAL32); builder.put(TINYINT, DECIMAL64); builder.put(TINYINT, DECIMAL128); + builder.put(TINYINT, DECIMAL256); builder.put(TINYINT, VARCHAR); builder.put(TINYINT, STRING); builder.put(TINYINT, TIME); @@ -190,6 +193,7 @@ public static ImmutableSetMultimap getImplicitCast builder.put(SMALLINT, DECIMAL32); builder.put(SMALLINT, DECIMAL64); builder.put(SMALLINT, DECIMAL128); + builder.put(SMALLINT, DECIMAL256); builder.put(SMALLINT, VARCHAR); builder.put(SMALLINT, STRING); builder.put(SMALLINT, TIME); @@ -211,6 +215,7 @@ public static ImmutableSetMultimap getImplicitCast builder.put(INT, DECIMAL32); builder.put(INT, DECIMAL64); builder.put(INT, DECIMAL128); + builder.put(INT, DECIMAL256); builder.put(INT, VARCHAR); builder.put(INT, STRING); builder.put(INT, TIME); @@ -232,6 +237,7 @@ public static ImmutableSetMultimap getImplicitCast builder.put(BIGINT, DECIMAL32); builder.put(BIGINT, DECIMAL64); builder.put(BIGINT, DECIMAL128); + builder.put(BIGINT, DECIMAL256); builder.put(BIGINT, VARCHAR); builder.put(BIGINT, STRING); builder.put(BIGINT, TIME); @@ -253,6 +259,7 @@ public static ImmutableSetMultimap getImplicitCast builder.put(LARGEINT, DECIMAL32); builder.put(LARGEINT, DECIMAL64); builder.put(LARGEINT, DECIMAL128); + builder.put(LARGEINT, DECIMAL256); builder.put(LARGEINT, VARCHAR); builder.put(LARGEINT, STRING); builder.put(LARGEINT, TIME); @@ -274,6 +281,7 @@ public static ImmutableSetMultimap getImplicitCast builder.put(FLOAT, DECIMAL32); builder.put(FLOAT, DECIMAL64); builder.put(FLOAT, DECIMAL128); + builder.put(FLOAT, DECIMAL256); builder.put(FLOAT, VARCHAR); builder.put(FLOAT, STRING); builder.put(FLOAT, TIME); @@ -295,6 +303,7 @@ public static ImmutableSetMultimap getImplicitCast builder.put(DOUBLE, DECIMAL32); builder.put(DOUBLE, DECIMAL64); builder.put(DOUBLE, DECIMAL128); + builder.put(DOUBLE, DECIMAL256); builder.put(DOUBLE, VARCHAR); builder.put(DOUBLE, STRING); builder.put(DOUBLE, TIME); @@ -316,6 +325,7 @@ public static ImmutableSetMultimap getImplicitCast builder.put(DATE, DECIMAL32); builder.put(DATE, DECIMAL64); builder.put(DATE, DECIMAL128); + builder.put(DATE, DECIMAL256); builder.put(DATE, VARCHAR); builder.put(DATE, STRING); // Datetime @@ -335,6 +345,7 @@ public static ImmutableSetMultimap getImplicitCast builder.put(DATETIME, DECIMAL32); builder.put(DATETIME, DECIMAL64); builder.put(DATETIME, DECIMAL128); + builder.put(DATETIME, DECIMAL256); builder.put(DATETIME, VARCHAR); builder.put(DATETIME, STRING); // DateV2 @@ -354,6 +365,7 @@ public static ImmutableSetMultimap getImplicitCast builder.put(DATEV2, DECIMAL32); builder.put(DATEV2, DECIMAL64); builder.put(DATEV2, DECIMAL128); + builder.put(DATEV2, DECIMAL256); builder.put(DATEV2, VARCHAR); builder.put(DATEV2, STRING); // DatetimeV2 @@ -373,6 +385,7 @@ public static ImmutableSetMultimap getImplicitCast builder.put(DATETIMEV2, DECIMAL32); builder.put(DATETIMEV2, DECIMAL64); builder.put(DATETIMEV2, DECIMAL128); + builder.put(DATETIMEV2, DECIMAL256); builder.put(DATETIMEV2, VARCHAR); builder.put(DATETIMEV2, STRING); // Char @@ -393,6 +406,7 @@ public static ImmutableSetMultimap getImplicitCast builder.put(CHAR, DECIMAL32); builder.put(CHAR, DECIMAL64); builder.put(CHAR, DECIMAL128); + builder.put(CHAR, DECIMAL256); builder.put(CHAR, VARCHAR); builder.put(CHAR, STRING); builder.put(CHAR, TIME); @@ -414,6 +428,7 @@ public static ImmutableSetMultimap getImplicitCast builder.put(VARCHAR, DECIMAL32); builder.put(VARCHAR, DECIMAL64); builder.put(VARCHAR, DECIMAL128); + builder.put(VARCHAR, DECIMAL256); builder.put(VARCHAR, VARCHAR); builder.put(VARCHAR, JSONB); builder.put(VARCHAR, VARIANT); @@ -438,6 +453,7 @@ public static ImmutableSetMultimap getImplicitCast builder.put(STRING, DECIMAL32); builder.put(STRING, DECIMAL64); builder.put(STRING, DECIMAL128); + builder.put(STRING, DECIMAL256); builder.put(STRING, VARCHAR); builder.put(STRING, JSONB); builder.put(STRING, VARIANT); @@ -458,6 +474,7 @@ public static ImmutableSetMultimap getImplicitCast builder.put(DECIMALV2, DECIMAL32); builder.put(DECIMALV2, DECIMAL64); builder.put(DECIMALV2, DECIMAL128); + builder.put(DECIMALV2, DECIMAL256); builder.put(DECIMALV2, VARCHAR); builder.put(DECIMALV2, STRING); @@ -473,6 +490,7 @@ public static ImmutableSetMultimap getImplicitCast builder.put(DECIMAL32, DECIMAL32); builder.put(DECIMAL32, DECIMAL64); builder.put(DECIMAL32, DECIMAL128); + builder.put(DECIMAL32, DECIMAL256); builder.put(DECIMAL32, VARCHAR); builder.put(DECIMAL32, STRING); @@ -488,6 +506,7 @@ public static ImmutableSetMultimap getImplicitCast builder.put(DECIMAL64, DECIMAL32); builder.put(DECIMAL64, DECIMAL64); builder.put(DECIMAL64, DECIMAL128); + builder.put(DECIMAL64, DECIMAL256); builder.put(DECIMAL64, VARCHAR); builder.put(DECIMAL64, STRING); @@ -503,9 +522,27 @@ public static ImmutableSetMultimap getImplicitCast builder.put(DECIMAL128, DECIMAL32); builder.put(DECIMAL128, DECIMAL64); builder.put(DECIMAL128, DECIMAL128); + builder.put(DECIMAL128, DECIMAL256); builder.put(DECIMAL128, VARCHAR); builder.put(DECIMAL128, STRING); + // decimal256 + builder.put(DECIMAL256, BOOLEAN); + builder.put(DECIMAL256, TINYINT); + builder.put(DECIMAL256, SMALLINT); + builder.put(DECIMAL256, INT); + builder.put(DECIMAL256, BIGINT); + builder.put(DECIMAL256, LARGEINT); + builder.put(DECIMAL256, FLOAT); + builder.put(DECIMAL256, DOUBLE); + builder.put(DECIMAL256, DECIMALV2); + builder.put(DECIMAL256, DECIMAL32); + builder.put(DECIMAL256, DECIMAL64); + builder.put(DECIMAL256, DECIMAL128); + builder.put(DECIMAL256, DECIMAL256); + builder.put(DECIMAL256, VARCHAR); + builder.put(DECIMAL256, STRING); + // JSONB builder.put(JSONB, BOOLEAN); builder.put(JSONB, TINYINT); @@ -519,6 +556,7 @@ public static ImmutableSetMultimap getImplicitCast builder.put(JSONB, DECIMAL32); builder.put(JSONB, DECIMAL64); builder.put(JSONB, DECIMAL128); + // builder.put(JSONB, DECIMAL256); builder.put(JSONB, VARCHAR); builder.put(JSONB, STRING); builder.put(JSONB, VARIANT); diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java index 618aecb996966be..7271429b66d6845 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java @@ -72,7 +72,6 @@ public class ScalarType extends Type { public static final int MAX_JSONB_LENGTH = 0x7fffffff - 4; // Hive, mysql, sql server standard. - public static final int MAX_PRECISION = 38; public static final int MAX_DECIMALV2_PRECISION = 27; public static final int MAX_DECIMALV2_SCALE = 9; public static final int MAX_DECIMAL32_PRECISION = 9; @@ -81,6 +80,7 @@ public class ScalarType extends Type { public static final int MAX_DECIMAL256_PRECISION = 76; public static final int DEFAULT_MIN_AVG_DECIMAL128_SCALE = 4; public static final int MAX_DATETIMEV2_SCALE = 6; + public static final int MAX_PRECISION = MAX_DECIMAL256_PRECISION; private long byteSize = -1; @@ -139,6 +139,7 @@ public static ScalarType createType(PrimitiveType type, int len, int precision, case DECIMAL32: case DECIMAL64: case DECIMAL128: + case DECIMAL256: return createDecimalV3Type(precision, scale); case DECIMALV2: return createDecimalType(precision, scale); @@ -211,6 +212,8 @@ public static ScalarType createType(PrimitiveType type) { return DEFAULT_DECIMAL64; case DECIMAL128: return DEFAULT_DECIMAL128; + case DECIMAL256: + return DEFAULT_DECIMAL256; case DECIMALV2: return DEFAULT_DECIMALV2; case LARGEINT: @@ -386,8 +389,10 @@ public static PrimitiveType getSuitableDecimalType(int precision, boolean decima return PrimitiveType.DECIMAL32; } else if (precision <= MAX_DECIMAL64_PRECISION) { return PrimitiveType.DECIMAL64; - } else { + } else if (precision <= MAX_DECIMAL128_PRECISION) { return PrimitiveType.DECIMAL128; + } else { + return PrimitiveType.DECIMAL256; } } @@ -470,22 +475,6 @@ public static Type getDefaultDateType(Type type) { } } - /** - * create a wider decimal type. - */ - public static ScalarType createWiderDecimalV3Type(int precision, int scale) { - ScalarType type = new ScalarType(getSuitableDecimalType(precision, false)); - if (precision <= MAX_DECIMAL32_PRECISION) { - type.precision = MAX_DECIMAL32_PRECISION; - } else if (precision <= MAX_DECIMAL64_PRECISION) { - type.precision = MAX_DECIMAL64_PRECISION; - } else { - type.precision = MAX_DECIMAL128_PRECISION; - } - type.scale = scale; - return type; - } - public static ScalarType createVarcharType(int len) { // length checked in analysis ScalarType type = new ScalarType(PrimitiveType.VARCHAR); @@ -612,6 +601,7 @@ public String toSql(int depth) { case DECIMAL32: case DECIMAL64: case DECIMAL128: + case DECIMAL256: String typeName = "decimalv3"; if (Strings.isNullOrEmpty(precisionStr)) { stringBuilder.append(typeName).append("(").append(precision) @@ -702,6 +692,7 @@ public void toThrift(TTypeDesc container) { case DECIMAL32: case DECIMAL64: case DECIMAL128: + case DECIMAL256: case DATETIMEV2: { Preconditions.checkArgument(precision >= scale, String.format("given precision %d is out of scale bound %d", precision, scale)); @@ -725,14 +716,16 @@ public void toThrift(TTypeDesc container) { public int decimalPrecision() { Preconditions.checkState(type == PrimitiveType.DECIMALV2 || type == PrimitiveType.DATETIMEV2 || type == PrimitiveType.TIMEV2 || type == PrimitiveType.DECIMAL32 - || type == PrimitiveType.DECIMAL64 || type == PrimitiveType.DECIMAL128); + || type == PrimitiveType.DECIMAL64 || type == PrimitiveType.DECIMAL128 + || type == PrimitiveType.DECIMAL256); return precision; } public int decimalScale() { Preconditions.checkState(type == PrimitiveType.DECIMALV2 || type == PrimitiveType.DATETIMEV2 || type == PrimitiveType.TIMEV2 || type == PrimitiveType.DECIMAL32 - || type == PrimitiveType.DECIMAL64 || type == PrimitiveType.DECIMAL128); + || type == PrimitiveType.DECIMAL64 || type == PrimitiveType.DECIMAL128 + || type == PrimitiveType.DECIMAL256); return scale; } @@ -943,9 +936,9 @@ public ScalarType getMinResolutionDecimal() { case BIGINT: return createDecimalType(19); case FLOAT: - return createDecimalTypeInternal(MAX_PRECISION, 9, false); + return createDecimalTypeInternal(MAX_DECIMAL128_PRECISION, 9, false); case DOUBLE: - return createDecimalTypeInternal(MAX_PRECISION, 17, false); + return createDecimalTypeInternal(MAX_DECIMAL128_PRECISION, 17, false); default: return ScalarType.INVALID; } diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java index d0e44bf27c7eb65..64d14350ae915bd 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java @@ -154,6 +154,7 @@ public abstract class Type { numericTypes.add(DECIMAL32); numericTypes.add(DECIMAL64); numericTypes.add(DECIMAL128); + numericTypes.add(DECIMAL256); numericDateTimeTypes = Lists.newArrayList(); numericDateTimeTypes.add(DATE); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DecimalV3Type.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DecimalV3Type.java index c30b6c9024ff39d..2e18ba49ed80625 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DecimalV3Type.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DecimalV3Type.java @@ -31,15 +31,11 @@ import java.util.Map; import java.util.Objects; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - /** * Decimal type in Nereids. */ @Developing public class DecimalV3Type extends FractionalType { - private static final Logger LOG = LoggerFactory.getLogger(DecimalV3Type.class); public static final int MAX_DECIMAL32_PRECISION = 9; public static final int MAX_DECIMAL64_PRECISION = 18; @@ -114,7 +110,8 @@ public static DecimalV3Type createDecimalV3Type(int precision, int scale) { + " but precision is " + precision, ", scale is " + scale); boolean enableDecimal256 = ConnectContext.get().getSessionVariable().enableDecimal256(); if (precision > MAX_DECIMAL128_PRECISION && !enableDecimal256) { - throw new NotSupportedException("Datatype DecimalV3 with precision " + precision + ", which is greater than 38 is disabled by default. set enable_decimal256 = true to enable it."); + throw new NotSupportedException("Datatype DecimalV3 with precision " + precision + + ", which is greater than 38 is disabled by default. set enable_decimal256 = true to enable it."); } else { return new DecimalV3Type(precision, scale); } @@ -139,10 +136,6 @@ private static DataType widerDecimalV3Type( int range = Math.max(leftPrecision - leftScale, rightPrecision - rightScale); boolean enableDecimal256 = ConnectContext.get().getSessionVariable().enableDecimal256(); if (range + scale > MAX_DECIMAL128_PRECISION && overflowToDouble && !enableDecimal256) { - LOG.warn("sum test widerDecimalV3Type return double"); - for (StackTraceElement ste : Thread.currentThread().getStackTrace()) { - LOG.warn(ste.toString()); - } return DoubleType.INSTANCE; } return DecimalV3Type.createDecimalV3Type(range + scale, scale); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java index 26e8cb7f5e0b13c..5640a8c034c74fd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java @@ -313,7 +313,7 @@ private void handleQuery(MysqlCommand mysqlCommand) { return; } catch (Exception e) { // TODO: We should catch all exception here until we support all query syntax. - LOG.warn("Nereids parse sql failed. Reason: {}. Statement: \"{}\".", + LOG.debug("Nereids parse sql failed. Reason: {}. Statement: \"{}\".", e.getMessage(), originStmt); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index b33a2175bfd13ba..751ef6b8f815e92 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -1263,8 +1263,8 @@ public void setIgnoreShapePlanNodes(String ignoreShapePlanNodes) { "the plan node type which is ignored in 'explain shape plan' command"}) public String ignoreShapePlanNodes = ""; - @VariableMgr.VarAttr(name = ENABLE_DECIMAL256, - description = {"控制是否在计算过程中使用Decimal256类型", "Set to true to enable Decimal256 type"}) + @VariableMgr.VarAttr(name = ENABLE_DECIMAL256, description = { "控制是否在计算过程中使用Decimal256类型", + "Set to true to enable Decimal256 type" }) public boolean enableDecimal256 = false; // If this fe is in fuzzy mode, then will use initFuzzyModeVariables to generate some variables, diff --git a/regression-test/suites/datatype_p0/decimalv3/test_decimalv3_overflow.groovy b/regression-test/suites/datatype_p0/decimalv3/test_decimalv3_overflow.groovy index bf4b1ef220b0fa3..bb4b4ba42d736ed 100644 --- a/regression-test/suites/datatype_p0/decimalv3/test_decimalv3_overflow.groovy +++ b/regression-test/suites/datatype_p0/decimalv3/test_decimalv3_overflow.groovy @@ -42,7 +42,6 @@ suite("test_decimalv3_overflow") { ); """ sql "insert into ${tblName2} values('2022-08-01', 705091149953414452.46)" - // modify case qt_sql """ select c2 / 10000 * c1 from ${tblName1}, ${tblName2}; """ sql """ set check_overflow_for_decimal=true; """