From bdab3bd4d7268c22fa0531788c8b911b1fd93656 Mon Sep 17 00:00:00 2001 From: lihangyu Date: Tue, 7 Jan 2025 16:10:51 +0800 Subject: [PATCH] [Fix](Variant) variant should not implicit be short key column when create mv (#46444) When creating mv, the variant column should not be part of short key since it's not supported sorting --- .../java/org/apache/doris/catalog/Type.java | 12 +++++++ .../doris/alter/MaterializedViewHandler.java | 3 +- .../analysis/CreateMaterializedViewStmt.java | 6 ++-- .../doris/analysis/CreateTableStmt.java | 11 +------ .../java/org/apache/doris/catalog/Env.java | 2 +- .../plans/commands/info/CreateMTMVInfo.java | 8 ++--- .../plans/commands/info/CreateTableInfo.java | 3 +- .../data/variant_p0/mv/multi_slot.out | 3 ++ .../test_create_mv_complex_type.groovy | 8 ++--- .../test_materialized_view_array.groovy | 2 +- .../test_materialized_view_struct.groovy | 2 +- .../suites/variant_p0/mv/multi_slot.groovy | 31 +++++++++++++++++-- 12 files changed, 62 insertions(+), 29 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java index 7dfcfd15ebec84..d3218203bf6462 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java @@ -382,6 +382,18 @@ public boolean supportSubType(Type subType) { return false; } + /** + * Return true if this type can be as short key + */ + public boolean couldBeShortKey() { + return !(isFloatingPointType() + || getPrimitiveType() == PrimitiveType.STRING + || isJsonbType() + || isComplexType() + || isObjectStored() + || isVariantType()); + } + /** * The output of this is stored directly in the hive metastore as the column type. * The string must match exactly. diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java index a6f1cae9987678..64a747e99e4d6a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java @@ -795,10 +795,9 @@ public List checkAndPrepareMaterializedView(AddRollupClause addRollupCla break; } } - if (column.getType().isFloatingPointType()) { + if (!column.getType().couldBeShortKey()) { break; } - column.setIsKey(true); if (column.getType().getPrimitiveType() == PrimitiveType.VARCHAR) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java index 68ec82387020ab..2922595fbce00e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java @@ -476,7 +476,7 @@ private void supplyOrderColumn() throws AnalysisException { } break; } - if (column.getType().isFloatingPointType()) { + if (!column.getType().couldBeShortKey()) { break; } if (column.getType().getPrimitiveType() == PrimitiveType.VARCHAR) { @@ -487,7 +487,9 @@ private void supplyOrderColumn() throws AnalysisException { column.setIsKey(true); } if (theBeginIndexOfValue == 0) { - throw new AnalysisException("The first column could not be float or double type, use decimal instead"); + throw new AnalysisException( + "The first column could not be float, double or complex " + + "type like array, struct, map, json, variant."); } // supply value for (; theBeginIndexOfValue < mvColumnItemList.size(); theBeginIndexOfValue++) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java index 5f173b4a243702..9232d66050a30e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java @@ -375,16 +375,7 @@ public void analyze(Analyzer analyzer) throws UserException { } break; } - if (columnDef.getType().isFloatingPointType()) { - break; - } - if (columnDef.getType().getPrimitiveType() == PrimitiveType.STRING) { - break; - } - if (columnDef.getType().getPrimitiveType() == PrimitiveType.JSONB) { - break; - } - if (columnDef.getType().isComplexType()) { + if (!columnDef.getType().couldBeShortKey()) { break; } if (columnDef.getType().getPrimitiveType() == PrimitiveType.VARCHAR) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 19865446456967..505970e104c26f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -4699,7 +4699,7 @@ public static short calcShortKeyColumnCount(List columns, Map NULL] not support to create materialized view" + exception "errCode = 2, detailMessage = The first column could not be float, double or complex type like array, struct, map, json, variant" } } finally { try_sql("DROP TABLE IF EXISTS ${tableName}") diff --git a/regression-test/suites/rollup_p0/test_materialized_view_struct.groovy b/regression-test/suites/rollup_p0/test_materialized_view_struct.groovy index 61e8415cacc68b..2a05b89538433f 100644 --- a/regression-test/suites/rollup_p0/test_materialized_view_struct.groovy +++ b/regression-test/suites/rollup_p0/test_materialized_view_struct.groovy @@ -59,7 +59,7 @@ suite("test_materialized_view_struct", "rollup") { create_test_table.call(tableName) test { sql "CREATE MATERIALIZED VIEW idx AS select k2,k1, k3, k4, k5 from ${tableName}" - exception "errCode = 2, detailMessage = The STRUCT column[`mv_k2` struct NULL] not support to create materialized view" + exception "errCode = 2, detailMessage = The first column could not be float, double or complex type like array, struct, map, json, variant." } } finally { try_sql("DROP TABLE IF EXISTS ${tableName}") diff --git a/regression-test/suites/variant_p0/mv/multi_slot.groovy b/regression-test/suites/variant_p0/mv/multi_slot.groovy index fd1727c571df55..1490c30e58778b 100644 --- a/regression-test/suites/variant_p0/mv/multi_slot.groovy +++ b/regression-test/suites/variant_p0/mv/multi_slot.groovy @@ -50,8 +50,35 @@ suite ("multi_slot") { order_qt_select_star "select abs(cast(v['k1'] as int))+cast(v['k2'] as int)+1,abs(cast(v['k2'] as int)+2)+cast(v['k3'] as int)+3 from multi_slot;" order_qt_select_star "select * from multi_slot order by cast(v['k1'] as int);" - // TODO fix and remove enable_rewrite_element_at_to_slot - order_qt_select_star "select /*+SET_VAR(enable_rewrite_element_at_to_slot=false) */ abs(cast(v['k4']['k44'] as int)), sum(abs(cast(v['k2'] as int)+2)+cast(v['k3'] as int)+3) from multi_slot group by abs(cast(v['k4']['k44'] as int))" + order_qt_select_star "select abs(cast(v['k4']['k44'] as int)), sum(abs(cast(v['k2'] as int)+2)+cast(v['k3'] as int)+3) from multi_slot group by abs(cast(v['k4']['k44'] as int))" + + sql "drop table if exists test_mv" + sql """ + CREATE TABLE `test_mv` ( + `handle_time` datetime NOT NULL , + `client_request` variant NULL, + `status` int NULL + ) + DISTRIBUTED BY HASH(`handle_time`) + BUCKETS 10 PROPERTIES ( + "is_being_synced" = "false", + "storage_medium" = "hdd", + "storage_format" = "V2", + "inverted_index_storage_format" = "V1", + "light_schema_change" = "true", + "disable_auto_compaction" = "false", + "enable_single_replica_compaction" = "false", + "replication_num" = "1" + ); + """ + sql """insert into test_mv values ('2021-01-01 11:11:11', '{"url" : "http://xxx.xxx.xxx"}', 12)""" + createMV("create materialized view mv_1 as select `handle_time`, `client_request`['url'] as `uri`, `status` from test_mv") + qt_sql "select `handle_time`, `client_request`['url'] as `uri`, `status` from test_mv" + test { + sql "create materialized view mv_x as select `client_request`['url'] as `uri`, `status` from test_mv" + exception("The first column could not be float, double or complex type like array, struct, map, json, variant.") + } + // def retry_times = 60 // for (def i = 0; i < retry_times; ++i) {