From 6295f4052ffb461fcdb09f1dabf4ee415b4f5f0d Mon Sep 17 00:00:00 2001 From: Lijia Liu Date: Wed, 31 Jul 2024 22:48:24 +0800 Subject: [PATCH] [fix](mtmv) Choose a valid partition column when there are both valid and invalid expressions (#38367) Choose a partition column when there are both valid and invalid expressions. --- .../exploration/mv/MaterializedViewUtils.java | 14 +++--- .../mv/MaterializedViewUtilsTest.java | 49 +++++++++++++++++++ 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java index 1a487791398a50..835b01c00b25ea 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java @@ -580,7 +580,8 @@ private SlotReference getContextPartitionColumn(IncrementCheckerContext context) private static boolean checkPartition(Collection expressionsToCheck, Plan plan, IncrementCheckerContext context) { NamedExpression partitionColumn = context.getMvPartitionColumn(); - for (Expression projectSlot : expressionsToCheck) { + + OUTER_CHECK: for (Expression projectSlot : expressionsToCheck) { if (projectSlot.isColumnFromTable() && projectSlot.equals(partitionColumn.toSlot())) { continue; } @@ -612,7 +613,7 @@ private static boolean checkPartition(Collection expressio String.format("partition expression use more than one slot reference, invalid " + "expressionToCheckColumns is %s, partitionColumnDateColumns is %s", expressionToCheckColumns, partitionColumns)); - return false; + continue; } List expressions = expressionToCheck.collectToList(Expression.class::isInstance); for (Expression expression : expressions) { @@ -621,7 +622,7 @@ private static boolean checkPartition(Collection expressio context.addFailReason( String.format("column to check use invalid implicit expression, invalid " + "expression is %s", expression)); - return false; + continue OUTER_CHECK; } } List partitionExpressions = partitionExpression.collectToList( @@ -632,7 +633,7 @@ private static boolean checkPartition(Collection expressio context.addFailReason( String.format("partition column use invalid implicit expression, invalid " + "expression is %s", expression)); - return false; + continue OUTER_CHECK; } } List expressionToCheckDataTruncList = @@ -643,7 +644,7 @@ private static boolean checkPartition(Collection expressio // mv time unit level is little then query context.addFailReason("partition column time unit level should be " + "greater than sql select column"); - return false; + continue; } if (!partitionColumn.isColumnFromTable()) { context.setMvPartitionColumn(partitionColumns.iterator().next()); @@ -651,8 +652,9 @@ private static boolean checkPartition(Collection expressio if (!context.getPartitionExpression().isPresent()) { context.setPartitionExpression(partitionExpression); } + return true; } - return true; + return context.getMvPartitionColumn().isColumnFromTable(); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java index 89fe34876ae476..9ac6d7d3a753ed 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java @@ -234,6 +234,19 @@ protected void runBeforeAll() throws Exception { + " \"replication_num\" = \"1\"\n" + ");\n" ); + createTable("CREATE TABLE `test3` (\n" + + " `id` VARCHAR(36) NOT NULL COMMENT 'id',\n" + + " `created_time` DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT ''\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`id`)\n" + + "COMMENT ''\n" + + "PARTITION BY RANGE(`created_time`)\n" + + "(PARTITION P_2024071713 VALUES [('2024-07-17 13:00:00'), ('2024-07-17 14:00:00')),\n" + + "PARTITION P_2024071714 VALUES [('2024-07-17 14:00:00'), ('2024-07-17 15:00:00')))\n" + + "DISTRIBUTED BY HASH(`id`) BUCKETS AUTO\n" + + "PROPERTIES (\n" + + "\"replication_allocation\" = \"tag.location.default: 1\"\n" + + ");\n"); // Should not make scan to empty relation when the table used by materialized view has no data connectContext.getSessionVariable().setDisableNereidsRules("OLAP_SCAN_PARTITION_PRUNE,PRUNE_EMPTY_PARTITION"); } @@ -809,6 +822,42 @@ public void containTableQueryOperatorWithoutOperatorTest() { }); } + @Test + public void getRelatedTableInfoWhenMultiPartitionExprs() { + PlanChecker.from(connectContext) + .checkExplain("select id, date_trunc(created_time, 'minute') as created_time_minute," + + " min(created_time) as start_time," + + " if(count(id) > 0, 1, 0) as status\n" + + " from test3 \n" + + " group by id, date_trunc(created_time, 'minute')", + nereidsPlanner -> { + Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan(); + RelatedTableInfo relatedTableInfo = + MaterializedViewUtils.getRelatedTableInfo("created_time_minute", + "day", rewrittenPlan, nereidsPlanner.getCascadesContext()); + checkRelatedTableInfo(relatedTableInfo, + "test3", + "created_time", + true); + }); + PlanChecker.from(connectContext) + .checkExplain("select id, date_trunc(created_time, 'hour') as created_time_hour," + + " min(created_time) as start_time\n" + + " from test3 \n" + + " group by id, date_trunc(created_time, 'minute')," + + " date_trunc(created_time, 'hour');", + nereidsPlanner -> { + Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan(); + RelatedTableInfo relatedTableInfo = + MaterializedViewUtils.getRelatedTableInfo("created_time_hour", + "day", rewrittenPlan, nereidsPlanner.getCascadesContext()); + checkRelatedTableInfo(relatedTableInfo, + "test3", + "created_time", + true); + }); + } + private void checkRelatedTableInfo(RelatedTableInfo relatedTableInfo, String expectTableName, String expectColumnName,