diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java index 13ecddd150ac36..b4fb931ec1e799 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java @@ -222,8 +222,12 @@ public EvaluateRangeResult visit(Expression expr, EvaluateRangeInput context) { if (expr.getDataType() instanceof BooleanType && !(expr instanceof Literal) && result.childrenResult.stream().anyMatch(childResult -> childResult.columnRanges.values().stream().anyMatch(ColumnRange::isEmptyRange))) { + // this assumes that for expression: func(A) + // if A reject partition, then func(A) reject partition. + // implement visitFunc for Func if Func does not satisfy the above assumption. return new EvaluateRangeResult(BooleanLiteral.FALSE, result.columnRanges, result.childrenResult); } + // assumption: for func(A), if A accept range (n, m), then func(A) accept range (n, m). return result; } @@ -342,13 +346,16 @@ public EvaluateRangeResult visitEqualTo(EqualTo equalTo, EvaluateRangeInput cont if (!(result.result instanceof EqualTo)) { return result; } - equalTo = (EqualTo) result.result; + boolean isRejectNot = false; if (equalTo.left() instanceof Slot && equalTo.right() instanceof Literal) { Slot slot = (Slot) equalTo.left(); if (isPartitionSlot(slot)) { Map leftColumnRanges = result.childrenResult.get(0).columnRanges; ColumnRange atLeastRange = ColumnRange.singleton((Literal) equalTo.right()); result = intersectSlotRange(result, leftColumnRanges, slot, atLeastRange); + if (leftColumnRanges.get(slot).isSingleton()) { + isRejectNot = true; + } } } else if (equalTo.left() instanceof Literal && equalTo.right() instanceof Slot) { Slot slot = (Slot) equalTo.right(); @@ -356,7 +363,15 @@ public EvaluateRangeResult visitEqualTo(EqualTo equalTo, EvaluateRangeInput cont Map rightColumnRanges = result.childrenResult.get(1).columnRanges; ColumnRange atMostRange = ColumnRange.singleton((Literal) equalTo.left()); result = intersectSlotRange(result, rightColumnRanges, slot, atMostRange); + if (rightColumnRanges.get(slot).isSingleton()) { + isRejectNot = true; + } } + } else { + isRejectNot = false; + } + if (!isRejectNot) { + result = result.withRejectNot(false); } return result; } @@ -379,6 +394,7 @@ public EvaluateRangeResult visitInPredicate(InPredicate inPredicate, EvaluateRan Map slotRanges = result.childrenResult.get(0).columnRanges; result = intersectSlotRange(result, slotRanges, slot, unionLiteralRange); } + result = result.withRejectNot(false); return result; } @@ -388,14 +404,15 @@ public EvaluateRangeResult visitIsNull(IsNull isNull, EvaluateRangeInput context if (!(result.result instanceof IsNull)) { return result; } - + result = result.withRejectNot(false); Expression child = isNull.child(); if (!(child instanceof Slot) || !isPartitionSlot((Slot) child)) { return result; } if (!partitionSlotContainsNull.get((Slot) child)) { - return new EvaluateRangeResult(BooleanLiteral.FALSE, result.columnRanges, result.childrenResult); + return new EvaluateRangeResult(BooleanLiteral.FALSE, + result.columnRanges, result.childrenResult, false); } return result; } @@ -430,12 +447,16 @@ public EvaluateRangeResult visitOr(Or or, EvaluateRangeInput context) { @Override public EvaluateRangeResult visitNot(Not not, EvaluateRangeInput context) { EvaluateRangeResult result = evaluateChildrenThenThis(not, context); - - Map newRanges = result.childrenResult.get(0).columnRanges.entrySet() - .stream() - .map(slotToRange -> Pair.of(slotToRange.getKey(), slotToRange.getValue().complete())) - .collect(ImmutableMap.toImmutableMap(Pair::key, Pair::value)); - result = new EvaluateRangeResult(result.result, newRanges, result.childrenResult); + if (result.isRejectNot()) { + Map newRanges = Maps.newHashMap(); + for (Map.Entry entry : result.childrenResult.get(0).columnRanges.entrySet()) { + Slot slot = entry.getKey(); + ColumnRange childRange = entry.getValue(); + ColumnRange partitionRange = result.columnRanges.get(slot); + newRanges.put(slot, partitionRange.intersect(childRange.complete())); + } + result = new EvaluateRangeResult(result.result, newRanges, result.childrenResult); + } return returnFalseIfExistEmptyRange(result); } @@ -658,11 +679,37 @@ public static class EvaluateRangeResult { private final Map columnRanges; private final List childrenResult; + // rejectNot = true, if \exist e \in R, pred(e)=true, then we have \forAll e \in R, !pred(e)=false + // that is, if pred holds true over R, then !pred does not hold true over R. + // example 1. rejectNot=false + // R=(1,10), pred: k = 5. "k = 5" holds true over R, and "NOT k = 5" holds true over R. + // example 2. rejectNot=false + // R=(1,10), pred: k = 11. "k=10" dose not holds over R + // example 3. rejectNot=false + // R=(1,10), pred: k in (4, 5). "k in (4, 5)" holds true over R, and "NOT k in (4, 5)" holds over R + // example 3. rejectNot=true + // R=(1,10), pred: k < 11. "k<11" holds true over R, and "NOT k<11" dose not hold over R + private final boolean rejectNot; + public EvaluateRangeResult(Expression result, Map columnRanges, - List childrenResult) { + List childrenResult, boolean rejectNot) { this.result = result; this.columnRanges = columnRanges; this.childrenResult = childrenResult; + this.rejectNot = rejectNot; + } + + public EvaluateRangeResult(Expression result, Map columnRanges, + List childrenResult) { + this(result, columnRanges, childrenResult, childrenResult.stream().allMatch(r -> r.isRejectNot())); + } + + public EvaluateRangeResult withRejectNot(boolean rejectNot) { + return new EvaluateRangeResult(result, columnRanges, childrenResult, rejectNot); + } + + public boolean isRejectNot() { + return rejectNot; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java index afee004507a8a2..d7c869943ac6da 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -1218,8 +1218,11 @@ public String getNodeExplainString(String prefix, TExplainLevel detailLevel) { output.append(getRuntimeFilterExplainString(false)); } - output.append(prefix).append(String.format("partitions=%s/%s, tablets=%s/%s", selectedPartitionNum, - olapTable.getPartitions().size(), selectedTabletsNum, totalTabletsNum)); + String selectedPartitions = getSelectedPartitionIds().stream().sorted() + .map(id -> olapTable.getPartition(id).getName()) + .collect(Collectors.joining(",")); + output.append(prefix).append(String.format("partitions=%s/%s (%s), tablets=%s/%s", selectedPartitionNum, + olapTable.getPartitions().size(), selectedPartitions, selectedTabletsNum, totalTabletsNum)); // We print up to 3 tablet, and we print "..." if the number is more than 3 if (scanTabletIds.size() > 3) { List firstTenTabletIds = scanTabletIds.subList(0, 3); diff --git a/regression-test/data/performance_p0/redundant_conjuncts.out b/regression-test/data/performance_p0/redundant_conjuncts.out index 4c432cf6d2b3f7..06003d12d2f0cf 100644 --- a/regression-test/data/performance_p0/redundant_conjuncts.out +++ b/regression-test/data/performance_p0/redundant_conjuncts.out @@ -12,7 +12,7 @@ PLAN FRAGMENT 0 0:VOlapScanNode TABLE: default_cluster:regression_test_performance_p0.redundant_conjuncts(redundant_conjuncts), PREAGGREGATION: OFF. Reason: No AggregateInfo PREDICATES: `k1` = 1 - partitions=0/1, tablets=0/0, tabletList= + partitions=0/1 (), tablets=0/0, tabletList= cardinality=0, avgRowSize=8.0, numNodes=1 pushAggOp=NONE @@ -29,7 +29,7 @@ PLAN FRAGMENT 0 0:VOlapScanNode TABLE: default_cluster:regression_test_performance_p0.redundant_conjuncts(redundant_conjuncts), PREAGGREGATION: OFF. Reason: No AggregateInfo PREDICATES: `k1` = 1 OR `k1` = 2 - partitions=0/1, tablets=0/0, tabletList= + partitions=0/1 (), tablets=0/0, tabletList= cardinality=0, avgRowSize=8.0, numNodes=1 pushAggOp=NONE diff --git a/regression-test/suites/nereids_rules_p0/partition_prune/test_multi_range_partition.groovy b/regression-test/suites/nereids_rules_p0/partition_prune/test_multi_range_partition.groovy new file mode 100644 index 00000000000000..d24972c6b5f777 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/partition_prune/test_multi_range_partition.groovy @@ -0,0 +1,260 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_multi_range_partition") { + String db = context.config.getDbNameByFile(context.file) + sql "use ${db}" + sql "SET enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + sql "set partition_pruning_expand_threshold=10;" + sql "drop table if exists pt" + sql """ + CREATE TABLE `pt` ( + `k1` int(11) NULL COMMENT "", + `k2` int(11) NULL COMMENT "", + `k3` int(11) NULL COMMENT "" + ) + PARTITION BY RANGE(`k1`, `k2`) + (PARTITION p1 VALUES LESS THAN ("3", "1"), + PARTITION p2 VALUES [("3", "1"), ("7", "10")), + PARTITION p3 VALUES [("7", "10"), ("10", "15"))) + DISTRIBUTED BY HASH(`k1`) BUCKETS 10 + PROPERTIES ('replication_num' = '1'); + """ + sql "insert into pt values (7, 0, 0);" + sql "insert into pt(k1) values (7);" + sql "insert into pt(k1) values (0);" + sql "insert into pt values (7, 11, 0);" + + // basic test + explain{ + sql "select * from pt where k1=7;" + contains "partitions=2/3 (p2,p3)" + } + + explain{ + sql "select * from pt where k1=10;" + contains "partitions=1/3 (p3)" + } + + explain{ + sql "select * from pt where k1=-1;" + contains "partitions=1/3 (p1)" + } + + // ===============function(part_key)=================== + explain{ + sql "select * from pt where 2*k1=20; --漏裁 p1" + contains "partitions=2/3 (p1,p3)" + } + explain{ + sql "select * from pt where 2*(k1+1)=22; --漏裁 p1" + contains "partitions=2/3 (p1,p3)" + } + explain{ + sql "select * from pt where sin(k1)=0" + contains "artitions=3/3 (p1,p2,p3)" + } + + // fix BUG: p1 missed + explain{ + sql "select * from pt where sin(k1)<>0" + contains "partitions=3/3 (p1,p2,p3)" + } + + // ============= in predicate ====================== + explain{ + sql "select * from pt where k1 in (7, 8);" + contains "partitions=2/3 (p2,p3)" + } + + // =========== is null =================== + explain{ + sql "select * from pt where k1 is null" + contains "partitions=1/3 (p1)" + } + + // fix BUG: p1 missed + explain{ + sql "select * from pt where k1 is not null" + contains "partitions=3/3 (p1,p2,p3)" + } + + //======== the second partition key ========= + explain{ + sql "select * from pt where k1=7 and (k1>k2);" + contains "partitions=1/3 (p2)" + } + + explain { + sql "select * from pt where k1=7 and not (k1>k2);" + contains "partitions=2/3 (p2,p3)" + } + + // p3 NOT pruned + explain { + sql "select * from pt where k1=7 and (k1k2" + contains "partitions=2/3 (p2,p3)" + } + + //p3 NOT pruned + explain { + sql "select * from pt where k1=7 and (k1 > cast(k2 as bigint));" + contains "partitions=2/3 (p2,p3)" + } + + //fix BUG: p2 missed + explain { + sql "select * from pt where k1=7 and not (k2 is null);" + contains "partitions=2/3 (p2,p3)" + } + + //p3 NOT pruned + explain { + sql "select * from pt where k1=7 and not (k2 is not null);" + contains "partitions=2/3 (p2,p3)" + } + + //fix BUG: p2 missed + explain { + sql "select * from pt where k1=7 and k2 not in (1, 2);" + contains "partitions=2/3 (p2,p3)" + } + + explain { + sql "select * from pt where k1=7 and k2 in (1, 12);" + contains "partitions=2/3 (p2,p3)" + } + + //fix BUG: p2,p3 pruned + explain { + sql "select * from pt where k1=7 and k2 not in (1, 12)" + contains "partitions=2/3 (p2,p3)" + } + + explain { + sql" select * from pt where k1=7 and k2 in (0);" + contains "partitions=1/3 (p2)" + } + + explain { + sql "select * from pt where k1=7 and k2 in (11)" + contains "partitions=1/3 (p3)" + } + + explain { + sql "select * from pt where k1=7 and k2 in (null);" + contains "partitions=2/3 (p2,p3)" + } + + explain { + sql "select * from pt where k1=7 and k2 not in (null);" + contains "partitions=2/3 (p2,p3)" + } + + explain { + sql "select * from pt where k1=7 and k1 > k3;" + contains "partitions=2/3 (p2,p3)" + } + + explain { + sql "select * from pt where k1=7 and k1 <> k3;" + contains "partitions=2/3 (p2,p3)" + } + + explain { + sql "select * from pt where k2 in (null);" + contains "partitions=3/3 (p1,p2,p3)" + } + + // p1/p2/p3 NOT pruned + explain { + sql "select * from pt where k2 not in (null)" + contains "partitions=3/3 (p1,p2,p3)" + } + + explain { + sql "select * from pt where k2 in (0)" + contains "partitions=3/3 (p1,p2,p3)" + } + + explain { + sql "select * from pt where k2 > 100" + contains "partitions=3/3 (p1,p2,p3)" + } + + explain { + sql "select * from pt where k2 < -1" + contains "partitions=3/3 (p1,p2,p3)" + } + + explain { + sql "select * from pt where k1=7 and (k3 is null)" + contains "partitions=2/3 (p2,p3)" + } + + explain { + sql "select * from pt where k1=7 and not (k3 is null);" + contains "partitions=2/3 (p2,p3)" + } + + + // test if a range is not sliced to multiple single point + // for example: range [3,7) is sliced to [3,3], [4,4],[5,5],[6,6] + sql "set partition_pruning_expand_threshold=1;" + + explain { + sql "select * from pt where k1 < 5;" + contains "partitions=2/3 (p1,p2)" + } + + explain { + sql "select * from pt where not k1 < 5;" + contains "partitions=2/3 (p2,p3)" + } + + explain { + sql "select * from pt where k2 < 5;" + contains "partitions=3/3 (p1,p2,p3)" + } + + explain { + sql "select * from pt where not k2 < 5;" + contains "partitions=3/3 (p1,p2,p3)" + } + + explain { + sql "select * from pt where k3 < 5;" + contains "partitions=3/3 (p1,p2,p3)" + } + + explain { + sql "select * from pt where not k3 < 5;" + contains "partitions=3/3 (p1,p2,p3)" + } +}