[Pick](nereids) Pick: partition prune fails in case of NOT expression (…

…apache#27047) (apache#27507)

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java

@@ -222,8 +222,12 @@ public EvaluateRangeResult visit(Expression expr, EvaluateRangeInput context) {
         if (expr.getDataType() instanceof BooleanType && !(expr instanceof Literal)
                 && result.childrenResult.stream().anyMatch(childResult ->
                 childResult.columnRanges.values().stream().anyMatch(ColumnRange::isEmptyRange))) {
+            // this assumes that for expression: func(A)
+            // if A reject partition, then func(A) reject partition.
+            // implement visitFunc for Func if Func does not satisfy the above assumption.
             return new EvaluateRangeResult(BooleanLiteral.FALSE, result.columnRanges, result.childrenResult);
         }
+        // assumption: for func(A), if A accept range (n, m), then func(A) accept range (n, m).
         return result;
     }
 
@@ -342,21 +346,32 @@ public EvaluateRangeResult visitEqualTo(EqualTo equalTo, EvaluateRangeInput cont
         if (!(result.result instanceof EqualTo)) {
             return result;
         }
-        equalTo = (EqualTo) result.result;
+        boolean isRejectNot = false;
         if (equalTo.left() instanceof Slot && equalTo.right() instanceof Literal) {
             Slot slot = (Slot) equalTo.left();
             if (isPartitionSlot(slot)) {
                 Map<Slot, ColumnRange> leftColumnRanges = result.childrenResult.get(0).columnRanges;
                 ColumnRange atLeastRange = ColumnRange.singleton((Literal) equalTo.right());
                 result = intersectSlotRange(result, leftColumnRanges, slot, atLeastRange);
+                if (leftColumnRanges.get(slot).isSingleton()) {
+                    isRejectNot = true;
+                }
             }
         } else if (equalTo.left() instanceof Literal && equalTo.right() instanceof Slot) {
             Slot slot = (Slot) equalTo.right();
             if (isPartitionSlot(slot)) {
                 Map<Slot, ColumnRange> rightColumnRanges = result.childrenResult.get(1).columnRanges;
                 ColumnRange atMostRange = ColumnRange.singleton((Literal) equalTo.left());
                 result = intersectSlotRange(result, rightColumnRanges, slot, atMostRange);
+                if (rightColumnRanges.get(slot).isSingleton()) {
+                    isRejectNot = true;
+                }
             }
+        } else {
+            isRejectNot = false;
+        }
+        if (!isRejectNot) {
+            result = result.withRejectNot(false);
         }
         return result;
     }
@@ -379,6 +394,7 @@ public EvaluateRangeResult visitInPredicate(InPredicate inPredicate, EvaluateRan
             Map<Slot, ColumnRange> slotRanges = result.childrenResult.get(0).columnRanges;
             result = intersectSlotRange(result, slotRanges, slot, unionLiteralRange);
         }
+        result = result.withRejectNot(false);
         return result;
     }
 
@@ -388,14 +404,15 @@ public EvaluateRangeResult visitIsNull(IsNull isNull, EvaluateRangeInput context
         if (!(result.result instanceof IsNull)) {
             return result;
         }
-
+        result = result.withRejectNot(false);
         Expression child = isNull.child();
         if (!(child instanceof Slot) || !isPartitionSlot((Slot) child)) {
             return result;
         }
 
         if (!partitionSlotContainsNull.get((Slot) child)) {
-            return new EvaluateRangeResult(BooleanLiteral.FALSE, result.columnRanges, result.childrenResult);
+            return new EvaluateRangeResult(BooleanLiteral.FALSE,
+                    result.columnRanges, result.childrenResult, false);
         }
         return result;
     }
@@ -430,12 +447,16 @@ public EvaluateRangeResult visitOr(Or or, EvaluateRangeInput context) {
     @Override
     public EvaluateRangeResult visitNot(Not not, EvaluateRangeInput context) {
         EvaluateRangeResult result = evaluateChildrenThenThis(not, context);
-
-        Map<Slot, ColumnRange> newRanges = result.childrenResult.get(0).columnRanges.entrySet()
-                .stream()
-                .map(slotToRange -> Pair.of(slotToRange.getKey(), slotToRange.getValue().complete()))
-                .collect(ImmutableMap.toImmutableMap(Pair::key, Pair::value));
-        result = new EvaluateRangeResult(result.result, newRanges, result.childrenResult);
+        if (result.isRejectNot()) {
+            Map<Slot, ColumnRange> newRanges = Maps.newHashMap();
+            for (Map.Entry<Slot, ColumnRange> entry : result.childrenResult.get(0).columnRanges.entrySet()) {
+                Slot slot = entry.getKey();
+                ColumnRange childRange = entry.getValue();
+                ColumnRange partitionRange = result.columnRanges.get(slot);
+                newRanges.put(slot, partitionRange.intersect(childRange.complete()));
+            }
+            result = new EvaluateRangeResult(result.result, newRanges, result.childrenResult);
+        }
         return returnFalseIfExistEmptyRange(result);
     }
 
@@ -658,11 +679,37 @@ public static class EvaluateRangeResult {
         private final Map<Slot, ColumnRange> columnRanges;
         private final List<EvaluateRangeResult> childrenResult;
 
+        // rejectNot = true, if \exist e \in R, pred(e)=true, then we have \forAll e \in R, !pred(e)=false
+        // that is, if pred holds true over R, then !pred does not hold true over R.
+        // example 1. rejectNot=false
+        //      R=(1,10), pred: k = 5. "k = 5" holds true over R, and "NOT k = 5" holds true over R.
+        // example 2. rejectNot=false
+        //      R=(1,10), pred: k = 11. "k=10" dose not holds over R
+        // example 3. rejectNot=false
+        //      R=(1,10), pred: k in (4, 5). "k in (4, 5)" holds true over R, and "NOT k in (4, 5)" holds over R
+        // example 3. rejectNot=true
+        //      R=(1,10), pred: k < 11. "k<11" holds true over R, and "NOT k<11" dose not hold over R
+        private final boolean rejectNot;
+
         public EvaluateRangeResult(Expression result, Map<Slot, ColumnRange> columnRanges,
-                List<EvaluateRangeResult> childrenResult) {
+                                   List<EvaluateRangeResult> childrenResult, boolean rejectNot) {
             this.result = result;
             this.columnRanges = columnRanges;
             this.childrenResult = childrenResult;
+            this.rejectNot = rejectNot;
+        }
+
+        public EvaluateRangeResult(Expression result, Map<Slot, ColumnRange> columnRanges,
+                List<EvaluateRangeResult> childrenResult) {
+            this(result, columnRanges, childrenResult, childrenResult.stream().allMatch(r -> r.isRejectNot()));
+        }
+
+        public EvaluateRangeResult withRejectNot(boolean rejectNot) {
+            return new EvaluateRangeResult(result, columnRanges, childrenResult, rejectNot);
+        }
+
+        public boolean isRejectNot() {
+            return rejectNot;
         }
     }
 

fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java

@@ -1218,8 +1218,11 @@ public String getNodeExplainString(String prefix, TExplainLevel detailLevel) {
             output.append(getRuntimeFilterExplainString(false));
         }
 
-        output.append(prefix).append(String.format("partitions=%s/%s, tablets=%s/%s", selectedPartitionNum,
-                olapTable.getPartitions().size(), selectedTabletsNum, totalTabletsNum));
+        String selectedPartitions = getSelectedPartitionIds().stream().sorted()
+                .map(id -> olapTable.getPartition(id).getName())
+                .collect(Collectors.joining(","));
+        output.append(prefix).append(String.format("partitions=%s/%s (%s), tablets=%s/%s", selectedPartitionNum,
+                olapTable.getPartitions().size(), selectedPartitions, selectedTabletsNum, totalTabletsNum));
         // We print up to 3 tablet, and we print "..." if the number is more than 3
         if (scanTabletIds.size() > 3) {
             List<Long> firstTenTabletIds = scanTabletIds.subList(0, 3);

regression-test/data/performance_p0/redundant_conjuncts.out

@@ -12,7 +12,7 @@ PLAN FRAGMENT 0
   0:VOlapScanNode
      TABLE: default_cluster:regression_test_performance_p0.redundant_conjuncts(redundant_conjuncts), PREAGGREGATION: OFF. Reason: No AggregateInfo
      PREDICATES: `k1` = 1
-     partitions=0/1, tablets=0/0, tabletList=
+     partitions=0/1 (), tablets=0/0, tabletList=
      cardinality=0, avgRowSize=8.0, numNodes=1
      pushAggOp=NONE
 
@@ -29,7 +29,7 @@ PLAN FRAGMENT 0
   0:VOlapScanNode
      TABLE: default_cluster:regression_test_performance_p0.redundant_conjuncts(redundant_conjuncts), PREAGGREGATION: OFF. Reason: No AggregateInfo
      PREDICATES: `k1` = 1 OR `k1` = 2
-     partitions=0/1, tablets=0/0, tabletList=
+     partitions=0/1 (), tablets=0/0, tabletList=
      cardinality=0, avgRowSize=8.0, numNodes=1
      pushAggOp=NONE