From 1fa3cd01cf04f405d70e4b938c029ba80bcd4bf7 Mon Sep 17 00:00:00 2001 From: Seaven Date: Thu, 16 Jan 2025 10:17:04 +0800 Subject: [PATCH] [Refactor] refactor pattern logical (#55048) Signed-off-by: Seaven --- .../sql/optimizer/MaterializationContext.java | 4 +- .../starrocks/sql/optimizer/Optimizer.java | 10 +- .../operator/pattern/AnyPattern.java | 33 +++++ .../operator/pattern/MultiJoinPattern.java | 33 +++++ .../operator/pattern/MultiLeafPattern.java | 33 +++++ .../operator/pattern/MultiOpPattern.java | 62 +++++++++ .../optimizer/operator/pattern/OpPattern.java | 46 +++++++ .../optimizer/operator/pattern/Pattern.java | 119 +++++------------- .../starrocks/sql/optimizer/rule/Binder.java | 25 ++-- .../starrocks/sql/optimizer/rule/RuleSet.java | 94 ++------------ .../rule/transformation/CombinationRule.java | 15 +-- .../ExternalScanPartitionPruneRule.java | 41 +++--- .../transformation/MergeLimitDirectRule.java | 62 ++++----- .../transformation/PruneEmptyScanRule.java | 24 ++-- .../PruneHDFSScanColumnRule.java | 37 +++--- .../transformation/PruneScanColumnRule.java | 22 ++-- .../PushDownLimitDirectRule.java | 16 +-- .../PushDownPredicateScanRule.java | 63 ++++------ ...hDownPredicateToExternalTableScanRule.java | 15 +-- .../RewriteSimpleAggToHDFSScanRule.java | 39 +++--- .../transformation/SkewJoinOptimizeRule.java | 7 ++ .../sql/optimizer/task/RewriteTreeTask.java | 4 +- .../sql/optimizer/task/TaskScheduler.java | 3 +- .../connector/paimon/PaimonMetadataTest.java | 2 +- .../PruneHDFSScanColumnRuleTest.java | 6 +- .../transformation/PushDownScanRuleTest.java | 2 +- .../com/starrocks/sql/plan/SkewJoinTest.java | 4 +- 27 files changed, 431 insertions(+), 390 deletions(-) create mode 100644 fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/pattern/AnyPattern.java create mode 100644 fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/pattern/MultiJoinPattern.java create mode 100644 fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/pattern/MultiLeafPattern.java create mode 100644 fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/pattern/MultiOpPattern.java create mode 100644 fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/pattern/OpPattern.java diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MaterializationContext.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MaterializationContext.java index 6c22ad99947f5..7eabf32ca15e3 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MaterializationContext.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MaterializationContext.java @@ -31,7 +31,7 @@ import com.starrocks.sql.optimizer.operator.logical.LogicalAggregationOperator; import com.starrocks.sql.optimizer.operator.logical.LogicalOlapScanOperator; import com.starrocks.sql.optimizer.operator.logical.LogicalScanOperator; -import com.starrocks.sql.optimizer.operator.pattern.Pattern; +import com.starrocks.sql.optimizer.operator.pattern.MultiOpPattern; import com.starrocks.sql.optimizer.operator.scalar.ColumnRefOperator; import com.starrocks.sql.optimizer.operator.scalar.ScalarOperator; import com.starrocks.sql.optimizer.rule.transformation.materialization.MaterializedViewRewriter; @@ -323,7 +323,7 @@ private static int getOperatorOrdering(OperatorType op) { return 1; } else if (op == OperatorType.LOGICAL_JOIN) { return 2; - } else if (Pattern.isScanOperator(op)) { + } else if (MultiOpPattern.ALL_SCAN_TYPES.contains(op)) { return 3; } else { return 4; diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/Optimizer.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/Optimizer.java index 60a5634ed611e..eaf325e43c288 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/Optimizer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/Optimizer.java @@ -695,9 +695,7 @@ private OptExpression logicalRuleRewrite( ruleBasedMaterializedViewRewrite(tree, rootTaskContext, requiredColumns); // this rewrite rule should be after mv. - scheduler.rewriteIterative(tree, rootTaskContext, RewriteSimpleAggToHDFSScanRule.HIVE_SCAN_NO_PROJECT); - scheduler.rewriteIterative(tree, rootTaskContext, RewriteSimpleAggToHDFSScanRule.ICEBERG_SCAN_NO_PROJECT); - scheduler.rewriteIterative(tree, rootTaskContext, RewriteSimpleAggToHDFSScanRule.FILE_SCAN_NO_PROJECT); + scheduler.rewriteOnce(tree, rootTaskContext, RewriteSimpleAggToHDFSScanRule.SCAN_NO_PROJECT); // NOTE: This rule should be after MV Rewrite because MV Rewrite cannot handle // select count(distinct c) from t group by a, b @@ -842,16 +840,12 @@ private OptExpression pushDownAggregation(OptExpression tree, TaskContext rootTa } private void skewJoinOptimize(OptExpression tree, TaskContext rootTaskContext) { - SkewJoinOptimizeRule rule = new SkewJoinOptimizeRule(); if (context.getSessionVariable().isEnableStatsToOptimizeSkewJoin()) { // merge projects before calculate statistics scheduler.rewriteOnce(tree, rootTaskContext, new MergeTwoProjectRule()); Utils.calculateStatistics(tree, rootTaskContext.getOptimizerContext()); } - if (scheduler.rewriteOnce(tree, rootTaskContext, rule)) { - // skew join generate new join and on predicate, need to push down join on expression to child project again - scheduler.rewriteOnce(tree, rootTaskContext, new PushDownJoinOnExpressionToChildProject()); - } + scheduler.rewriteOnce(tree, rootTaskContext, new SkewJoinOptimizeRule()); } private OptExpression pruneSubfield(OptExpression tree, TaskContext rootTaskContext, ColumnRefSet requiredColumns) { diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/pattern/AnyPattern.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/pattern/AnyPattern.java new file mode 100644 index 0000000000000..a3139cee6902a --- /dev/null +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/pattern/AnyPattern.java @@ -0,0 +1,33 @@ +// Copyright 2021-present StarRocks, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.starrocks.sql.optimizer.operator.pattern; + +import com.starrocks.sql.optimizer.operator.OperatorType; + +public class AnyPattern extends OpPattern { + protected AnyPattern() { + super(OperatorType.PATTERN_LEAF); + } + + @Override + public boolean isFixedPattern() { + return false; + } + + @Override + protected boolean matchWithoutChild(OperatorType op) { + return true; + } +} diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/pattern/MultiJoinPattern.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/pattern/MultiJoinPattern.java new file mode 100644 index 0000000000000..119d7ff8f3d72 --- /dev/null +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/pattern/MultiJoinPattern.java @@ -0,0 +1,33 @@ +// Copyright 2021-present StarRocks, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.starrocks.sql.optimizer.operator.pattern; + +import com.starrocks.sql.optimizer.operator.OperatorType; + +public class MultiJoinPattern extends OpPattern { + protected MultiJoinPattern() { + super(OperatorType.PATTERN_MULTIJOIN); + } + + @Override + public boolean isFixedPattern() { + return false; + } + + @Override + protected boolean matchWithoutChild(OperatorType op) { + return op.equals(OperatorType.LOGICAL_JOIN) || MultiOpPattern.ALL_SCAN_TYPES.contains(op); + } +} diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/pattern/MultiLeafPattern.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/pattern/MultiLeafPattern.java new file mode 100644 index 0000000000000..59e58e9aec9f8 --- /dev/null +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/pattern/MultiLeafPattern.java @@ -0,0 +1,33 @@ +// Copyright 2021-present StarRocks, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.starrocks.sql.optimizer.operator.pattern; + +import com.starrocks.sql.optimizer.operator.OperatorType; + +public class MultiLeafPattern extends OpPattern { + protected MultiLeafPattern() { + super(OperatorType.PATTERN_MULTI_LEAF); + } + + @Override + public boolean isFixedPattern() { + return false; + } + + @Override + protected boolean matchWithoutChild(OperatorType op) { + return true; + } +} diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/pattern/MultiOpPattern.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/pattern/MultiOpPattern.java new file mode 100644 index 0000000000000..19017b7545608 --- /dev/null +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/pattern/MultiOpPattern.java @@ -0,0 +1,62 @@ +// Copyright 2021-present StarRocks, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.starrocks.sql.optimizer.operator.pattern; + +import com.google.common.collect.ImmutableSet; +import com.starrocks.sql.optimizer.operator.OperatorType; + +import java.util.Set; + +public class MultiOpPattern extends Pattern { + public static final ImmutableSet ALL_SCAN_TYPES = ImmutableSet.builder() + .add(OperatorType.LOGICAL_OLAP_SCAN) + .add(OperatorType.LOGICAL_HIVE_SCAN) + .add(OperatorType.LOGICAL_ICEBERG_SCAN) + .add(OperatorType.LOGICAL_HUDI_SCAN) + .add(OperatorType.LOGICAL_FILE_SCAN) + .add(OperatorType.LOGICAL_SCHEMA_SCAN) + .add(OperatorType.LOGICAL_MYSQL_SCAN) + .add(OperatorType.LOGICAL_ES_SCAN) + .add(OperatorType.LOGICAL_META_SCAN) + .add(OperatorType.LOGICAL_JDBC_SCAN) + .add(OperatorType.LOGICAL_BINLOG_SCAN) + .add(OperatorType.LOGICAL_VIEW_SCAN) + .add(OperatorType.LOGICAL_PAIMON_SCAN) + .add(OperatorType.PATTERN_SCAN) + .build(); + + private final Set ops; + protected MultiOpPattern(Set ops) { + super(); + this.ops = ops; + } + + @Override + protected boolean matchWithoutChild(OperatorType op) { + return ops.contains(op); + } + + public static Pattern ofAllScan() { + return of(ALL_SCAN_TYPES); + } + + public static Pattern of(OperatorType... types) { + return new MultiOpPattern(Set.of(types)); + } + + public static Pattern of(Set types) { + return new MultiOpPattern(types); + } +} diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/pattern/OpPattern.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/pattern/OpPattern.java new file mode 100644 index 0000000000000..626e5ba599fd3 --- /dev/null +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/pattern/OpPattern.java @@ -0,0 +1,46 @@ +// Copyright 2021-present StarRocks, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.starrocks.sql.optimizer.operator.pattern; + +import com.starrocks.sql.optimizer.operator.OperatorType; + +public class OpPattern extends Pattern { + + protected final OperatorType opType; + + protected OpPattern(OperatorType opType) { + super(); + this.opType = opType; + } + + public OperatorType getOpType() { + return opType; + } + + @Override + public boolean is(OperatorType opType) { + return this.opType.equals(opType); + } + + @Override + public boolean isFixedPattern() { + return true; + } + + @Override + protected boolean matchWithoutChild(OperatorType op) { + return opType.equals(op); + } +} diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/pattern/Pattern.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/pattern/Pattern.java index 35921e0a04683..bc39cdf2b9350 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/pattern/Pattern.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/pattern/Pattern.java @@ -15,7 +15,6 @@ package com.starrocks.sql.optimizer.operator.pattern; import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.starrocks.sql.optimizer.GroupExpression; import com.starrocks.sql.optimizer.OptExpression; @@ -23,47 +22,26 @@ import java.util.Arrays; import java.util.List; +import java.util.Map; +import java.util.function.Function; /** * Pattern is used in rules as a placeholder for group */ -public class Pattern { - public static final ImmutableSet ALL_SCAN_TYPES = ImmutableSet.builder() - .add(OperatorType.LOGICAL_OLAP_SCAN) - .add(OperatorType.LOGICAL_HIVE_SCAN) - .add(OperatorType.LOGICAL_ICEBERG_SCAN) - .add(OperatorType.LOGICAL_HUDI_SCAN) - .add(OperatorType.LOGICAL_FILE_SCAN) - .add(OperatorType.LOGICAL_SCHEMA_SCAN) - .add(OperatorType.LOGICAL_MYSQL_SCAN) - .add(OperatorType.LOGICAL_ES_SCAN) - .add(OperatorType.LOGICAL_META_SCAN) - .add(OperatorType.LOGICAL_JDBC_SCAN) - .add(OperatorType.LOGICAL_BINLOG_SCAN) - .add(OperatorType.LOGICAL_VIEW_SCAN) - .add(OperatorType.LOGICAL_PAIMON_SCAN) - .build(); +public abstract class Pattern { + private static final Map> PATTERN_MAP = Map.of( + OperatorType.PATTERN_LEAF, p -> new AnyPattern(), + OperatorType.PATTERN_MULTI_LEAF, p -> new MultiLeafPattern(), + OperatorType.PATTERN_SCAN, p -> MultiOpPattern.ofAllScan(), + OperatorType.PATTERN_MULTIJOIN, p -> new MultiJoinPattern() + ); - private final OperatorType opType; private final List children; - protected Pattern(OperatorType opType) { - this.opType = opType; + protected Pattern() { this.children = Lists.newArrayList(); } - public OperatorType getOpType() { - return opType; - } - - public static Pattern create(OperatorType type, OperatorType... children) { - Pattern p = new Pattern(type); - for (OperatorType child : children) { - p.addChildren(new Pattern(child)); - } - return p; - } - public List children() { return children; } @@ -73,89 +51,50 @@ public Pattern childAt(int i) { } public Pattern addChildren(Pattern... children) { - Preconditions.checkArgument(opType != OperatorType.PATTERN_MULTIJOIN, - "MULTI_JOIN cannot has children"); this.children.addAll(Arrays.asList(children)); return this; } - public boolean isPatternLeaf() { - return OperatorType.PATTERN_LEAF.equals(opType); + public boolean is(OperatorType opType) { + return false; } - public boolean isPatternMultiLeaf() { - return OperatorType.PATTERN_MULTI_LEAF.equals(opType); + public boolean isFixedPattern() { + return false; } - public boolean isPatternScan() { - return OperatorType.PATTERN_SCAN.equals(opType); - } - - public boolean isPatternMultiJoin() { - return OperatorType.PATTERN_MULTIJOIN.equals(opType); - } - - public static boolean isScanOperator(OperatorType operatorType) { - return ALL_SCAN_TYPES.contains(operatorType); - } + protected abstract boolean matchWithoutChild(OperatorType op); public boolean matchWithoutChild(GroupExpression expression) { if (expression == null) { return false; } - if (expression.getInputs().size() < children.size() - && children.stream().noneMatch(p -> OperatorType.PATTERN_MULTI_LEAF.equals(p.getOpType()))) { + && children.stream().noneMatch(p -> p.is(OperatorType.PATTERN_MULTI_LEAF))) { return false; } - - if (OperatorType.PATTERN_LEAF.equals(getOpType()) || OperatorType.PATTERN_MULTI_LEAF.equals(getOpType())) { - return true; - } - - if (isPatternScan() && ALL_SCAN_TYPES.contains(expression.getOp().getOpType())) { - return true; - } - - if (isPatternMultiJoin() && isMultiJoin(expression.getOp().getOpType())) { - return true; - } - - return getOpType().equals(expression.getOp().getOpType()); + return matchWithoutChild(expression.getOp().getOpType()); } public boolean matchWithoutChild(OptExpression expression) { - if (expression == null) { - return false; - } - + Preconditions.checkNotNull(expression); if (expression.getInputs().size() < this.children().size() - && children.stream().noneMatch(p -> OperatorType.PATTERN_MULTI_LEAF.equals(p.getOpType()))) { + && children.stream().noneMatch(p -> p.is(OperatorType.PATTERN_MULTI_LEAF))) { return false; } - - if (OperatorType.PATTERN_LEAF.equals(getOpType()) || OperatorType.PATTERN_MULTI_LEAF.equals(getOpType())) { - return true; - } - - if (isPatternScan() && ALL_SCAN_TYPES.contains(expression.getOp().getOpType())) { - return true; - } - - if (isPatternMultiJoin() && isMultiJoin(expression.getOp().getOpType())) { - return true; - } - - return getOpType().equals(expression.getOp().getOpType()); + return matchWithoutChild(expression.getOp().getOpType()); } - private boolean isMultiJoin(OperatorType operatorType) { - if (ALL_SCAN_TYPES.contains(operatorType)) { - return true; - } else if (operatorType.equals(OperatorType.LOGICAL_JOIN)) { - return true; + public static Pattern create(OperatorType type, OperatorType... children) { + Pattern p; + if (PATTERN_MAP.containsKey(type)) { + p = PATTERN_MAP.get(type).apply(null); } else { - return false; + p = new OpPattern(type); + } + for (OperatorType child : children) { + p.addChildren(create(child)); } + return p; } } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/Binder.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/Binder.java index 179eaba2ab819..eaf34a5ceba8c 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/Binder.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/Binder.java @@ -23,6 +23,7 @@ import com.starrocks.sql.optimizer.OptExpression; import com.starrocks.sql.optimizer.OptimizerContext; import com.starrocks.sql.optimizer.operator.OperatorType; +import com.starrocks.sql.optimizer.operator.pattern.MultiOpPattern; import com.starrocks.sql.optimizer.operator.pattern.Pattern; import java.util.List; @@ -30,7 +31,6 @@ // Used to extract matched expression from GroupExpression public class Binder { - private final OptimizerContext optimizerContext; private final Pattern pattern; private final GroupExpression groupExpression; // binder status @@ -57,7 +57,6 @@ public class Binder { */ public Binder(OptimizerContext optimizerContext, Pattern pattern, GroupExpression groupExpression, Stopwatch stopwatch) { - this.optimizerContext = optimizerContext; this.pattern = pattern; this.groupExpression = groupExpression; this.groupExpressionIndex = Lists.newArrayList(0); @@ -65,9 +64,9 @@ public Binder(OptimizerContext optimizerContext, Pattern pattern, this.multiJoinBinder = new MultiJoinBinder(optimizerContext, stopwatch); // MULTI_JOIN is a special pattern which can contain children groups if the input group expression // is not a scan node. - this.isPatternWithoutChildren = pattern.isPatternMultiJoin() - ? Pattern.ALL_SCAN_TYPES.contains(groupExpression.getOp().getOpType()) - : pattern.children().size() == 0; + this.isPatternWithoutChildren = pattern.is(OperatorType.PATTERN_MULTIJOIN) + ? MultiOpPattern.ALL_SCAN_TYPES.contains(groupExpression.getOp().getOpType()) + : pattern.children().isEmpty(); } /* @@ -108,12 +107,11 @@ public OptExpression next() { return expression; } - /** * Pattern tree match groupExpression tree */ private OptExpression match(Pattern pattern, GroupExpression groupExpression) { - if (pattern.isPatternMultiJoin()) { + if (pattern.is(OperatorType.PATTERN_MULTIJOIN)) { return multiJoinBinder.match(groupExpression); } @@ -142,7 +140,7 @@ private OptExpression match(Pattern pattern, GroupExpression groupExpression) { resultInputs.add(opt); } - if (!(childPattern.isPatternMultiLeaf() && + if (!(childPattern.is(OperatorType.PATTERN_MULTI_LEAF) && geSize - groupExpressionIndex > patternSize - patternIndex)) { patternIndex++; } @@ -165,7 +163,7 @@ private void trace() { */ private GroupExpression extractGroupExpression(Pattern pattern, Group group) { final int valueIndex = groupExpressionIndex.get(groupTraceKey); - if (pattern.isPatternLeaf() || pattern.isPatternMultiLeaf()) { + if (pattern.is(OperatorType.PATTERN_LEAF) || pattern.is(OperatorType.PATTERN_MULTI_LEAF)) { if (valueIndex > 0) { groupExpressionIndex.remove(groupTraceKey); return null; @@ -190,7 +188,6 @@ private GroupExpression extractGroupExpression(Pattern pattern, Group group) { * binding state and check the expression at the same time. But MULTI_JOIN could enumerate the GE without any check */ private class MultiJoinBinder { - private final SessionVariable sessionVariable; // Stopwatch to void infinite loop private final Stopwatch watch; // Time limit for the entire optimization @@ -200,7 +197,7 @@ private class MultiJoinBinder { private long loopCount = 0; public MultiJoinBinder(OptimizerContext optimizerContext, Stopwatch stopwatch) { - this.sessionVariable = optimizerContext.getSessionVariable(); + SessionVariable sessionVariable = optimizerContext.getSessionVariable(); this.watch = stopwatch; this.timeLimit = Math.min(sessionVariable.getOptimizerMaterializedViewTimeLimitMillis(), sessionVariable.getOptimizerExecuteTimeout()); @@ -276,7 +273,7 @@ private GroupExpression extractGroupExpression(Group group) { } // shortcut for no child group expression - if (valueIndex > 0 && Pattern.ALL_SCAN_TYPES.contains(next.getOp().getOpType())) { + if (valueIndex > 0 && MultiOpPattern.ALL_SCAN_TYPES.contains(next.getOp().getOpType())) { groupExpressionIndex.remove(groupTraceKey); return null; } @@ -307,7 +304,8 @@ private GroupExpression extractGroupExpression(Group group) { private boolean isMultiJoinOp(GroupExpression ge) { OperatorType operatorType = ge.getOp().getOpType(); - return operatorType.equals(OperatorType.LOGICAL_JOIN) || Pattern.ALL_SCAN_TYPES.contains(operatorType); + return operatorType.equals(OperatorType.LOGICAL_JOIN) || + MultiOpPattern.ALL_SCAN_TYPES.contains(operatorType); } private boolean isMultiJoin(GroupExpression ge) { @@ -342,6 +340,7 @@ private boolean isMultiJoinRecursive(GroupExpression ge) { /** * Check Group's logical expressions except the first has already been rewritten by mv rules. + * * @param g : Group to check whether it has been rewritten by mv rules. * @return : true if the Group has GroupExpression which is rewritten by mv rules. */ diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/RuleSet.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/RuleSet.java index 99eaf59e7b63f..b5887591b6afe 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/RuleSet.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/RuleSet.java @@ -222,47 +222,14 @@ public class RuleSet { new PushDownLimitCTEAnchor(), new PushDownLimitUnionRule(), new MergeLimitWithLimitRule(), - PushDownLimitDirectRule.PROJECT, - PushDownLimitDirectRule.ASSERT_ONE_ROW, - PushDownLimitDirectRule.CTE_CONSUME, - MergeLimitDirectRule.AGGREGATE, - MergeLimitDirectRule.OLAP_SCAN, - MergeLimitDirectRule.VIEW_SCAN, - MergeLimitDirectRule.HIVE_SCAN, - MergeLimitDirectRule.ICEBERG_SCAN, - MergeLimitDirectRule.HUDI_SCAN, - MergeLimitDirectRule.DELTALAKE_SCAN, - MergeLimitDirectRule.FILE_SCAN, - MergeLimitDirectRule.PAIMON_SCAN, - MergeLimitDirectRule.ODPS_SCAN, - MergeLimitDirectRule.KUDU_SCAN, - MergeLimitDirectRule.SCHEMA_SCAN, - MergeLimitDirectRule.MYSQL_SCAN, - MergeLimitDirectRule.ES_SCAN, - MergeLimitDirectRule.JDBC_SCAN, - MergeLimitDirectRule.ICEBERG_METADATA_SCAN, - MergeLimitDirectRule.WINDOW, - MergeLimitDirectRule.INTERSECT, - MergeLimitDirectRule.EXCEPT, - MergeLimitDirectRule.VALUES, - MergeLimitDirectRule.FILTER, - MergeLimitDirectRule.CTE_CONSUMER, - MergeLimitDirectRule.TABLE_FUNCTION, - MergeLimitDirectRule.TABLE_FUNCTION_TABLE_SCAN + new PushDownLimitDirectRule(), + new MergeLimitDirectRule() )); public static final Rule PARTITION_PRUNE_RULES = new CombinationRule(RuleType.GP_PARTITION_PRUNE, ImmutableList.of( new PartitionPruneRule(), new DistributionPruneRule(), - ExternalScanPartitionPruneRule.HIVE_SCAN, - ExternalScanPartitionPruneRule.HUDI_SCAN, - ExternalScanPartitionPruneRule.ICEBERG_SCAN, - ExternalScanPartitionPruneRule.DELTALAKE_SCAN, - ExternalScanPartitionPruneRule.FILE_SCAN, - ExternalScanPartitionPruneRule.ES_SCAN, - ExternalScanPartitionPruneRule.PAIMON_SCAN, - ExternalScanPartitionPruneRule.ODPS_SCAN, - ExternalScanPartitionPruneRule.KUDU_SCAN, + new ExternalScanPartitionPruneRule(), new LimitPruneTabletsRule() )); @@ -271,22 +238,8 @@ public class RuleSet { )); public static final Rule PRUNE_COLUMNS_RULES = new CombinationRule(RuleType.GP_PRUNE_COLUMNS, ImmutableList.of( - PruneScanColumnRule.OLAP_SCAN, - PruneScanColumnRule.SCHEMA_SCAN, - PruneScanColumnRule.MYSQL_SCAN, - PruneScanColumnRule.ES_SCAN, - PruneHDFSScanColumnRule.HIVE_SCAN, - PruneHDFSScanColumnRule.ICEBERG_SCAN, - PruneHDFSScanColumnRule.DELTALAKE_SCAN, - PruneHDFSScanColumnRule.FILE_SCAN, - PruneHDFSScanColumnRule.HUDI_SCAN, - PruneHDFSScanColumnRule.TABLE_FUNCTION_TABLE_SCAN, - PruneHDFSScanColumnRule.ICEBERG_METADATA_SCAN, - PruneHDFSScanColumnRule.PAIMON_SCAN, - PruneHDFSScanColumnRule.ODPS_SCAN, - PruneScanColumnRule.KUDU_SCAN, - PruneScanColumnRule.JDBC_SCAN, - PruneScanColumnRule.BINLOG_SCAN, + new PruneScanColumnRule(), + new PruneHDFSScanColumnRule(), new PruneProjectColumnsRule(), new PruneFilterColumnsRule(), new PruneUKFKGroupByKeysRule(), // Put this before PruneAggregateColumnsRule @@ -309,22 +262,7 @@ public class RuleSet { new CastToEmptyRule(), new PruneTrueFilterRule(), new PushDownPredicateCTEAnchor(), - PushDownPredicateScanRule.OLAP_SCAN, - PushDownPredicateScanRule.HIVE_SCAN, - PushDownPredicateScanRule.ICEBERG_SCAN, - PushDownPredicateScanRule.HUDI_SCAN, - PushDownPredicateScanRule.DELTALAKE_SCAN, - PushDownPredicateScanRule.FILE_SCAN, - PushDownPredicateScanRule.PAIMON_SCAN, - PushDownPredicateScanRule.ICEBERG_METADATA_SCAN, - PushDownPredicateScanRule.ICEBERG_EQUALITY_DELETE_SCAN, - PushDownPredicateScanRule.KUDU_SCAN, - PushDownPredicateScanRule.SCHEMA_SCAN, - PushDownPredicateScanRule.ES_SCAN, - PushDownPredicateScanRule.META_SCAN, - PushDownPredicateScanRule.BINLOG_SCAN, - PushDownPredicateScanRule.TABLE_FUNCTION_TABLE_SCAN, - PushDownPredicateScanRule.VIEW_SCAN, + new PushDownPredicateScanRule(), new PushDownPredicateAggRule(), new PushDownPredicateWindowRule(), new PushDownPredicateJoinRule(), @@ -336,9 +274,7 @@ public class RuleSet { new PushDownPredicateTableFunctionRule(), new PushDownPredicateRepeatRule(), - PushDownPredicateToExternalTableScanRule.MYSQL_SCAN, - PushDownPredicateToExternalTableScanRule.JDBC_SCAN, - PushDownPredicateToExternalTableScanRule.ODPS_SCAN, + new PushDownPredicateToExternalTableScanRule(), new MergeTwoFiltersRule(), new PushDownPredicateCTEConsumeRule() )); @@ -439,13 +375,7 @@ public class RuleSet { public static final Rule PRUNE_EMPTY_OPERATOR_RULES = new CombinationRule(RuleType.GP_PRUNE_EMPTY_OPERATOR, ImmutableList.of( - PruneEmptyScanRule.OLAP_SCAN, - PruneEmptyScanRule.HIVE_SCAN, - PruneEmptyScanRule.HUDI_SCAN, - PruneEmptyScanRule.ICEBERG_SCAN, - PruneEmptyScanRule.PAIMON_SCAN, - PruneEmptyScanRule.ODPS_SCAN, - PruneEmptyScanRule.KUDU_SCAN, + new PruneEmptyScanRule(), PruneEmptyJoinRule.JOIN_LEFT_EMPTY, PruneEmptyJoinRule.JOIN_RIGHT_EMPTY, new PruneEmptyDirectRule(), @@ -459,10 +389,10 @@ public class RuleSet { new CombinationRule(RuleType.GP_SHORT_CIRCUIT_SET, ImmutableList.of( new PruneTrueFilterRule(), new PushDownPredicateProjectRule(), - PushDownPredicateScanRule.OLAP_SCAN, + new PushDownPredicateScanRule(), new CastToEmptyRule(), new PruneProjectColumnsRule(), - PruneScanColumnRule.OLAP_SCAN, + new PruneScanColumnRule(), new PruneProjectEmptyRule(), new MergeTwoProjectRule(), new PruneProjectRule(), @@ -486,9 +416,7 @@ public class RuleSet { new PushDownAggToMetaScanRule(), new PushDownFlatJsonMetaToMetaScanRule(), new RewriteSimpleAggToMetaScanRule(), - RewriteSimpleAggToHDFSScanRule.FILE_SCAN, - RewriteSimpleAggToHDFSScanRule.HIVE_SCAN, - RewriteSimpleAggToHDFSScanRule.ICEBERG_SCAN, + new RewriteSimpleAggToHDFSScanRule(), new MinMaxCountOptOnScanRule() )); diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/CombinationRule.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/CombinationRule.java index 74eba698a47f6..460b16e0ed64a 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/CombinationRule.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/CombinationRule.java @@ -14,37 +14,38 @@ package com.starrocks.sql.optimizer.rule.transformation; +import com.google.common.collect.Sets; import com.starrocks.sql.optimizer.OptExpression; import com.starrocks.sql.optimizer.OptimizerContext; import com.starrocks.sql.optimizer.operator.OperatorType; +import com.starrocks.sql.optimizer.operator.pattern.OpPattern; import com.starrocks.sql.optimizer.operator.pattern.Pattern; import com.starrocks.sql.optimizer.rule.Rule; import com.starrocks.sql.optimizer.rule.RuleType; import java.util.Collections; import java.util.List; +import java.util.Set; public class CombinationRule extends TransformationRule { private final List rules; - private int patternHash = 0; + private final Set ops = Sets.newHashSet(); public CombinationRule(RuleType ruleType, List rules) { super(ruleType, Pattern.create(OperatorType.PATTERN_LEAF)); this.rules = rules; - if (rules.stream().anyMatch(rule -> rule.getPattern().getOpType().ordinal() > OperatorType.PATTERN.ordinal())) { - patternHash = Integer.MAX_VALUE; - } else { + if (rules.stream().allMatch(rule -> rule.getPattern().isFixedPattern())) { for (Rule rule : rules) { - OperatorType type = rule.getPattern().getOpType(); - patternHash = patternHash | type.hashCode(); + OperatorType type = ((OpPattern) rule.getPattern()).getOpType(); + ops.add(type); } } } @Override public boolean check(OptExpression input, OptimizerContext context) { - return (patternHash & input.getOp().getOpType().hashCode()) != 0; + return ops.isEmpty() || ops.contains(input.getOp().getOpType()); } @Override diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/ExternalScanPartitionPruneRule.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/ExternalScanPartitionPruneRule.java index b5a1570322c80..994fb17107298 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/ExternalScanPartitionPruneRule.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/ExternalScanPartitionPruneRule.java @@ -15,16 +15,15 @@ package com.starrocks.sql.optimizer.rule.transformation; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.starrocks.sql.optimizer.OptExpression; import com.starrocks.sql.optimizer.OptimizerContext; import com.starrocks.sql.optimizer.operator.OperatorType; import com.starrocks.sql.optimizer.operator.logical.LogicalScanOperator; -import com.starrocks.sql.optimizer.operator.pattern.Pattern; +import com.starrocks.sql.optimizer.operator.pattern.MultiOpPattern; import com.starrocks.sql.optimizer.rewrite.OptExternalPartitionPruner; import com.starrocks.sql.optimizer.rule.RuleType; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import java.util.Collections; import java.util.List; @@ -32,29 +31,21 @@ import static com.starrocks.sql.optimizer.operator.OpRuleBit.OP_PARTITION_PRUNED; public class ExternalScanPartitionPruneRule extends TransformationRule { - private static final Logger LOG = LogManager.getLogger(ExternalScanPartitionPruneRule.class); - public static final ExternalScanPartitionPruneRule HIVE_SCAN = - new ExternalScanPartitionPruneRule(OperatorType.LOGICAL_HIVE_SCAN); - public static final ExternalScanPartitionPruneRule HUDI_SCAN = - new ExternalScanPartitionPruneRule(OperatorType.LOGICAL_HUDI_SCAN); - public static final ExternalScanPartitionPruneRule ICEBERG_SCAN = - new ExternalScanPartitionPruneRule(OperatorType.LOGICAL_ICEBERG_SCAN); - public static final ExternalScanPartitionPruneRule DELTALAKE_SCAN = - new ExternalScanPartitionPruneRule(OperatorType.LOGICAL_DELTALAKE_SCAN); - public static final ExternalScanPartitionPruneRule FILE_SCAN = - new ExternalScanPartitionPruneRule(OperatorType.LOGICAL_FILE_SCAN); - public static final ExternalScanPartitionPruneRule ES_SCAN = - new ExternalScanPartitionPruneRule(OperatorType.LOGICAL_ES_SCAN); - public static final ExternalScanPartitionPruneRule PAIMON_SCAN = - new ExternalScanPartitionPruneRule(OperatorType.LOGICAL_PAIMON_SCAN); - public static final ExternalScanPartitionPruneRule ODPS_SCAN = - new ExternalScanPartitionPruneRule(OperatorType.LOGICAL_ODPS_SCAN); - public static final ExternalScanPartitionPruneRule KUDU_SCAN = - new ExternalScanPartitionPruneRule(OperatorType.LOGICAL_KUDU_SCAN); - - public ExternalScanPartitionPruneRule(OperatorType logicalOperatorType) { - super(RuleType.TF_PARTITION_PRUNE, Pattern.create(logicalOperatorType)); + private static final ImmutableSet SUPPORT = ImmutableSet.of( + OperatorType.LOGICAL_HIVE_SCAN, + OperatorType.LOGICAL_HUDI_SCAN, + OperatorType.LOGICAL_ICEBERG_SCAN, + OperatorType.LOGICAL_DELTALAKE_SCAN, + OperatorType.LOGICAL_FILE_SCAN, + OperatorType.LOGICAL_ES_SCAN, + OperatorType.LOGICAL_PAIMON_SCAN, + OperatorType.LOGICAL_ODPS_SCAN, + OperatorType.LOGICAL_KUDU_SCAN + ); + + public ExternalScanPartitionPruneRule() { + super(RuleType.TF_PARTITION_PRUNE, MultiOpPattern.of(SUPPORT)); } @Override diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/MergeLimitDirectRule.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/MergeLimitDirectRule.java index 0c39a64ab1d41..cbbbcaba6a40c 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/MergeLimitDirectRule.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/MergeLimitDirectRule.java @@ -16,50 +16,52 @@ package com.starrocks.sql.optimizer.rule.transformation; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.starrocks.sql.optimizer.OptExpression; import com.starrocks.sql.optimizer.OptimizerContext; import com.starrocks.sql.optimizer.operator.OperatorType; import com.starrocks.sql.optimizer.operator.logical.LogicalLimitOperator; import com.starrocks.sql.optimizer.operator.logical.LogicalOperator; +import com.starrocks.sql.optimizer.operator.pattern.MultiOpPattern; import com.starrocks.sql.optimizer.operator.pattern.Pattern; import com.starrocks.sql.optimizer.rule.RuleType; import java.util.List; +import java.util.Set; public class MergeLimitDirectRule extends TransformationRule { - public static final MergeLimitDirectRule AGGREGATE = new MergeLimitDirectRule(OperatorType.LOGICAL_AGGR); - public static final MergeLimitDirectRule OLAP_SCAN = new MergeLimitDirectRule(OperatorType.LOGICAL_OLAP_SCAN); - public static final MergeLimitDirectRule VIEW_SCAN = new MergeLimitDirectRule(OperatorType.LOGICAL_VIEW_SCAN); - public static final MergeLimitDirectRule HIVE_SCAN = new MergeLimitDirectRule(OperatorType.LOGICAL_HIVE_SCAN); - public static final MergeLimitDirectRule ICEBERG_SCAN = new MergeLimitDirectRule(OperatorType.LOGICAL_ICEBERG_SCAN); - public static final MergeLimitDirectRule HUDI_SCAN = new MergeLimitDirectRule(OperatorType.LOGICAL_HUDI_SCAN); - public static final MergeLimitDirectRule DELTALAKE_SCAN = new MergeLimitDirectRule(OperatorType.LOGICAL_DELTALAKE_SCAN); - public static final MergeLimitDirectRule FILE_SCAN = new MergeLimitDirectRule(OperatorType.LOGICAL_FILE_SCAN); - public static final MergeLimitDirectRule PAIMON_SCAN = new MergeLimitDirectRule(OperatorType.LOGICAL_PAIMON_SCAN); - public static final MergeLimitDirectRule ODPS_SCAN = new MergeLimitDirectRule(OperatorType.LOGICAL_ODPS_SCAN); - public static final MergeLimitDirectRule KUDU_SCAN = new MergeLimitDirectRule(OperatorType.LOGICAL_KUDU_SCAN); - public static final MergeLimitDirectRule SCHEMA_SCAN = new MergeLimitDirectRule(OperatorType.LOGICAL_SCHEMA_SCAN); - public static final MergeLimitDirectRule MYSQL_SCAN = new MergeLimitDirectRule(OperatorType.LOGICAL_MYSQL_SCAN); - public static final MergeLimitDirectRule ES_SCAN = new MergeLimitDirectRule(OperatorType.LOGICAL_ES_SCAN); - public static final MergeLimitDirectRule JDBC_SCAN = new MergeLimitDirectRule(OperatorType.LOGICAL_JDBC_SCAN); - public static final MergeLimitDirectRule ICEBERG_METADATA_SCAN = new MergeLimitDirectRule( - OperatorType.LOGICAL_ICEBERG_METADATA_SCAN); - public static final MergeLimitDirectRule WINDOW = new MergeLimitDirectRule(OperatorType.LOGICAL_WINDOW); - public static final MergeLimitDirectRule INTERSECT = new MergeLimitDirectRule(OperatorType.LOGICAL_INTERSECT); - public static final MergeLimitDirectRule EXCEPT = new MergeLimitDirectRule(OperatorType.LOGICAL_EXCEPT); - public static final MergeLimitDirectRule VALUES = new MergeLimitDirectRule(OperatorType.LOGICAL_VALUES); - public static final MergeLimitDirectRule FILTER = new MergeLimitDirectRule(OperatorType.LOGICAL_FILTER); - public static final MergeLimitDirectRule TABLE_FUNCTION = - new MergeLimitDirectRule(OperatorType.LOGICAL_TABLE_FUNCTION); - public static final MergeLimitDirectRule TABLE_FUNCTION_TABLE_SCAN = - new MergeLimitDirectRule(OperatorType.LOGICAL_TABLE_FUNCTION_TABLE_SCAN); - public static final MergeLimitDirectRule CTE_CONSUMER = - new MergeLimitDirectRule(OperatorType.LOGICAL_CTE_CONSUME); + private static final Set SUPPORT_OPERATOR = ImmutableSet.builder() + .add(OperatorType.LOGICAL_OLAP_SCAN) + .add(OperatorType.LOGICAL_VIEW_SCAN) + .add(OperatorType.LOGICAL_HIVE_SCAN) + .add(OperatorType.LOGICAL_ICEBERG_SCAN) + .add(OperatorType.LOGICAL_HUDI_SCAN) + .add(OperatorType.LOGICAL_DELTALAKE_SCAN) + .add(OperatorType.LOGICAL_FILE_SCAN) + .add(OperatorType.LOGICAL_PAIMON_SCAN) + .add(OperatorType.LOGICAL_ODPS_SCAN) + .add(OperatorType.LOGICAL_KUDU_SCAN) + .add(OperatorType.LOGICAL_SCHEMA_SCAN) + .add(OperatorType.LOGICAL_MYSQL_SCAN) + .add(OperatorType.LOGICAL_ES_SCAN) + .add(OperatorType.LOGICAL_JDBC_SCAN) + .add(OperatorType.LOGICAL_ICEBERG_METADATA_SCAN) + .add(OperatorType.LOGICAL_AGGR) + .add(OperatorType.LOGICAL_WINDOW) + .add(OperatorType.LOGICAL_INTERSECT) + .add(OperatorType.LOGICAL_EXCEPT) + .add(OperatorType.LOGICAL_VALUES) + .add(OperatorType.LOGICAL_FILTER) + .add(OperatorType.LOGICAL_TABLE_FUNCTION) + .add(OperatorType.LOGICAL_TABLE_FUNCTION_TABLE_SCAN) + .add(OperatorType.LOGICAL_CTE_CONSUME) + .build(); - private MergeLimitDirectRule(OperatorType logicalOperatorType) { + public MergeLimitDirectRule() { super(RuleType.TF_MERGE_LIMIT_DIRECT, Pattern.create(OperatorType.LOGICAL_LIMIT) - .addChildren(Pattern.create(logicalOperatorType, OperatorType.PATTERN_MULTI_LEAF))); + .addChildren(MultiOpPattern.of(SUPPORT_OPERATOR) + .addChildren(Pattern.create(OperatorType.PATTERN_MULTI_LEAF)))); } @Override diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PruneEmptyScanRule.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PruneEmptyScanRule.java index ef11e121f516a..f909a95d09c6d 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PruneEmptyScanRule.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PruneEmptyScanRule.java @@ -21,23 +21,27 @@ import com.starrocks.sql.optimizer.operator.OperatorType; import com.starrocks.sql.optimizer.operator.logical.LogicalScanOperator; import com.starrocks.sql.optimizer.operator.logical.LogicalValuesOperator; -import com.starrocks.sql.optimizer.operator.pattern.Pattern; +import com.starrocks.sql.optimizer.operator.pattern.MultiOpPattern; import com.starrocks.sql.optimizer.operator.scalar.ColumnRefOperator; import com.starrocks.sql.optimizer.rule.RuleType; import java.util.List; +import java.util.Set; // transform empty scan to empty values public class PruneEmptyScanRule extends TransformationRule { - public static final PruneEmptyScanRule OLAP_SCAN = new PruneEmptyScanRule(OperatorType.LOGICAL_OLAP_SCAN); - public static final PruneEmptyScanRule HIVE_SCAN = new PruneEmptyScanRule(OperatorType.LOGICAL_HIVE_SCAN); - public static final PruneEmptyScanRule HUDI_SCAN = new PruneEmptyScanRule(OperatorType.LOGICAL_HUDI_SCAN); - public static final PruneEmptyScanRule ICEBERG_SCAN = new PruneEmptyScanRule(OperatorType.LOGICAL_ICEBERG_SCAN); - public static final PruneEmptyScanRule PAIMON_SCAN = new PruneEmptyScanRule(OperatorType.LOGICAL_PAIMON_SCAN); - public static final PruneEmptyScanRule ODPS_SCAN = new PruneEmptyScanRule(OperatorType.LOGICAL_ODPS_SCAN); - public static final PruneEmptyScanRule KUDU_SCAN = new PruneEmptyScanRule(OperatorType.LOGICAL_KUDU_SCAN); - private PruneEmptyScanRule(OperatorType logicalOperatorType) { - super(RuleType.TF_PRUNE_EMPTY_SCAN, Pattern.create(logicalOperatorType)); + private static final Set SUPPORTED = Set.of( + OperatorType.LOGICAL_OLAP_SCAN, + OperatorType.LOGICAL_HIVE_SCAN, + OperatorType.LOGICAL_HUDI_SCAN, + OperatorType.LOGICAL_ICEBERG_SCAN, + OperatorType.LOGICAL_PAIMON_SCAN, + OperatorType.LOGICAL_ODPS_SCAN, + OperatorType.LOGICAL_KUDU_SCAN + ); + + public PruneEmptyScanRule() { + super(RuleType.TF_PRUNE_EMPTY_SCAN, MultiOpPattern.of(SUPPORTED)); } @Override diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PruneHDFSScanColumnRule.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PruneHDFSScanColumnRule.java index 8a896b31f7009..0c20071d9b682 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PruneHDFSScanColumnRule.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PruneHDFSScanColumnRule.java @@ -29,7 +29,7 @@ import com.starrocks.sql.optimizer.operator.OperatorType; import com.starrocks.sql.optimizer.operator.logical.LogicalIcebergScanOperator; import com.starrocks.sql.optimizer.operator.logical.LogicalScanOperator; -import com.starrocks.sql.optimizer.operator.pattern.Pattern; +import com.starrocks.sql.optimizer.operator.pattern.MultiOpPattern; import com.starrocks.sql.optimizer.operator.scalar.ColumnRefOperator; import com.starrocks.sql.optimizer.operator.scalar.ScalarOperator; import com.starrocks.sql.optimizer.rule.RuleType; @@ -45,27 +45,20 @@ import static java.util.function.UnaryOperator.identity; public class PruneHDFSScanColumnRule extends TransformationRule { - public static final PruneHDFSScanColumnRule HIVE_SCAN = new PruneHDFSScanColumnRule(OperatorType.LOGICAL_HIVE_SCAN); - public static final PruneHDFSScanColumnRule ICEBERG_SCAN = - new PruneHDFSScanColumnRule(OperatorType.LOGICAL_ICEBERG_SCAN); - public static final PruneHDFSScanColumnRule HUDI_SCAN = new PruneHDFSScanColumnRule(OperatorType.LOGICAL_HUDI_SCAN); - public static final PruneHDFSScanColumnRule DELTALAKE_SCAN = - new PruneHDFSScanColumnRule(OperatorType.LOGICAL_DELTALAKE_SCAN); - public static final PruneHDFSScanColumnRule FILE_SCAN = - new PruneHDFSScanColumnRule(OperatorType.LOGICAL_FILE_SCAN); - public static final PruneHDFSScanColumnRule PAIMON_SCAN = - new PruneHDFSScanColumnRule(OperatorType.LOGICAL_PAIMON_SCAN); - public static final PruneHDFSScanColumnRule ODPS_SCAN = - new PruneHDFSScanColumnRule(OperatorType.LOGICAL_ODPS_SCAN); - - public static final PruneHDFSScanColumnRule TABLE_FUNCTION_TABLE_SCAN = - new PruneHDFSScanColumnRule(OperatorType.LOGICAL_TABLE_FUNCTION_TABLE_SCAN); - - public static final PruneHDFSScanColumnRule ICEBERG_METADATA_SCAN = - new PruneHDFSScanColumnRule(OperatorType.LOGICAL_ICEBERG_METADATA_SCAN); - - public PruneHDFSScanColumnRule(OperatorType logicalOperatorType) { - super(RuleType.TF_PRUNE_OLAP_SCAN_COLUMNS, Pattern.create(logicalOperatorType)); + private static final Set SUPPORTED = Set.of( + OperatorType.LOGICAL_HIVE_SCAN, + OperatorType.LOGICAL_ICEBERG_SCAN, + OperatorType.LOGICAL_HUDI_SCAN, + OperatorType.LOGICAL_DELTALAKE_SCAN, + OperatorType.LOGICAL_FILE_SCAN, + OperatorType.LOGICAL_PAIMON_SCAN, + OperatorType.LOGICAL_ODPS_SCAN, + OperatorType.LOGICAL_TABLE_FUNCTION_TABLE_SCAN, + OperatorType.LOGICAL_ICEBERG_METADATA_SCAN + ); + + public PruneHDFSScanColumnRule() { + super(RuleType.TF_PRUNE_OLAP_SCAN_COLUMNS, MultiOpPattern.of(SUPPORTED)); } @Override diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PruneScanColumnRule.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PruneScanColumnRule.java index b10ec6cb245a2..b872cf18184a0 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PruneScanColumnRule.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PruneScanColumnRule.java @@ -25,7 +25,7 @@ import com.starrocks.sql.optimizer.operator.OperatorType; import com.starrocks.sql.optimizer.operator.logical.LogicalOlapScanOperator; import com.starrocks.sql.optimizer.operator.logical.LogicalScanOperator; -import com.starrocks.sql.optimizer.operator.pattern.Pattern; +import com.starrocks.sql.optimizer.operator.pattern.MultiOpPattern; import com.starrocks.sql.optimizer.operator.scalar.ColumnRefOperator; import com.starrocks.sql.optimizer.operator.scalar.ScalarOperator; import com.starrocks.sql.optimizer.rule.RuleType; @@ -40,16 +40,18 @@ import static java.util.function.Function.identity; public class PruneScanColumnRule extends TransformationRule { - public static final PruneScanColumnRule OLAP_SCAN = new PruneScanColumnRule(OperatorType.LOGICAL_OLAP_SCAN); - public static final PruneScanColumnRule SCHEMA_SCAN = new PruneScanColumnRule(OperatorType.LOGICAL_SCHEMA_SCAN); - public static final PruneScanColumnRule MYSQL_SCAN = new PruneScanColumnRule(OperatorType.LOGICAL_MYSQL_SCAN); - public static final PruneScanColumnRule ES_SCAN = new PruneScanColumnRule(OperatorType.LOGICAL_ES_SCAN); - public static final PruneScanColumnRule JDBC_SCAN = new PruneScanColumnRule(OperatorType.LOGICAL_JDBC_SCAN); - public static final PruneScanColumnRule BINLOG_SCAN = new PruneScanColumnRule(OperatorType.LOGICAL_BINLOG_SCAN); - public static final PruneScanColumnRule KUDU_SCAN = new PruneScanColumnRule(OperatorType.LOGICAL_KUDU_SCAN); + private static final Set SUPPORTED = Set.of( + OperatorType.LOGICAL_OLAP_SCAN, + OperatorType.LOGICAL_SCHEMA_SCAN, + OperatorType.LOGICAL_MYSQL_SCAN, + OperatorType.LOGICAL_ES_SCAN, + OperatorType.LOGICAL_JDBC_SCAN, + OperatorType.LOGICAL_BINLOG_SCAN, + OperatorType.LOGICAL_KUDU_SCAN + ); - public PruneScanColumnRule(OperatorType logicalOperatorType) { - super(RuleType.TF_PRUNE_OLAP_SCAN_COLUMNS, Pattern.create(logicalOperatorType)); + public PruneScanColumnRule() { + super(RuleType.TF_PRUNE_OLAP_SCAN_COLUMNS, MultiOpPattern.of(SUPPORTED)); } @Override diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PushDownLimitDirectRule.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PushDownLimitDirectRule.java index 853fe6c286c63..76f91709e77da 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PushDownLimitDirectRule.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PushDownLimitDirectRule.java @@ -22,21 +22,23 @@ import com.starrocks.sql.optimizer.operator.OperatorType; import com.starrocks.sql.optimizer.operator.logical.LogicalLimitOperator; import com.starrocks.sql.optimizer.operator.logical.LogicalOperator; +import com.starrocks.sql.optimizer.operator.pattern.MultiOpPattern; import com.starrocks.sql.optimizer.operator.pattern.Pattern; import com.starrocks.sql.optimizer.rule.RuleType; import java.util.List; +import java.util.Set; public class PushDownLimitDirectRule extends TransformationRule { - public static final PushDownLimitDirectRule PROJECT = new PushDownLimitDirectRule(OperatorType.LOGICAL_PROJECT); - public static final PushDownLimitDirectRule ASSERT_ONE_ROW = - new PushDownLimitDirectRule(OperatorType.LOGICAL_ASSERT_ONE_ROW); - public static final PushDownLimitDirectRule CTE_CONSUME = - new PushDownLimitDirectRule(OperatorType.LOGICAL_CTE_CONSUME); + private static final Set SUPPORTED_OPERATOR = Set.of( + OperatorType.LOGICAL_PROJECT, + OperatorType.LOGICAL_ASSERT_ONE_ROW, + OperatorType.LOGICAL_CTE_CONSUME + ); - public PushDownLimitDirectRule(OperatorType logicalOperatorType) { + public PushDownLimitDirectRule() { super(RuleType.TF_PUSH_DOWN_LIMIT, Pattern.create(OperatorType.LOGICAL_LIMIT) - .addChildren(Pattern.create(logicalOperatorType, OperatorType.PATTERN_LEAF))); + .addChildren(MultiOpPattern.of(SUPPORTED_OPERATOR))); } @Override diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PushDownPredicateScanRule.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PushDownPredicateScanRule.java index d699d810f573b..4373e2d38b528 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PushDownPredicateScanRule.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PushDownPredicateScanRule.java @@ -15,6 +15,7 @@ package com.starrocks.sql.optimizer.rule.transformation; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.starrocks.sql.optimizer.OptExpression; import com.starrocks.sql.optimizer.OptimizerContext; @@ -25,6 +26,7 @@ import com.starrocks.sql.optimizer.operator.logical.LogicalFilterOperator; import com.starrocks.sql.optimizer.operator.logical.LogicalProjectOperator; import com.starrocks.sql.optimizer.operator.logical.LogicalScanOperator; +import com.starrocks.sql.optimizer.operator.pattern.MultiOpPattern; import com.starrocks.sql.optimizer.operator.pattern.Pattern; import com.starrocks.sql.optimizer.operator.scalar.ColumnRefOperator; import com.starrocks.sql.optimizer.operator.scalar.ScalarOperator; @@ -38,44 +40,28 @@ import java.util.stream.Collectors; public class PushDownPredicateScanRule extends TransformationRule { - public static final PushDownPredicateScanRule OLAP_SCAN = - new PushDownPredicateScanRule(OperatorType.LOGICAL_OLAP_SCAN); - public static final PushDownPredicateScanRule HIVE_SCAN = - new PushDownPredicateScanRule(OperatorType.LOGICAL_HIVE_SCAN); - public static final PushDownPredicateScanRule ICEBERG_SCAN = - new PushDownPredicateScanRule(OperatorType.LOGICAL_ICEBERG_SCAN); - public static final PushDownPredicateScanRule HUDI_SCAN = - new PushDownPredicateScanRule(OperatorType.LOGICAL_HUDI_SCAN); - public static final PushDownPredicateScanRule DELTALAKE_SCAN = - new PushDownPredicateScanRule(OperatorType.LOGICAL_DELTALAKE_SCAN); - public static final PushDownPredicateScanRule FILE_SCAN = - new PushDownPredicateScanRule(OperatorType.LOGICAL_FILE_SCAN); - public static final PushDownPredicateScanRule PAIMON_SCAN = - new PushDownPredicateScanRule(OperatorType.LOGICAL_PAIMON_SCAN); - public static final PushDownPredicateScanRule ICEBERG_METADATA_SCAN = - new PushDownPredicateScanRule(OperatorType.LOGICAL_ICEBERG_METADATA_SCAN); + private static final ImmutableSet SUPPORT = ImmutableSet.of( + OperatorType.LOGICAL_OLAP_SCAN, + OperatorType.LOGICAL_HIVE_SCAN, + OperatorType.LOGICAL_ICEBERG_SCAN, + OperatorType.LOGICAL_HUDI_SCAN, + OperatorType.LOGICAL_DELTALAKE_SCAN, + OperatorType.LOGICAL_FILE_SCAN, + OperatorType.LOGICAL_PAIMON_SCAN, + OperatorType.LOGICAL_ICEBERG_METADATA_SCAN, + OperatorType.LOGICAL_ICEBERG_EQUALITY_DELETE_SCAN, + OperatorType.LOGICAL_KUDU_SCAN, + OperatorType.LOGICAL_SCHEMA_SCAN, + OperatorType.LOGICAL_ES_SCAN, + OperatorType.LOGICAL_META_SCAN, + OperatorType.LOGICAL_BINLOG_SCAN, + OperatorType.LOGICAL_VIEW_SCAN, + OperatorType.LOGICAL_TABLE_FUNCTION_TABLE_SCAN + ); - public static final PushDownPredicateScanRule ICEBERG_EQUALITY_DELETE_SCAN = - new PushDownPredicateScanRule(OperatorType.LOGICAL_ICEBERG_EQUALITY_DELETE_SCAN); - public static final PushDownPredicateScanRule KUDU_SCAN = - new PushDownPredicateScanRule(OperatorType.LOGICAL_KUDU_SCAN); - public static final PushDownPredicateScanRule SCHEMA_SCAN = - new PushDownPredicateScanRule(OperatorType.LOGICAL_SCHEMA_SCAN); - public static final PushDownPredicateScanRule ES_SCAN = new PushDownPredicateScanRule(OperatorType.LOGICAL_ES_SCAN); - public static final PushDownPredicateScanRule META_SCAN = - new PushDownPredicateScanRule(OperatorType.LOGICAL_META_SCAN); - public static final PushDownPredicateScanRule JDBC_SCAN = - new PushDownPredicateScanRule(OperatorType.LOGICAL_JDBC_SCAN); - public static final PushDownPredicateScanRule BINLOG_SCAN = - new PushDownPredicateScanRule(OperatorType.LOGICAL_BINLOG_SCAN); - public static final PushDownPredicateScanRule VIEW_SCAN = - new PushDownPredicateScanRule(OperatorType.LOGICAL_VIEW_SCAN); - - public static final PushDownPredicateScanRule TABLE_FUNCTION_TABLE_SCAN = - new PushDownPredicateScanRule(OperatorType.LOGICAL_TABLE_FUNCTION_TABLE_SCAN); - - public PushDownPredicateScanRule(OperatorType type) { - super(RuleType.TF_PUSH_DOWN_PREDICATE_SCAN, Pattern.create(OperatorType.LOGICAL_FILTER, type)); + public PushDownPredicateScanRule() { + super(RuleType.TF_PUSH_DOWN_PREDICATE_SCAN, Pattern.create(OperatorType.LOGICAL_FILTER).addChildren( + MultiOpPattern.of(SUPPORT))); } @Override @@ -85,9 +71,6 @@ public List transform(OptExpression input, OptimizerContext conte OptExpression scan = input.getInputs().get(0); LogicalScanOperator logicalScanOperator = (LogicalScanOperator) scan.getOp(); - ScalarOperator scanPredicate = logicalScanOperator.getPredicate(); - ScalarOperator filterPredicate = lfo.getPredicate(); - ScalarOperatorRewriter scalarOperatorRewriter = new ScalarOperatorRewriter(); ScalarOperator predicates = Utils.compoundAnd(lfo.getPredicate(), logicalScanOperator.getPredicate()); diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PushDownPredicateToExternalTableScanRule.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PushDownPredicateToExternalTableScanRule.java index 01d68e5586bde..a4d88404bb3f3 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PushDownPredicateToExternalTableScanRule.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PushDownPredicateToExternalTableScanRule.java @@ -25,6 +25,7 @@ import com.starrocks.sql.optimizer.operator.logical.LogicalFilterOperator; import com.starrocks.sql.optimizer.operator.logical.LogicalProjectOperator; import com.starrocks.sql.optimizer.operator.logical.LogicalScanOperator; +import com.starrocks.sql.optimizer.operator.pattern.MultiOpPattern; import com.starrocks.sql.optimizer.operator.pattern.Pattern; import com.starrocks.sql.optimizer.operator.scalar.ColumnRefOperator; import com.starrocks.sql.optimizer.operator.scalar.ScalarOperator; @@ -41,16 +42,12 @@ // Because the external table may not support the functions in StarRocks, // to be on the safe side, we only push down partial predicates to the external table public class PushDownPredicateToExternalTableScanRule extends TransformationRule { - public static final PushDownPredicateToExternalTableScanRule MYSQL_SCAN = - new PushDownPredicateToExternalTableScanRule(OperatorType.LOGICAL_MYSQL_SCAN); - public static final PushDownPredicateToExternalTableScanRule JDBC_SCAN = - new PushDownPredicateToExternalTableScanRule(OperatorType.LOGICAL_JDBC_SCAN); - public static final PushDownPredicateToExternalTableScanRule ODPS_SCAN = - new PushDownPredicateToExternalTableScanRule(OperatorType.LOGICAL_ODPS_SCAN); - - public PushDownPredicateToExternalTableScanRule(OperatorType type) { + public PushDownPredicateToExternalTableScanRule() { super(RuleType.TF_PUSH_DOWN_PREDICATE_TO_EXTERNAL_TABLE_SCAN, - Pattern.create(OperatorType.LOGICAL_FILTER, type)); + Pattern.create(OperatorType.LOGICAL_FILTER) + .addChildren(MultiOpPattern.of(OperatorType.LOGICAL_MYSQL_SCAN, + OperatorType.LOGICAL_JDBC_SCAN, + OperatorType.LOGICAL_ODPS_SCAN))); } @Override diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/RewriteSimpleAggToHDFSScanRule.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/RewriteSimpleAggToHDFSScanRule.java index efca2737230d6..1617b54876a41 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/RewriteSimpleAggToHDFSScanRule.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/RewriteSimpleAggToHDFSScanRule.java @@ -34,6 +34,7 @@ import com.starrocks.sql.optimizer.operator.logical.LogicalHiveScanOperator; import com.starrocks.sql.optimizer.operator.logical.LogicalIcebergScanOperator; import com.starrocks.sql.optimizer.operator.logical.LogicalScanOperator; +import com.starrocks.sql.optimizer.operator.pattern.MultiOpPattern; import com.starrocks.sql.optimizer.operator.pattern.Pattern; import com.starrocks.sql.optimizer.operator.scalar.CallOperator; import com.starrocks.sql.optimizer.operator.scalar.ColumnRefOperator; @@ -45,40 +46,32 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; public class RewriteSimpleAggToHDFSScanRule extends TransformationRule { private static final Logger LOG = LogManager.getLogger(RewriteSimpleAggToHDFSScanRule.class); - public static final RewriteSimpleAggToHDFSScanRule HIVE_SCAN_NO_PROJECT = - new RewriteSimpleAggToHDFSScanRule(OperatorType.LOGICAL_HIVE_SCAN, true); - public static final RewriteSimpleAggToHDFSScanRule ICEBERG_SCAN_NO_PROJECT = - new RewriteSimpleAggToHDFSScanRule(OperatorType.LOGICAL_ICEBERG_SCAN, true); - public static final RewriteSimpleAggToHDFSScanRule FILE_SCAN_NO_PROJECT = - new RewriteSimpleAggToHDFSScanRule(OperatorType.LOGICAL_FILE_SCAN, true); + private static final Set SUPPORTED = Set.of(OperatorType.LOGICAL_HIVE_SCAN, + OperatorType.LOGICAL_ICEBERG_SCAN, + OperatorType.LOGICAL_FILE_SCAN + ); - public static final RewriteSimpleAggToHDFSScanRule HIVE_SCAN = - new RewriteSimpleAggToHDFSScanRule(OperatorType.LOGICAL_HIVE_SCAN); - public static final RewriteSimpleAggToHDFSScanRule ICEBERG_SCAN = - new RewriteSimpleAggToHDFSScanRule(OperatorType.LOGICAL_ICEBERG_SCAN); - public static final RewriteSimpleAggToHDFSScanRule FILE_SCAN = - new RewriteSimpleAggToHDFSScanRule(OperatorType.LOGICAL_FILE_SCAN); + public static final RewriteSimpleAggToHDFSScanRule SCAN_NO_PROJECT = + new RewriteSimpleAggToHDFSScanRule(false); - final OperatorType scanOperatorType; - final boolean hasProjectOperator; + private final boolean hasProjectOperator; - private RewriteSimpleAggToHDFSScanRule(OperatorType logicalOperatorType, boolean withoutProject) { + private RewriteSimpleAggToHDFSScanRule(boolean withoutProject) { super(RuleType.TF_REWRITE_SIMPLE_AGG, Pattern.create(OperatorType.LOGICAL_AGGR) - .addChildren(Pattern.create(logicalOperatorType))); - hasProjectOperator = false; - scanOperatorType = logicalOperatorType; + .addChildren(MultiOpPattern.of(SUPPORTED))); + hasProjectOperator = withoutProject; } - private RewriteSimpleAggToHDFSScanRule(OperatorType logicalOperatorType) { + public RewriteSimpleAggToHDFSScanRule() { super(RuleType.TF_REWRITE_SIMPLE_AGG, Pattern.create(OperatorType.LOGICAL_AGGR) - .addChildren(Pattern.create(OperatorType.LOGICAL_PROJECT, logicalOperatorType))); + .addChildren(Pattern.create(OperatorType.LOGICAL_PROJECT).addChildren(MultiOpPattern.of(SUPPORTED)))); hasProjectOperator = true; - scanOperatorType = logicalOperatorType; } private OptExpression buildAggScanOperator(LogicalAggregationOperator aggregationOperator, @@ -97,7 +90,7 @@ private OptExpression buildAggScanOperator(LogicalAggregationOperator aggregatio if (tableRelationId == -1) { tableRelationId = relationId; } else if (tableRelationId != relationId) { - LOG.warn("Table relationIds are different in columns, tableRelationId = %d, relationId = %d", + LOG.warn("Table relationIds are different in columns, tableRelationId = {}, relationId = {}", tableRelationId, relationId); return null; } @@ -222,7 +215,7 @@ public boolean check(final OptExpression input, OptimizerContext context) { return false; } - if (scanOperatorType == OperatorType.LOGICAL_ICEBERG_SCAN) { + if (scanOperator.getOpType() == OperatorType.LOGICAL_ICEBERG_SCAN) { IcebergTable icebergTable = (IcebergTable) scanOperator.getTable(); if (!icebergTable.isUnPartitioned() && !icebergTable.isAllPartitionColumnsAlwaysIdentity()) { return false; diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/SkewJoinOptimizeRule.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/SkewJoinOptimizeRule.java index 0716f0bbceead..db4da33b23e8b 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/SkewJoinOptimizeRule.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/SkewJoinOptimizeRule.java @@ -51,6 +51,7 @@ import com.starrocks.sql.optimizer.operator.scalar.ScalarOperator; import com.starrocks.sql.optimizer.rewrite.ReplaceColumnRefRewriter; import com.starrocks.sql.optimizer.rewrite.ScalarOperatorRewriter; +import com.starrocks.sql.optimizer.rule.Rule; import com.starrocks.sql.optimizer.rule.RuleType; import com.starrocks.sql.optimizer.statistics.ColumnStatistic; import com.starrocks.sql.optimizer.statistics.Statistics; @@ -96,6 +97,12 @@ public SkewJoinOptimizeRule() { Pattern.create(OperatorType.LOGICAL_JOIN, OperatorType.PATTERN_LEAF, OperatorType.PATTERN_LEAF)); } + @Override + public List successorRules() { + // skew join generate new join and on predicate, need to push down join on expression to child project again + return Lists.newArrayList(new PushDownJoinOnExpressionToChildProject()); + } + @Override public boolean check(OptExpression input, OptimizerContext context) { // respect the join hint diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/RewriteTreeTask.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/RewriteTreeTask.java index 7f50091d12eae..17380d201a7ff 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/RewriteTreeTask.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/RewriteTreeTask.java @@ -124,7 +124,7 @@ protected boolean match(Pattern pattern, OptExpression root) { } if (!pattern.children().isEmpty() && pattern.children().size() != root.getInputs().size() && - pattern.children().stream().noneMatch(Pattern::isPatternMultiLeaf)) { + pattern.children().stream().noneMatch(p -> p.is(OperatorType.PATTERN_MULTI_LEAF))) { return false; } int patternIndex = 0; @@ -138,7 +138,7 @@ protected boolean match(Pattern pattern, OptExpression root) { return false; } - if (!(childPattern.isPatternMultiLeaf() && (root.getInputs().size() - childIndex) > + if (!(childPattern.is(OperatorType.PATTERN_MULTI_LEAF) && (root.getInputs().size() - childIndex) > (pattern.children().size() - patternIndex))) { patternIndex++; } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/TaskScheduler.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/TaskScheduler.java index a9a81a5f5a8e0..a5eb700b9648a 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/TaskScheduler.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/TaskScheduler.java @@ -79,11 +79,10 @@ public void rewriteIterative(OptExpression tree, TaskContext rootTaskContext, Ru executeTasks(rootTaskContext); } - public boolean rewriteOnce(OptExpression tree, TaskContext rootTaskContext, Rule rule) { + public void rewriteOnce(OptExpression tree, TaskContext rootTaskContext, Rule rule) { RewriteTreeTask rewriteTreeTask = new RewriteTreeTask(rootTaskContext, tree, rule, true); pushTask(rewriteTreeTask); executeTasks(rootTaskContext); - return rewriteTreeTask.hasChange(); } public void rewriteAtMostOnce(OptExpression tree, TaskContext rootTaskContext, Rule rule) { diff --git a/fe/fe-core/src/test/java/com/starrocks/connector/paimon/PaimonMetadataTest.java b/fe/fe-core/src/test/java/com/starrocks/connector/paimon/PaimonMetadataTest.java index 71d777ae4293f..57372c09a3b3c 100644 --- a/fe/fe-core/src/test/java/com/starrocks/connector/paimon/PaimonMetadataTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/connector/paimon/PaimonMetadataTest.java @@ -427,7 +427,7 @@ public long getTableCreateTime(String dbName, String tblName) { PaimonTable paimonTable = (PaimonTable) metadata.getTable("db1", "tbl1"); - ExternalScanPartitionPruneRule rule0 = ExternalScanPartitionPruneRule.PAIMON_SCAN; + ExternalScanPartitionPruneRule rule0 = new ExternalScanPartitionPruneRule(); ColumnRefOperator colRef1 = new ColumnRefOperator(1, Type.INT, "f2", true); Column col1 = new Column("f2", Type.INT, true); diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/PruneHDFSScanColumnRuleTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/PruneHDFSScanColumnRuleTest.java index 2166cbb8e47cd..213248337ff16 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/PruneHDFSScanColumnRuleTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/PruneHDFSScanColumnRuleTest.java @@ -45,9 +45,9 @@ import java.util.stream.Collectors; public class PruneHDFSScanColumnRuleTest { - private PruneHDFSScanColumnRule icebergRule = PruneHDFSScanColumnRule.ICEBERG_SCAN; - private PruneHDFSScanColumnRule hudiRule = PruneHDFSScanColumnRule.HUDI_SCAN; - private PruneHDFSScanColumnRule odpsRule = PruneHDFSScanColumnRule.ODPS_SCAN; + private PruneHDFSScanColumnRule icebergRule = new PruneHDFSScanColumnRule(); + private PruneHDFSScanColumnRule hudiRule = new PruneHDFSScanColumnRule(); + private PruneHDFSScanColumnRule odpsRule = new PruneHDFSScanColumnRule(); ColumnRefOperator intColumnOperator = new ColumnRefOperator(1, Type.INT, "id", true); ColumnRefOperator strColumnOperator = new ColumnRefOperator(2, Type.STRING, "name", true); diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/PushDownScanRuleTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/PushDownScanRuleTest.java index 837baefd6ed72..c56e0ca90bddc 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/PushDownScanRuleTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/PushDownScanRuleTest.java @@ -42,7 +42,7 @@ public class PushDownScanRuleTest { @Test public void transform(@Mocked OlapTable table) { - PushDownPredicateScanRule rule = PushDownPredicateScanRule.OLAP_SCAN; + PushDownPredicateScanRule rule = new PushDownPredicateScanRule(); OptExpression optExpression = new OptExpression(new LogicalFilterOperator( new BinaryPredicateOperator(BinaryType.EQ, diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/plan/SkewJoinTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/plan/SkewJoinTest.java index e89844cc4b506..064470f08ba21 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/plan/SkewJoinTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/plan/SkewJoinTest.java @@ -220,8 +220,8 @@ public void testSkewJoinWithStructType() throws Exception { " | colocate: false, reason: \n" + " | equal join conjunct: 10: rand_col = 17: rand_col\n" + " | equal join conjunct: 9: cast = 5: v1", - " : CASE WHEN 2: c1.a[true] IS NULL THEN 24: round WHEN 2: c1.a[true] IN (1, 2) THEN 24: " + - "round ELSE 0 END"); + " : CASE WHEN 2: c1.a[true] IS NULL THEN 23: round WHEN 2: c1.a[true] IN (1, 2) THEN " + + "23: round ELSE 0 END"); } @Test