From c56d3237e8e70c161f3812cab2d8ff97f887bb0e Mon Sep 17 00:00:00 2001 From: morrySnow <101034200+morrySnow@users.noreply.github.com> Date: Mon, 18 Sep 2023 12:22:33 +0800 Subject: [PATCH 01/12] [opt](Nereids) remove canEliminate flag on LogicalProject (#24362) since we have three infrastructure to ensure changing input column order not lead to wrong result, we could remove this flag on LogicalProject to eliminate project as mush as possible and let code clear. 1. output list in ResultSink node 2. regular children output in SetOperation node 3. producer to consumer slot id map in CteConsumer --- .../org/apache/doris/nereids/memo/Memo.java | 13 ++- .../rewrite/EliminateUnnecessaryProject.java | 56 +++-------- .../nereids/rules/rewrite/MergeProjects.java | 9 +- .../trees/plans/logical/LogicalProject.java | 45 +++------ .../EliminateUnnecessaryProjectTest.java | 6 +- .../nereids/trees/plans/PlanToStringTest.java | 2 +- .../doris/nereids/util/PlanChecker.java | 7 ++ .../nereids_ssb_shape_sf100_p0/shape/q2.1.out | 33 +++---- .../nereids_ssb_shape_sf100_p0/shape/q2.2.out | 45 +++++---- .../nereids_ssb_shape_sf100_p0/shape/q2.3.out | 45 +++++---- .../shape/query10.out | 70 ++++++------- .../shape/query16.out | 61 ++++++------ .../shape/query19.out | 59 ++++++----- .../shape/query3.out | 33 +++---- .../shape/query31.out | 48 +++++---- .../shape/query35.out | 80 +++++++-------- .../shape/query38.out | 99 +++++++++---------- .../shape/query52.out | 29 +++--- .../shape/query55.out | 29 +++--- .../shape/query59.out | 19 ++-- .../shape/query71.out | 83 ++++++++-------- .../shape/query87.out | 99 +++++++++---------- .../shape/query94.out | 53 +++++----- .../shape/query95.out | 49 +++++---- .../shape/q10.out | 41 ++++---- .../nereids_tpch_shape_sf1000_p0/shape/q3.out | 33 +++---- .../nereids_tpch_shape_sf500_p0/shape/q10.out | 41 ++++---- .../nereids_tpch_shape_sf500_p0/shape/q3.out | 33 +++---- .../rf/ds_rf10.groovy | 2 +- .../rf/ds_rf35.groovy | 2 +- .../rf/ds_rf95.groovy | 2 +- 31 files changed, 577 insertions(+), 649 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java index d3a5d8a4889983..41e2cd912b7e4b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java @@ -138,8 +138,8 @@ public void removePhysicalExpression() { private Plan skipProject(Plan plan, Group targetGroup) { // Some top project can't be eliminated - if (plan instanceof LogicalProject && ((LogicalProject) plan).canEliminate()) { - LogicalProject logicalProject = (LogicalProject) plan; + if (plan instanceof LogicalProject) { + LogicalProject logicalProject = (LogicalProject) plan; if (targetGroup != root) { if (logicalProject.getOutputSet().equals(logicalProject.child().getOutputSet())) { return skipProject(logicalProject.child(), targetGroup); @@ -155,7 +155,7 @@ private Plan skipProject(Plan plan, Group targetGroup) { private Plan skipProjectGetChild(Plan plan) { if (plan instanceof LogicalProject) { - LogicalProject logicalProject = (LogicalProject) plan; + LogicalProject logicalProject = (LogicalProject) plan; Plan child = logicalProject.child(); if (logicalProject.getOutputSet().equals(child.getOutputSet())) { return skipProjectGetChild(child); @@ -915,7 +915,7 @@ private PhysicalPlan unrankGroup(Group group, PhysicalProperties prop, long rank int prefix = 0; for (GroupExpression groupExpression : extractGroupExpressionContainsProp(group, prop)) { List> possiblePlans = rankGroupExpression(groupExpression, prop); - if (possiblePlans.size() != 0 && rank - prefix <= possiblePlans.get(possiblePlans.size() - 1).first) { + if (!possiblePlans.isEmpty() && rank - prefix <= possiblePlans.get(possiblePlans.size() - 1).first) { return unrankGroupExpression(groupExpression, prop, rank - prefix); } prefix += possiblePlans.size(); @@ -944,10 +944,9 @@ private PhysicalPlan unrankGroupExpression(GroupExpression groupExpression, childrenPlan.add(unrankGroup(groupExpression.child(i), properties.get(i), childrenRanks.get(i))); } Plan plan = groupExpression.getPlan().withChildren(childrenPlan); - PhysicalPlan physicalPlan = ((PhysicalPlan) plan).withPhysicalPropertiesAndStats( + return ((PhysicalPlan) plan).withPhysicalPropertiesAndStats( groupExpression.getOutputProperties(prop), groupExpression.getOwnerGroup().getStatistics()); - return physicalPlan; } /** @@ -957,7 +956,7 @@ private PhysicalPlan unrankGroupExpression(GroupExpression groupExpression, * 2: [2%1, 2%(1*2)] */ private List extractChildRanks(long rank, List>> children) { - Preconditions.checkArgument(children.size() > 0); + Preconditions.checkArgument(!children.isEmpty(), "children should not empty in extractChildRanks"); int factor = children.get(0).size(); List indices = new ArrayList<>(); for (int i = 0; i < children.size() - 1; i++) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateUnnecessaryProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateUnnecessaryProject.java index adb355f94dbe28..852c6f5ea21515 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateUnnecessaryProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateUnnecessaryProject.java @@ -23,8 +23,6 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; -import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation; -import org.apache.doris.nereids.trees.plans.logical.OutputSavePoint; import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter; import java.util.ArrayList; @@ -39,62 +37,34 @@ public class EliminateUnnecessaryProject implements CustomRewriter { @Override public Plan rewriteRoot(Plan plan, JobContext jobContext) { - return rewrite(plan, false); + return rewrite(plan); } - private Plan rewrite(Plan plan, boolean outputSavePoint) { - if (plan instanceof LogicalSetOperation) { - return rewriteLogicalSetOperation((LogicalSetOperation) plan, outputSavePoint); - } else if (plan instanceof LogicalProject) { - return rewriteProject((LogicalProject) plan, outputSavePoint); - } else if (plan instanceof OutputSavePoint) { - return rewriteChildren(plan, true); + private Plan rewrite(Plan plan) { + if (plan instanceof LogicalProject) { + return rewriteProject((LogicalProject) plan); } else { - return rewriteChildren(plan, outputSavePoint); + return rewriteChildren(plan); } } - private Plan rewriteProject(LogicalProject project, boolean outputSavePoint) { + private Plan rewriteProject(LogicalProject project) { if (project.child() instanceof LogicalEmptyRelation) { // eliminate unnecessary project return new LogicalEmptyRelation(StatementScopeIdGenerator.newRelationId(), project.getProjects()); - } else if (project.canEliminate() && outputSavePoint - && project.getOutputSet().equals(project.child().getOutputSet())) { + } else if (project.getOutputSet().equals(project.child().getOutputSet())) { // eliminate unnecessary project - return rewrite(project.child(), outputSavePoint); - } else if (project.canEliminate() && project.getOutput().equals(project.child().getOutput())) { - // eliminate unnecessary project - return rewrite(project.child(), outputSavePoint); + return rewrite(project.child()); } else { - return rewriteChildren(project, true); + return rewriteChildren(project); } } - private Plan rewriteLogicalSetOperation(LogicalSetOperation set, boolean outputSavePoint) { - if (set.arity() == 2) { - Plan left = set.child(0); - Plan right = set.child(1); - boolean changed = false; - if (isCanEliminateProject(left)) { - changed = true; - left = ((LogicalProject) left).withEliminate(false); - } - if (isCanEliminateProject(right)) { - changed = true; - right = ((LogicalProject) right).withEliminate(false); - } - if (changed) { - set = (LogicalSetOperation) set.withChildren(left, right); - } - } - return rewriteChildren(set, outputSavePoint); - } - - private Plan rewriteChildren(Plan plan, boolean outputSavePoint) { + private Plan rewriteChildren(Plan plan) { List newChildren = new ArrayList<>(); boolean hasNewChildren = false; for (Plan child : plan.children()) { - Plan newChild = rewrite(child, outputSavePoint); + Plan newChild = rewrite(child); if (newChild != child) { hasNewChildren = true; } @@ -102,8 +72,4 @@ private Plan rewriteChildren(Plan plan, boolean outputSavePoint) { } return hasNewChildren ? plan.withChildren(newChildren) : plan; } - - private static boolean isCanEliminateProject(Plan plan) { - return plan instanceof LogicalProject && ((LogicalProject) plan).canEliminate(); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeProjects.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeProjects.java index 96e506ec144e04..6452b4e0ba61ff 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeProjects.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeProjects.java @@ -45,14 +45,13 @@ public Rule build() { return logicalProject(logicalProject()) .whenNot(project -> containsWindowExpression(project.getProjects()) && containsWindowExpression(project.child().getProjects())) - .then(project -> mergeProjects(project)).toRule(RuleType.MERGE_PROJECTS); + .then(MergeProjects::mergeProjects).toRule(RuleType.MERGE_PROJECTS); } - public static Plan mergeProjects(LogicalProject project) { - LogicalProject childProject = (LogicalProject) project.child(); + public static Plan mergeProjects(LogicalProject project) { + LogicalProject childProject = (LogicalProject) project.child(); List projectExpressions = project.mergeProjections(childProject); - LogicalProject newProject = childProject.canEliminate() ? project : childProject; - return newProject.withProjectsAndChild(projectExpressions, childProject.child(0)); + return project.withProjectsAndChild(projectExpressions, childProject.child(0)); } private boolean containsWindowExpression(List expressions) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java index 5b58c305b16a9f..283d0976730107 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java @@ -49,37 +49,30 @@ public class LogicalProject extends LogicalUnary projects; private final List excepts; private final boolean isDistinct; - // For project nodes under union, erasure cannot be configured, so add this flag. - private final boolean canEliminate; public LogicalProject(List projects, CHILD_TYPE child) { - this(projects, ImmutableList.of(), false, true, child); + this(projects, ImmutableList.of(), false, child); } /** * only for test. */ public LogicalProject(List projects, List excepts, CHILD_TYPE child) { - this(projects, excepts, false, true, child); + this(projects, excepts, false, child); } public LogicalProject(List projects, boolean isDistinct, CHILD_TYPE child) { - this(projects, ImmutableList.of(), isDistinct, true, child); + this(projects, ImmutableList.of(), isDistinct, child); } public LogicalProject(List projects, List excepts, boolean isDistinct, CHILD_TYPE child) { - this(projects, excepts, isDistinct, true, child); - } - - private LogicalProject(List projects, List excepts, - boolean isDistinct, boolean canEliminate, CHILD_TYPE child) { - this(projects, excepts, isDistinct, canEliminate, Optional.empty(), Optional.empty(), child); + this(projects, excepts, isDistinct, Optional.empty(), Optional.empty(), child); } private LogicalProject(List projects, List excepts, boolean isDistinct, - boolean canEliminate, Optional groupExpression, - Optional logicalProperties, CHILD_TYPE child) { + Optional groupExpression, Optional logicalProperties, + CHILD_TYPE child) { super(PlanType.LOGICAL_PROJECT, groupExpression, logicalProperties, child); Preconditions.checkArgument(projects != null, "projects can not be null"); // only ColumnPrune rule may produce empty projects, this happens in rewrite phase @@ -91,7 +84,6 @@ private LogicalProject(List projects, List exc : projects; this.excepts = ImmutableList.copyOf(excepts); this.isDistinct = isDistinct; - this.canEliminate = canEliminate; } /** @@ -124,8 +116,7 @@ public String toString() { return Utils.toSqlString("LogicalProject[" + id.asInt() + "]", "distinct", isDistinct, "projects", projects, - "excepts", excepts, - "canEliminate", canEliminate + "excepts", excepts ); } @@ -150,7 +141,6 @@ public boolean equals(Object o) { LogicalProject that = (LogicalProject) o; boolean equal = projects.equals(that.projects) && excepts.equals(that.excepts) - && canEliminate == that.canEliminate && isDistinct == that.isDistinct; // TODO: should add exprId for UnBoundStar and BoundStar for equality comparison if (!projects.isEmpty() && (projects.get(0) instanceof UnboundStar || projects.get(0) instanceof BoundStar)) { @@ -161,18 +151,18 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(projects, canEliminate); + return Objects.hash(projects); } @Override public LogicalProject withChildren(List children) { Preconditions.checkArgument(children.size() == 1); - return new LogicalProject<>(projects, excepts, isDistinct, canEliminate, children.get(0)); + return new LogicalProject<>(projects, excepts, isDistinct, children.get(0)); } @Override public LogicalProject withGroupExpression(Optional groupExpression) { - return new LogicalProject<>(projects, excepts, isDistinct, canEliminate, + return new LogicalProject<>(projects, excepts, isDistinct, groupExpression, Optional.of(getLogicalProperties()), child()); } @@ -180,30 +170,22 @@ public LogicalProject withGroupExpression(Optional groupE public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { Preconditions.checkArgument(children.size() == 1); - return new LogicalProject<>(projects, excepts, isDistinct, canEliminate, + return new LogicalProject<>(projects, excepts, isDistinct, groupExpression, logicalProperties, children.get(0)); } - public LogicalProject withEliminate(boolean isEliminate) { - return new LogicalProject<>(projects, excepts, isDistinct, isEliminate, child()); - } - public LogicalProject withProjects(List projects) { - return new LogicalProject<>(projects, excepts, isDistinct, canEliminate, child()); + return new LogicalProject<>(projects, excepts, isDistinct, child()); } public LogicalProject withProjectsAndChild(List projects, Plan child) { - return new LogicalProject<>(projects, excepts, isDistinct, canEliminate, child); + return new LogicalProject<>(projects, excepts, isDistinct, child); } public boolean isDistinct() { return isDistinct; } - public boolean canEliminate() { - return canEliminate; - } - @Override public List getOutputs() { return projects; @@ -220,7 +202,6 @@ public JSONObject toJson() { JSONObject properties = new JSONObject(); properties.put("Projects", projects.toString()); properties.put("Excepts", excepts.toString()); - properties.put("CanEliminate", canEliminate); properties.put("IsDistinct", isDistinct); logicalProject.put("Properties", properties); return logicalProject; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateUnnecessaryProjectTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateUnnecessaryProjectTest.java index 548a7dd0689532..15eccdd8f23560 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateUnnecessaryProjectTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateUnnecessaryProjectTest.java @@ -61,7 +61,7 @@ void testEliminateNonTopUnnecessaryProject() { PlanChecker.from(MemoTestUtils.createConnectContext(), unnecessaryProject) .customRewrite(new EliminateUnnecessaryProject()) - .matchesFromRoot(logicalFilter(logicalProject())); + .matchesFromRoot(logicalFilter(logicalOlapScan())); } @Test @@ -76,14 +76,14 @@ void testEliminateTopUnnecessaryProject() { } @Test - void testNotEliminateTopProjectWhenOutputNotEquals() { + void testEliminateTopProjectWhenOutputNotEquals() { LogicalPlan necessaryProject = new LogicalPlanBuilder(PlanConstructor.newLogicalOlapScan(0, "t1", 0)) .project(ImmutableList.of(1, 0)) .build(); PlanChecker.from(MemoTestUtils.createConnectContext(), necessaryProject) .customRewrite(new EliminateUnnecessaryProject()) - .matchesFromRoot(logicalProject()); + .notMatchesFromRoot(logicalProject()); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanToStringTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanToStringTest.java index 7d9594c73ba607..53c938d8493140 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanToStringTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanToStringTest.java @@ -91,7 +91,7 @@ public void testLogicalProject(@Mocked Plan child) { LogicalProject plan = new LogicalProject<>(ImmutableList.of( new SlotReference(new ExprId(0), "a", BigIntType.INSTANCE, true, Lists.newArrayList())), child); - Assertions.assertTrue(plan.toString().matches("LogicalProject\\[\\d+\\] \\( distinct=false, projects=\\[a#\\d+], excepts=\\[], canEliminate=true \\)")); + Assertions.assertTrue(plan.toString().matches("LogicalProject\\[\\d+\\] \\( distinct=false, projects=\\[a#\\d+], excepts=\\[] \\)")); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java index 254b6d5a9814fe..9274fdd0abd4dc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java @@ -457,6 +457,13 @@ public PlanChecker matchesFromRoot(PatternDescriptor patternDesc return this; } + public PlanChecker notMatchesFromRoot(PatternDescriptor patternDesc) { + Memo memo = cascadesContext.getMemo(); + assertMatches(memo, () -> !(new GroupExpressionMatching(patternDesc.pattern, + memo.getRoot().getLogicalExpression()).iterator().hasNext())); + return this; + } + public PlanChecker matches(PatternDescriptor patternDesc) { Memo memo = cascadesContext.getMemo(); checkSlotFromChildren(memo); diff --git a/regression-test/data/nereids_ssb_shape_sf100_p0/shape/q2.1.out b/regression-test/data/nereids_ssb_shape_sf100_p0/shape/q2.1.out index f8cbe1787f6aef..44e6976230157e 100644 --- a/regression-test/data/nereids_ssb_shape_sf100_p0/shape/q2.1.out +++ b/regression-test/data/nereids_ssb_shape_sf100_p0/shape/q2.1.out @@ -4,25 +4,24 @@ PhysicalResultSink --PhysicalQuickSort ----PhysicalDistribute ------PhysicalQuickSort ---------PhysicalProject -----------hashAgg[GLOBAL] -------------PhysicalDistribute ---------------hashAgg[LOCAL] -----------------PhysicalProject -------------------hashJoin[INNER_JOIN](lineorder.lo_orderdate = dates.d_datekey) ---------------------hashJoin[INNER_JOIN](lineorder.lo_suppkey = supplier.s_suppkey) -----------------------hashJoin[INNER_JOIN](lineorder.lo_partkey = part.p_partkey) -------------------------PhysicalProject ---------------------------PhysicalOlapScan[lineorder] -------------------------PhysicalDistribute ---------------------------PhysicalProject -----------------------------filter((part.p_category = 'MFGR#12')) -------------------------------PhysicalOlapScan[part] +--------hashAgg[GLOBAL] +----------PhysicalDistribute +------------hashAgg[LOCAL] +--------------PhysicalProject +----------------hashJoin[INNER_JOIN](lineorder.lo_orderdate = dates.d_datekey) +------------------hashJoin[INNER_JOIN](lineorder.lo_suppkey = supplier.s_suppkey) +--------------------hashJoin[INNER_JOIN](lineorder.lo_partkey = part.p_partkey) +----------------------PhysicalProject +------------------------PhysicalOlapScan[lineorder] ----------------------PhysicalDistribute ------------------------PhysicalProject ---------------------------filter((supplier.s_region = 'AMERICA')) -----------------------------PhysicalOlapScan[supplier] +--------------------------filter((part.p_category = 'MFGR#12')) +----------------------------PhysicalOlapScan[part] --------------------PhysicalDistribute ----------------------PhysicalProject -------------------------PhysicalOlapScan[dates] +------------------------filter((supplier.s_region = 'AMERICA')) +--------------------------PhysicalOlapScan[supplier] +------------------PhysicalDistribute +--------------------PhysicalProject +----------------------PhysicalOlapScan[dates] diff --git a/regression-test/data/nereids_ssb_shape_sf100_p0/shape/q2.2.out b/regression-test/data/nereids_ssb_shape_sf100_p0/shape/q2.2.out index de0b1d9ce05100..7f5570465b93a2 100644 --- a/regression-test/data/nereids_ssb_shape_sf100_p0/shape/q2.2.out +++ b/regression-test/data/nereids_ssb_shape_sf100_p0/shape/q2.2.out @@ -4,28 +4,27 @@ PhysicalResultSink --PhysicalQuickSort ----PhysicalDistribute ------PhysicalQuickSort ---------PhysicalProject -----------hashAgg[GLOBAL] -------------PhysicalDistribute ---------------hashAgg[LOCAL] -----------------PhysicalProject -------------------hashJoin[INNER_JOIN](lineorder.lo_orderdate = dates.d_datekey) ---------------------PhysicalDistribute -----------------------PhysicalProject -------------------------hashJoin[INNER_JOIN](lineorder.lo_suppkey = supplier.s_suppkey) ---------------------------PhysicalDistribute -----------------------------hashJoin[INNER_JOIN](lineorder.lo_partkey = part.p_partkey) -------------------------------PhysicalProject ---------------------------------PhysicalOlapScan[lineorder] -------------------------------PhysicalDistribute ---------------------------------PhysicalProject -----------------------------------filter((part.p_brand >= 'MFGR#2221')(part.p_brand <= 'MFGR#2228')) -------------------------------------PhysicalOlapScan[part] ---------------------------PhysicalDistribute +--------hashAgg[GLOBAL] +----------PhysicalDistribute +------------hashAgg[LOCAL] +--------------PhysicalProject +----------------hashJoin[INNER_JOIN](lineorder.lo_orderdate = dates.d_datekey) +------------------PhysicalDistribute +--------------------PhysicalProject +----------------------hashJoin[INNER_JOIN](lineorder.lo_suppkey = supplier.s_suppkey) +------------------------PhysicalDistribute +--------------------------hashJoin[INNER_JOIN](lineorder.lo_partkey = part.p_partkey) ----------------------------PhysicalProject -------------------------------filter((supplier.s_region = 'ASIA')) ---------------------------------PhysicalOlapScan[supplier] ---------------------PhysicalDistribute -----------------------PhysicalProject -------------------------PhysicalOlapScan[dates] +------------------------------PhysicalOlapScan[lineorder] +----------------------------PhysicalDistribute +------------------------------PhysicalProject +--------------------------------filter((part.p_brand >= 'MFGR#2221')(part.p_brand <= 'MFGR#2228')) +----------------------------------PhysicalOlapScan[part] +------------------------PhysicalDistribute +--------------------------PhysicalProject +----------------------------filter((supplier.s_region = 'ASIA')) +------------------------------PhysicalOlapScan[supplier] +------------------PhysicalDistribute +--------------------PhysicalProject +----------------------PhysicalOlapScan[dates] diff --git a/regression-test/data/nereids_ssb_shape_sf100_p0/shape/q2.3.out b/regression-test/data/nereids_ssb_shape_sf100_p0/shape/q2.3.out index 17b70822a93148..10955aaf2295d3 100644 --- a/regression-test/data/nereids_ssb_shape_sf100_p0/shape/q2.3.out +++ b/regression-test/data/nereids_ssb_shape_sf100_p0/shape/q2.3.out @@ -4,28 +4,27 @@ PhysicalResultSink --PhysicalQuickSort ----PhysicalDistribute ------PhysicalQuickSort ---------PhysicalProject -----------hashAgg[GLOBAL] -------------PhysicalDistribute ---------------hashAgg[LOCAL] -----------------PhysicalProject -------------------hashJoin[INNER_JOIN](lineorder.lo_orderdate = dates.d_datekey) ---------------------PhysicalDistribute -----------------------PhysicalProject -------------------------hashJoin[INNER_JOIN](lineorder.lo_suppkey = supplier.s_suppkey) ---------------------------PhysicalDistribute -----------------------------hashJoin[INNER_JOIN](lineorder.lo_partkey = part.p_partkey) -------------------------------PhysicalProject ---------------------------------PhysicalOlapScan[lineorder] -------------------------------PhysicalDistribute ---------------------------------PhysicalProject -----------------------------------filter((part.p_brand = 'MFGR#2239')) -------------------------------------PhysicalOlapScan[part] ---------------------------PhysicalDistribute +--------hashAgg[GLOBAL] +----------PhysicalDistribute +------------hashAgg[LOCAL] +--------------PhysicalProject +----------------hashJoin[INNER_JOIN](lineorder.lo_orderdate = dates.d_datekey) +------------------PhysicalDistribute +--------------------PhysicalProject +----------------------hashJoin[INNER_JOIN](lineorder.lo_suppkey = supplier.s_suppkey) +------------------------PhysicalDistribute +--------------------------hashJoin[INNER_JOIN](lineorder.lo_partkey = part.p_partkey) ----------------------------PhysicalProject -------------------------------filter((supplier.s_region = 'EUROPE')) ---------------------------------PhysicalOlapScan[supplier] ---------------------PhysicalDistribute -----------------------PhysicalProject -------------------------PhysicalOlapScan[dates] +------------------------------PhysicalOlapScan[lineorder] +----------------------------PhysicalDistribute +------------------------------PhysicalProject +--------------------------------filter((part.p_brand = 'MFGR#2239')) +----------------------------------PhysicalOlapScan[part] +------------------------PhysicalDistribute +--------------------------PhysicalProject +----------------------------filter((supplier.s_region = 'EUROPE')) +------------------------------PhysicalOlapScan[supplier] +------------------PhysicalDistribute +--------------------PhysicalProject +----------------------PhysicalOlapScan[dates] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query10.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query10.out index df38625cf1552b..f9fa345be63011 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query10.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query10.out @@ -9,26 +9,26 @@ PhysicalResultSink ------------PhysicalDistribute --------------hashAgg[LOCAL] ----------------PhysicalProject -------------------filter(($c$1 OR $c$2)) ---------------------hashJoin[LEFT_SEMI_JOIN](c.c_customer_sk = catalog_sales.cs_ship_customer_sk) -----------------------hashJoin[LEFT_SEMI_JOIN](c.c_customer_sk = web_sales.ws_bill_customer_sk) -------------------------PhysicalDistribute +------------------hashJoin[RIGHT_SEMI_JOIN](c.c_customer_sk = store_sales.ss_customer_sk) +--------------------PhysicalDistribute +----------------------PhysicalProject +------------------------hashJoin[INNER_JOIN](store_sales.ss_sold_date_sk = date_dim.d_date_sk) --------------------------PhysicalProject -----------------------------hashJoin[INNER_JOIN](customer_demographics.cd_demo_sk = c.c_current_cdemo_sk) -------------------------------PhysicalOlapScan[customer_demographics] -------------------------------PhysicalDistribute ---------------------------------PhysicalProject -----------------------------------hashJoin[RIGHT_SEMI_JOIN](c.c_customer_sk = store_sales.ss_customer_sk) -------------------------------------PhysicalDistribute ---------------------------------------PhysicalProject -----------------------------------------hashJoin[INNER_JOIN](store_sales.ss_sold_date_sk = date_dim.d_date_sk) -------------------------------------------PhysicalProject ---------------------------------------------PhysicalOlapScan[store_sales] -------------------------------------------PhysicalDistribute ---------------------------------------------PhysicalProject -----------------------------------------------filter((date_dim.d_moy <= 4)(date_dim.d_moy >= 1)(date_dim.d_year = 2001)) -------------------------------------------------PhysicalOlapScan[date_dim] -------------------------------------PhysicalDistribute +----------------------------PhysicalOlapScan[store_sales] +--------------------------PhysicalDistribute +----------------------------PhysicalProject +------------------------------filter((date_dim.d_moy >= 1)(date_dim.d_moy <= 4)(date_dim.d_year = 2001)) +--------------------------------PhysicalOlapScan[date_dim] +--------------------PhysicalProject +----------------------filter(($c$1 OR $c$2)) +------------------------hashJoin[LEFT_SEMI_JOIN](c.c_customer_sk = catalog_sales.cs_ship_customer_sk) +--------------------------hashJoin[LEFT_SEMI_JOIN](c.c_customer_sk = web_sales.ws_bill_customer_sk) +----------------------------PhysicalDistribute +------------------------------PhysicalProject +--------------------------------hashJoin[INNER_JOIN](customer_demographics.cd_demo_sk = c.c_current_cdemo_sk) +----------------------------------PhysicalOlapScan[customer_demographics] +----------------------------------PhysicalDistribute +------------------------------------PhysicalProject --------------------------------------hashJoin[INNER_JOIN](c.c_current_addr_sk = ca.ca_address_sk) ----------------------------------------PhysicalProject ------------------------------------------PhysicalOlapScan[customer] @@ -36,22 +36,22 @@ PhysicalResultSink ------------------------------------------PhysicalProject --------------------------------------------filter(ca_county IN ('Storey County', 'Marquette County', 'Warren County', 'Cochran County', 'Kandiyohi County')) ----------------------------------------------PhysicalOlapScan[customer_address] -------------------------PhysicalDistribute ---------------------------PhysicalProject -----------------------------hashJoin[INNER_JOIN](web_sales.ws_sold_date_sk = date_dim.d_date_sk) -------------------------------PhysicalProject ---------------------------------PhysicalOlapScan[web_sales] -------------------------------PhysicalDistribute ---------------------------------PhysicalProject -----------------------------------filter((date_dim.d_moy <= 4)(date_dim.d_moy >= 1)(date_dim.d_year = 2001)) -------------------------------------PhysicalOlapScan[date_dim] -----------------------PhysicalDistribute -------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN](catalog_sales.cs_sold_date_sk = date_dim.d_date_sk) -----------------------------PhysicalProject -------------------------------PhysicalOlapScan[catalog_sales] ----------------------------PhysicalDistribute ------------------------------PhysicalProject ---------------------------------filter((date_dim.d_moy >= 1)(date_dim.d_moy <= 4)(date_dim.d_year = 2001)) -----------------------------------PhysicalOlapScan[date_dim] +--------------------------------hashJoin[INNER_JOIN](web_sales.ws_sold_date_sk = date_dim.d_date_sk) +----------------------------------PhysicalProject +------------------------------------PhysicalOlapScan[web_sales] +----------------------------------PhysicalDistribute +------------------------------------PhysicalProject +--------------------------------------filter((date_dim.d_moy >= 1)(date_dim.d_year = 2001)(date_dim.d_moy <= 4)) +----------------------------------------PhysicalOlapScan[date_dim] +--------------------------PhysicalDistribute +----------------------------PhysicalProject +------------------------------hashJoin[INNER_JOIN](catalog_sales.cs_sold_date_sk = date_dim.d_date_sk) +--------------------------------PhysicalProject +----------------------------------PhysicalOlapScan[catalog_sales] +--------------------------------PhysicalDistribute +----------------------------------PhysicalProject +------------------------------------filter((date_dim.d_moy >= 1)(date_dim.d_moy <= 4)(date_dim.d_year = 2001)) +--------------------------------------PhysicalOlapScan[date_dim] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query16.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query16.out index 43abdade0fe88c..7167ef0b0c35c3 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query16.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query16.out @@ -3,37 +3,36 @@ PhysicalResultSink --PhysicalTopN ----PhysicalTopN -------PhysicalProject ---------hashAgg[DISTINCT_GLOBAL] -----------PhysicalDistribute -------------hashAgg[DISTINCT_LOCAL] ---------------hashAgg[GLOBAL] -----------------hashAgg[LOCAL] -------------------PhysicalProject ---------------------hashJoin[RIGHT_SEMI_JOIN](cs1.cs_order_number = cs2.cs_order_number)( not (cs_warehouse_sk = cs_warehouse_sk)) -----------------------PhysicalDistribute -------------------------PhysicalProject ---------------------------PhysicalOlapScan[catalog_sales] -----------------------hashJoin[INNER_JOIN](cs1.cs_call_center_sk = call_center.cc_call_center_sk) -------------------------hashJoin[RIGHT_ANTI_JOIN](cs1.cs_order_number = cr1.cr_order_number) ---------------------------PhysicalDistribute -----------------------------PhysicalProject -------------------------------PhysicalOlapScan[catalog_returns] ---------------------------PhysicalDistribute -----------------------------hashJoin[INNER_JOIN](cs1.cs_ship_date_sk = date_dim.d_date_sk) -------------------------------hashJoin[INNER_JOIN](cs1.cs_ship_addr_sk = customer_address.ca_address_sk) ---------------------------------PhysicalProject -----------------------------------PhysicalOlapScan[catalog_sales] ---------------------------------PhysicalDistribute -----------------------------------PhysicalProject -------------------------------------filter((cast(ca_state as VARCHAR(*)) = 'WV')) ---------------------------------------PhysicalOlapScan[customer_address] -------------------------------PhysicalDistribute ---------------------------------PhysicalProject -----------------------------------filter((date_dim.d_date <= 2002-05-31)(date_dim.d_date >= 2002-04-01)) -------------------------------------PhysicalOlapScan[date_dim] +------hashAgg[DISTINCT_GLOBAL] +--------PhysicalDistribute +----------hashAgg[DISTINCT_LOCAL] +------------hashAgg[GLOBAL] +--------------hashAgg[LOCAL] +----------------PhysicalProject +------------------hashJoin[RIGHT_SEMI_JOIN](cs1.cs_order_number = cs2.cs_order_number)( not (cs_warehouse_sk = cs_warehouse_sk)) +--------------------PhysicalDistribute +----------------------PhysicalProject +------------------------PhysicalOlapScan[catalog_sales] +--------------------hashJoin[INNER_JOIN](cs1.cs_call_center_sk = call_center.cc_call_center_sk) +----------------------hashJoin[RIGHT_ANTI_JOIN](cs1.cs_order_number = cr1.cr_order_number) ------------------------PhysicalDistribute --------------------------PhysicalProject -----------------------------filter(cc_county IN ('Ziebach County', 'Luce County', 'Richland County', 'Daviess County', 'Barrow County')) -------------------------------PhysicalOlapScan[call_center] +----------------------------PhysicalOlapScan[catalog_returns] +------------------------PhysicalDistribute +--------------------------hashJoin[INNER_JOIN](cs1.cs_ship_date_sk = date_dim.d_date_sk) +----------------------------hashJoin[INNER_JOIN](cs1.cs_ship_addr_sk = customer_address.ca_address_sk) +------------------------------PhysicalProject +--------------------------------PhysicalOlapScan[catalog_sales] +------------------------------PhysicalDistribute +--------------------------------PhysicalProject +----------------------------------filter((cast(ca_state as VARCHAR(*)) = 'WV')) +------------------------------------PhysicalOlapScan[customer_address] +----------------------------PhysicalDistribute +------------------------------PhysicalProject +--------------------------------filter((date_dim.d_date <= 2002-05-31)(date_dim.d_date >= 2002-04-01)) +----------------------------------PhysicalOlapScan[date_dim] +----------------------PhysicalDistribute +------------------------PhysicalProject +--------------------------filter(cc_county IN ('Ziebach County', 'Luce County', 'Richland County', 'Daviess County', 'Barrow County')) +----------------------------PhysicalOlapScan[call_center] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query19.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query19.out index f01f7b5a8e57fa..5164ce7840a79b 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query19.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query19.out @@ -4,37 +4,36 @@ PhysicalResultSink --PhysicalTopN ----PhysicalDistribute ------PhysicalTopN ---------PhysicalProject -----------hashAgg[GLOBAL] -------------PhysicalDistribute ---------------hashAgg[LOCAL] -----------------PhysicalProject -------------------hashJoin[INNER_JOIN](store_sales.ss_store_sk = store.s_store_sk)( not (substring(ca_zip, 1, 5) = substring(s_zip, 1, 5))) ---------------------PhysicalProject -----------------------hashJoin[INNER_JOIN](customer.c_current_addr_sk = customer_address.ca_address_sk) +--------hashAgg[GLOBAL] +----------PhysicalDistribute +------------hashAgg[LOCAL] +--------------PhysicalProject +----------------hashJoin[INNER_JOIN](store_sales.ss_store_sk = store.s_store_sk)( not (substring(ca_zip, 1, 5) = substring(s_zip, 1, 5))) +------------------PhysicalProject +--------------------hashJoin[INNER_JOIN](customer.c_current_addr_sk = customer_address.ca_address_sk) +----------------------PhysicalProject +------------------------PhysicalOlapScan[customer_address] +----------------------PhysicalDistribute ------------------------PhysicalProject ---------------------------PhysicalOlapScan[customer_address] -------------------------PhysicalDistribute ---------------------------PhysicalProject -----------------------------hashJoin[INNER_JOIN](store_sales.ss_customer_sk = customer.c_customer_sk) -------------------------------PhysicalDistribute ---------------------------------PhysicalProject -----------------------------------PhysicalOlapScan[customer] -------------------------------PhysicalDistribute ---------------------------------PhysicalProject -----------------------------------hashJoin[INNER_JOIN](date_dim.d_date_sk = store_sales.ss_sold_date_sk) -------------------------------------hashJoin[INNER_JOIN](store_sales.ss_item_sk = item.i_item_sk) ---------------------------------------PhysicalProject -----------------------------------------PhysicalOlapScan[store_sales] ---------------------------------------PhysicalDistribute -----------------------------------------PhysicalProject -------------------------------------------filter((item.i_manager_id = 2)) ---------------------------------------------PhysicalOlapScan[item] +--------------------------hashJoin[INNER_JOIN](store_sales.ss_customer_sk = customer.c_customer_sk) +----------------------------PhysicalDistribute +------------------------------PhysicalProject +--------------------------------PhysicalOlapScan[customer] +----------------------------PhysicalDistribute +------------------------------PhysicalProject +--------------------------------hashJoin[INNER_JOIN](date_dim.d_date_sk = store_sales.ss_sold_date_sk) +----------------------------------hashJoin[INNER_JOIN](store_sales.ss_item_sk = item.i_item_sk) +------------------------------------PhysicalProject +--------------------------------------PhysicalOlapScan[store_sales] ------------------------------------PhysicalDistribute --------------------------------------PhysicalProject -----------------------------------------filter((date_dim.d_moy = 12)(date_dim.d_year = 1999)) -------------------------------------------PhysicalOlapScan[date_dim] ---------------------PhysicalDistribute -----------------------PhysicalProject -------------------------PhysicalOlapScan[store] +----------------------------------------filter((item.i_manager_id = 2)) +------------------------------------------PhysicalOlapScan[item] +----------------------------------PhysicalDistribute +------------------------------------PhysicalProject +--------------------------------------filter((date_dim.d_moy = 12)(date_dim.d_year = 1999)) +----------------------------------------PhysicalOlapScan[date_dim] +------------------PhysicalDistribute +--------------------PhysicalProject +----------------------PhysicalOlapScan[store] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query3.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query3.out index fc17a5bab55f63..60ed4b442ce24f 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query3.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query3.out @@ -4,22 +4,21 @@ PhysicalResultSink --PhysicalTopN ----PhysicalDistribute ------PhysicalTopN ---------PhysicalProject -----------hashAgg[GLOBAL] -------------PhysicalDistribute ---------------hashAgg[LOCAL] -----------------PhysicalProject -------------------hashJoin[INNER_JOIN](dt.d_date_sk = store_sales.ss_sold_date_sk) ---------------------PhysicalDistribute -----------------------hashJoin[INNER_JOIN](store_sales.ss_item_sk = item.i_item_sk) -------------------------PhysicalProject ---------------------------PhysicalOlapScan[store_sales] -------------------------PhysicalDistribute ---------------------------PhysicalProject -----------------------------filter((item.i_manufact_id = 816)) -------------------------------PhysicalOlapScan[item] ---------------------PhysicalDistribute +--------hashAgg[GLOBAL] +----------PhysicalDistribute +------------hashAgg[LOCAL] +--------------PhysicalProject +----------------hashJoin[INNER_JOIN](dt.d_date_sk = store_sales.ss_sold_date_sk) +------------------PhysicalDistribute +--------------------hashJoin[INNER_JOIN](store_sales.ss_item_sk = item.i_item_sk) ----------------------PhysicalProject -------------------------filter((dt.d_moy = 11)) ---------------------------PhysicalOlapScan[date_dim] +------------------------PhysicalOlapScan[store_sales] +----------------------PhysicalDistribute +------------------------PhysicalProject +--------------------------filter((item.i_manufact_id = 816)) +----------------------------PhysicalOlapScan[item] +------------------PhysicalDistribute +--------------------PhysicalProject +----------------------filter((dt.d_moy = 11)) +------------------------PhysicalOlapScan[date_dim] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query31.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query31.out index 953ca425f06285..83184d7b60981f 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query31.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query31.out @@ -2,44 +2,42 @@ -- !ds_shape_31 -- PhysicalCteAnchor ( cteId=CTEId#0 ) --PhysicalCteProducer ( cteId=CTEId#0 ) -----PhysicalProject +----hashAgg[GLOBAL] +------PhysicalDistribute +--------hashAgg[LOCAL] +----------PhysicalProject +------------hashJoin[INNER_JOIN](store_sales.ss_addr_sk = customer_address.ca_address_sk) +--------------PhysicalDistribute +----------------PhysicalProject +------------------hashJoin[INNER_JOIN](store_sales.ss_sold_date_sk = date_dim.d_date_sk) +--------------------PhysicalProject +----------------------PhysicalOlapScan[store_sales] +--------------------PhysicalDistribute +----------------------PhysicalProject +------------------------filter(d_qoy IN (1, 2, 3)(ss.d_year = 2000)) +--------------------------PhysicalOlapScan[date_dim] +--------------PhysicalDistribute +----------------PhysicalProject +------------------PhysicalOlapScan[customer_address] +--PhysicalCteAnchor ( cteId=CTEId#1 ) +----PhysicalCteProducer ( cteId=CTEId#1 ) ------hashAgg[GLOBAL] --------PhysicalDistribute ----------hashAgg[LOCAL] ------------PhysicalProject ---------------hashJoin[INNER_JOIN](store_sales.ss_addr_sk = customer_address.ca_address_sk) +--------------hashJoin[INNER_JOIN](web_sales.ws_bill_addr_sk = customer_address.ca_address_sk) ----------------PhysicalDistribute ------------------PhysicalProject ---------------------hashJoin[INNER_JOIN](store_sales.ss_sold_date_sk = date_dim.d_date_sk) +--------------------hashJoin[INNER_JOIN](web_sales.ws_sold_date_sk = date_dim.d_date_sk) ----------------------PhysicalProject -------------------------PhysicalOlapScan[store_sales] +------------------------PhysicalOlapScan[web_sales] ----------------------PhysicalDistribute ------------------------PhysicalProject ---------------------------filter(d_qoy IN (1, 2, 3)(ss.d_year = 2000)) +--------------------------filter((ws.d_year = 2000)d_qoy IN (1, 2, 3)) ----------------------------PhysicalOlapScan[date_dim] ----------------PhysicalDistribute ------------------PhysicalProject --------------------PhysicalOlapScan[customer_address] ---PhysicalCteAnchor ( cteId=CTEId#1 ) -----PhysicalCteProducer ( cteId=CTEId#1 ) -------PhysicalProject ---------hashAgg[GLOBAL] -----------PhysicalDistribute -------------hashAgg[LOCAL] ---------------PhysicalProject -----------------hashJoin[INNER_JOIN](web_sales.ws_bill_addr_sk = customer_address.ca_address_sk) -------------------PhysicalDistribute ---------------------PhysicalProject -----------------------hashJoin[INNER_JOIN](web_sales.ws_sold_date_sk = date_dim.d_date_sk) -------------------------PhysicalProject ---------------------------PhysicalOlapScan[web_sales] -------------------------PhysicalDistribute ---------------------------PhysicalProject -----------------------------filter((ws.d_year = 2000)d_qoy IN (1, 2, 3)) -------------------------------PhysicalOlapScan[date_dim] -------------------PhysicalDistribute ---------------------PhysicalProject -----------------------PhysicalOlapScan[customer_address] ----PhysicalResultSink ------PhysicalQuickSort --------PhysicalDistribute diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query35.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query35.out index 1a1d022d75b7b9..8b3fcceebe9448 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query35.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query35.out @@ -9,51 +9,51 @@ PhysicalResultSink ------------PhysicalDistribute --------------hashAgg[LOCAL] ----------------PhysicalProject -------------------filter(($c$1 OR $c$2)) ---------------------hashJoin[LEFT_SEMI_JOIN](c.c_customer_sk = catalog_sales.cs_ship_customer_sk) -----------------------hashJoin[LEFT_SEMI_JOIN](c.c_customer_sk = web_sales.ws_bill_customer_sk) -------------------------PhysicalDistribute +------------------hashJoin[RIGHT_SEMI_JOIN](c.c_customer_sk = store_sales.ss_customer_sk) +--------------------PhysicalDistribute +----------------------PhysicalProject +------------------------hashJoin[INNER_JOIN](store_sales.ss_sold_date_sk = date_dim.d_date_sk) --------------------------PhysicalProject -----------------------------hashJoin[INNER_JOIN](customer_demographics.cd_demo_sk = c.c_current_cdemo_sk) -------------------------------PhysicalDistribute ---------------------------------PhysicalProject -----------------------------------hashJoin[INNER_JOIN](c.c_current_addr_sk = ca.ca_address_sk) -------------------------------------PhysicalDistribute ---------------------------------------hashJoin[RIGHT_SEMI_JOIN](c.c_customer_sk = store_sales.ss_customer_sk) +----------------------------PhysicalOlapScan[store_sales] +--------------------------PhysicalDistribute +----------------------------PhysicalProject +------------------------------filter((date_dim.d_qoy < 4)(date_dim.d_year = 2001)) +--------------------------------PhysicalOlapScan[date_dim] +--------------------PhysicalProject +----------------------filter(($c$1 OR $c$2)) +------------------------hashJoin[LEFT_SEMI_JOIN](c.c_customer_sk = catalog_sales.cs_ship_customer_sk) +--------------------------hashJoin[LEFT_SEMI_JOIN](c.c_customer_sk = web_sales.ws_bill_customer_sk) +----------------------------PhysicalDistribute +------------------------------PhysicalProject +--------------------------------hashJoin[INNER_JOIN](customer_demographics.cd_demo_sk = c.c_current_cdemo_sk) +----------------------------------PhysicalDistribute +------------------------------------PhysicalProject +--------------------------------------hashJoin[INNER_JOIN](c.c_current_addr_sk = ca.ca_address_sk) ----------------------------------------PhysicalDistribute ------------------------------------------PhysicalProject ---------------------------------------------hashJoin[INNER_JOIN](store_sales.ss_sold_date_sk = date_dim.d_date_sk) -----------------------------------------------PhysicalProject -------------------------------------------------PhysicalOlapScan[store_sales] -----------------------------------------------PhysicalDistribute -------------------------------------------------PhysicalProject ---------------------------------------------------filter((date_dim.d_qoy < 4)(date_dim.d_year = 2001)) -----------------------------------------------------PhysicalOlapScan[date_dim] +--------------------------------------------PhysicalOlapScan[customer] ----------------------------------------PhysicalDistribute ------------------------------------------PhysicalProject ---------------------------------------------PhysicalOlapScan[customer] -------------------------------------PhysicalDistribute ---------------------------------------PhysicalProject -----------------------------------------PhysicalOlapScan[customer_address] -------------------------------PhysicalDistribute ---------------------------------PhysicalProject -----------------------------------PhysicalOlapScan[customer_demographics] -------------------------PhysicalDistribute ---------------------------PhysicalProject -----------------------------hashJoin[INNER_JOIN](web_sales.ws_sold_date_sk = date_dim.d_date_sk) -------------------------------PhysicalProject ---------------------------------PhysicalOlapScan[web_sales] -------------------------------PhysicalDistribute ---------------------------------PhysicalProject -----------------------------------filter((date_dim.d_qoy < 4)(date_dim.d_year = 2001)) -------------------------------------PhysicalOlapScan[date_dim] -----------------------PhysicalDistribute -------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN](catalog_sales.cs_sold_date_sk = date_dim.d_date_sk) -----------------------------PhysicalProject -------------------------------PhysicalOlapScan[catalog_sales] +--------------------------------------------PhysicalOlapScan[customer_address] +----------------------------------PhysicalDistribute +------------------------------------PhysicalProject +--------------------------------------PhysicalOlapScan[customer_demographics] ----------------------------PhysicalDistribute ------------------------------PhysicalProject ---------------------------------filter((date_dim.d_qoy < 4)(date_dim.d_year = 2001)) -----------------------------------PhysicalOlapScan[date_dim] +--------------------------------hashJoin[INNER_JOIN](web_sales.ws_sold_date_sk = date_dim.d_date_sk) +----------------------------------PhysicalProject +------------------------------------PhysicalOlapScan[web_sales] +----------------------------------PhysicalDistribute +------------------------------------PhysicalProject +--------------------------------------filter((date_dim.d_qoy < 4)(date_dim.d_year = 2001)) +----------------------------------------PhysicalOlapScan[date_dim] +--------------------------PhysicalDistribute +----------------------------PhysicalProject +------------------------------hashJoin[INNER_JOIN](catalog_sales.cs_sold_date_sk = date_dim.d_date_sk) +--------------------------------PhysicalProject +----------------------------------PhysicalOlapScan[catalog_sales] +--------------------------------PhysicalDistribute +----------------------------------PhysicalProject +------------------------------------filter((date_dim.d_qoy < 4)(date_dim.d_year = 2001)) +--------------------------------------PhysicalOlapScan[date_dim] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query38.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query38.out index 3b8e5596cc2be6..d8ce696005c514 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query38.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query38.out @@ -8,58 +8,55 @@ PhysicalResultSink ----------hashAgg[LOCAL] ------------PhysicalProject --------------PhysicalIntersect -----------------PhysicalProject -------------------hashAgg[GLOBAL] ---------------------PhysicalDistribute -----------------------hashAgg[LOCAL] -------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN](store_sales.ss_customer_sk = customer.c_customer_sk) -----------------------------PhysicalDistribute -------------------------------PhysicalProject ---------------------------------hashJoin[INNER_JOIN](store_sales.ss_sold_date_sk = date_dim.d_date_sk) +----------------hashAgg[GLOBAL] +------------------PhysicalDistribute +--------------------hashAgg[LOCAL] +----------------------PhysicalProject +------------------------hashJoin[INNER_JOIN](store_sales.ss_customer_sk = customer.c_customer_sk) +--------------------------PhysicalDistribute +----------------------------PhysicalProject +------------------------------hashJoin[INNER_JOIN](store_sales.ss_sold_date_sk = date_dim.d_date_sk) +--------------------------------PhysicalProject +----------------------------------PhysicalOlapScan[store_sales] +--------------------------------PhysicalDistribute ----------------------------------PhysicalProject -------------------------------------PhysicalOlapScan[store_sales] -----------------------------------PhysicalDistribute -------------------------------------PhysicalProject ---------------------------------------filter((date_dim.d_month_seq <= 1194)(date_dim.d_month_seq >= 1183)) -----------------------------------------PhysicalOlapScan[date_dim] -----------------------------PhysicalDistribute -------------------------------PhysicalProject ---------------------------------PhysicalOlapScan[customer] -----------------PhysicalProject -------------------hashAgg[GLOBAL] ---------------------PhysicalDistribute -----------------------hashAgg[LOCAL] -------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN](catalog_sales.cs_bill_customer_sk = customer.c_customer_sk) -----------------------------PhysicalDistribute -------------------------------PhysicalProject ---------------------------------hashJoin[INNER_JOIN](catalog_sales.cs_sold_date_sk = date_dim.d_date_sk) +------------------------------------filter((date_dim.d_month_seq <= 1194)(date_dim.d_month_seq >= 1183)) +--------------------------------------PhysicalOlapScan[date_dim] +--------------------------PhysicalDistribute +----------------------------PhysicalProject +------------------------------PhysicalOlapScan[customer] +----------------hashAgg[GLOBAL] +------------------PhysicalDistribute +--------------------hashAgg[LOCAL] +----------------------PhysicalProject +------------------------hashJoin[INNER_JOIN](catalog_sales.cs_bill_customer_sk = customer.c_customer_sk) +--------------------------PhysicalDistribute +----------------------------PhysicalProject +------------------------------hashJoin[INNER_JOIN](catalog_sales.cs_sold_date_sk = date_dim.d_date_sk) +--------------------------------PhysicalProject +----------------------------------PhysicalOlapScan[catalog_sales] +--------------------------------PhysicalDistribute ----------------------------------PhysicalProject -------------------------------------PhysicalOlapScan[catalog_sales] -----------------------------------PhysicalDistribute -------------------------------------PhysicalProject ---------------------------------------filter((date_dim.d_month_seq <= 1194)(date_dim.d_month_seq >= 1183)) -----------------------------------------PhysicalOlapScan[date_dim] -----------------------------PhysicalDistribute -------------------------------PhysicalProject ---------------------------------PhysicalOlapScan[customer] -----------------PhysicalProject -------------------hashAgg[GLOBAL] ---------------------PhysicalDistribute -----------------------hashAgg[LOCAL] -------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN](web_sales.ws_bill_customer_sk = customer.c_customer_sk) -----------------------------PhysicalDistribute -------------------------------PhysicalProject ---------------------------------hashJoin[INNER_JOIN](web_sales.ws_sold_date_sk = date_dim.d_date_sk) +------------------------------------filter((date_dim.d_month_seq <= 1194)(date_dim.d_month_seq >= 1183)) +--------------------------------------PhysicalOlapScan[date_dim] +--------------------------PhysicalDistribute +----------------------------PhysicalProject +------------------------------PhysicalOlapScan[customer] +----------------hashAgg[GLOBAL] +------------------PhysicalDistribute +--------------------hashAgg[LOCAL] +----------------------PhysicalProject +------------------------hashJoin[INNER_JOIN](web_sales.ws_bill_customer_sk = customer.c_customer_sk) +--------------------------PhysicalDistribute +----------------------------PhysicalProject +------------------------------hashJoin[INNER_JOIN](web_sales.ws_sold_date_sk = date_dim.d_date_sk) +--------------------------------PhysicalProject +----------------------------------PhysicalOlapScan[web_sales] +--------------------------------PhysicalDistribute ----------------------------------PhysicalProject -------------------------------------PhysicalOlapScan[web_sales] -----------------------------------PhysicalDistribute -------------------------------------PhysicalProject ---------------------------------------filter((date_dim.d_month_seq <= 1194)(date_dim.d_month_seq >= 1183)) -----------------------------------------PhysicalOlapScan[date_dim] -----------------------------PhysicalDistribute -------------------------------PhysicalProject ---------------------------------PhysicalOlapScan[customer] +------------------------------------filter((date_dim.d_month_seq <= 1194)(date_dim.d_month_seq >= 1183)) +--------------------------------------PhysicalOlapScan[date_dim] +--------------------------PhysicalDistribute +----------------------------PhysicalProject +------------------------------PhysicalOlapScan[customer] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query52.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query52.out index 4bf2ceed3dcbef..66ea8b1873475e 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query52.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query52.out @@ -4,21 +4,20 @@ PhysicalResultSink --PhysicalTopN ----PhysicalDistribute ------PhysicalTopN ---------PhysicalProject -----------hashAgg[GLOBAL] -------------PhysicalDistribute ---------------hashAgg[LOCAL] -----------------PhysicalProject -------------------hashJoin[INNER_JOIN](dt.d_date_sk = store_sales.ss_sold_date_sk) ---------------------hashJoin[INNER_JOIN](store_sales.ss_item_sk = item.i_item_sk) -----------------------PhysicalProject -------------------------PhysicalOlapScan[store_sales] -----------------------PhysicalDistribute -------------------------PhysicalProject ---------------------------filter((item.i_manager_id = 1)) -----------------------------PhysicalOlapScan[item] +--------hashAgg[GLOBAL] +----------PhysicalDistribute +------------hashAgg[LOCAL] +--------------PhysicalProject +----------------hashJoin[INNER_JOIN](dt.d_date_sk = store_sales.ss_sold_date_sk) +------------------hashJoin[INNER_JOIN](store_sales.ss_item_sk = item.i_item_sk) +--------------------PhysicalProject +----------------------PhysicalOlapScan[store_sales] --------------------PhysicalDistribute ----------------------PhysicalProject -------------------------filter((dt.d_moy = 12)(dt.d_year = 2002)) ---------------------------PhysicalOlapScan[date_dim] +------------------------filter((item.i_manager_id = 1)) +--------------------------PhysicalOlapScan[item] +------------------PhysicalDistribute +--------------------PhysicalProject +----------------------filter((dt.d_moy = 12)(dt.d_year = 2002)) +------------------------PhysicalOlapScan[date_dim] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query55.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query55.out index 90db0b7ca05f11..1215eb4637bf8e 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query55.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query55.out @@ -4,21 +4,20 @@ PhysicalResultSink --PhysicalTopN ----PhysicalDistribute ------PhysicalTopN ---------PhysicalProject -----------hashAgg[GLOBAL] -------------PhysicalDistribute ---------------hashAgg[LOCAL] -----------------PhysicalProject -------------------hashJoin[INNER_JOIN](date_dim.d_date_sk = store_sales.ss_sold_date_sk) ---------------------hashJoin[INNER_JOIN](store_sales.ss_item_sk = item.i_item_sk) -----------------------PhysicalProject -------------------------PhysicalOlapScan[store_sales] -----------------------PhysicalDistribute -------------------------PhysicalProject ---------------------------filter((item.i_manager_id = 100)) -----------------------------PhysicalOlapScan[item] +--------hashAgg[GLOBAL] +----------PhysicalDistribute +------------hashAgg[LOCAL] +--------------PhysicalProject +----------------hashJoin[INNER_JOIN](date_dim.d_date_sk = store_sales.ss_sold_date_sk) +------------------hashJoin[INNER_JOIN](store_sales.ss_item_sk = item.i_item_sk) +--------------------PhysicalProject +----------------------PhysicalOlapScan[store_sales] --------------------PhysicalDistribute ----------------------PhysicalProject -------------------------filter((date_dim.d_moy = 12)(date_dim.d_year = 2000)) ---------------------------PhysicalOlapScan[date_dim] +------------------------filter((item.i_manager_id = 100)) +--------------------------PhysicalOlapScan[item] +------------------PhysicalDistribute +--------------------PhysicalProject +----------------------filter((date_dim.d_moy = 12)(date_dim.d_year = 2000)) +------------------------PhysicalOlapScan[date_dim] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query59.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query59.out index 9f57ca69941365..74f0058cc1fb92 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query59.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query59.out @@ -2,17 +2,16 @@ -- !ds_shape_59 -- PhysicalCteAnchor ( cteId=CTEId#0 ) --PhysicalCteProducer ( cteId=CTEId#0 ) -----PhysicalProject -------hashAgg[GLOBAL] ---------PhysicalDistribute -----------hashAgg[LOCAL] -------------PhysicalProject ---------------hashJoin[INNER_JOIN](date_dim.d_date_sk = store_sales.ss_sold_date_sk) +----hashAgg[GLOBAL] +------PhysicalDistribute +--------hashAgg[LOCAL] +----------PhysicalProject +------------hashJoin[INNER_JOIN](date_dim.d_date_sk = store_sales.ss_sold_date_sk) +--------------PhysicalProject +----------------PhysicalOlapScan[store_sales] +--------------PhysicalDistribute ----------------PhysicalProject -------------------PhysicalOlapScan[store_sales] -----------------PhysicalDistribute -------------------PhysicalProject ---------------------PhysicalOlapScan[date_dim] +------------------PhysicalOlapScan[date_dim] --PhysicalResultSink ----PhysicalTopN ------PhysicalDistribute diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query71.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query71.out index 54922e2623df98..d52d1ff6ae7b57 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query71.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query71.out @@ -4,49 +4,48 @@ PhysicalResultSink --PhysicalQuickSort ----PhysicalDistribute ------PhysicalQuickSort ---------PhysicalProject -----------hashAgg[GLOBAL] -------------PhysicalDistribute ---------------hashAgg[LOCAL] -----------------PhysicalProject -------------------hashJoin[INNER_JOIN](tmp.time_sk = time_dim.t_time_sk) ---------------------PhysicalDistribute -----------------------hashJoin[INNER_JOIN](tmp.sold_item_sk = item.i_item_sk) -------------------------PhysicalDistribute ---------------------------PhysicalUnion -----------------------------PhysicalDistribute -------------------------------PhysicalProject ---------------------------------hashJoin[INNER_JOIN](date_dim.d_date_sk = web_sales.ws_sold_date_sk) +--------hashAgg[GLOBAL] +----------PhysicalDistribute +------------hashAgg[LOCAL] +--------------PhysicalProject +----------------hashJoin[INNER_JOIN](tmp.time_sk = time_dim.t_time_sk) +------------------PhysicalDistribute +--------------------hashJoin[INNER_JOIN](tmp.sold_item_sk = item.i_item_sk) +----------------------PhysicalDistribute +------------------------PhysicalUnion +--------------------------PhysicalDistribute +----------------------------PhysicalProject +------------------------------hashJoin[INNER_JOIN](date_dim.d_date_sk = web_sales.ws_sold_date_sk) +--------------------------------PhysicalProject +----------------------------------PhysicalOlapScan[web_sales] +--------------------------------PhysicalDistribute ----------------------------------PhysicalProject -------------------------------------PhysicalOlapScan[web_sales] -----------------------------------PhysicalDistribute -------------------------------------PhysicalProject ---------------------------------------filter((date_dim.d_moy = 12)(date_dim.d_year = 1998)) -----------------------------------------PhysicalOlapScan[date_dim] -----------------------------PhysicalDistribute -------------------------------PhysicalProject ---------------------------------hashJoin[INNER_JOIN](date_dim.d_date_sk = catalog_sales.cs_sold_date_sk) +------------------------------------filter((date_dim.d_moy = 12)(date_dim.d_year = 1998)) +--------------------------------------PhysicalOlapScan[date_dim] +--------------------------PhysicalDistribute +----------------------------PhysicalProject +------------------------------hashJoin[INNER_JOIN](date_dim.d_date_sk = catalog_sales.cs_sold_date_sk) +--------------------------------PhysicalProject +----------------------------------PhysicalOlapScan[catalog_sales] +--------------------------------PhysicalDistribute ----------------------------------PhysicalProject -------------------------------------PhysicalOlapScan[catalog_sales] -----------------------------------PhysicalDistribute -------------------------------------PhysicalProject ---------------------------------------filter((date_dim.d_moy = 12)(date_dim.d_year = 1998)) -----------------------------------------PhysicalOlapScan[date_dim] -----------------------------PhysicalDistribute -------------------------------PhysicalProject ---------------------------------hashJoin[INNER_JOIN](date_dim.d_date_sk = store_sales.ss_sold_date_sk) +------------------------------------filter((date_dim.d_moy = 12)(date_dim.d_year = 1998)) +--------------------------------------PhysicalOlapScan[date_dim] +--------------------------PhysicalDistribute +----------------------------PhysicalProject +------------------------------hashJoin[INNER_JOIN](date_dim.d_date_sk = store_sales.ss_sold_date_sk) +--------------------------------PhysicalProject +----------------------------------PhysicalOlapScan[store_sales] +--------------------------------PhysicalDistribute ----------------------------------PhysicalProject -------------------------------------PhysicalOlapScan[store_sales] -----------------------------------PhysicalDistribute -------------------------------------PhysicalProject ---------------------------------------filter((date_dim.d_moy = 12)(date_dim.d_year = 1998)) -----------------------------------------PhysicalOlapScan[date_dim] -------------------------PhysicalDistribute ---------------------------PhysicalProject -----------------------------filter((item.i_manager_id = 1)) -------------------------------PhysicalOlapScan[item] ---------------------PhysicalDistribute -----------------------PhysicalProject -------------------------filter(((cast(t_meal_time as VARCHAR(*)) = 'breakfast') OR (cast(t_meal_time as VARCHAR(*)) = 'dinner'))) ---------------------------PhysicalOlapScan[time_dim] +------------------------------------filter((date_dim.d_moy = 12)(date_dim.d_year = 1998)) +--------------------------------------PhysicalOlapScan[date_dim] +----------------------PhysicalDistribute +------------------------PhysicalProject +--------------------------filter((item.i_manager_id = 1)) +----------------------------PhysicalOlapScan[item] +------------------PhysicalDistribute +--------------------PhysicalProject +----------------------filter(((cast(t_meal_time as VARCHAR(*)) = 'breakfast') OR (cast(t_meal_time as VARCHAR(*)) = 'dinner'))) +------------------------PhysicalOlapScan[time_dim] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query87.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query87.out index c42693e374ca2f..fcf6c2b80ac4c8 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query87.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query87.out @@ -6,58 +6,55 @@ PhysicalResultSink ------hashAgg[LOCAL] --------PhysicalProject ----------PhysicalExcept -------------PhysicalProject ---------------hashAgg[GLOBAL] -----------------PhysicalDistribute -------------------hashAgg[LOCAL] ---------------------PhysicalProject -----------------------hashJoin[INNER_JOIN](store_sales.ss_customer_sk = customer.c_customer_sk) -------------------------PhysicalDistribute ---------------------------PhysicalProject -----------------------------hashJoin[INNER_JOIN](store_sales.ss_sold_date_sk = date_dim.d_date_sk) +------------hashAgg[GLOBAL] +--------------PhysicalDistribute +----------------hashAgg[LOCAL] +------------------PhysicalProject +--------------------hashJoin[INNER_JOIN](store_sales.ss_customer_sk = customer.c_customer_sk) +----------------------PhysicalDistribute +------------------------PhysicalProject +--------------------------hashJoin[INNER_JOIN](store_sales.ss_sold_date_sk = date_dim.d_date_sk) +----------------------------PhysicalProject +------------------------------PhysicalOlapScan[store_sales] +----------------------------PhysicalDistribute ------------------------------PhysicalProject ---------------------------------PhysicalOlapScan[store_sales] -------------------------------PhysicalDistribute ---------------------------------PhysicalProject -----------------------------------filter((date_dim.d_month_seq >= 1184)(date_dim.d_month_seq <= 1195)) -------------------------------------PhysicalOlapScan[date_dim] -------------------------PhysicalDistribute ---------------------------PhysicalProject -----------------------------PhysicalOlapScan[customer] -------------PhysicalProject ---------------hashAgg[GLOBAL] -----------------PhysicalDistribute -------------------hashAgg[LOCAL] ---------------------PhysicalProject -----------------------hashJoin[INNER_JOIN](catalog_sales.cs_bill_customer_sk = customer.c_customer_sk) -------------------------PhysicalDistribute ---------------------------PhysicalProject -----------------------------hashJoin[INNER_JOIN](catalog_sales.cs_sold_date_sk = date_dim.d_date_sk) +--------------------------------filter((date_dim.d_month_seq >= 1184)(date_dim.d_month_seq <= 1195)) +----------------------------------PhysicalOlapScan[date_dim] +----------------------PhysicalDistribute +------------------------PhysicalProject +--------------------------PhysicalOlapScan[customer] +------------hashAgg[GLOBAL] +--------------PhysicalDistribute +----------------hashAgg[LOCAL] +------------------PhysicalProject +--------------------hashJoin[INNER_JOIN](catalog_sales.cs_bill_customer_sk = customer.c_customer_sk) +----------------------PhysicalDistribute +------------------------PhysicalProject +--------------------------hashJoin[INNER_JOIN](catalog_sales.cs_sold_date_sk = date_dim.d_date_sk) +----------------------------PhysicalProject +------------------------------PhysicalOlapScan[catalog_sales] +----------------------------PhysicalDistribute ------------------------------PhysicalProject ---------------------------------PhysicalOlapScan[catalog_sales] -------------------------------PhysicalDistribute ---------------------------------PhysicalProject -----------------------------------filter((date_dim.d_month_seq >= 1184)(date_dim.d_month_seq <= 1195)) -------------------------------------PhysicalOlapScan[date_dim] -------------------------PhysicalDistribute ---------------------------PhysicalProject -----------------------------PhysicalOlapScan[customer] -------------PhysicalProject ---------------hashAgg[GLOBAL] -----------------PhysicalDistribute -------------------hashAgg[LOCAL] ---------------------PhysicalProject -----------------------hashJoin[INNER_JOIN](web_sales.ws_bill_customer_sk = customer.c_customer_sk) -------------------------PhysicalDistribute ---------------------------PhysicalProject -----------------------------hashJoin[INNER_JOIN](web_sales.ws_sold_date_sk = date_dim.d_date_sk) +--------------------------------filter((date_dim.d_month_seq >= 1184)(date_dim.d_month_seq <= 1195)) +----------------------------------PhysicalOlapScan[date_dim] +----------------------PhysicalDistribute +------------------------PhysicalProject +--------------------------PhysicalOlapScan[customer] +------------hashAgg[GLOBAL] +--------------PhysicalDistribute +----------------hashAgg[LOCAL] +------------------PhysicalProject +--------------------hashJoin[INNER_JOIN](web_sales.ws_bill_customer_sk = customer.c_customer_sk) +----------------------PhysicalDistribute +------------------------PhysicalProject +--------------------------hashJoin[INNER_JOIN](web_sales.ws_sold_date_sk = date_dim.d_date_sk) +----------------------------PhysicalProject +------------------------------PhysicalOlapScan[web_sales] +----------------------------PhysicalDistribute ------------------------------PhysicalProject ---------------------------------PhysicalOlapScan[web_sales] -------------------------------PhysicalDistribute ---------------------------------PhysicalProject -----------------------------------filter((date_dim.d_month_seq <= 1195)(date_dim.d_month_seq >= 1184)) -------------------------------------PhysicalOlapScan[date_dim] -------------------------PhysicalDistribute ---------------------------PhysicalProject -----------------------------PhysicalOlapScan[customer] +--------------------------------filter((date_dim.d_month_seq <= 1195)(date_dim.d_month_seq >= 1184)) +----------------------------------PhysicalOlapScan[date_dim] +----------------------PhysicalDistribute +------------------------PhysicalProject +--------------------------PhysicalOlapScan[customer] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query94.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query94.out index 863a0879b241c5..970cc6eb7c4e54 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query94.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query94.out @@ -3,37 +3,36 @@ PhysicalResultSink --PhysicalTopN ----PhysicalTopN -------PhysicalProject ---------hashAgg[DISTINCT_GLOBAL] -----------PhysicalDistribute -------------hashAgg[DISTINCT_LOCAL] ---------------hashAgg[GLOBAL] -----------------hashAgg[LOCAL] -------------------PhysicalProject ---------------------hashJoin[RIGHT_SEMI_JOIN](ws1.ws_order_number = ws2.ws_order_number)( not (ws_warehouse_sk = ws_warehouse_sk)) +------hashAgg[DISTINCT_GLOBAL] +--------PhysicalDistribute +----------hashAgg[DISTINCT_LOCAL] +------------hashAgg[GLOBAL] +--------------hashAgg[LOCAL] +----------------PhysicalProject +------------------hashJoin[RIGHT_SEMI_JOIN](ws1.ws_order_number = ws2.ws_order_number)( not (ws_warehouse_sk = ws_warehouse_sk)) +--------------------PhysicalDistribute +----------------------PhysicalProject +------------------------PhysicalOlapScan[web_sales] +--------------------hashJoin[RIGHT_ANTI_JOIN](ws1.ws_order_number = wr1.wr_order_number) ----------------------PhysicalDistribute ------------------------PhysicalProject ---------------------------PhysicalOlapScan[web_sales] -----------------------hashJoin[RIGHT_ANTI_JOIN](ws1.ws_order_number = wr1.wr_order_number) -------------------------PhysicalDistribute ---------------------------PhysicalProject -----------------------------PhysicalOlapScan[web_returns] -------------------------PhysicalDistribute ---------------------------hashJoin[INNER_JOIN](ws1.ws_web_site_sk = web_site.web_site_sk) -----------------------------hashJoin[INNER_JOIN](ws1.ws_ship_date_sk = date_dim.d_date_sk) -------------------------------hashJoin[INNER_JOIN](ws1.ws_ship_addr_sk = customer_address.ca_address_sk) ---------------------------------PhysicalProject -----------------------------------PhysicalOlapScan[web_sales] ---------------------------------PhysicalDistribute -----------------------------------PhysicalProject -------------------------------------filter((cast(ca_state as VARCHAR(*)) = 'OK')) ---------------------------------------PhysicalOlapScan[customer_address] +--------------------------PhysicalOlapScan[web_returns] +----------------------PhysicalDistribute +------------------------hashJoin[INNER_JOIN](ws1.ws_web_site_sk = web_site.web_site_sk) +--------------------------hashJoin[INNER_JOIN](ws1.ws_ship_date_sk = date_dim.d_date_sk) +----------------------------hashJoin[INNER_JOIN](ws1.ws_ship_addr_sk = customer_address.ca_address_sk) +------------------------------PhysicalProject +--------------------------------PhysicalOlapScan[web_sales] ------------------------------PhysicalDistribute --------------------------------PhysicalProject -----------------------------------filter((date_dim.d_date <= 2000-04-01)(date_dim.d_date >= 2000-02-01)) -------------------------------------PhysicalOlapScan[date_dim] +----------------------------------filter((cast(ca_state as VARCHAR(*)) = 'OK')) +------------------------------------PhysicalOlapScan[customer_address] ----------------------------PhysicalDistribute ------------------------------PhysicalProject ---------------------------------filter((cast(web_company_name as VARCHAR(*)) = 'pri')) -----------------------------------PhysicalOlapScan[web_site] +--------------------------------filter((date_dim.d_date <= 2000-04-01)(date_dim.d_date >= 2000-02-01)) +----------------------------------PhysicalOlapScan[date_dim] +--------------------------PhysicalDistribute +----------------------------PhysicalProject +------------------------------filter((cast(web_company_name as VARCHAR(*)) = 'pri')) +--------------------------------PhysicalOlapScan[web_site] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query95.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query95.out index 2c6ebe0d7be728..f4b0cac33931bd 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query95.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query95.out @@ -13,13 +13,16 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --PhysicalResultSink ----PhysicalTopN ------PhysicalTopN ---------PhysicalProject -----------hashAgg[DISTINCT_GLOBAL] -------------PhysicalDistribute ---------------hashAgg[DISTINCT_LOCAL] -----------------hashAgg[GLOBAL] -------------------hashAgg[LOCAL] ---------------------PhysicalProject +--------hashAgg[DISTINCT_GLOBAL] +----------PhysicalDistribute +------------hashAgg[DISTINCT_LOCAL] +--------------hashAgg[GLOBAL] +----------------hashAgg[LOCAL] +------------------PhysicalProject +--------------------hashJoin[RIGHT_SEMI_JOIN](ws1.ws_order_number = ws_wh.ws_order_number) +----------------------PhysicalDistribute +------------------------PhysicalProject +--------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ----------------------hashJoin[RIGHT_SEMI_JOIN](ws1.ws_order_number = web_returns.wr_order_number) ------------------------PhysicalProject --------------------------hashJoin[INNER_JOIN](web_returns.wr_order_number = ws_wh.ws_order_number) @@ -29,26 +32,22 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------------PhysicalDistribute ------------------------------PhysicalProject --------------------------------PhysicalOlapScan[web_returns] -------------------------hashJoin[RIGHT_SEMI_JOIN](ws1.ws_order_number = ws_wh.ws_order_number) ---------------------------PhysicalDistribute -----------------------------PhysicalProject -------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ---------------------------PhysicalDistribute -----------------------------hashJoin[INNER_JOIN](ws1.ws_web_site_sk = web_site.web_site_sk) -------------------------------hashJoin[INNER_JOIN](ws1.ws_ship_date_sk = date_dim.d_date_sk) ---------------------------------hashJoin[INNER_JOIN](ws1.ws_ship_addr_sk = customer_address.ca_address_sk) -----------------------------------PhysicalProject -------------------------------------PhysicalOlapScan[web_sales] -----------------------------------PhysicalDistribute -------------------------------------PhysicalProject ---------------------------------------filter((cast(ca_state as VARCHAR(*)) = 'NC')) -----------------------------------------PhysicalOlapScan[customer_address] +------------------------PhysicalDistribute +--------------------------hashJoin[INNER_JOIN](ws1.ws_web_site_sk = web_site.web_site_sk) +----------------------------hashJoin[INNER_JOIN](ws1.ws_ship_date_sk = date_dim.d_date_sk) +------------------------------hashJoin[INNER_JOIN](ws1.ws_ship_addr_sk = customer_address.ca_address_sk) +--------------------------------PhysicalProject +----------------------------------PhysicalOlapScan[web_sales] --------------------------------PhysicalDistribute ----------------------------------PhysicalProject -------------------------------------filter((date_dim.d_date >= 1999-02-01)(date_dim.d_date <= 1999-04-02)) ---------------------------------------PhysicalOlapScan[date_dim] +------------------------------------filter((cast(ca_state as VARCHAR(*)) = 'NC')) +--------------------------------------PhysicalOlapScan[customer_address] ------------------------------PhysicalDistribute --------------------------------PhysicalProject -----------------------------------filter((cast(web_company_name as VARCHAR(*)) = 'pri')) -------------------------------------PhysicalOlapScan[web_site] +----------------------------------filter((date_dim.d_date >= 1999-02-01)(date_dim.d_date <= 1999-04-02)) +------------------------------------PhysicalOlapScan[date_dim] +----------------------------PhysicalDistribute +------------------------------PhysicalProject +--------------------------------filter((cast(web_company_name as VARCHAR(*)) = 'pri')) +----------------------------------PhysicalOlapScan[web_site] diff --git a/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q10.out b/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q10.out index bbf108f0aedc48..d71a0ef61c5f05 100644 --- a/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q10.out +++ b/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q10.out @@ -4,26 +4,25 @@ PhysicalResultSink --PhysicalTopN ----PhysicalDistribute ------PhysicalTopN ---------PhysicalProject -----------hashAgg[GLOBAL] -------------PhysicalDistribute ---------------hashAgg[LOCAL] -----------------PhysicalProject -------------------hashJoin[INNER_JOIN](lineitem.l_orderkey = orders.o_orderkey) ---------------------PhysicalProject -----------------------filter((lineitem.l_returnflag = 'R')) -------------------------PhysicalOlapScan[lineitem] ---------------------PhysicalDistribute -----------------------hashJoin[INNER_JOIN](customer.c_nationkey = nation.n_nationkey) -------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN](customer.c_custkey = orders.o_custkey) -----------------------------PhysicalProject -------------------------------PhysicalOlapScan[customer] -----------------------------PhysicalDistribute -------------------------------PhysicalProject ---------------------------------filter((orders.o_orderdate < 1994-01-01)(orders.o_orderdate >= 1993-10-01)) -----------------------------------PhysicalOlapScan[orders] -------------------------PhysicalDistribute +--------hashAgg[GLOBAL] +----------PhysicalDistribute +------------hashAgg[LOCAL] +--------------PhysicalProject +----------------hashJoin[INNER_JOIN](lineitem.l_orderkey = orders.o_orderkey) +------------------PhysicalProject +--------------------filter((lineitem.l_returnflag = 'R')) +----------------------PhysicalOlapScan[lineitem] +------------------PhysicalDistribute +--------------------hashJoin[INNER_JOIN](customer.c_nationkey = nation.n_nationkey) +----------------------PhysicalProject +------------------------hashJoin[INNER_JOIN](customer.c_custkey = orders.o_custkey) --------------------------PhysicalProject -----------------------------PhysicalOlapScan[nation] +----------------------------PhysicalOlapScan[customer] +--------------------------PhysicalDistribute +----------------------------PhysicalProject +------------------------------filter((orders.o_orderdate < 1994-01-01)(orders.o_orderdate >= 1993-10-01)) +--------------------------------PhysicalOlapScan[orders] +----------------------PhysicalDistribute +------------------------PhysicalProject +--------------------------PhysicalOlapScan[nation] diff --git a/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q3.out b/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q3.out index da651e12cda6bc..04cec030a0f017 100644 --- a/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q3.out +++ b/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q3.out @@ -4,22 +4,21 @@ PhysicalResultSink --PhysicalTopN ----PhysicalDistribute ------PhysicalTopN ---------PhysicalProject -----------hashAgg[LOCAL] -------------PhysicalProject ---------------hashJoin[INNER_JOIN](lineitem.l_orderkey = orders.o_orderkey) +--------hashAgg[LOCAL] +----------PhysicalProject +------------hashJoin[INNER_JOIN](lineitem.l_orderkey = orders.o_orderkey) +--------------PhysicalProject +----------------filter((lineitem.l_shipdate > 1995-03-15)) +------------------PhysicalOlapScan[lineitem] +--------------PhysicalDistribute ----------------PhysicalProject -------------------filter((lineitem.l_shipdate > 1995-03-15)) ---------------------PhysicalOlapScan[lineitem] -----------------PhysicalDistribute -------------------PhysicalProject ---------------------hashJoin[INNER_JOIN](customer.c_custkey = orders.o_custkey) -----------------------PhysicalDistribute -------------------------PhysicalProject ---------------------------filter((orders.o_orderdate < 1995-03-15)) -----------------------------PhysicalOlapScan[orders] -----------------------PhysicalDistribute -------------------------PhysicalProject ---------------------------filter((customer.c_mktsegment = 'BUILDING')) -----------------------------PhysicalOlapScan[customer] +------------------hashJoin[INNER_JOIN](customer.c_custkey = orders.o_custkey) +--------------------PhysicalDistribute +----------------------PhysicalProject +------------------------filter((orders.o_orderdate < 1995-03-15)) +--------------------------PhysicalOlapScan[orders] +--------------------PhysicalDistribute +----------------------PhysicalProject +------------------------filter((customer.c_mktsegment = 'BUILDING')) +--------------------------PhysicalOlapScan[customer] diff --git a/regression-test/data/nereids_tpch_shape_sf500_p0/shape/q10.out b/regression-test/data/nereids_tpch_shape_sf500_p0/shape/q10.out index bbf108f0aedc48..d71a0ef61c5f05 100644 --- a/regression-test/data/nereids_tpch_shape_sf500_p0/shape/q10.out +++ b/regression-test/data/nereids_tpch_shape_sf500_p0/shape/q10.out @@ -4,26 +4,25 @@ PhysicalResultSink --PhysicalTopN ----PhysicalDistribute ------PhysicalTopN ---------PhysicalProject -----------hashAgg[GLOBAL] -------------PhysicalDistribute ---------------hashAgg[LOCAL] -----------------PhysicalProject -------------------hashJoin[INNER_JOIN](lineitem.l_orderkey = orders.o_orderkey) ---------------------PhysicalProject -----------------------filter((lineitem.l_returnflag = 'R')) -------------------------PhysicalOlapScan[lineitem] ---------------------PhysicalDistribute -----------------------hashJoin[INNER_JOIN](customer.c_nationkey = nation.n_nationkey) -------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN](customer.c_custkey = orders.o_custkey) -----------------------------PhysicalProject -------------------------------PhysicalOlapScan[customer] -----------------------------PhysicalDistribute -------------------------------PhysicalProject ---------------------------------filter((orders.o_orderdate < 1994-01-01)(orders.o_orderdate >= 1993-10-01)) -----------------------------------PhysicalOlapScan[orders] -------------------------PhysicalDistribute +--------hashAgg[GLOBAL] +----------PhysicalDistribute +------------hashAgg[LOCAL] +--------------PhysicalProject +----------------hashJoin[INNER_JOIN](lineitem.l_orderkey = orders.o_orderkey) +------------------PhysicalProject +--------------------filter((lineitem.l_returnflag = 'R')) +----------------------PhysicalOlapScan[lineitem] +------------------PhysicalDistribute +--------------------hashJoin[INNER_JOIN](customer.c_nationkey = nation.n_nationkey) +----------------------PhysicalProject +------------------------hashJoin[INNER_JOIN](customer.c_custkey = orders.o_custkey) --------------------------PhysicalProject -----------------------------PhysicalOlapScan[nation] +----------------------------PhysicalOlapScan[customer] +--------------------------PhysicalDistribute +----------------------------PhysicalProject +------------------------------filter((orders.o_orderdate < 1994-01-01)(orders.o_orderdate >= 1993-10-01)) +--------------------------------PhysicalOlapScan[orders] +----------------------PhysicalDistribute +------------------------PhysicalProject +--------------------------PhysicalOlapScan[nation] diff --git a/regression-test/data/nereids_tpch_shape_sf500_p0/shape/q3.out b/regression-test/data/nereids_tpch_shape_sf500_p0/shape/q3.out index da651e12cda6bc..04cec030a0f017 100644 --- a/regression-test/data/nereids_tpch_shape_sf500_p0/shape/q3.out +++ b/regression-test/data/nereids_tpch_shape_sf500_p0/shape/q3.out @@ -4,22 +4,21 @@ PhysicalResultSink --PhysicalTopN ----PhysicalDistribute ------PhysicalTopN ---------PhysicalProject -----------hashAgg[LOCAL] -------------PhysicalProject ---------------hashJoin[INNER_JOIN](lineitem.l_orderkey = orders.o_orderkey) +--------hashAgg[LOCAL] +----------PhysicalProject +------------hashJoin[INNER_JOIN](lineitem.l_orderkey = orders.o_orderkey) +--------------PhysicalProject +----------------filter((lineitem.l_shipdate > 1995-03-15)) +------------------PhysicalOlapScan[lineitem] +--------------PhysicalDistribute ----------------PhysicalProject -------------------filter((lineitem.l_shipdate > 1995-03-15)) ---------------------PhysicalOlapScan[lineitem] -----------------PhysicalDistribute -------------------PhysicalProject ---------------------hashJoin[INNER_JOIN](customer.c_custkey = orders.o_custkey) -----------------------PhysicalDistribute -------------------------PhysicalProject ---------------------------filter((orders.o_orderdate < 1995-03-15)) -----------------------------PhysicalOlapScan[orders] -----------------------PhysicalDistribute -------------------------PhysicalProject ---------------------------filter((customer.c_mktsegment = 'BUILDING')) -----------------------------PhysicalOlapScan[customer] +------------------hashJoin[INNER_JOIN](customer.c_custkey = orders.o_custkey) +--------------------PhysicalDistribute +----------------------PhysicalProject +------------------------filter((orders.o_orderdate < 1995-03-15)) +--------------------------PhysicalOlapScan[orders] +--------------------PhysicalDistribute +----------------------PhysicalProject +------------------------filter((customer.c_mktsegment = 'BUILDING')) +--------------------------PhysicalOlapScan[customer] diff --git a/regression-test/suites/nereids_tpcds_shape_sf100_p0/rf/ds_rf10.groovy b/regression-test/suites/nereids_tpcds_shape_sf100_p0/rf/ds_rf10.groovy index 015f5555c07e40..124c3be5d6df24 100644 --- a/regression-test/suites/nereids_tpcds_shape_sf100_p0/rf/ds_rf10.groovy +++ b/regression-test/suites/nereids_tpcds_shape_sf100_p0/rf/ds_rf10.groovy @@ -109,5 +109,5 @@ limit 100; // File file = new File(outFile) // file.write(getRuntimeFilters(plan)) - assertEquals("RF5[c_current_cdemo_sk->[cd_demo_sk],RF4[c_customer_sk->[ss_customer_sk],RF3[d_date_sk->[ss_sold_date_sk],RF2[ca_address_sk->[c_current_addr_sk],RF1[d_date_sk->[ws_sold_date_sk],RF0[d_date_sk->[cs_sold_date_sk]", getRuntimeFilters(plan)) + assertEquals("RF5[c_customer_sk->[ss_customer_sk],RF4[d_date_sk->[ss_sold_date_sk],RF3[c_current_cdemo_sk->[cd_demo_sk],RF2[ca_address_sk->[c_current_addr_sk],RF1[d_date_sk->[ws_sold_date_sk],RF0[d_date_sk->[cs_sold_date_sk]", getRuntimeFilters(plan)) } diff --git a/regression-test/suites/nereids_tpcds_shape_sf100_p0/rf/ds_rf35.groovy b/regression-test/suites/nereids_tpcds_shape_sf100_p0/rf/ds_rf35.groovy index 34ea54beea197d..d4110b005ce1c1 100644 --- a/regression-test/suites/nereids_tpcds_shape_sf100_p0/rf/ds_rf35.groovy +++ b/regression-test/suites/nereids_tpcds_shape_sf100_p0/rf/ds_rf35.groovy @@ -108,5 +108,5 @@ suite("ds_rf35") { // File file = new File(outFile) // file.write(getRuntimeFilters(plan)) - assertEquals("RF5[cd_demo_sk->[c_current_cdemo_sk],RF4[ca_address_sk->[c_current_addr_sk],RF3[c_customer_sk->[ss_customer_sk],RF2[d_date_sk->[ss_sold_date_sk],RF1[d_date_sk->[ws_sold_date_sk],RF0[d_date_sk->[cs_sold_date_sk]", getRuntimeFilters(plan)) + assertEquals("RF5[c_customer_sk->[ss_customer_sk],RF4[d_date_sk->[ss_sold_date_sk],RF3[cd_demo_sk->[c_current_cdemo_sk],RF2[ca_address_sk->[c_current_addr_sk],RF1[d_date_sk->[ws_sold_date_sk],RF0[d_date_sk->[cs_sold_date_sk]", getRuntimeFilters(plan)) } diff --git a/regression-test/suites/nereids_tpcds_shape_sf100_p0/rf/ds_rf95.groovy b/regression-test/suites/nereids_tpcds_shape_sf100_p0/rf/ds_rf95.groovy index 075cbfcdfa337f..689c0467102c9d 100644 --- a/regression-test/suites/nereids_tpcds_shape_sf100_p0/rf/ds_rf95.groovy +++ b/regression-test/suites/nereids_tpcds_shape_sf100_p0/rf/ds_rf95.groovy @@ -82,5 +82,5 @@ limit 100; // File file = new File(outFile) // file.write(getRuntimeFilters(plan)) - assertEquals("RF0[ws_order_number->[ws_order_number],RF5[wr_order_number->[ws_order_number],RF7[wr_order_number->[ws_order_number, ws_order_number],RF4[ws_order_number->[ws_order_number],RF6[ws_order_number->[ws_order_number, ws_order_number],RF3[web_site_sk->[ws_web_site_sk],RF2[d_date_sk->[ws_ship_date_sk],RF1[ca_address_sk->[ws_ship_addr_sk]", getRuntimeFilters(plan)) + assertEquals("RF0[ws_order_number->[ws_order_number],RF5[ws_order_number->[wr_order_number],RF4[wr_order_number->[ws_order_number],RF3[web_site_sk->[ws_web_site_sk],RF2[d_date_sk->[ws_ship_date_sk],RF1[ca_address_sk->[ws_ship_addr_sk]", getRuntimeFilters(plan)) } From 79fbc2e8197d8bfccb4b3185a81b84570a9e5016 Mon Sep 17 00:00:00 2001 From: mch_ucchi <41606806+sohardforaname@users.noreply.github.com> Date: Mon, 18 Sep 2023 12:24:54 +0800 Subject: [PATCH 02/12] [regression-test](planner)add test for insert default values (#23559) Co-authored-by: Yongqiang YANG <98214048+dataroaring@users.noreply.github.com> --- .../suites/load_p0/insert/test_insert.groovy | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/regression-test/suites/load_p0/insert/test_insert.groovy b/regression-test/suites/load_p0/insert/test_insert.groovy index 0010676ddca523..08e02d5f4b8bab 100644 --- a/regression-test/suites/load_p0/insert/test_insert.groovy +++ b/regression-test/suites/load_p0/insert/test_insert.groovy @@ -88,4 +88,21 @@ suite("test_insert") { sql """ insert into ${insert_tbl_dft} values() """ qt_select """ select k1, k2, k3, k4, k5, k6, k8, k10, k11, k12 from ${insert_tbl_dft} """ + + sql 'drop table if exists f3' + sql ''' + create table t3 ( + id int default "10" + ) distributed by hash(id) buckets 13 + properties( + 'replication_num'='1' + ); + ''' + + sql 'insert into t3 values(default)' + + test { + sql 'select * from t3' + result([[10]]) + } } From f3e350e8ec964d323e49097ad9a6ba58d8bd6738 Mon Sep 17 00:00:00 2001 From: Jibing-Li <64681310+Jibing-Li@users.noreply.github.com> Date: Mon, 18 Sep 2023 13:36:41 +0800 Subject: [PATCH 03/12] [Improvement](statistics)Improve statistics user experience (#24414) Two improvements: 1. Move the `Job_id` column for the return info of `Analyze table` command to the first column. To keep consistent with `show analyze`. ``` mysql> analyze table hive.tpch100.region; +--------+--------------+-------------------------+------------+--------------------------------+ | Job_Id | Catalog_Name | DB_Name | Table_Name | Columns | +--------+--------------+-------------------------+------------+--------------------------------+ | 14403 | hive | default_cluster:tpch100 | region | [r_regionkey,r_comment,r_name] | +--------+--------------+-------------------------+------------+--------------------------------+ 1 row in set (0.03 sec) ``` 2. Add `analyze_timeout` session variable, to control `analyze table/database with sync` timeout. --- .../org/apache/doris/qe/ConnectContext.java | 2 + .../org/apache/doris/qe/SessionVariable.java | 17 ++++++ .../org/apache/doris/qe/StmtExecutor.java | 7 +++ .../doris/statistics/AnalysisManager.java | 4 +- gensrc/thrift/PaloInternalService.thrift | 1 + .../hive/test_hive_statistic_timeout.out | 7 +++ .../hive/test_hive_statistic_timeout.groovy | 54 +++++++++++++++++++ 7 files changed, 90 insertions(+), 2 deletions(-) create mode 100644 regression-test/data/external_table_p2/hive/test_hive_statistic_timeout.out create mode 100644 regression-test/suites/external_table_p2/hive/test_hive_statistic_timeout.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java index 7c7aa7e2da91a2..bf7c84a2c4db6c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java @@ -742,6 +742,8 @@ public int getExecTimeout() { if (executor != null && executor.isInsertStmt()) { // particular for insert stmt, we can expand other type of timeout in the same way return Math.max(sessionVariable.getInsertTimeoutS(), sessionVariable.getQueryTimeoutS()); + } else if (executor != null && executor.isAnalyzeStmt()) { + return sessionVariable.getAnalyzeTimeoutS(); } else { // normal query stmt return sessionVariable.getQueryTimeoutS(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 5b9cb1b8075374..4859fea95c800e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -68,6 +68,7 @@ public class SessionVariable implements Serializable, Writable { public static final String EXEC_MEM_LIMIT = "exec_mem_limit"; public static final String SCAN_QUEUE_MEM_LIMIT = "scan_queue_mem_limit"; public static final String QUERY_TIMEOUT = "query_timeout"; + public static final String ANALYZE_TIMEOUT = "analyze_timeout"; public static final String MAX_EXECUTION_TIME = "max_execution_time"; public static final String INSERT_TIMEOUT = "insert_timeout"; @@ -453,6 +454,10 @@ public class SessionVariable implements Serializable, Writable { @VariableMgr.VarAttr(name = QUERY_TIMEOUT) public int queryTimeoutS = 900; + // query timeout in second. + @VariableMgr.VarAttr(name = ANALYZE_TIMEOUT, needForward = true) + public int analyzeTimeoutS = 43200; + // The global max_execution_time value provides the default for the session value for new connections. // The session value applies to SELECT executions executed within the session that include // no MAX_EXECUTION_TIME(N) optimizer hint or for which N is 0. @@ -1373,6 +1378,10 @@ public int getQueryTimeoutS() { return queryTimeoutS; } + public int getAnalyzeTimeoutS() { + return analyzeTimeoutS; + } + public void setEnableTwoPhaseReadOpt(boolean enable) { enableTwoPhaseReadOpt = enable; } @@ -1552,6 +1561,10 @@ public void setQueryTimeoutS(int queryTimeoutS) { this.queryTimeoutS = queryTimeoutS; } + public void setAnalyzeTimeoutS(int analyzeTimeoutS) { + this.analyzeTimeoutS = analyzeTimeoutS; + } + public void setMaxExecutionTimeMS(int maxExecutionTimeMS) { this.maxExecutionTimeMS = maxExecutionTimeMS; this.queryTimeoutS = this.maxExecutionTimeMS / 1000; @@ -2486,6 +2499,9 @@ public void setForwardedSessionVariables(TQueryOptions queryOptions) { if (queryOptions.isSetInsertTimeout()) { setInsertTimeoutS(queryOptions.getInsertTimeout()); } + if (queryOptions.isSetAnalyzeTimeout()) { + setAnalyzeTimeoutS(queryOptions.getAnalyzeTimeout()); + } } /** @@ -2497,6 +2513,7 @@ public TQueryOptions getQueryOptionVariables() { queryOptions.setScanQueueMemLimit(Math.min(maxScanQueueMemByte, maxExecMemByte / 20)); queryOptions.setQueryTimeout(queryTimeoutS); queryOptions.setInsertTimeout(insertTimeoutS); + queryOptions.setAnalyzeTimeout(analyzeTimeoutS); return queryOptions; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index b7b633a4b3d493..114080036ed7a2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -405,6 +405,13 @@ public boolean isInsertStmt() { return parsedStmt instanceof InsertStmt; } + public boolean isAnalyzeStmt() { + if (parsedStmt == null) { + return false; + } + return parsedStmt instanceof AnalyzeStmt; + } + /** * Used for audit in ConnectProcessor. *

diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java index a83f9795ffe201..131f650e4dbabd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java @@ -384,11 +384,11 @@ protected AnalysisInfo buildAndAssignJob(AnalyzeTblStmt stmt) throws DdlExceptio private void sendJobId(List analysisInfos, boolean proxy) { List columns = new ArrayList<>(); + columns.add(new Column("Job_Id", ScalarType.createVarchar(19))); columns.add(new Column("Catalog_Name", ScalarType.createVarchar(1024))); columns.add(new Column("DB_Name", ScalarType.createVarchar(1024))); columns.add(new Column("Table_Name", ScalarType.createVarchar(1024))); columns.add(new Column("Columns", ScalarType.createVarchar(1024))); - columns.add(new Column("Job_Id", ScalarType.createVarchar(19))); ShowResultSetMetaData commonResultSetMetaData = new ShowResultSetMetaData(columns); List> resultRows = new ArrayList<>(); for (AnalysisInfo analysisInfo : analysisInfos) { @@ -396,11 +396,11 @@ private void sendJobId(List analysisInfos, boolean proxy) { continue; } List row = new ArrayList<>(); + row.add(String.valueOf(analysisInfo.jobId)); row.add(analysisInfo.catalogName); row.add(analysisInfo.dbName); row.add(analysisInfo.tblName); row.add(analysisInfo.colName); - row.add(String.valueOf(analysisInfo.jobId)); resultRows.add(row); } ShowResultSet commonResultSet = new ShowResultSet(commonResultSetMetaData, resultRows); diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift index 5dcf961f9a6ffb..2588996cd49102 100644 --- a/gensrc/thrift/PaloInternalService.thrift +++ b/gensrc/thrift/PaloInternalService.thrift @@ -246,6 +246,7 @@ struct TQueryOptions { // use is_report_success any more 84: optional bool enable_profile = false; 85: optional bool enable_page_cache = false; + 86: optional i32 analyze_timeout = 43200 } diff --git a/regression-test/data/external_table_p2/hive/test_hive_statistic_timeout.out b/regression-test/data/external_table_p2/hive/test_hive_statistic_timeout.out new file mode 100644 index 00000000000000..e906deea593915 --- /dev/null +++ b/regression-test/data/external_table_p2/hive/test_hive_statistic_timeout.out @@ -0,0 +1,7 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !01 -- +p_container 200000000 40 0 JUMBO BAG WRAP PKG +p_partkey 200000000 200778064 0 1 200000000 +p_retailprice 200000000 120014 0 900.00 2099.00 +p_type 200000000 150 0 ECONOMY ANODIZED BRASS STANDARD POLISHED TIN + diff --git a/regression-test/suites/external_table_p2/hive/test_hive_statistic_timeout.groovy b/regression-test/suites/external_table_p2/hive/test_hive_statistic_timeout.groovy new file mode 100644 index 00000000000000..a3754d9d2a4a02 --- /dev/null +++ b/regression-test/suites/external_table_p2/hive/test_hive_statistic_timeout.groovy @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_hive_statistic_timeout", "p2,external,hive,external_remote,external_remote_hive") { + String enabled = context.config.otherConfigs.get("enableExternalHiveTest") + if (enabled != null && enabled.equalsIgnoreCase("true")) { + String extHiveHmsHost = context.config.otherConfigs.get("extHiveHmsHost") + String extHiveHmsPort = context.config.otherConfigs.get("extHiveHmsPort") + String catalog_name = "test_hive_statistic_timeout" + sql """drop catalog if exists ${catalog_name};""" + sql """ + create catalog if not exists ${catalog_name} properties ( + 'type'='hms', + 'hadoop.username' = 'hadoop', + 'hive.metastore.uris' = 'thrift://${extHiveHmsHost}:${extHiveHmsPort}' + ); + """ + logger.info("catalog " + catalog_name + " created") + + sql """use ${catalog_name}.tpch_1000_parquet""" + sql """set query_timeout=1""" + sql """analyze table part (p_partkey, p_container, p_type, p_retailprice) with sync;""" + + def result = sql """show column stats part""" + assertTrue(result.size() == 4) + + def ctlId + result = sql """show proc '/catalogs'""" + + for (int i = 0; i < result.size(); i++) { + if (result[i][1] == catalog_name) { + ctlId = result[i][0] + } + } + + qt_01 """select col_id, count, ndv, null_count, min, max from internal.__internal_schema.column_statistics where catalog_id='$ctlId' order by col_id;""" + sql """drop catalog ${catalog_name}"""; + } +} + From 115390789766d7167a0be59ca196af469732f01d Mon Sep 17 00:00:00 2001 From: minghong Date: Mon, 18 Sep 2023 13:47:37 +0800 Subject: [PATCH 04/12] [opt](nereids)add explanation why we always update col stats in StatsCalculator. --- .../apache/doris/nereids/stats/StatsCalculator.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java index 93792e62b6cab9..c3b187aad7c229 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java @@ -227,6 +227,17 @@ private void estimate() { groupExpression.getOwnerGroup().setStatistics(newStats); groupExpression.setEstOutputRowCount(newStats.getRowCount()); } else { + // the reason why we update col stats here. + // consider join between 3 tables: A/B/C with join condition: A.id=B.id=C.id and a filter: C.id=1 + // in the final join result, the ndv of A.id/B.id/C.id should be 1 + // suppose we have 2 candidate plans + // plan1: (A join B on A.id=B.id) join C on B.id=C.id + // plan2:(B join C)join A + // suppose plan1 is estimated before plan2 + // + // after estimate the outer join of plan1 (join C), we update B.id.ndv=1, but A.id.ndv is not updated + // then we estimate plan2. the stats of plan2 is denoted by stats2. obviously, stats2.A.id.ndv is 1 + // now we update OwnerGroup().getStatistics().A.id.ndv to 1 groupExpression.getOwnerGroup().getStatistics().updateNdv(newStats); } groupExpression.setStatDerived(true); From b4432ce577b9f7cd38c8202b2f492b4623bc6ce7 Mon Sep 17 00:00:00 2001 From: Jibing-Li <64681310+Jibing-Li@users.noreply.github.com> Date: Mon, 18 Sep 2023 14:59:26 +0800 Subject: [PATCH 05/12] [Feature](statistics)Support external table analyze partition (#24154) Enable collect partition level stats for hive external table. --- docs/en/docs/query-acceleration/statistics.md | 6 +- .../docs/query-acceleration/statistics.md | 4 +- fe/fe-core/src/main/cup/sql_parser.cup | 9 + .../apache/doris/analysis/AnalyzeTblStmt.java | 32 +++- .../apache/doris/analysis/PartitionNames.java | 38 ++++- .../catalog/external/HMSExternalTable.java | 7 + .../apache/doris/statistics/AnalysisInfo.java | 16 +- .../doris/statistics/AnalysisInfoBuilder.java | 48 +++++- .../doris/statistics/AnalysisManager.java | 16 +- .../doris/statistics/ColumnStatistic.java | 8 +- .../statistics/ColumnStatisticBuilder.java | 2 +- .../doris/statistics/HMSAnalysisTask.java | 154 +++++++++++------- .../doris/statistics/StatisticsCache.java | 2 +- .../doris/statistics/StatisticsCleaner.java | 4 +- .../statistics/StatisticsRepository.java | 8 +- .../org/apache/doris/statistics/StatsId.java | 8 +- .../doris/statistics/util/StatisticsUtil.java | 16 +- fe/fe-core/src/main/jflex/sql_scanner.flex | 1 + .../hive/test_hive_partition_statistic.out | 87 ++++++++++ .../hive/test_hive_partition_statistic.groovy | 53 ++++++ 20 files changed, 420 insertions(+), 99 deletions(-) create mode 100644 regression-test/data/external_table_p2/hive/test_hive_partition_statistic.out create mode 100644 regression-test/suites/external_table_p2/hive/test_hive_partition_statistic.groovy diff --git a/docs/en/docs/query-acceleration/statistics.md b/docs/en/docs/query-acceleration/statistics.md index 4d37d278eef4f9..9cf75cdd5ec329 100644 --- a/docs/en/docs/query-acceleration/statistics.md +++ b/docs/en/docs/query-acceleration/statistics.md @@ -69,7 +69,7 @@ Column statistics collection syntax: ```SQL ANALYZE < TABLE | DATABASE table_name | db_name > - [ PARTITIONS (partition_name [, ...]) ] + [ PARTITIONS [(*) | (partition_name [, ...]) | WITH RECENT COUNT] ] [ (column_name [, ...]) ] [ [ WITH SYNC ] [ WITH INCREMENTAL ] [ WITH SAMPLE PERCENT | ROWS ] [ WITH PERIOD ]] [ PROPERTIES ("key" = "value", ...) ]; @@ -78,7 +78,7 @@ ANALYZE < TABLE | DATABASE table_name | db_name > Explanation: - Table_name: The target table for the specified. It can be a `db_name.table_name` form. -- partition_name: The specified target partitions（for hive external table only）。Must be partitions exist in `table_name`. Multiple partition names are separated by commas. e.g. (nation=US/city=Washington) +- partition_name: The specified target partitions（for hive external table only）。Must be partitions exist in `table_name`. Multiple partition names are separated by commas. e.g. for single level partition: PARTITIONS(`event_date=20230706`), for multi level partition: PARTITIONS(`nation=US/city=Washington`). PARTITIONS(*) specifies all partitions, PARTITIONS WITH RECENT 30 specifies the latest 30 partitions. - Column_name: The specified target column. Must be `table_name` a column that exists in. Multiple column names are separated by commas. - Sync: Synchronizes the collection of statistics. Return after collection. If not specified, it will be executed asynchronously and the job ID will be returned. - Incremental: Incrementally gather statistics. @@ -673,4 +673,4 @@ When executing ANALYZE, statistical data is written to the internal table __inte ### ANALYZE Failure for Large Tables -Due to the strict resource limitations for ANALYZE, the ANALYZE operation for large tables might experience timeouts or exceed the memory limit of BE. In such cases, it's recommended to use ANALYZE ... WITH SAMPLE.... Additionally, for scenarios involving dynamic partitioned tables, it's highly recommended to use ANALYZE ... WITH INCREMENTAL.... This statement processes only the partitions with updated data incrementally, avoiding redundant computations and improving efficiency. \ No newline at end of file +Due to the strict resource limitations for ANALYZE, the ANALYZE operation for large tables might experience timeouts or exceed the memory limit of BE. In such cases, it's recommended to use ANALYZE ... WITH SAMPLE.... Additionally, for scenarios involving dynamic partitioned tables, it's highly recommended to use ANALYZE ... WITH INCREMENTAL.... This statement processes only the partitions with updated data incrementally, avoiding redundant computations and improving efficiency. diff --git a/docs/zh-CN/docs/query-acceleration/statistics.md b/docs/zh-CN/docs/query-acceleration/statistics.md index ad14c1c9504df0..784dc0def7c928 100644 --- a/docs/zh-CN/docs/query-acceleration/statistics.md +++ b/docs/zh-CN/docs/query-acceleration/statistics.md @@ -66,7 +66,7 @@ Doris 查询优化器使用统计信息来确定查询最有效的执行计划 ```SQL ANALYZE < TABLE | DATABASE table_name | db_name > - [ PARTITIONS (partition_name [, ...]) ] + [ PARTITIONS [(*) | (partition_name [, ...]) | WITH RECENT COUNT] ] [ (column_name [, ...]) ] [ [ WITH SYNC ] [WITH INCREMENTAL] [ WITH SAMPLE PERCENT | ROWS ] [ WITH PERIOD ] ] [ PROPERTIES ("key" = "value", ...) ]; @@ -75,7 +75,7 @@ ANALYZE < TABLE | DATABASE table_name | db_name > 其中： - table_name: 指定的的目标表。可以是  `db_name.table_name`  形式。 -- partition_name: 指定的目标分区（目前只针对Hive外表）。必须是  `table_name`  中存在的分区，多个列名称用逗号分隔。分区名样例:event_date=20230706, nation=CN/city=Beijing +- partition_name: 指定的目标分区（目前只针对Hive外表）。必须是  `table_name`  中存在的分区，多个列名称用逗号分隔。分区名样例: 单层分区PARTITIONS(`event_date=20230706`)，多层分区PARTITIONS(`nation=CN/city=Beijing`)。PARTITIONS (*)指定所有分区，PARTITIONS WITH RECENT 100指定最新的100个分区。 - column_name: 指定的目标列。必须是  `table_name`  中存在的列，多个列名称用逗号分隔。 - sync：同步收集统计信息。收集完后返回。若不指定则异步执行并返回任务 ID。 - period：周期性收集统计信息。单位为秒，指定后会定期收集相应的统计信息。 diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index f0e83f09646f12..baec3cd18d00b4 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -528,6 +528,7 @@ terminal String KW_QUOTA, KW_RANDOM, KW_RANGE, + KW_RECENT, KW_READ, KW_REBALANCE, KW_RECOVER, @@ -5900,6 +5901,14 @@ partition_names ::= {: RESULT = new PartitionNames(true, Lists.newArrayList(partName)); :} + | KW_PARTITIONS LPAREN STAR RPAREN + {: + RESULT = new PartitionNames(true); + :} + | KW_PARTITIONS KW_WITH KW_RECENT INTEGER_LITERAL:count + {: + RESULT = new PartitionNames(count); + :} ; opt_table_sample ::= diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeTblStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeTblStmt.java index ed5dda22498a36..874b83b280d886 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeTblStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeTblStmt.java @@ -84,7 +84,7 @@ public class AnalyzeTblStmt extends AnalyzeStmt { private final TableName tableName; private List columnNames; - private List partitionNames; + private PartitionNames partitionNames; private boolean isAllColumns; // after analyzed @@ -97,7 +97,7 @@ public AnalyzeTblStmt(TableName tableName, AnalyzeProperties properties) { super(properties); this.tableName = tableName; - this.partitionNames = partitionNames == null ? null : partitionNames.getPartitionNames(); + this.partitionNames = partitionNames; this.columnNames = columnNames; this.analyzeProperties = properties; this.isAllColumns = columnNames == null; @@ -240,14 +240,34 @@ public Set getColumnNames() { } public Set getPartitionNames() { - Set partitions = partitionNames == null ? table.getPartitionNames() : Sets.newHashSet(partitionNames); - if (isSamplingPartition()) { - int partNum = ConnectContext.get().getSessionVariable().getExternalTableAnalyzePartNum(); - partitions = partitions.stream().limit(partNum).collect(Collectors.toSet()); + if (partitionNames == null || partitionNames.getPartitionNames() == null) { + return null; } + Set partitions = Sets.newHashSet(); + partitions.addAll(partitionNames.getPartitionNames()); + /* + if (isSamplingPartition()) { + int partNum = ConnectContext.get().getSessionVariable().getExternalTableAnalyzePartNum(); + partitions = partitions.stream().limit(partNum).collect(Collectors.toSet()); + } + */ return partitions; } + public boolean isAllPartitions() { + if (partitionNames == null) { + return false; + } + return partitionNames.isAllPartitions(); + } + + public long getPartitionCount() { + if (partitionNames == null) { + return 0; + } + return partitionNames.getCount(); + } + public boolean isPartitionOnly() { return partitionNames != null; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionNames.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionNames.java index 1140dfc6777641..ca26a2978e0e54 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionNames.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionNames.java @@ -48,15 +48,37 @@ public class PartitionNames implements ParseNode, Writable { // true if these partitions are temp partitions @SerializedName(value = "isTemp") private final boolean isTemp; + private final boolean allPartitions; + private final long count; + // Default partition count to collect statistic for external table. + private static final long DEFAULT_PARTITION_COUNT = 100; public PartitionNames(boolean isTemp, List partitionNames) { this.partitionNames = partitionNames; this.isTemp = isTemp; + this.allPartitions = false; + this.count = 0; } public PartitionNames(PartitionNames other) { this.partitionNames = Lists.newArrayList(other.partitionNames); this.isTemp = other.isTemp; + this.allPartitions = other.allPartitions; + this.count = 0; + } + + public PartitionNames(boolean allPartitions) { + this.partitionNames = null; + this.isTemp = false; + this.allPartitions = allPartitions; + this.count = 0; + } + + public PartitionNames(long partitionCount) { + this.partitionNames = null; + this.isTemp = false; + this.allPartitions = false; + this.count = partitionCount; } public List getPartitionNames() { @@ -67,9 +89,23 @@ public boolean isTemp() { return isTemp; } + public boolean isAllPartitions() { + return allPartitions; + } + + public long getCount() { + return count; + } + @Override public void analyze(Analyzer analyzer) throws AnalysisException { - if (partitionNames.isEmpty()) { + if (allPartitions && count > 0) { + throw new AnalysisException("All partition and partition count couldn't be set at the same time."); + } + if (allPartitions || count > 0) { + return; + } + if (partitionNames == null || partitionNames.isEmpty()) { throw new AnalysisException("No partition specified in partition lists"); } // check if partition name is not empty string diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java index c1de1ea98d7555..d691c0c6e5f4c7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java @@ -57,6 +57,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; import java.time.LocalDate; @@ -628,6 +629,12 @@ private void setStatData(Column col, ColumnStatisticsData data, ColumnStatisticB builder.setMaxValue(Double.MAX_VALUE); } } + + @Override + public void gsonPostProcess() throws IOException { + super.gsonPostProcess(); + estimatedRowCount = -1; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java index 00b8c7cdaaea50..ec59c61fffe4d0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java @@ -155,13 +155,19 @@ public enum ScheduleType { // True means this task is a table level task for external table. // This kind of task is mainly to collect the number of rows of a table. @SerializedName("externalTableLevelTask") - public boolean externalTableLevelTask; + public final boolean externalTableLevelTask; @SerializedName("partitionOnly") - public boolean partitionOnly; + public final boolean partitionOnly; @SerializedName("samplingPartition") - public boolean samplingPartition; + public final boolean samplingPartition; + + @SerializedName("isAllPartition") + public final boolean isAllPartition; + + @SerializedName("partitionCount") + public final long partitionCount; // For serialize @SerializedName("cronExpr") @@ -181,7 +187,7 @@ public AnalysisInfo(long jobId, long taskId, List taskIds, String catalogN int samplePercent, int sampleRows, int maxBucketNum, long periodTimeInMs, String message, long lastExecTimeInMs, long timeCostInMs, AnalysisState state, ScheduleType scheduleType, boolean isExternalTableLevelTask, boolean partitionOnly, boolean samplingPartition, - CronExpression cronExpression, boolean forceFull) { + boolean isAllPartition, long partitionCount, CronExpression cronExpression, boolean forceFull) { this.jobId = jobId; this.taskId = taskId; this.taskIds = taskIds; @@ -208,6 +214,8 @@ public AnalysisInfo(long jobId, long taskId, List taskIds, String catalogN this.externalTableLevelTask = isExternalTableLevelTask; this.partitionOnly = partitionOnly; this.samplingPartition = samplingPartition; + this.isAllPartition = isAllPartition; + this.partitionCount = partitionCount; this.cronExpression = cronExpression; if (cronExpression != null) { this.cronExprStr = cronExpression.getCronExpression(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java index 0c296ace91da14..c17bbc69d91c6d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java @@ -56,6 +56,8 @@ public class AnalysisInfoBuilder { private boolean externalTableLevelTask; private boolean partitionOnly; private boolean samplingPartition; + private boolean isAllPartition; + private long partitionCount; private CronExpression cronExpression; @@ -91,6 +93,8 @@ public AnalysisInfoBuilder(AnalysisInfo info) { externalTableLevelTask = info.externalTableLevelTask; partitionOnly = info.partitionOnly; samplingPartition = info.samplingPartition; + isAllPartition = info.isAllPartition; + partitionCount = info.partitionCount; cronExpression = info.cronExpression; forceFull = info.forceFull; } @@ -225,6 +229,16 @@ public AnalysisInfoBuilder setSamplingPartition(boolean samplingPartition) { return this; } + public AnalysisInfoBuilder setAllPartition(boolean isAllPartition) { + this.isAllPartition = isAllPartition; + return this; + } + + public AnalysisInfoBuilder setPartitionCount(long partitionCount) { + this.partitionCount = partitionCount; + return this; + } + public void setCronExpression(CronExpression cronExpression) { this.cronExpression = cronExpression; } @@ -237,6 +251,38 @@ public AnalysisInfo build() { return new AnalysisInfo(jobId, taskId, taskIds, catalogName, dbName, tblName, colToPartitions, partitionNames, colName, indexId, jobType, analysisMode, analysisMethod, analysisType, samplePercent, sampleRows, maxBucketNum, periodTimeInMs, message, lastExecTimeInMs, timeCostInMs, state, scheduleType, - externalTableLevelTask, partitionOnly, samplingPartition, cronExpression, forceFull); + externalTableLevelTask, partitionOnly, samplingPartition, isAllPartition, partitionCount, + cronExpression, forceFull); + } + + public AnalysisInfoBuilder copy() { + return new AnalysisInfoBuilder() + .setJobId(jobId) + .setTaskId(taskId) + .setTaskIds(taskIds) + .setCatalogName(catalogName) + .setDbName(dbName) + .setTblName(tblName) + .setColToPartitions(colToPartitions) + .setColName(colName) + .setIndexId(indexId) + .setJobType(jobType) + .setAnalysisMode(analysisMode) + .setAnalysisMethod(analysisMethod) + .setAnalysisType(analysisType) + .setSamplePercent(samplePercent) + .setSampleRows(sampleRows) + .setPeriodTimeInMs(periodTimeInMs) + .setMaxBucketNum(maxBucketNum) + .setMessage(message) + .setLastExecTimeInMs(lastExecTimeInMs) + .setTimeCostInMs(timeCostInMs) + .setState(state) + .setScheduleType(scheduleType) + .setExternalTableLevelTask(externalTableLevelTask) + .setSamplingPartition(samplingPartition) + .setPartitionOnly(partitionOnly) + .setAllPartition(isAllPartition) + .setPartitionCount(partitionCount); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java index 131f650e4dbabd..83c3cc84e494bd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java @@ -363,7 +363,7 @@ protected AnalysisInfo buildAndAssignJob(AnalyzeTblStmt stmt) throws DdlExceptio boolean isSync = stmt.isSync(); Map analysisTaskInfos = new HashMap<>(); createTaskForEachColumns(jobInfo, analysisTaskInfos, isSync); - if (stmt.isAllColumns() + if (!jobInfo.partitionOnly && stmt.isAllColumns() && StatisticsUtil.isExternalTable(jobInfo.catalogName, jobInfo.dbName, jobInfo.tblName)) { createTableLevelTaskForExternalTable(jobInfo, analysisTaskInfos, isSync); } @@ -453,7 +453,7 @@ private Map> validateAndGetPartitions(TableIf table, Set> existColAndPartsForStats = StatisticsRepository + Map> existColAndPartsForStats = StatisticsRepository .fetchColAndPartsForStats(tableId); if (existColAndPartsForStats.isEmpty()) { @@ -461,12 +461,12 @@ private Map> validateAndGetPartitions(TableIf table, Set existPartIdsForStats = new HashSet<>(); + Set existPartIdsForStats = new HashSet<>(); existColAndPartsForStats.values().forEach(existPartIdsForStats::addAll); - Map idToPartition = StatisticsUtil.getPartitionIdToName(table); + Set idToPartition = StatisticsUtil.getPartitionIds(table); // Get an invalid set of partitions (those partitions were deleted) - Set invalidPartIds = existPartIdsForStats.stream() - .filter(id -> !idToPartition.containsKey(id)).collect(Collectors.toSet()); + Set invalidPartIds = existPartIdsForStats.stream() + .filter(id -> !idToPartition.contains(id)).collect(Collectors.toSet()); if (!invalidPartIds.isEmpty()) { // Delete invalid partition statistics to avoid affecting table statistics @@ -496,6 +496,8 @@ public AnalysisInfo buildAnalysisJobInfo(AnalyzeTblStmt stmt) throws DdlExceptio Set partitionNames = stmt.getPartitionNames(); boolean partitionOnly = stmt.isPartitionOnly(); boolean isSamplingPartition = stmt.isSamplingPartition(); + boolean isAllPartition = stmt.isAllPartitions(); + long partitionCount = stmt.getPartitionCount(); int samplePercent = stmt.getSamplePercent(); int sampleRows = stmt.getSampleRows(); AnalysisType analysisType = stmt.getAnalysisType(); @@ -516,6 +518,8 @@ public AnalysisInfo buildAnalysisJobInfo(AnalyzeTblStmt stmt) throws DdlExceptio infoBuilder.setPartitionNames(partitionNames); infoBuilder.setPartitionOnly(partitionOnly); infoBuilder.setSamplingPartition(isSamplingPartition); + infoBuilder.setAllPartition(isAllPartition); + infoBuilder.setPartitionCount(partitionCount); infoBuilder.setJobType(JobType.MANUAL); infoBuilder.setState(AnalysisState.PENDING); infoBuilder.setLastExecTimeInMs(System.currentTimeMillis()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java index 84f8d5bfbfe0fc..67bc308bcaa833 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java @@ -93,7 +93,7 @@ public class ColumnStatistic { public final Histogram histogram; @SerializedName("partitionIdToColStats") - public final Map partitionIdToColStats = new HashMap<>(); + public final Map partitionIdToColStats = new HashMap<>(); public final String updatedTime; @@ -120,7 +120,7 @@ public ColumnStatistic(double count, double ndv, ColumnStatistic original, doubl } public static ColumnStatistic fromResultRow(List resultRows) { - Map partitionIdToColStats = new HashMap<>(); + Map partitionIdToColStats = new HashMap<>(); ColumnStatistic columnStatistic = null; try { for (ResultRow resultRow : resultRows) { @@ -128,7 +128,7 @@ public static ColumnStatistic fromResultRow(List resultRows) { if (partId == null) { columnStatistic = fromResultRow(resultRow); } else { - partitionIdToColStats.put(Long.parseLong(partId), fromResultRow(resultRow)); + partitionIdToColStats.put(partId, fromResultRow(resultRow)); } } } catch (Throwable t) { @@ -392,7 +392,7 @@ public boolean isUnKnown() { return isUnKnown; } - public void putPartStats(long partId, ColumnStatistic columnStatistic) { + public void putPartStats(String partId, ColumnStatistic columnStatistic) { this.partitionIdToColStats.put(partId, columnStatistic); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticBuilder.java index 4a3af054df7382..fa4cf7ebc99cb4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticBuilder.java @@ -40,7 +40,7 @@ public class ColumnStatisticBuilder { private ColumnStatistic original; - private Map partitionIdToColStats = new HashMap<>(); + private Map partitionIdToColStats = new HashMap<>(); private String updatedTime; diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/HMSAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/HMSAnalysisTask.java index 512aa9982ffed5..973e5e76aa010a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/HMSAnalysisTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/HMSAnalysisTask.java @@ -26,7 +26,7 @@ import org.apache.doris.qe.StmtExecutor; import org.apache.doris.statistics.util.StatisticsUtil; -import org.apache.commons.lang3.StringUtils; +import com.google.common.collect.Lists; import org.apache.commons.text.StringSubstitutor; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -35,10 +35,13 @@ import java.time.LocalDateTime; import java.time.ZoneId; import java.util.ArrayList; +import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.StringJoiner; +import java.util.stream.Collectors; public class HMSAnalysisTask extends BaseAnalysisTask { private static final Logger LOG = LogManager.getLogger(HMSAnalysisTask.class); @@ -48,7 +51,7 @@ public class HMSAnalysisTask extends BaseAnalysisTask { public static final String NUM_FILES = "numFiles"; public static final String TIMESTAMP = "transient_lastDdlTime"; - private static final String ANALYZE_SQL_TABLE_TEMPLATE = "INSERT INTO " + private static final String ANALYZE_TABLE_TEMPLATE = "INSERT INTO " + "${internalDB}.${columnStatTbl}" + " SELECT " + "CONCAT(${tblId}, '-', ${idxId}, '-', '${colId}') AS id, " @@ -67,10 +70,8 @@ public class HMSAnalysisTask extends BaseAnalysisTask { + "NOW() " + "FROM `${catalogName}`.`${dbName}`.`${tblName}`"; - private static final String ANALYZE_SQL_PARTITION_TEMPLATE = "INSERT INTO " - + "${internalDB}.${columnStatTbl}" - + " SELECT " - + "CONCAT(${tblId}, '-', ${idxId}, '-', '${colId}') AS id, " + private static final String ANALYZE_PARTITION_TEMPLATE = " SELECT " + + "CONCAT(${tblId}, '-', ${idxId}, '-', '${colId}', '-', ${partId}) AS id, " + "${catalogId} AS catalog_id, " + "${dbId} AS db_id, " + "${tblId} AS tbl_id, " @@ -83,22 +84,22 @@ public class HMSAnalysisTask extends BaseAnalysisTask { + "MIN(`${colName}`) AS min, " + "MAX(`${colName}`) AS max, " + "${dataSizeFunction} AS data_size, " - + "NOW() " - + "FROM `${catalogName}`.`${dbName}`.`${tblName}`"; + + "NOW() FROM `${catalogName}`.`${dbName}`.`${tblName}` where "; private static final String ANALYZE_TABLE_COUNT_TEMPLATE = "SELECT COUNT(1) as rowCount " + "FROM `${catalogName}`.`${dbName}`.`${tblName}`"; + // cache stats for each partition, it would be inserted into column_statistics in a batch. + private final List> buf = new ArrayList<>(); + private final boolean isTableLevelTask; - private final boolean isSamplingPartition; private final boolean isPartitionOnly; - private final Set partitionNames; + private Set partitionNames; private HMSExternalTable table; public HMSAnalysisTask(AnalysisInfo info) { super(info); isTableLevelTask = info.externalTableLevelTask; - isSamplingPartition = info.samplingPartition; isPartitionOnly = info.partitionOnly; partitionNames = info.partitionNames; table = (HMSExternalTable) tbl; @@ -113,7 +114,7 @@ public void doExecute() throws Exception { } /** - * Get table row count and insert the result to __internal_schema.table_statistics + * Get table row count */ private void getTableStats() throws Exception { Map params = buildTableStatsParams(null); @@ -147,55 +148,15 @@ private void getTableColumnStats() throws Exception { // 0 AS data_size, // NOW() FROM `hive`.`tpch100`.`region` if (isPartitionOnly) { - for (String partId : partitionNames) { - StringBuilder sb = new StringBuilder(); - sb.append(ANALYZE_SQL_TABLE_TEMPLATE); - sb.append(" where "); - String[] splits = partId.split("/"); - for (int i = 0; i < splits.length; i++) { - String value = splits[i].split("=")[1]; - splits[i] = splits[i].replace(value, "\'" + value + "\'"); - } - sb.append(StringUtils.join(splits, " and ")); - Map params = buildTableStatsParams(partId); - params.put("internalDB", FeConstants.INTERNAL_DB_NAME); - params.put("columnStatTbl", StatisticConstants.STATISTIC_TBL_NAME); - params.put("colName", col.getName()); - params.put("colId", info.colName); - params.put("dataSizeFunction", getDataSizeFunction(col)); - StringSubstitutor stringSubstitutor = new StringSubstitutor(params); - String sql = stringSubstitutor.replace(sb.toString()); - executeInsertSql(sql); + getPartitionNames(); + List partitionAnalysisSQLs = new ArrayList<>(); + for (String partId : this.partitionNames) { + partitionAnalysisSQLs.add(generateSqlForPartition(partId)); } + execSQLs(partitionAnalysisSQLs); } else { StringBuilder sb = new StringBuilder(); - sb.append(ANALYZE_SQL_TABLE_TEMPLATE); - if (isSamplingPartition) { - sb.append(" where 1=1 "); - String[] splitExample = partitionNames.stream().findFirst().get().split("/"); - int parts = splitExample.length; - List partNames = new ArrayList<>(); - for (String split : splitExample) { - partNames.add(split.split("=")[0]); - } - List> valueLists = new ArrayList<>(); - for (int i = 0; i < parts; i++) { - valueLists.add(new ArrayList<>()); - } - for (String partId : partitionNames) { - String[] partIds = partId.split("/"); - for (int i = 0; i < partIds.length; i++) { - valueLists.get(i).add("\'" + partIds[i].split("=")[1] + "\'"); - } - } - for (int i = 0; i < parts; i++) { - sb.append(" and "); - sb.append(partNames.get(i)); - sb.append(" in ("); - sb.append(StringUtils.join(valueLists.get(i), ",")); - sb.append(") "); - } - } + sb.append(ANALYZE_TABLE_TEMPLATE); Map params = buildTableStatsParams("NULL"); params.put("internalDB", FeConstants.INTERNAL_DB_NAME); params.put("columnStatTbl", StatisticConstants.STATISTIC_TBL_NAME); @@ -208,6 +169,80 @@ private void getTableColumnStats() throws Exception { } } + private void getPartitionNames() { + if (partitionNames == null) { + if (info.isAllPartition) { + partitionNames = table.getPartitionNames(); + } else if (info.partitionCount > 0) { + partitionNames = table.getPartitionNames().stream() + .limit(info.partitionCount).collect(Collectors.toSet()); + } + if (partitionNames == null || partitionNames.isEmpty()) { + throw new RuntimeException("Not a partition table or no partition specified."); + } + } + } + + private String generateSqlForPartition(String partId) { + StringBuilder sb = new StringBuilder(); + sb.append(ANALYZE_PARTITION_TEMPLATE); + String[] splits = partId.split("/"); + for (int i = 0; i < splits.length; i++) { + String[] kv = splits[i].split("="); + sb.append(kv[0]); + sb.append("='"); + sb.append(kv[1]); + sb.append("'"); + if (i < splits.length - 1) { + sb.append(" and "); + } + } + Map params = buildTableStatsParams(partId); + params.put("internalDB", FeConstants.INTERNAL_DB_NAME); + params.put("columnStatTbl", StatisticConstants.STATISTIC_TBL_NAME); + params.put("colName", col.getName()); + params.put("colId", info.colName); + params.put("dataSizeFunction", getDataSizeFunction(col)); + return new StringSubstitutor(params).replace(sb.toString()); + } + + public void execSQLs(List partitionAnalysisSQLs) throws Exception { + long startTime = System.currentTimeMillis(); + LOG.debug("analyze task {} start at {}", info.toString(), new Date()); + try (AutoCloseConnectContext r = StatisticsUtil.buildConnectContext()) { + List> sqlGroups = Lists.partition(partitionAnalysisSQLs, StatisticConstants.UNION_ALL_LIMIT); + for (List group : sqlGroups) { + if (killed) { + return; + } + StringJoiner partitionCollectSQL = new StringJoiner("UNION ALL"); + group.forEach(partitionCollectSQL::add); + stmtExecutor = new StmtExecutor(r.connectContext, partitionCollectSQL.toString()); + buf.add(stmtExecutor.executeInternalQuery() + .stream().map(ColStatsData::new).collect(Collectors.toList())); + QueryState queryState = r.connectContext.getState(); + if (queryState.getStateType().equals(QueryState.MysqlStateType.ERR)) { + throw new RuntimeException(String.format("Failed to analyze %s.%s.%s, error: %s sql: %s", + info.catalogName, info.dbName, info.colName, partitionCollectSQL, + queryState.getErrorMessage())); + } + } + for (List colStatsDataList : buf) { + StringBuilder batchInsertSQL = + new StringBuilder("INSERT INTO " + StatisticConstants.FULL_QUALIFIED_STATS_TBL_NAME + + " VALUES "); + StringJoiner sj = new StringJoiner(","); + colStatsDataList.forEach(c -> sj.add(c.toSQL(true))); + batchInsertSQL.append(sj); + stmtExecutor = new StmtExecutor(r.connectContext, batchInsertSQL.toString()); + executeWithExceptionOnFail(stmtExecutor); + } + } finally { + LOG.debug("analyze task {} end. cost {}ms", info, System.currentTimeMillis() - startTime); + } + + } + private void executeInsertSql(String sql) throws Exception { long startTime = System.currentTimeMillis(); try (AutoCloseConnectContext r = StatisticsUtil.buildConnectContext()) { @@ -270,7 +305,8 @@ private void setParameterData(Map parameters, Map key StatsId statsId = new StatsId(r); long tblId = statsId.tblId; long idxId = statsId.idxId; - long partId = statsId.partId; + String partId = statsId.partId; String colId = statsId.colId; ColumnStatistic partStats = ColumnStatistic.fromResultRow(r); keyToColStats.get(new StatisticsCacheKey(tblId, idxId, colId)).putPartStats(partId, partStats); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java index 849b68fe94ad4f..6521a8b4a5999b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java @@ -228,11 +228,11 @@ private long findExpiredStats(OlapTable statsTbl, ExpiredStats expiredStats, lon continue; } OlapTable olapTable = (OlapTable) t; - Long partId = statsId.partId; + String partId = statsId.partId; if (partId == null) { continue; } - if (!olapTable.getPartitionIds().contains(partId)) { + if (!olapTable.getPartitionIds().contains(Long.parseLong(partId))) { expiredStats.ids.add(id); } } catch (Exception e) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java index f9b18f41e4531d..cd3cc67f3c91c7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java @@ -179,7 +179,7 @@ private static String constructId(Object... params) { return stringJoiner.toString(); } - public static void dropStatistics(Set partIds) throws DdlException { + public static void dropStatistics(Set partIds) throws DdlException { dropStatisticsByPartId(partIds, StatisticConstants.STATISTIC_TBL_NAME); } @@ -202,7 +202,7 @@ public static void dropStatisticsByColName(long tblId, Set colNames, Str } } - public static void dropStatisticsByPartId(Set partIds, String statsTblName) throws DdlException { + public static void dropStatisticsByPartId(Set partIds, String statsTblName) throws DdlException { Map params = new HashMap<>(); String right = StatisticsUtil.joinElementsToString(partIds, ","); String inPredicate = String.format(" part_id IN (%s)", right); @@ -296,14 +296,14 @@ public static List fetchStatsFullName(long limit, long offset) { return StatisticsUtil.execStatisticQuery(new StringSubstitutor(params).replace(FETCH_STATS_FULL_NAME)); } - public static Map> fetchColAndPartsForStats(long tblId) { + public static Map> fetchColAndPartsForStats(long tblId) { Map params = Maps.newHashMap(); params.put("tblId", String.valueOf(tblId)); StringSubstitutor stringSubstitutor = new StringSubstitutor(params); String partSql = stringSubstitutor.replace(FETCH_STATS_PART_ID); List resultRows = StatisticsUtil.execStatisticQuery(partSql); - Map> columnToPartitions = Maps.newHashMap(); + Map> columnToPartitions = Maps.newHashMap(); resultRows.forEach(row -> { try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatsId.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatsId.java index c7af03a8d9e2c3..3f9b2641b75224 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatsId.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatsId.java @@ -32,7 +32,7 @@ public class StatsId { public final String colId; // nullable - public final Long partId; + public final String partId; public StatsId(ResultRow row) { this.id = row.get(0); @@ -41,7 +41,7 @@ public StatsId(ResultRow row) { this.tblId = Long.parseLong(row.get(3)); this.idxId = Long.parseLong(row.get(4)); this.colId = row.get(5); - this.partId = row.get(6) == null ? null : Long.parseLong(row.get(6)); + this.partId = row.get(6); } public String toSQL() { @@ -51,8 +51,8 @@ public String toSQL() { sj.add(String.valueOf(dbId)); sj.add(String.valueOf(tblId)); sj.add(String.valueOf(idxId)); - sj.add(StatisticsUtil.quote(String.valueOf(colId))); - sj.add(String.valueOf(partId)); + sj.add(StatisticsUtil.quote(colId)); + sj.add(StatisticsUtil.quote(partId)); return sj.toString(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java index ed95f4bd1f3645..3d2d0b171882a5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java @@ -103,6 +103,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.StringJoiner; import java.util.UUID; import java.util.function.Function; @@ -439,6 +440,15 @@ public static Map getPartitionIdToName(TableIf table) { )); } + public static Set getPartitionIds(TableIf table) { + if (table instanceof OlapTable) { + return ((OlapTable) table).getPartitionIds().stream().map(String::valueOf).collect(Collectors.toSet()); + } else if (table instanceof ExternalTable) { + return table.getPartitionNames(); + } + throw new RuntimeException(String.format("Not supported Table %s", table.getClass().getName())); + } + public static String joinElementsToString(Collection values, String delimiter) { StringJoiner builder = new StringJoiner(delimiter); values.forEach(v -> builder.add(String.valueOf(v))); @@ -512,7 +522,11 @@ public static long getHiveRowCount(HMSExternalTable table, boolean isInit) { } // Table parameters contains row count, simply get and return it. if (parameters.containsKey(NUM_ROWS)) { - return Long.parseLong(parameters.get(NUM_ROWS)); + long rows = Long.parseLong(parameters.get(NUM_ROWS)); + // Sometimes, the NUM_ROWS in hms is 0 but actually is not. Need to check TOTAL_SIZE if NUM_ROWS is 0. + if (rows != 0) { + return rows; + } } if (!parameters.containsKey(TOTAL_SIZE) || isInit) { return -1; diff --git a/fe/fe-core/src/main/jflex/sql_scanner.flex b/fe/fe-core/src/main/jflex/sql_scanner.flex index 9845c2c8df5dc4..0f8eaa5d9bc4d4 100644 --- a/fe/fe-core/src/main/jflex/sql_scanner.flex +++ b/fe/fe-core/src/main/jflex/sql_scanner.flex @@ -381,6 +381,7 @@ import org.apache.doris.qe.SqlModeHelper; keywordMap.put("read", new Integer(SqlParserSymbols.KW_READ)); keywordMap.put("real", new Integer(SqlParserSymbols.KW_DOUBLE)); keywordMap.put("rebalance", new Integer(SqlParserSymbols.KW_REBALANCE)); + keywordMap.put("recent", new Integer(SqlParserSymbols.KW_RECENT)); keywordMap.put("recover", new Integer(SqlParserSymbols.KW_RECOVER)); keywordMap.put("recycle", new Integer(SqlParserSymbols.KW_RECYCLE)); keywordMap.put("refresh", new Integer(SqlParserSymbols.KW_REFRESH)); diff --git a/regression-test/data/external_table_p2/hive/test_hive_partition_statistic.out b/regression-test/data/external_table_p2/hive/test_hive_partition_statistic.out new file mode 100644 index 00000000000000..0e32ebe4775654 --- /dev/null +++ b/regression-test/data/external_table_p2/hive/test_hive_partition_statistic.out @@ -0,0 +1,87 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !01 -- +event_day=1956-09-07 39 +event_day=2008-09-25 39 + +-- !1 -- +event_day=2008-09-25 10000 1 0 0 0 +event_day=2008-09-25 10000 1 0 2008-09-25 2008-09-25 +event_day=2008-09-25 10000 11 0 0 10 +event_day=2008-09-25 10000 13 0 MFGR#12 MFGR#52 +event_day=2008-09-25 10000 13 0 antique wheat +event_day=2008-09-25 10000 16 0 JUMBO BAG WRAP PACK +event_day=2008-09-25 10000 17 0 1 48 +event_day=2008-09-25 10000 17 0 64078 113087 +event_day=2008-09-25 10000 17 0 754035 763603 +event_day=2008-09-25 10000 17 0 ECONOMY ANODIZED BRASS STANDARD POLISHED TIN +event_day=2008-09-25 10000 17 0 MFGR#1221 MFGR#528 +event_day=2008-09-25 10000 17 0 burnished drab violet firebrick +event_day=2008-09-25 10000 2362 0 19920101 19980802 +event_day=2008-09-25 10000 2382 0 19920203 19981027 +event_day=2008-09-25 10000 25 0 ALGERIA VIETNAM +event_day=2008-09-25 10000 25 0 ALGERIA VIETNAM +event_day=2008-09-25 10000 250 0 ALGERIA 0 VIETNAM 9 +event_day=2008-09-25 10000 250 0 ALGERIA 0 VIETNAM 9 +event_day=2008-09-25 10000 5 0 1-URGENT 5-LOW +event_day=2008-09-25 10000 5 0 AFRICA MIDDLE EAST +event_day=2008-09-25 10000 5 0 AFRICA MIDDLE EAST +event_day=2008-09-25 10000 5 0 AUTOMOBILE MACHINERY +event_day=2008-09-25 10000 5 0 MFGR#1 MFGR#5 +event_day=2008-09-25 10000 50 0 1 50 +event_day=2008-09-25 10000 6074 0 96748 9388900 +event_day=2008-09-25 10000 7 0 1 7 +event_day=2008-09-25 10000 7 0 AIR TRUCK +event_day=2008-09-25 10000 845 0 106797 9423950 +event_day=2008-09-25 10000 9 0 0 8 +event_day=2008-09-25 10000 9775 0 119 2999848 +event_day=2008-09-25 10000 9794 0 107970 45833194 +event_day=2008-09-25 10000 9837 0 MGHV8XBriO zzlztYTFMFW +event_day=2008-09-25 10000 9846 0 Customer#000000119 Customer#002999848 +event_day=2008-09-25 10000 9861 0 13091 599962401 +event_day=2008-09-25 10000 9879 0 10-100-337-6599 34-999-684-2905 +event_day=2008-09-25 10000 9883 0 Supplier#000000001 Supplier#000199983 +event_day=2008-09-25 10000 9896 0 B5YhCdkaxR232CrXx zyxtAvAViHMabnr,1UQybiW +event_day=2008-09-25 10000 9927 0 10-105-800-9296 34-998-982-7450 +event_day=2008-09-25 10000 9971 0 1 199983 + +-- !2 -- +event_day=1956-09-07 10000 1 0 0 0 +event_day=1956-09-07 10000 1 0 1956-09-07 1956-09-07 +event_day=1956-09-07 10000 11 0 0 10 +event_day=1956-09-07 10000 13 0 MFGR#12 MFGR#52 +event_day=1956-09-07 10000 13 0 antique wheat +event_day=1956-09-07 10000 16 0 JUMBO BAG WRAP PACK +event_day=1956-09-07 10000 17 0 1 48 +event_day=1956-09-07 10000 17 0 64078 113087 +event_day=1956-09-07 10000 17 0 754035 763603 +event_day=1956-09-07 10000 17 0 ECONOMY ANODIZED BRASS STANDARD POLISHED TIN +event_day=1956-09-07 10000 17 0 MFGR#1221 MFGR#528 +event_day=1956-09-07 10000 17 0 burnished drab violet firebrick +event_day=1956-09-07 10000 2362 0 19920101 19980802 +event_day=1956-09-07 10000 2382 0 19920203 19981027 +event_day=1956-09-07 10000 25 0 ALGERIA VIETNAM +event_day=1956-09-07 10000 25 0 ALGERIA VIETNAM +event_day=1956-09-07 10000 250 0 ALGERIA 0 VIETNAM 9 +event_day=1956-09-07 10000 250 0 ALGERIA 0 VIETNAM 9 +event_day=1956-09-07 10000 5 0 1-URGENT 5-LOW +event_day=1956-09-07 10000 5 0 AFRICA MIDDLE EAST +event_day=1956-09-07 10000 5 0 AFRICA MIDDLE EAST +event_day=1956-09-07 10000 5 0 AUTOMOBILE MACHINERY +event_day=1956-09-07 10000 5 0 MFGR#1 MFGR#5 +event_day=1956-09-07 10000 50 0 1 50 +event_day=1956-09-07 10000 6074 0 96748 9388900 +event_day=1956-09-07 10000 7 0 1 7 +event_day=1956-09-07 10000 7 0 AIR TRUCK +event_day=1956-09-07 10000 845 0 106797 9423950 +event_day=1956-09-07 10000 9 0 0 8 +event_day=1956-09-07 10000 9775 0 119 2999848 +event_day=1956-09-07 10000 9794 0 107970 45833194 +event_day=1956-09-07 10000 9837 0 MGHV8XBriO zzlztYTFMFW +event_day=1956-09-07 10000 9846 0 Customer#000000119 Customer#002999848 +event_day=1956-09-07 10000 9861 0 13091 599962401 +event_day=1956-09-07 10000 9879 0 10-100-337-6599 34-999-684-2905 +event_day=1956-09-07 10000 9883 0 Supplier#000000001 Supplier#000199983 +event_day=1956-09-07 10000 9896 0 B5YhCdkaxR232CrXx zyxtAvAViHMabnr,1UQybiW +event_day=1956-09-07 10000 9927 0 10-105-800-9296 34-998-982-7450 +event_day=1956-09-07 10000 9971 0 1 199983 + diff --git a/regression-test/suites/external_table_p2/hive/test_hive_partition_statistic.groovy b/regression-test/suites/external_table_p2/hive/test_hive_partition_statistic.groovy new file mode 100644 index 00000000000000..9f4b462237fa3f --- /dev/null +++ b/regression-test/suites/external_table_p2/hive/test_hive_partition_statistic.groovy @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_hive_partition_statistic", "p2,external,hive,external_remote,external_remote_hive") { + String enabled = context.config.otherConfigs.get("enableExternalHiveTest") + if (enabled != null && enabled.equalsIgnoreCase("true")) { + String extHiveHmsHost = context.config.otherConfigs.get("extHiveHmsHost") + String extHiveHmsPort = context.config.otherConfigs.get("extHiveHmsPort") + String catalog_name = "test_hive_partition_statistic" + sql """drop catalog if exists ${catalog_name};""" + sql """ + create catalog if not exists ${catalog_name} properties ( + 'type'='hms', + 'hadoop.username' = 'hadoop', + 'hive.metastore.uris' = 'thrift://${extHiveHmsHost}:${extHiveHmsPort}' + ); + """ + logger.info("catalog " + catalog_name + " created") + + sql """use ${catalog_name}.multi_partition""" + sql """analyze table multi_partition_orc partitions (`event_day=2008-09-25`, `event_day=1956-09-07`) with sync""" + + def ctlId + def result = sql """show proc '/catalogs'""" + + for (int i = 0; i < result.size(); i++) { + if (result[i][1] == catalog_name) { + ctlId = result[i][0] + } + } + + qt_01 """select part_id, count(*) from internal.__internal_schema.column_statistics where catalog_id='$ctlId' group by part_id order by part_id;""" + order_qt_1 """select part_id, count, ndv, null_count, min, max from internal.__internal_schema.column_statistics where catalog_id='$ctlId' and part_id='event_day=2008-09-25'""" + order_qt_2 """select part_id, count, ndv, null_count, min, max from internal.__internal_schema.column_statistics where catalog_id='$ctlId' and part_id='event_day=1956-09-07'""" + + sql """drop catalog ${catalog_name}"""; + } +} + From 1d1eeaec4a940c8982fa4aeb731b9a755b8b4d4b Mon Sep 17 00:00:00 2001 From: Pxl Date: Mon, 18 Sep 2023 15:30:20 +0800 Subject: [PATCH 06/12] [Chore](checks) fix Project root configuration file: NONE (#24533) fix Project root configuration file: NONE --- sonar-project.properties => sonar-scanner.properties | 1 - 1 file changed, 1 deletion(-) rename sonar-project.properties => sonar-scanner.properties (95%) diff --git a/ sonar-project.properties b/sonar-scanner.properties similarity index 95% rename from sonar-project.properties rename to sonar-scanner.properties index bc2baada2c69a2..b47307940afe5a 100644 --- a/ sonar-project.properties +++ b/sonar-scanner.properties @@ -16,6 +16,5 @@ # under the License. # -sonar.host.url=https://sonarcloud.io sonar.projectKey=apache_incubator-doris sonar.organization=apache From b9f1ac153afc70c5032c3473c1702ed0590ee5a3 Mon Sep 17 00:00:00 2001 From: Mryange <59914473+Mryange@users.noreply.github.com> Date: Mon, 18 Sep 2023 15:31:19 +0800 Subject: [PATCH 07/12] [improvement](profile) do not remove value 0 counter (#24487) do not remove value 0 counter --- .../main/java/org/apache/doris/common/util/Counter.java | 9 +++++++++ .../org/apache/doris/common/util/RuntimeProfile.java | 8 ++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/Counter.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/Counter.java index 5bc4aa2ead0f77..77baf3ecd83b4b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/Counter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/Counter.java @@ -23,6 +23,7 @@ public class Counter { private volatile long value; private volatile int type; + private volatile boolean remove = false; public long getValue() { return value; @@ -68,4 +69,12 @@ public boolean isTimeType() { TUnit ttype = TUnit.findByValue(type); return ttype == TUnit.TIME_MS || ttype == TUnit.TIME_NS || ttype == TUnit.TIME_S; } + + public void setCanRemove() { + this.remove = true; + } + + public boolean isRemove() { + return this.remove; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/RuntimeProfile.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/RuntimeProfile.java index a69ecb5d100736..1b8a1f0238039c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/RuntimeProfile.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/RuntimeProfile.java @@ -399,7 +399,7 @@ private static void mergeProfileCounter(RuntimeProfile src, String counterName, mergeProfileCounter(src, childCounterName, rhs); mergeCounter(src, childCounterName, counter, rhsCounter); - removeZeroeCounter(childCounterSet, childCounterName, counter); + removeCounter(childCounterSet, childCounterName, counter); } } @@ -423,8 +423,8 @@ private static void mergeProfileInfoStr(RuntimeProfile src, LinkedList childCounterSet, String childCounterName, Counter counter) { - if (counter.getValue() == 0) { + private static void removeCounter(Set childCounterSet, String childCounterName, Counter counter) { + if (counter.isRemove()) { childCounterSet.remove(childCounterName); } } @@ -476,7 +476,7 @@ private static void mergeCounter(RuntimeProfile src, String counterName, Counter + MIN_TIME_PRE + printCounter(minCounter.getValue(), minCounter.getType()) + " ]"; src.infoStrings.put(counterName, infoString); } - counter.setValue(0); // value 0 will remove in removeZeroeCounter + counter.setCanRemove(); // value will remove in removeCounter } else { if (rhsCounter.size() == 0) { return; From ef4ab106d8dc20b5cad2a70951bc27771b5286e0 Mon Sep 17 00:00:00 2001 From: xu tao Date: Mon, 18 Sep 2023 15:55:43 +0800 Subject: [PATCH 08/12] [fix](security): non-static inner class should not implement serialized interface, or when it is serialized it will contain outer class info, which is not safe #24454 fix: non-static inner class should not implement serialized interface, or when it is serialized it will contain outer class info, which is not safe And in this scenario, the class does not use info of outer class, which should use static class instead --- .../src/main/java/org/apache/doris/load/DppScheduler.java | 2 +- fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/DppScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/load/DppScheduler.java index c037f25b606f40..f777cdfa747348 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/DppScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/DppScheduler.java @@ -552,7 +552,7 @@ public static String getEtlOutputPath(String fsDefaultName, String outputPath, l return String.format(ETL_OUTPUT_PATH, fsDefaultName, outputPath, dbId, loadLabel, etlOutputDir); } - private class InputSizeInvalidException extends LoadException { + private static class InputSizeInvalidException extends LoadException { public InputSizeInvalidException(String msg) { super(msg); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java index f601faba683728..48f71915310603 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java @@ -2476,12 +2476,12 @@ public boolean isDone() { // map from a BE host address to the per-node assigned scan ranges; // records scan range assignment for a single fragment - class FragmentScanRangeAssignment + static class FragmentScanRangeAssignment extends HashMap>> { } // Bucket sequence -> (scan node id -> list of TScanRangeParams) - class BucketSeqToScanRange extends HashMap>> { + static class BucketSeqToScanRange extends HashMap>> { } From 96f197114cfbe5ec116ce1e29b9702ebc256caef Mon Sep 17 00:00:00 2001 From: amory Date: Mon, 18 Sep 2023 16:05:30 +0800 Subject: [PATCH 09/12] [Improve](explode) improve explode func with array nested other type (#24455) improve explode func with array nested other type --- be/src/vec/exprs/table_function/vexplode.cpp | 12 +- .../sql_functions/table_function/am.json | 25 +++ .../sql_functions/table_function/explode.out | 189 ++++++++++++++++++ .../table_function/explode.groovy | 67 +++++++ 4 files changed, 291 insertions(+), 2 deletions(-) create mode 100644 regression-test/data/query_p0/sql_functions/table_function/am.json diff --git a/be/src/vec/exprs/table_function/vexplode.cpp b/be/src/vec/exprs/table_function/vexplode.cpp index 6f13a710e27a2a..912b7acb5de026 100644 --- a/be/src/vec/exprs/table_function/vexplode.cpp +++ b/be/src/vec/exprs/table_function/vexplode.cpp @@ -79,8 +79,16 @@ void VExplodeTableFunction::get_value(MutableColumnPtr& column) { if (current_empty() || (_detail.nested_nullmap_data && _detail.nested_nullmap_data[pos])) { column->insert_default(); } else { - column->insert_data(const_cast(_detail.nested_col->get_data_at(pos).data), - _detail.nested_col->get_data_at(pos).size); + if (_is_nullable) { + assert_cast(column.get()) + ->get_nested_column_ptr() + ->insert_from(*_detail.nested_col, pos); + assert_cast( + assert_cast(column.get())->get_null_map_column_ptr().get()) + ->insert_default(); + } else { + column->insert_from(*_detail.nested_col, pos); + } } } diff --git a/regression-test/data/query_p0/sql_functions/table_function/am.json b/regression-test/data/query_p0/sql_functions/table_function/am.json new file mode 100644 index 00000000000000..c97d389b93db8d --- /dev/null +++ b/regression-test/data/query_p0/sql_functions/table_function/am.json @@ -0,0 +1,25 @@ +{"id":0,"arr_arr":[["feHR6W-ttEtC3v-RvY-RyY3","omWUhV-SlIJJv4-kEW-VDBJ","NZh0JU-TJDXU9B-og5-0J61","V6fdEU-jkJevQT-dWN-sIIY"],null,["SSe3dY-I23acFP-rlr-Xify","XUEhSe-HOgsNt9-RlT-H8JA","zCEmCf-gozXHWE-aI4-y9Dq","qcZATo-a7Czu4Y-KuY-eSX9"],["ADAoxE-ejB7xJn-1zn-Xgpq","DFEZOm-EC6331F-9pF-4ox2","GZ7P0Q-WB5YcBq-af3-HWxg","BPHZHn-faeylOR-PYM-x6gN"]],"arr_map":[{"swpp0q-vgNTgMG-U1h-GrtK":5,"e0bNvm-ZuCo9v7-Gu1-9qsJ":9,"CW2OBW-ET71l4k-wke-B8pK":3,"ydO2p2-64OmnPD-JT1-Xe9q":6,"b9P9Pj-8o3jgFn-E5G-M6Wx":3},{"swpp0q-vgNTgMG-U1h-GrtK":5,"e0bNvm-ZuCo9v7-Gu1-9qsJ":9,"CW2OBW-ET71l4k-wke-B8pK":3,"ydO2p2-64OmnPD-JT1-Xe9q":6,"b9P9Pj-8o3jgFn-E5G-M6Wx":3},{"swpp0q-vgNTgMG-U1h-GrtK":5,"e0bNvm-ZuCo9v7-Gu1-9qsJ":9,"CW2OBW-ET71l4k-wke-B8pK":3,"ydO2p2-64OmnPD-JT1-Xe9q":6,"b9P9Pj-8o3jgFn-E5G-M6Wx":3},{"swpp0q-vgNTgMG-U1h-GrtK":5,"e0bNvm-ZuCo9v7-Gu1-9qsJ":9,"CW2OBW-ET71l4k-wke-B8pK":3,"ydO2p2-64OmnPD-JT1-Xe9q":6,"b9P9Pj-8o3jgFn-E5G-M6Wx":3},{"swpp0q-vgNTgMG-U1h-GrtK":5,"e0bNvm-ZuCo9v7-Gu1-9qsJ":9,"CW2OBW-ET71l4k-wke-B8pK":3,"ydO2p2-64OmnPD-JT1-Xe9q":6,"b9P9Pj-8o3jgFn-E5G-M6Wx":3},{"swpp0q-vgNTgMG-U1h-GrtK":5,"e0bNvm-ZuCo9v7-Gu1-9qsJ":9,"CW2OBW-ET71l4k-wke-B8pK":3,"ydO2p2-64OmnPD-JT1-Xe9q":6,"b9P9Pj-8o3jgFn-E5G-M6Wx":3},{"swpp0q-vgNTgMG-U1h-GrtK":5,"e0bNvm-ZuCo9v7-Gu1-9qsJ":9,"CW2OBW-ET71l4k-wke-B8pK":3,"ydO2p2-64OmnPD-JT1-Xe9q":6,"b9P9Pj-8o3jgFn-E5G-M6Wx":3}]} +{"id":1,"arr_arr":[["ZGRUhx-eadzlDE-JFF-BH3P","vrhIVi-tIBN9fc-tBe-4FUC","v006kw-wIB8x6E-WWt-Li23"],["Q3UE7a-yBfPvoZ-q5e-xtc3","b4Is4f-1Q17TLQ-3Nh-E9XK","1bkMir-QC3ddA8-Wy6-dcgr"],null],"arr_map":[{"nKWQnv-HolfdQc-tqs-Trnb":0,"GEAmQi-bMHzvyG-MGd-GBjA":5,"KtMT9e-jKcWd9t-9GG-8mG6":3,"znKFYo-4abMgt6-k4A-L4Ki":4,"IpXgBa-FGiyskO-R5R-T3CY":6},{"nKWQnv-HolfdQc-tqs-Trnb":0,"GEAmQi-bMHzvyG-MGd-GBjA":5,"KtMT9e-jKcWd9t-9GG-8mG6":3,"znKFYo-4abMgt6-k4A-L4Ki":4,"IpXgBa-FGiyskO-R5R-T3CY":6},{"nKWQnv-HolfdQc-tqs-Trnb":0,"GEAmQi-bMHzvyG-MGd-GBjA":5,"KtMT9e-jKcWd9t-9GG-8mG6":3,"znKFYo-4abMgt6-k4A-L4Ki":4,"IpXgBa-FGiyskO-R5R-T3CY":6},{"nKWQnv-HolfdQc-tqs-Trnb":0,"GEAmQi-bMHzvyG-MGd-GBjA":5,"KtMT9e-jKcWd9t-9GG-8mG6":3,"znKFYo-4abMgt6-k4A-L4Ki":4,"IpXgBa-FGiyskO-R5R-T3CY":6},{"nKWQnv-HolfdQc-tqs-Trnb":0,"GEAmQi-bMHzvyG-MGd-GBjA":5,"KtMT9e-jKcWd9t-9GG-8mG6":3,"znKFYo-4abMgt6-k4A-L4Ki":4,"IpXgBa-FGiyskO-R5R-T3CY":6},{"nKWQnv-HolfdQc-tqs-Trnb":0,"GEAmQi-bMHzvyG-MGd-GBjA":5,"KtMT9e-jKcWd9t-9GG-8mG6":3,"znKFYo-4abMgt6-k4A-L4Ki":4,"IpXgBa-FGiyskO-R5R-T3CY":6}]} +{"id":2,"arr_arr":[["lWGk4M-ba0tBFf-JVl-qYen","2O1nVO-8pdtYoX-jaz-9Lyv"],null],"arr_map":[{"N1isOk-HqyCfZ3-Iqu-sHC4":8,"zoEAUW-yYiiBxG-xER-4AVu":0,"y6OK1G-wIHaG2A-oWB-WQel":3,"oWys2y-FLQKf53-5mm-3BWr":9,"Kkjpbh-vOvz3Hu-WdS-UVI6":4},{"N1isOk-HqyCfZ3-Iqu-sHC4":8,"zoEAUW-yYiiBxG-xER-4AVu":0,"y6OK1G-wIHaG2A-oWB-WQel":3,"oWys2y-FLQKf53-5mm-3BWr":9,"Kkjpbh-vOvz3Hu-WdS-UVI6":4}]} +{"id":3,"arr_arr":[],"arr_map":[{"sAAkg4-TsxcHvG-PU5-reih":0,"gLZA8c-BegqvvI-c8X-k2sG":0,"XsjVET-cVWzizh-IQL-oD9Z":3,"mn5po7-sdVBNfT-D1h-J9cr":1,"VhWPpg-MjnLQxg-fQW-HSEW":5},{"sAAkg4-TsxcHvG-PU5-reih":0,"gLZA8c-BegqvvI-c8X-k2sG":0,"XsjVET-cVWzizh-IQL-oD9Z":3,"mn5po7-sdVBNfT-D1h-J9cr":1,"VhWPpg-MjnLQxg-fQW-HSEW":5},{"sAAkg4-TsxcHvG-PU5-reih":0,"gLZA8c-BegqvvI-c8X-k2sG":0,"XsjVET-cVWzizh-IQL-oD9Z":3,"mn5po7-sdVBNfT-D1h-J9cr":1,"VhWPpg-MjnLQxg-fQW-HSEW":5}]} +{"id":4,"arr_arr":[],"arr_map":[{"GW98C8-Xv6AWLP-ivi-FLtW":0,"Q635Ar-OMGGYWX-Dhg-7HgP":8,"TbeKcq-4AUGF5H-DzO-KwWg":6,"fKDLAp-tPmFPle-lF6-FGNZ":1,"W5qTCQ-Lfqxf29-Z3J-9Yrz":9},{"GW98C8-Xv6AWLP-ivi-FLtW":0,"Q635Ar-OMGGYWX-Dhg-7HgP":8,"TbeKcq-4AUGF5H-DzO-KwWg":6,"fKDLAp-tPmFPle-lF6-FGNZ":1,"W5qTCQ-Lfqxf29-Z3J-9Yrz":9}]} +{"id":5,"arr_arr":[null,["YaKNfI-gV6lMRG-dio-Jg9D","TC7wdG-1YvyDHG-HVV-rdRR","8dMmjA-lA9oHx9-dSS-Fl86"],["4Rff3K-TATMSg1-b6R-SmKN","t1j5tX-dTCrxmn-idk-rk24","7LW7Lq-cXYuy90-uyH-M6pK"]],"arr_map":[{"iLOhUA-n05O2OY-LHB-qS3H":8,"ATC5e8-MywwrP3-Vhc-QdxZ":1,"fST2pb-uAaS3Mb-Pl6-9i1J":0,"fppq1A-Q1WOvMj-Eod-xbTz":3,"ebkDD2-T3jSVTg-zZS-Bqnh":4},{"iLOhUA-n05O2OY-LHB-qS3H":8,"ATC5e8-MywwrP3-Vhc-QdxZ":1,"fST2pb-uAaS3Mb-Pl6-9i1J":0,"fppq1A-Q1WOvMj-Eod-xbTz":3,"ebkDD2-T3jSVTg-zZS-Bqnh":4},{"iLOhUA-n05O2OY-LHB-qS3H":8,"ATC5e8-MywwrP3-Vhc-QdxZ":1,"fST2pb-uAaS3Mb-Pl6-9i1J":0,"fppq1A-Q1WOvMj-Eod-xbTz":3,"ebkDD2-T3jSVTg-zZS-Bqnh":4}]} +{"id":6,"arr_arr":[],"arr_map":[{"eSB1Yl-jDIrIHV-7WE-qyvM":6,"4N8yoM-901A5FI-B9Q-3EpM":9,"pBdMp1-nVbMzpB-WJj-dhDd":4,"3jGdNP-NgyL5fW-9Ty-CLlY":2,"5gaoHt-7MmTVLs-Jm0-KpaG":2},{"eSB1Yl-jDIrIHV-7WE-qyvM":6,"4N8yoM-901A5FI-B9Q-3EpM":9,"pBdMp1-nVbMzpB-WJj-dhDd":4,"3jGdNP-NgyL5fW-9Ty-CLlY":2,"5gaoHt-7MmTVLs-Jm0-KpaG":2},{"eSB1Yl-jDIrIHV-7WE-qyvM":6,"4N8yoM-901A5FI-B9Q-3EpM":9,"pBdMp1-nVbMzpB-WJj-dhDd":4,"3jGdNP-NgyL5fW-9Ty-CLlY":2,"5gaoHt-7MmTVLs-Jm0-KpaG":2}]} +{"id":7,"arr_arr":[["UGG3Iy-bMW7c6T-fz9-WYOG","SWWur3-zdHlN1q-527-UtFT","XrCVm0-bbwQBay-lcn-vNTJ"],null,["1KN2Cf-UE1ruFq-sRm-pW7A","2lwurI-ANjel77-nhS-ahzI","DR5JH4-WM6QRe7-FgQ-qHnV"]],"arr_map":[{"yfDrEq-z2VBXi1-bSe-tgGW":7,"zLkwFE-i6GiJut-yHW-e4wJ":8,"pl1mqW-qSEYcv0-kTT-8Uuz":3,"cEbTSo-CoT1upF-XsY-gafn":8,"Shcq2l-iuYvkyo-ssN-okkW":7},{"yfDrEq-z2VBXi1-bSe-tgGW":7,"zLkwFE-i6GiJut-yHW-e4wJ":8,"pl1mqW-qSEYcv0-kTT-8Uuz":3,"cEbTSo-CoT1upF-XsY-gafn":8,"Shcq2l-iuYvkyo-ssN-okkW":7},{"yfDrEq-z2VBXi1-bSe-tgGW":7,"zLkwFE-i6GiJut-yHW-e4wJ":8,"pl1mqW-qSEYcv0-kTT-8Uuz":3,"cEbTSo-CoT1upF-XsY-gafn":8,"Shcq2l-iuYvkyo-ssN-okkW":7},{"yfDrEq-z2VBXi1-bSe-tgGW":7,"zLkwFE-i6GiJut-yHW-e4wJ":8,"pl1mqW-qSEYcv0-kTT-8Uuz":3,"cEbTSo-CoT1upF-XsY-gafn":8,"Shcq2l-iuYvkyo-ssN-okkW":7}]} +{"id":8,"arr_arr":[["ZfdXEt-opP9k3J-l2M-sIdX","CyLrQr-KniPD7H-VaF-y563","UtVD3f-NsX6D4j-UQt-Pgpg"],["RjHATY-RbQXO8o-9kC-yMN6","QIsZVT-SBeloLd-RgF-zws0","6d2byV-qKIHXRn-mDF-TM1g"],null],"arr_map":[{"lJFCyu-BhT7N4x-luz-9t85":2,"wGuvM7-zJNjFUz-4z3-Y6Gf":7,"F9KCc5-ZzE0IYN-c5t-VqUH":8,"FgXjJi-l10Pijy-jtK-WgMf":4,"uexWAB-xHkHMYL-vUi-4Umt":9},{"lJFCyu-BhT7N4x-luz-9t85":2,"wGuvM7-zJNjFUz-4z3-Y6Gf":7,"F9KCc5-ZzE0IYN-c5t-VqUH":8,"FgXjJi-l10Pijy-jtK-WgMf":4,"uexWAB-xHkHMYL-vUi-4Umt":9},{"lJFCyu-BhT7N4x-luz-9t85":2,"wGuvM7-zJNjFUz-4z3-Y6Gf":7,"F9KCc5-ZzE0IYN-c5t-VqUH":8,"FgXjJi-l10Pijy-jtK-WgMf":4,"uexWAB-xHkHMYL-vUi-4Umt":9},{"lJFCyu-BhT7N4x-luz-9t85":2,"wGuvM7-zJNjFUz-4z3-Y6Gf":7,"F9KCc5-ZzE0IYN-c5t-VqUH":8,"FgXjJi-l10Pijy-jtK-WgMf":4,"uexWAB-xHkHMYL-vUi-4Umt":9},{"lJFCyu-BhT7N4x-luz-9t85":2,"wGuvM7-zJNjFUz-4z3-Y6Gf":7,"F9KCc5-ZzE0IYN-c5t-VqUH":8,"FgXjJi-l10Pijy-jtK-WgMf":4,"uexWAB-xHkHMYL-vUi-4Umt":9},{"lJFCyu-BhT7N4x-luz-9t85":2,"wGuvM7-zJNjFUz-4z3-Y6Gf":7,"F9KCc5-ZzE0IYN-c5t-VqUH":8,"FgXjJi-l10Pijy-jtK-WgMf":4,"uexWAB-xHkHMYL-vUi-4Umt":9}]} +{"id":9,"arr_arr":[null,["xI0TNh-wAH12NI-6Px-MLib","Jmesk7-hJzDXz6-lFC-Egmp","YFtlSh-8pZVVxG-mNa-RYES","q5sDuB-WfmqXDr-WXJ-UPq6","3Toro7-4wzAPD9-G6e-ajl1","bE4dED-fNBqL16-JXt-0lZY"],["n14avO-lueGLXs-zxM-tOEZ","yj4iMW-h1Z5isV-jM4-0BRC","YXg40k-p00nu3H-DLQ-ASu2","KOg34p-E4TtdAs-xcl-ajzL","d0LfzZ-2huMiVN-4Fc-aOuz","2cJHzT-KjvF9cE-W5u-nOrI"],["CSwQYw-bvMIUzj-bqf-AXpo","tc0Rwz-uQ3Wdbv-z8f-qklW","X6X0G8-JlYcXKh-pay-D21m","n7RWXW-ZFh2HqS-6XE-v9TY","zAXTa3-zz5OkX6-jAM-ZuyD","UXdt9B-Mml6eBa-hN7-vQOp"],["Hi9Osx-9BogT7m-XBD-rtEV","xoF9Rq-MshvMLG-jEJ-8965","k5LrCO-6jQD17H-6jT-zYVY","UYAu4H-KgjxcmP-XyO-ZFQJ","eQVglX-V5YAuIJ-xC7-RYgL","QydLV7-7XqcCuO-rlG-w7ba"],["FxiSSg-4B2Ep4r-AoC-nwwf","QIafe6-b9ETKKv-WTq-wXnZ","vP0uXm-5lRFe5D-R6o-58Jt","UnzUJl-DXv294E-ozf-KgZF","0ikQZL-ORqwsFr-fGs-flzQ","Ltap6E-siSL3Hm-h0i-sKp2"]],"arr_map":[{"s4d0hU-uWynWWe-eIf-ym5C":9,"ANvepQ-oO7fMUr-qdl-XbUo":4,"G16nau-7qY3fb1-UYL-oiNd":5,"bVd4er-efUCDow-nzo-87fg":2,"7I6zLI-hQTTlLe-sBr-oCTh":7},{"s4d0hU-uWynWWe-eIf-ym5C":9,"ANvepQ-oO7fMUr-qdl-XbUo":4,"G16nau-7qY3fb1-UYL-oiNd":5,"bVd4er-efUCDow-nzo-87fg":2,"7I6zLI-hQTTlLe-sBr-oCTh":7},{"s4d0hU-uWynWWe-eIf-ym5C":9,"ANvepQ-oO7fMUr-qdl-XbUo":4,"G16nau-7qY3fb1-UYL-oiNd":5,"bVd4er-efUCDow-nzo-87fg":2,"7I6zLI-hQTTlLe-sBr-oCTh":7},{"s4d0hU-uWynWWe-eIf-ym5C":9,"ANvepQ-oO7fMUr-qdl-XbUo":4,"G16nau-7qY3fb1-UYL-oiNd":5,"bVd4er-efUCDow-nzo-87fg":2,"7I6zLI-hQTTlLe-sBr-oCTh":7},{"s4d0hU-uWynWWe-eIf-ym5C":9,"ANvepQ-oO7fMUr-qdl-XbUo":4,"G16nau-7qY3fb1-UYL-oiNd":5,"bVd4er-efUCDow-nzo-87fg":2,"7I6zLI-hQTTlLe-sBr-oCTh":7},{"s4d0hU-uWynWWe-eIf-ym5C":9,"ANvepQ-oO7fMUr-qdl-XbUo":4,"G16nau-7qY3fb1-UYL-oiNd":5,"bVd4er-efUCDow-nzo-87fg":2,"7I6zLI-hQTTlLe-sBr-oCTh":7},{"s4d0hU-uWynWWe-eIf-ym5C":9,"ANvepQ-oO7fMUr-qdl-XbUo":4,"G16nau-7qY3fb1-UYL-oiNd":5,"bVd4er-efUCDow-nzo-87fg":2,"7I6zLI-hQTTlLe-sBr-oCTh":7}]} +{"id":10,"arr_arr":[["PWNdZn-256Mzh0-Wxl-i1VN","oyLKti-7HvbWsn-MQ4-EMuK"],null],"arr_map":[{"kpjXle-CayOe2S-2W4-vqvG":4,"VIWvQo-q2swzEQ-loT-PpQA":9,"PnLRPK-AW1LRJj-QDh-WLfE":5,"vGVTU8-SbgpatL-PXh-6DPp":0,"YPH9gI-Uv5Ec0u-QHJ-5QKd":3},{"kpjXle-CayOe2S-2W4-vqvG":4,"VIWvQo-q2swzEQ-loT-PpQA":9,"PnLRPK-AW1LRJj-QDh-WLfE":5,"vGVTU8-SbgpatL-PXh-6DPp":0,"YPH9gI-Uv5Ec0u-QHJ-5QKd":3},{"kpjXle-CayOe2S-2W4-vqvG":4,"VIWvQo-q2swzEQ-loT-PpQA":9,"PnLRPK-AW1LRJj-QDh-WLfE":5,"vGVTU8-SbgpatL-PXh-6DPp":0,"YPH9gI-Uv5Ec0u-QHJ-5QKd":3},{"kpjXle-CayOe2S-2W4-vqvG":4,"VIWvQo-q2swzEQ-loT-PpQA":9,"PnLRPK-AW1LRJj-QDh-WLfE":5,"vGVTU8-SbgpatL-PXh-6DPp":0,"YPH9gI-Uv5Ec0u-QHJ-5QKd":3},{"kpjXle-CayOe2S-2W4-vqvG":4,"VIWvQo-q2swzEQ-loT-PpQA":9,"PnLRPK-AW1LRJj-QDh-WLfE":5,"vGVTU8-SbgpatL-PXh-6DPp":0,"YPH9gI-Uv5Ec0u-QHJ-5QKd":3}]} +{"id":11,"arr_arr":[["5fkZOk-BckdrAU-MM3-HwPF","vicmYn-wD21968-mbI-Wv3c"],null],"arr_map":[{"Mqzc3X-DzSmJor-o9C-g73I":4,"Xgdg1m-DZzCOqO-7pA-PKeS":6,"JzTEl1-CwOaAmt-yV3-n2k0":9,"pm0RBB-gdAl37a-8Pq-QLJV":4,"oN1HDX-xCm8dRv-0wk-MJeu":8},{"Mqzc3X-DzSmJor-o9C-g73I":4,"Xgdg1m-DZzCOqO-7pA-PKeS":6,"JzTEl1-CwOaAmt-yV3-n2k0":9,"pm0RBB-gdAl37a-8Pq-QLJV":4,"oN1HDX-xCm8dRv-0wk-MJeu":8},{"Mqzc3X-DzSmJor-o9C-g73I":4,"Xgdg1m-DZzCOqO-7pA-PKeS":6,"JzTEl1-CwOaAmt-yV3-n2k0":9,"pm0RBB-gdAl37a-8Pq-QLJV":4,"oN1HDX-xCm8dRv-0wk-MJeu":8}]} +{"id":12,"arr_arr":[["srDnEJ-ySU31lR-ul1-ROCJ","cMnWa5-bA1OMFg-kiY-AZMI","8iRcRL-3v72cNM-Ojm-MUpV"],["ydwzpg-kqjN5zO-Vb5-wo5P","Rpk4Cc-1MaKNzp-0GF-Hr8K","G64mgw-wLRtwwY-XFn-BlaK"],null],"arr_map":[{"GFBqsx-TvhMATX-6ZB-0Zva":2,"D6QtvK-8eTn7dE-CZn-h18O":2,"tdKmN3-ealR73D-PO4-6nh8":4,"veE9Ee-GBc3xZa-O1T-n8u3":5,"0P9fws-Pb03WDb-lOu-l4Ww":7},{"GFBqsx-TvhMATX-6ZB-0Zva":2,"D6QtvK-8eTn7dE-CZn-h18O":2,"tdKmN3-ealR73D-PO4-6nh8":4,"veE9Ee-GBc3xZa-O1T-n8u3":5,"0P9fws-Pb03WDb-lOu-l4Ww":7},{"GFBqsx-TvhMATX-6ZB-0Zva":2,"D6QtvK-8eTn7dE-CZn-h18O":2,"tdKmN3-ealR73D-PO4-6nh8":4,"veE9Ee-GBc3xZa-O1T-n8u3":5,"0P9fws-Pb03WDb-lOu-l4Ww":7}]} +{"id":13,"arr_arr":[],"arr_map":[{"kIpyuK-FUQet1S-6JW-PJjH":0,"KosGlG-X9weJpR-Olb-qNNY":4,"x8HUd5-IzLbruK-ruY-HPIv":8,"UaBaT4-ZKtEsWB-yJ7-ikwZ":2,"tyiuhU-08ulAju-XWn-fsID":6},{"kIpyuK-FUQet1S-6JW-PJjH":0,"KosGlG-X9weJpR-Olb-qNNY":4,"x8HUd5-IzLbruK-ruY-HPIv":8,"UaBaT4-ZKtEsWB-yJ7-ikwZ":2,"tyiuhU-08ulAju-XWn-fsID":6},{"kIpyuK-FUQet1S-6JW-PJjH":0,"KosGlG-X9weJpR-Olb-qNNY":4,"x8HUd5-IzLbruK-ruY-HPIv":8,"UaBaT4-ZKtEsWB-yJ7-ikwZ":2,"tyiuhU-08ulAju-XWn-fsID":6}]} +{"id":14,"arr_arr":[["hBG9CJ-Pkcv5Xv-y5o-SiDS","VAaAjn-bFJZdHR-NYt-2on0","T3pZPF-LB6LuqK-Zfg-CwRv","2yAunr-D0ywKAw-fA7-19qi","5t1kUf-nkasZdS-9cr-6J2b","7Yk0yJ-PuETSw0-nGi-yIET"],["EUKF7p-G5gNyzr-nTn-AlOt","NHve5R-8hB7e9z-1s4-AmPq","4rGDYS-KTrWNMD-Iur-dn0b","9RZDOx-oShNnLu-rQ1-uCmU","FA2HUr-CoTvZBz-ZlO-cEA3","XeSGqz-Lrl8p39-Hx6-HZQh"],["Zi6B6P-FFGNvXt-IN9-3MQg","QZ4Blu-xat1Zh7-Id2-emNh","zBJ4k4-qB9gGgf-uEF-Cuee","ndFSon-iwKJGIb-gQa-keG0","ZI14D4-NOhnTPA-DtG-xuhP","x0UxHB-lEmujK2-snM-q8ze"],["2ISsbI-Tpnywjw-dKc-awNI","KNq9c2-Yvp3aEc-geM-MQV0","oDkQ38-Eu9E6bj-Ei6-rwwq","678xMV-Tn0fVv6-O67-Jg01","c5Xufb-QGUezAE-8iM-3VBg","MhjSVE-UKjyuyZ-h71-q8fS"],["KdXHnA-bNv8rwO-4Ey-nVz6","a9J9b4-Yd2j674-3eB-Bqjv","ciwaAk-ktsHqMi-aug-tSgZ","U2TSkV-PrbEclO-tIt-ilN7","EDRbVY-6Y8vZgH-QYY-CPur","nKUwkr-WXn2hj1-xay-JeZA"],null],"arr_map":[{"Jfv2A1-Nh98daR-5eV-63xx":9,"iqv9Dc-AovBuni-nRH-XnEZ":1,"OQkgBo-duRmhW2-pTI-dSTS":0,"pfWZRg-8lXC9OV-NPT-nHjl":4,"Wso7dF-rmVTlDS-s9Q-MGtf":0},{"Jfv2A1-Nh98daR-5eV-63xx":9,"iqv9Dc-AovBuni-nRH-XnEZ":1,"OQkgBo-duRmhW2-pTI-dSTS":0,"pfWZRg-8lXC9OV-NPT-nHjl":4,"Wso7dF-rmVTlDS-s9Q-MGtf":0},{"Jfv2A1-Nh98daR-5eV-63xx":9,"iqv9Dc-AovBuni-nRH-XnEZ":1,"OQkgBo-duRmhW2-pTI-dSTS":0,"pfWZRg-8lXC9OV-NPT-nHjl":4,"Wso7dF-rmVTlDS-s9Q-MGtf":0},{"Jfv2A1-Nh98daR-5eV-63xx":9,"iqv9Dc-AovBuni-nRH-XnEZ":1,"OQkgBo-duRmhW2-pTI-dSTS":0,"pfWZRg-8lXC9OV-NPT-nHjl":4,"Wso7dF-rmVTlDS-s9Q-MGtf":0},{"Jfv2A1-Nh98daR-5eV-63xx":9,"iqv9Dc-AovBuni-nRH-XnEZ":1,"OQkgBo-duRmhW2-pTI-dSTS":0,"pfWZRg-8lXC9OV-NPT-nHjl":4,"Wso7dF-rmVTlDS-s9Q-MGtf":0},{"Jfv2A1-Nh98daR-5eV-63xx":9,"iqv9Dc-AovBuni-nRH-XnEZ":1,"OQkgBo-duRmhW2-pTI-dSTS":0,"pfWZRg-8lXC9OV-NPT-nHjl":4,"Wso7dF-rmVTlDS-s9Q-MGtf":0},{"Jfv2A1-Nh98daR-5eV-63xx":9,"iqv9Dc-AovBuni-nRH-XnEZ":1,"OQkgBo-duRmhW2-pTI-dSTS":0,"pfWZRg-8lXC9OV-NPT-nHjl":4,"Wso7dF-rmVTlDS-s9Q-MGtf":0},{"Jfv2A1-Nh98daR-5eV-63xx":9,"iqv9Dc-AovBuni-nRH-XnEZ":1,"OQkgBo-duRmhW2-pTI-dSTS":0,"pfWZRg-8lXC9OV-NPT-nHjl":4,"Wso7dF-rmVTlDS-s9Q-MGtf":0},{"Jfv2A1-Nh98daR-5eV-63xx":9,"iqv9Dc-AovBuni-nRH-XnEZ":1,"OQkgBo-duRmhW2-pTI-dSTS":0,"pfWZRg-8lXC9OV-NPT-nHjl":4,"Wso7dF-rmVTlDS-s9Q-MGtf":0}]} +{"id":15,"arr_arr":[["VMXWe8-sJGeEn4-qgR-82VU"]],"arr_map":[{"ZZoIk6-FCvllrX-ynY-WH9b":8,"Lo6mI2-FUuG0Kp-ALg-N9CN":2,"lFhxJ0-iaZhTSc-XKg-Ats9":9,"hiaVfD-qDapBgY-DBu-uCif":1,"i5zkzH-UFvf6zO-ubG-bDEb":9},{"ZZoIk6-FCvllrX-ynY-WH9b":8,"Lo6mI2-FUuG0Kp-ALg-N9CN":2,"lFhxJ0-iaZhTSc-XKg-Ats9":9,"hiaVfD-qDapBgY-DBu-uCif":1,"i5zkzH-UFvf6zO-ubG-bDEb":9},{"ZZoIk6-FCvllrX-ynY-WH9b":8,"Lo6mI2-FUuG0Kp-ALg-N9CN":2,"lFhxJ0-iaZhTSc-XKg-Ats9":9,"hiaVfD-qDapBgY-DBu-uCif":1,"i5zkzH-UFvf6zO-ubG-bDEb":9},{"ZZoIk6-FCvllrX-ynY-WH9b":8,"Lo6mI2-FUuG0Kp-ALg-N9CN":2,"lFhxJ0-iaZhTSc-XKg-Ats9":9,"hiaVfD-qDapBgY-DBu-uCif":1,"i5zkzH-UFvf6zO-ubG-bDEb":9}]} +{"id":16,"arr_arr":[["Q6ZUdh-aHSTJSP-qvd-z7a6","HsdNd0-VKHJL5f-6MU-yGb6","FZObTa-iuP04lW-tZw-D68t","5eaGJJ-uyU5Au4-dBC-5AR1"],["tgS4vz-pU4PS8m-E5R-DSU8","wxkjf8-Wwl6cjm-AC9-KcKi","US36ry-lTrkIhR-MTf-7YyD","xNmbYx-VDzJq4E-5Q2-DCvO"],["OIN1Oe-FI3wcrp-66H-fGKM","ijHMjw-V2O8F9E-OYq-NpkQ","aChpoK-FAtOsCI-kWI-oYnF","iE0YBR-zGoZeFx-nLL-CGMW"],null],"arr_map":[{"6HGUM0-YSULhQm-tNd-WrUB":6,"2iQyQQ-0Q79hSa-xas-mA1b":2,"kHyrvC-FG8yKzY-bJi-FTGR":5,"kCKYlJ-LlfcyEz-Zzc-pcuw":2,"D97U6o-rNR0M8j-7FD-yB2i":2},{"6HGUM0-YSULhQm-tNd-WrUB":6,"2iQyQQ-0Q79hSa-xas-mA1b":2,"kHyrvC-FG8yKzY-bJi-FTGR":5,"kCKYlJ-LlfcyEz-Zzc-pcuw":2,"D97U6o-rNR0M8j-7FD-yB2i":2},{"6HGUM0-YSULhQm-tNd-WrUB":6,"2iQyQQ-0Q79hSa-xas-mA1b":2,"kHyrvC-FG8yKzY-bJi-FTGR":5,"kCKYlJ-LlfcyEz-Zzc-pcuw":2,"D97U6o-rNR0M8j-7FD-yB2i":2},{"6HGUM0-YSULhQm-tNd-WrUB":6,"2iQyQQ-0Q79hSa-xas-mA1b":2,"kHyrvC-FG8yKzY-bJi-FTGR":5,"kCKYlJ-LlfcyEz-Zzc-pcuw":2,"D97U6o-rNR0M8j-7FD-yB2i":2},{"6HGUM0-YSULhQm-tNd-WrUB":6,"2iQyQQ-0Q79hSa-xas-mA1b":2,"kHyrvC-FG8yKzY-bJi-FTGR":5,"kCKYlJ-LlfcyEz-Zzc-pcuw":2,"D97U6o-rNR0M8j-7FD-yB2i":2},{"6HGUM0-YSULhQm-tNd-WrUB":6,"2iQyQQ-0Q79hSa-xas-mA1b":2,"kHyrvC-FG8yKzY-bJi-FTGR":5,"kCKYlJ-LlfcyEz-Zzc-pcuw":2,"D97U6o-rNR0M8j-7FD-yB2i":2},{"6HGUM0-YSULhQm-tNd-WrUB":6,"2iQyQQ-0Q79hSa-xas-mA1b":2,"kHyrvC-FG8yKzY-bJi-FTGR":5,"kCKYlJ-LlfcyEz-Zzc-pcuw":2,"D97U6o-rNR0M8j-7FD-yB2i":2}]} +{"id":17,"arr_arr":[["6h3u7b-znuJXzP-Y0y-67k5","kOW5Bh-iCk28Kl-qnX-YTEw","52e34r-ja3ecA3-f2X-tW5A","fbEZmh-SditnSh-1Nf-O939","ovmskR-d7Cb5GH-uTh-snqu"],["gaRzuW-8ljPxnB-sG4-EPyR","I1fCfi-dqF5jIV-xcO-qXxs","vteojI-nipd8nC-GbM-BuCz","OJtqcM-M4xJStL-DoO-pAVA","cHaELA-NYH7FiQ-JRw-Pplg"],null,["L1ROPG-CHHugVU-tQw-9PCs","7bZtwn-HrJkMD9-YFg-Fmcf","PVV4fi-5rlini0-2AI-97MK","ouWbwc-B2vQ7tR-DLi-H34g","R1crmv-lhkakVD-Gg4-LBIv"],["FACnXp-lxR724G-u2J-veYj","5iIj0x-NWC3OCy-zlU-8laP","2xSkQW-CxLzsjs-lPi-KhFu","GIIWc9-1gOT3Ql-zK9-aBK5","FFtM4W-pFhEp8x-D4s-AIX9"]],"arr_map":[{"pvJ17T-e8sUqJE-UAM-3HJC":1,"UqYyHo-6j72MWA-06u-ITXN":0,"ALehFX-Jdmwn4V-i9x-VnTD":6,"WRr0E1-5qGq3uV-MGG-8ugQ":1,"JKKvYY-N1BCDLk-JBF-K7FN":2},{"pvJ17T-e8sUqJE-UAM-3HJC":1,"UqYyHo-6j72MWA-06u-ITXN":0,"ALehFX-Jdmwn4V-i9x-VnTD":6,"WRr0E1-5qGq3uV-MGG-8ugQ":1,"JKKvYY-N1BCDLk-JBF-K7FN":2},{"pvJ17T-e8sUqJE-UAM-3HJC":1,"UqYyHo-6j72MWA-06u-ITXN":0,"ALehFX-Jdmwn4V-i9x-VnTD":6,"WRr0E1-5qGq3uV-MGG-8ugQ":1,"JKKvYY-N1BCDLk-JBF-K7FN":2},{"pvJ17T-e8sUqJE-UAM-3HJC":1,"UqYyHo-6j72MWA-06u-ITXN":0,"ALehFX-Jdmwn4V-i9x-VnTD":6,"WRr0E1-5qGq3uV-MGG-8ugQ":1,"JKKvYY-N1BCDLk-JBF-K7FN":2},{"pvJ17T-e8sUqJE-UAM-3HJC":1,"UqYyHo-6j72MWA-06u-ITXN":0,"ALehFX-Jdmwn4V-i9x-VnTD":6,"WRr0E1-5qGq3uV-MGG-8ugQ":1,"JKKvYY-N1BCDLk-JBF-K7FN":2},{"pvJ17T-e8sUqJE-UAM-3HJC":1,"UqYyHo-6j72MWA-06u-ITXN":0,"ALehFX-Jdmwn4V-i9x-VnTD":6,"WRr0E1-5qGq3uV-MGG-8ugQ":1,"JKKvYY-N1BCDLk-JBF-K7FN":2}]} +{"id":18,"arr_arr":[["eBNnB0-qIe9HyT-unn-r8Sg"]],"arr_map":[{"RSxqTF-B1xKGK9-9te-Z1xX":0,"BcHNcY-Pwmjwi9-ZcZ-YE0Z":8,"jLDmB5-YGodMV9-bTw-fa2T":9,"u66Ef1-CwR2ipD-PME-fSP8":1,"KDObx4-FHLBa7W-r44-z9tZ":7},{"RSxqTF-B1xKGK9-9te-Z1xX":0,"BcHNcY-Pwmjwi9-ZcZ-YE0Z":8,"jLDmB5-YGodMV9-bTw-fa2T":9,"u66Ef1-CwR2ipD-PME-fSP8":1,"KDObx4-FHLBa7W-r44-z9tZ":7},{"RSxqTF-B1xKGK9-9te-Z1xX":0,"BcHNcY-Pwmjwi9-ZcZ-YE0Z":8,"jLDmB5-YGodMV9-bTw-fa2T":9,"u66Ef1-CwR2ipD-PME-fSP8":1,"KDObx4-FHLBa7W-r44-z9tZ":7}]} +{"id":19,"arr_arr":[["164Ucc-2m33JDj-cRk-xvJr","3PFwGM-MbNcZmG-PsT-yVEO","LoBi6u-BrGnZ72-PC2-Q3Dh","4TJDnj-Fd9d8n6-epp-qcBk"],["7S1jZR-tqeJcs1-IpA-5u7g","Y9Pj9d-i8aWERJ-x04-e82r","6GzJ9a-9vDrDmO-OQ2-Hy6L","CojUHC-RSlsTY2-ldc-M8kY"],null,["z10C0S-v1Ly6UA-5fZ-N2Yz","vUqsQc-OZznYLV-Yfg-xOwn","rtj7wC-1lnLARh-D0t-HIlb","pVxrxW-2gjav4g-Gbq-DYRn"]],"arr_map":[{"hBvx9O-RHmsr9n-bBE-h38U":9,"tV1hKh-q5dhFb1-lfN-Mc3P":0,"39ZYSM-aId3Vm9-Mjw-darG":1,"v0P8jE-aSQ8EqB-Pld-3aSb":7,"Pg5lQ0-cujZNmk-J9F-h9s4":3},{"hBvx9O-RHmsr9n-bBE-h38U":9,"tV1hKh-q5dhFb1-lfN-Mc3P":0,"39ZYSM-aId3Vm9-Mjw-darG":1,"v0P8jE-aSQ8EqB-Pld-3aSb":7,"Pg5lQ0-cujZNmk-J9F-h9s4":3},{"hBvx9O-RHmsr9n-bBE-h38U":9,"tV1hKh-q5dhFb1-lfN-Mc3P":0,"39ZYSM-aId3Vm9-Mjw-darG":1,"v0P8jE-aSQ8EqB-Pld-3aSb":7,"Pg5lQ0-cujZNmk-J9F-h9s4":3},{"hBvx9O-RHmsr9n-bBE-h38U":9,"tV1hKh-q5dhFb1-lfN-Mc3P":0,"39ZYSM-aId3Vm9-Mjw-darG":1,"v0P8jE-aSQ8EqB-Pld-3aSb":7,"Pg5lQ0-cujZNmk-J9F-h9s4":3},{"hBvx9O-RHmsr9n-bBE-h38U":9,"tV1hKh-q5dhFb1-lfN-Mc3P":0,"39ZYSM-aId3Vm9-Mjw-darG":1,"v0P8jE-aSQ8EqB-Pld-3aSb":7,"Pg5lQ0-cujZNmk-J9F-h9s4":3},{"hBvx9O-RHmsr9n-bBE-h38U":9,"tV1hKh-q5dhFb1-lfN-Mc3P":0,"39ZYSM-aId3Vm9-Mjw-darG":1,"v0P8jE-aSQ8EqB-Pld-3aSb":7,"Pg5lQ0-cujZNmk-J9F-h9s4":3},{"hBvx9O-RHmsr9n-bBE-h38U":9,"tV1hKh-q5dhFb1-lfN-Mc3P":0,"39ZYSM-aId3Vm9-Mjw-darG":1,"v0P8jE-aSQ8EqB-Pld-3aSb":7,"Pg5lQ0-cujZNmk-J9F-h9s4":3}]} +{"id":20,"arr_arr":[],"arr_map":[{"HJs9wu-EjXKZA0-AGv-Toi8":4,"T7URbX-187t8gX-tYf-ks7L":3,"f4i0Uf-oBOVCl5-1qx-n9Ss":9,"sqsjvC-u9ge35C-KFD-86lp":3,"n19oZV-8Ov2kKD-KWf-zb2s":5},{"HJs9wu-EjXKZA0-AGv-Toi8":4,"T7URbX-187t8gX-tYf-ks7L":3,"f4i0Uf-oBOVCl5-1qx-n9Ss":9,"sqsjvC-u9ge35C-KFD-86lp":3,"n19oZV-8Ov2kKD-KWf-zb2s":5}]} +{"id":21,"arr_arr":[null,["2ru83o-BLrXeLD-FMI-AHCI","AC5OaC-yOGE24o-n89-qUmQ"]],"arr_map":[{"xwlOrk-MpiP6RS-Vm4-XKef":3,"PTSQoh-9NkO5Sr-Bj9-DaSB":3,"hC1kwZ-RkpFtX4-ALg-4nMp":8,"CJwJkc-Q3Gn7BI-M2Z-jM3Q":5,"MAug88-UBrKX5Q-W3w-ztwL":2},{"xwlOrk-MpiP6RS-Vm4-XKef":3,"PTSQoh-9NkO5Sr-Bj9-DaSB":3,"hC1kwZ-RkpFtX4-ALg-4nMp":8,"CJwJkc-Q3Gn7BI-M2Z-jM3Q":5,"MAug88-UBrKX5Q-W3w-ztwL":2}]} +{"id":22,"arr_arr":[["iM77UH-g1Pul89-YnQ-mOju","JJekpT-wJk9ZZy-aou-6HRB","AWo8ix-RbUagjG-UtD-zW75","Kq1sq2-vPn2q8e-xuN-E4cr","TBnH0L-56BECcO-tG9-Jo1F","zf0TLE-QMDr10f-RTw-WhtA"],["V20VMl-FHaVSEY-tRf-EVLY","TC4bsI-2dN4rbV-Bc4-nqLw","zouZEJ-IcTifEo-dHk-RzX9","o9pub3-4Dkuxvr-ATy-gsVM","7MvH7l-L69XzoQ-11N-uePw","ldifMl-rhQBFYS-WUP-x2sV"],["6kV5kA-oLIN98n-R9x-CbOC","FlQiW3-FMAuhIl-KUI-57m8","mLZ5P5-Lk5B1T2-TcS-Sb3b","D7hYvP-GD3AC4F-0NW-qKlY","4ZRaDG-KDaPsCf-vve-j19b","dWKEqu-eqVLEud-Oek-EXgF"],["5n8x3t-LCKiMt5-NFA-KPcT","Vem133-dr9w7ZR-55U-609k","ruAbsU-gKlQUeb-buH-0TDB","j0f0ze-cVWoRTm-Rta-XvdD","m075Dt-qWdRe3H-jes-VfHK","ePYs8Y-YSwUPE4-QQ5-zNQG"],["0gexiD-Rm0IsA2-y58-HicR","UJuCFr-5djHZIq-kDb-K99q","wlwWKt-SfWP3Zt-0cn-TWsU","RuzR9n-ebOPXLv-W34-cP0s","W9ymXF-BGJIKS0-OyN-HXxv","LvasIG-yAdylNu-YqF-vBLJ"],null],"arr_map":[{"rY0E6a-ZcJhbVo-uWR-GmOY":8,"Am9um1-LmHKPQy-pfo-wJcc":4,"YMpSaO-IPhoMxN-p83-kc9V":2,"rmWbeI-DqKZ4jH-NyE-7dW1":2,"iiPxMz-2yRjDuq-dkc-Axcq":5},{"rY0E6a-ZcJhbVo-uWR-GmOY":8,"Am9um1-LmHKPQy-pfo-wJcc":4,"YMpSaO-IPhoMxN-p83-kc9V":2,"rmWbeI-DqKZ4jH-NyE-7dW1":2,"iiPxMz-2yRjDuq-dkc-Axcq":5},{"rY0E6a-ZcJhbVo-uWR-GmOY":8,"Am9um1-LmHKPQy-pfo-wJcc":4,"YMpSaO-IPhoMxN-p83-kc9V":2,"rmWbeI-DqKZ4jH-NyE-7dW1":2,"iiPxMz-2yRjDuq-dkc-Axcq":5},{"rY0E6a-ZcJhbVo-uWR-GmOY":8,"Am9um1-LmHKPQy-pfo-wJcc":4,"YMpSaO-IPhoMxN-p83-kc9V":2,"rmWbeI-DqKZ4jH-NyE-7dW1":2,"iiPxMz-2yRjDuq-dkc-Axcq":5},{"rY0E6a-ZcJhbVo-uWR-GmOY":8,"Am9um1-LmHKPQy-pfo-wJcc":4,"YMpSaO-IPhoMxN-p83-kc9V":2,"rmWbeI-DqKZ4jH-NyE-7dW1":2,"iiPxMz-2yRjDuq-dkc-Axcq":5},{"rY0E6a-ZcJhbVo-uWR-GmOY":8,"Am9um1-LmHKPQy-pfo-wJcc":4,"YMpSaO-IPhoMxN-p83-kc9V":2,"rmWbeI-DqKZ4jH-NyE-7dW1":2,"iiPxMz-2yRjDuq-dkc-Axcq":5},{"rY0E6a-ZcJhbVo-uWR-GmOY":8,"Am9um1-LmHKPQy-pfo-wJcc":4,"YMpSaO-IPhoMxN-p83-kc9V":2,"rmWbeI-DqKZ4jH-NyE-7dW1":2,"iiPxMz-2yRjDuq-dkc-Axcq":5},{"rY0E6a-ZcJhbVo-uWR-GmOY":8,"Am9um1-LmHKPQy-pfo-wJcc":4,"YMpSaO-IPhoMxN-p83-kc9V":2,"rmWbeI-DqKZ4jH-NyE-7dW1":2,"iiPxMz-2yRjDuq-dkc-Axcq":5}]} +{"id":23,"arr_arr":[["JXHEFv-2kkSptQ-ds9-olbF","0FbJaV-hQsUOY0-cZy-g8yw","DDNYmH-rqqsupW-DSF-ASYa","oo3QSm-nleM2nm-7Mr-NckZ"],null,["yL4Enp-fyRljVc-FrV-jzA7","cAAuwr-Ex4inWX-PGI-O1Pb","P35UXH-Z4XfwS8-8bB-otWi","GCtp6w-WRen8SV-VKh-1Fb0"],["UfsfRH-VVH8UqO-y5A-XYB8","OiuYyd-lkJesJY-QGG-dQOY","TIAbsF-lREUyDE-ZUA-oBBS","LttgCp-vdSwX9h-EsY-Ycpf"]],"arr_map":[{"JGQBg2-5qtPR8C-5Re-twPm":6,"Uj4xhW-KbWI4mC-72t-eepp":3,"27tztk-g7ind7U-jKv-QuhC":3,"iOwSq6-VqdFNrm-2nm-hY1n":0,"bct8XY-dpPRzHE-k2n-kTH2":0},{"JGQBg2-5qtPR8C-5Re-twPm":6,"Uj4xhW-KbWI4mC-72t-eepp":3,"27tztk-g7ind7U-jKv-QuhC":3,"iOwSq6-VqdFNrm-2nm-hY1n":0,"bct8XY-dpPRzHE-k2n-kTH2":0},{"JGQBg2-5qtPR8C-5Re-twPm":6,"Uj4xhW-KbWI4mC-72t-eepp":3,"27tztk-g7ind7U-jKv-QuhC":3,"iOwSq6-VqdFNrm-2nm-hY1n":0,"bct8XY-dpPRzHE-k2n-kTH2":0},{"JGQBg2-5qtPR8C-5Re-twPm":6,"Uj4xhW-KbWI4mC-72t-eepp":3,"27tztk-g7ind7U-jKv-QuhC":3,"iOwSq6-VqdFNrm-2nm-hY1n":0,"bct8XY-dpPRzHE-k2n-kTH2":0},{"JGQBg2-5qtPR8C-5Re-twPm":6,"Uj4xhW-KbWI4mC-72t-eepp":3,"27tztk-g7ind7U-jKv-QuhC":3,"iOwSq6-VqdFNrm-2nm-hY1n":0,"bct8XY-dpPRzHE-k2n-kTH2":0},{"JGQBg2-5qtPR8C-5Re-twPm":6,"Uj4xhW-KbWI4mC-72t-eepp":3,"27tztk-g7ind7U-jKv-QuhC":3,"iOwSq6-VqdFNrm-2nm-hY1n":0,"bct8XY-dpPRzHE-k2n-kTH2":0},{"JGQBg2-5qtPR8C-5Re-twPm":6,"Uj4xhW-KbWI4mC-72t-eepp":3,"27tztk-g7ind7U-jKv-QuhC":3,"iOwSq6-VqdFNrm-2nm-hY1n":0,"bct8XY-dpPRzHE-k2n-kTH2":0}]} +{"id":24,"arr_arr":[["wAacAz-fjaxrWS-Qll-q64P","PFuBtw-gUkeoKF-mFD-ziIt","YpcoBs-FYBjK5r-Z2c-r4Wf"],null,["qbjB3i-xWRlUzF-YKl-Gbyt","5OhXnX-6SEsMaG-ctZ-B7tV","ZnGjr9-fxDj0iV-HtM-QwCc"]],"arr_map":[{"Avnja3-OUO7GZp-Bbb-9WJ6":5,"mLpE5W-beMQh4z-Z6P-jPjO":9,"Rd3wR9-CHe1XzT-jcD-UBF3":9,"gTx716-7et3HUP-jfk-oXWX":1,"nEC8vl-y54mkM0-LKD-s18Q":8},{"Avnja3-OUO7GZp-Bbb-9WJ6":5,"mLpE5W-beMQh4z-Z6P-jPjO":9,"Rd3wR9-CHe1XzT-jcD-UBF3":9,"gTx716-7et3HUP-jfk-oXWX":1,"nEC8vl-y54mkM0-LKD-s18Q":8},{"Avnja3-OUO7GZp-Bbb-9WJ6":5,"mLpE5W-beMQh4z-Z6P-jPjO":9,"Rd3wR9-CHe1XzT-jcD-UBF3":9,"gTx716-7et3HUP-jfk-oXWX":1,"nEC8vl-y54mkM0-LKD-s18Q":8},{"Avnja3-OUO7GZp-Bbb-9WJ6":5,"mLpE5W-beMQh4z-Z6P-jPjO":9,"Rd3wR9-CHe1XzT-jcD-UBF3":9,"gTx716-7et3HUP-jfk-oXWX":1,"nEC8vl-y54mkM0-LKD-s18Q":8},{"Avnja3-OUO7GZp-Bbb-9WJ6":5,"mLpE5W-beMQh4z-Z6P-jPjO":9,"Rd3wR9-CHe1XzT-jcD-UBF3":9,"gTx716-7et3HUP-jfk-oXWX":1,"nEC8vl-y54mkM0-LKD-s18Q":8},{"Avnja3-OUO7GZp-Bbb-9WJ6":5,"mLpE5W-beMQh4z-Z6P-jPjO":9,"Rd3wR9-CHe1XzT-jcD-UBF3":9,"gTx716-7et3HUP-jfk-oXWX":1,"nEC8vl-y54mkM0-LKD-s18Q":8}]} diff --git a/regression-test/data/query_p0/sql_functions/table_function/explode.out b/regression-test/data/query_p0/sql_functions/table_function/explode.out index 2c06dc5d81c99b..415abdf82378e7 100644 --- a/regression-test/data/query_p0/sql_functions/table_function/explode.out +++ b/regression-test/data/query_p0/sql_functions/table_function/explode.out @@ -275,3 +275,192 @@ 15 8 15 9 +-- !sql -- +0 \N +0 ["ADAoxE-ejB7xJn-1zn-Xgpq", "DFEZOm-EC6331F-9pF-4ox2", "GZ7P0Q-WB5YcBq-af3-HWxg", "BPHZHn-faeylOR-PYM-x6gN"] +0 ["SSe3dY-I23acFP-rlr-Xify", "XUEhSe-HOgsNt9-RlT-H8JA", "zCEmCf-gozXHWE-aI4-y9Dq", "qcZATo-a7Czu4Y-KuY-eSX9"] +0 ["feHR6W-ttEtC3v-RvY-RyY3", "omWUhV-SlIJJv4-kEW-VDBJ", "NZh0JU-TJDXU9B-og5-0J61", "V6fdEU-jkJevQT-dWN-sIIY"] +1 \N +1 ["Q3UE7a-yBfPvoZ-q5e-xtc3", "b4Is4f-1Q17TLQ-3Nh-E9XK", "1bkMir-QC3ddA8-Wy6-dcgr"] +1 ["ZGRUhx-eadzlDE-JFF-BH3P", "vrhIVi-tIBN9fc-tBe-4FUC", "v006kw-wIB8x6E-WWt-Li23"] +10 \N +10 ["PWNdZn-256Mzh0-Wxl-i1VN", "oyLKti-7HvbWsn-MQ4-EMuK"] +11 \N +11 ["5fkZOk-BckdrAU-MM3-HwPF", "vicmYn-wD21968-mbI-Wv3c"] +12 \N +12 ["srDnEJ-ySU31lR-ul1-ROCJ", "cMnWa5-bA1OMFg-kiY-AZMI", "8iRcRL-3v72cNM-Ojm-MUpV"] +12 ["ydwzpg-kqjN5zO-Vb5-wo5P", "Rpk4Cc-1MaKNzp-0GF-Hr8K", "G64mgw-wLRtwwY-XFn-BlaK"] +14 \N +14 ["2ISsbI-Tpnywjw-dKc-awNI", "KNq9c2-Yvp3aEc-geM-MQV0", "oDkQ38-Eu9E6bj-Ei6-rwwq", "678xMV-Tn0fVv6-O67-Jg01", "c5Xufb-QGUezAE-8iM-3VBg", "MhjSVE-UKjyuyZ-h71-q8fS"] +14 ["EUKF7p-G5gNyzr-nTn-AlOt", "NHve5R-8hB7e9z-1s4-AmPq", "4rGDYS-KTrWNMD-Iur-dn0b", "9RZDOx-oShNnLu-rQ1-uCmU", "FA2HUr-CoTvZBz-ZlO-cEA3", "XeSGqz-Lrl8p39-Hx6-HZQh"] +14 ["KdXHnA-bNv8rwO-4Ey-nVz6", "a9J9b4-Yd2j674-3eB-Bqjv", "ciwaAk-ktsHqMi-aug-tSgZ", "U2TSkV-PrbEclO-tIt-ilN7", "EDRbVY-6Y8vZgH-QYY-CPur", "nKUwkr-WXn2hj1-xay-JeZA"] +14 ["Zi6B6P-FFGNvXt-IN9-3MQg", "QZ4Blu-xat1Zh7-Id2-emNh", "zBJ4k4-qB9gGgf-uEF-Cuee", "ndFSon-iwKJGIb-gQa-keG0", "ZI14D4-NOhnTPA-DtG-xuhP", "x0UxHB-lEmujK2-snM-q8ze"] +14 ["hBG9CJ-Pkcv5Xv-y5o-SiDS", "VAaAjn-bFJZdHR-NYt-2on0", "T3pZPF-LB6LuqK-Zfg-CwRv", "2yAunr-D0ywKAw-fA7-19qi", "5t1kUf-nkasZdS-9cr-6J2b", "7Yk0yJ-PuETSw0-nGi-yIET"] +15 ["VMXWe8-sJGeEn4-qgR-82VU"] +16 \N +16 ["OIN1Oe-FI3wcrp-66H-fGKM", "ijHMjw-V2O8F9E-OYq-NpkQ", "aChpoK-FAtOsCI-kWI-oYnF", "iE0YBR-zGoZeFx-nLL-CGMW"] +16 ["Q6ZUdh-aHSTJSP-qvd-z7a6", "HsdNd0-VKHJL5f-6MU-yGb6", "FZObTa-iuP04lW-tZw-D68t", "5eaGJJ-uyU5Au4-dBC-5AR1"] +16 ["tgS4vz-pU4PS8m-E5R-DSU8", "wxkjf8-Wwl6cjm-AC9-KcKi", "US36ry-lTrkIhR-MTf-7YyD", "xNmbYx-VDzJq4E-5Q2-DCvO"] +17 \N +17 ["6h3u7b-znuJXzP-Y0y-67k5", "kOW5Bh-iCk28Kl-qnX-YTEw", "52e34r-ja3ecA3-f2X-tW5A", "fbEZmh-SditnSh-1Nf-O939", "ovmskR-d7Cb5GH-uTh-snqu"] +17 ["FACnXp-lxR724G-u2J-veYj", "5iIj0x-NWC3OCy-zlU-8laP", "2xSkQW-CxLzsjs-lPi-KhFu", "GIIWc9-1gOT3Ql-zK9-aBK5", "FFtM4W-pFhEp8x-D4s-AIX9"] +17 ["L1ROPG-CHHugVU-tQw-9PCs", "7bZtwn-HrJkMD9-YFg-Fmcf", "PVV4fi-5rlini0-2AI-97MK", "ouWbwc-B2vQ7tR-DLi-H34g", "R1crmv-lhkakVD-Gg4-LBIv"] +17 ["gaRzuW-8ljPxnB-sG4-EPyR", "I1fCfi-dqF5jIV-xcO-qXxs", "vteojI-nipd8nC-GbM-BuCz", "OJtqcM-M4xJStL-DoO-pAVA", "cHaELA-NYH7FiQ-JRw-Pplg"] +18 ["eBNnB0-qIe9HyT-unn-r8Sg"] +19 \N +19 ["164Ucc-2m33JDj-cRk-xvJr", "3PFwGM-MbNcZmG-PsT-yVEO", "LoBi6u-BrGnZ72-PC2-Q3Dh", "4TJDnj-Fd9d8n6-epp-qcBk"] +19 ["7S1jZR-tqeJcs1-IpA-5u7g", "Y9Pj9d-i8aWERJ-x04-e82r", "6GzJ9a-9vDrDmO-OQ2-Hy6L", "CojUHC-RSlsTY2-ldc-M8kY"] +19 ["z10C0S-v1Ly6UA-5fZ-N2Yz", "vUqsQc-OZznYLV-Yfg-xOwn", "rtj7wC-1lnLARh-D0t-HIlb", "pVxrxW-2gjav4g-Gbq-DYRn"] +2 \N +2 ["lWGk4M-ba0tBFf-JVl-qYen", "2O1nVO-8pdtYoX-jaz-9Lyv"] +21 \N +21 ["2ru83o-BLrXeLD-FMI-AHCI", "AC5OaC-yOGE24o-n89-qUmQ"] +22 \N +22 ["0gexiD-Rm0IsA2-y58-HicR", "UJuCFr-5djHZIq-kDb-K99q", "wlwWKt-SfWP3Zt-0cn-TWsU", "RuzR9n-ebOPXLv-W34-cP0s", "W9ymXF-BGJIKS0-OyN-HXxv", "LvasIG-yAdylNu-YqF-vBLJ"] +22 ["5n8x3t-LCKiMt5-NFA-KPcT", "Vem133-dr9w7ZR-55U-609k", "ruAbsU-gKlQUeb-buH-0TDB", "j0f0ze-cVWoRTm-Rta-XvdD", "m075Dt-qWdRe3H-jes-VfHK", "ePYs8Y-YSwUPE4-QQ5-zNQG"] +22 ["6kV5kA-oLIN98n-R9x-CbOC", "FlQiW3-FMAuhIl-KUI-57m8", "mLZ5P5-Lk5B1T2-TcS-Sb3b", "D7hYvP-GD3AC4F-0NW-qKlY", "4ZRaDG-KDaPsCf-vve-j19b", "dWKEqu-eqVLEud-Oek-EXgF"] +22 ["V20VMl-FHaVSEY-tRf-EVLY", "TC4bsI-2dN4rbV-Bc4-nqLw", "zouZEJ-IcTifEo-dHk-RzX9", "o9pub3-4Dkuxvr-ATy-gsVM", "7MvH7l-L69XzoQ-11N-uePw", "ldifMl-rhQBFYS-WUP-x2sV"] +22 ["iM77UH-g1Pul89-YnQ-mOju", "JJekpT-wJk9ZZy-aou-6HRB", "AWo8ix-RbUagjG-UtD-zW75", "Kq1sq2-vPn2q8e-xuN-E4cr", "TBnH0L-56BECcO-tG9-Jo1F", "zf0TLE-QMDr10f-RTw-WhtA"] +23 \N +23 ["JXHEFv-2kkSptQ-ds9-olbF", "0FbJaV-hQsUOY0-cZy-g8yw", "DDNYmH-rqqsupW-DSF-ASYa", "oo3QSm-nleM2nm-7Mr-NckZ"] +23 ["UfsfRH-VVH8UqO-y5A-XYB8", "OiuYyd-lkJesJY-QGG-dQOY", "TIAbsF-lREUyDE-ZUA-oBBS", "LttgCp-vdSwX9h-EsY-Ycpf"] +23 ["yL4Enp-fyRljVc-FrV-jzA7", "cAAuwr-Ex4inWX-PGI-O1Pb", "P35UXH-Z4XfwS8-8bB-otWi", "GCtp6w-WRen8SV-VKh-1Fb0"] +24 \N +24 ["qbjB3i-xWRlUzF-YKl-Gbyt", "5OhXnX-6SEsMaG-ctZ-B7tV", "ZnGjr9-fxDj0iV-HtM-QwCc"] +24 ["wAacAz-fjaxrWS-Qll-q64P", "PFuBtw-gUkeoKF-mFD-ziIt", "YpcoBs-FYBjK5r-Z2c-r4Wf"] +5 \N +5 ["4Rff3K-TATMSg1-b6R-SmKN", "t1j5tX-dTCrxmn-idk-rk24", "7LW7Lq-cXYuy90-uyH-M6pK"] +5 ["YaKNfI-gV6lMRG-dio-Jg9D", "TC7wdG-1YvyDHG-HVV-rdRR", "8dMmjA-lA9oHx9-dSS-Fl86"] +7 \N +7 ["1KN2Cf-UE1ruFq-sRm-pW7A", "2lwurI-ANjel77-nhS-ahzI", "DR5JH4-WM6QRe7-FgQ-qHnV"] +7 ["UGG3Iy-bMW7c6T-fz9-WYOG", "SWWur3-zdHlN1q-527-UtFT", "XrCVm0-bbwQBay-lcn-vNTJ"] +8 \N +8 ["RjHATY-RbQXO8o-9kC-yMN6", "QIsZVT-SBeloLd-RgF-zws0", "6d2byV-qKIHXRn-mDF-TM1g"] +8 ["ZfdXEt-opP9k3J-l2M-sIdX", "CyLrQr-KniPD7H-VaF-y563", "UtVD3f-NsX6D4j-UQt-Pgpg"] +9 \N +9 ["CSwQYw-bvMIUzj-bqf-AXpo", "tc0Rwz-uQ3Wdbv-z8f-qklW", "X6X0G8-JlYcXKh-pay-D21m", "n7RWXW-ZFh2HqS-6XE-v9TY", "zAXTa3-zz5OkX6-jAM-ZuyD", "UXdt9B-Mml6eBa-hN7-vQOp"] +9 ["FxiSSg-4B2Ep4r-AoC-nwwf", "QIafe6-b9ETKKv-WTq-wXnZ", "vP0uXm-5lRFe5D-R6o-58Jt", "UnzUJl-DXv294E-ozf-KgZF", "0ikQZL-ORqwsFr-fGs-flzQ", "Ltap6E-siSL3Hm-h0i-sKp2"] +9 ["Hi9Osx-9BogT7m-XBD-rtEV", "xoF9Rq-MshvMLG-jEJ-8965", "k5LrCO-6jQD17H-6jT-zYVY", "UYAu4H-KgjxcmP-XyO-ZFQJ", "eQVglX-V5YAuIJ-xC7-RYgL", "QydLV7-7XqcCuO-rlG-w7ba"] +9 ["n14avO-lueGLXs-zxM-tOEZ", "yj4iMW-h1Z5isV-jM4-0BRC", "YXg40k-p00nu3H-DLQ-ASu2", "KOg34p-E4TtdAs-xcl-ajzL", "d0LfzZ-2huMiVN-4Fc-aOuz", "2cJHzT-KjvF9cE-W5u-nOrI"] +9 ["xI0TNh-wAH12NI-6Px-MLib", "Jmesk7-hJzDXz6-lFC-Egmp", "YFtlSh-8pZVVxG-mNa-RYES", "q5sDuB-WfmqXDr-WXJ-UPq6", "3Toro7-4wzAPD9-G6e-ajl1", "bE4dED-fNBqL16-JXt-0lZY"] + +-- !sql -- +0 {"swpp0q-vgNTgMG-U1h-GrtK":"5", "e0bNvm-ZuCo9v7-Gu1-9qsJ":"9", "CW2OBW-ET71l4k-wke-B8pK":"3", "ydO2p2-64OmnPD-JT1-Xe9q":"6", "b9P9Pj-8o3jgFn-E5G-M6Wx":"3"} +0 {"swpp0q-vgNTgMG-U1h-GrtK":"5", "e0bNvm-ZuCo9v7-Gu1-9qsJ":"9", "CW2OBW-ET71l4k-wke-B8pK":"3", "ydO2p2-64OmnPD-JT1-Xe9q":"6", "b9P9Pj-8o3jgFn-E5G-M6Wx":"3"} +0 {"swpp0q-vgNTgMG-U1h-GrtK":"5", "e0bNvm-ZuCo9v7-Gu1-9qsJ":"9", "CW2OBW-ET71l4k-wke-B8pK":"3", "ydO2p2-64OmnPD-JT1-Xe9q":"6", "b9P9Pj-8o3jgFn-E5G-M6Wx":"3"} +0 {"swpp0q-vgNTgMG-U1h-GrtK":"5", "e0bNvm-ZuCo9v7-Gu1-9qsJ":"9", "CW2OBW-ET71l4k-wke-B8pK":"3", "ydO2p2-64OmnPD-JT1-Xe9q":"6", "b9P9Pj-8o3jgFn-E5G-M6Wx":"3"} +0 {"swpp0q-vgNTgMG-U1h-GrtK":"5", "e0bNvm-ZuCo9v7-Gu1-9qsJ":"9", "CW2OBW-ET71l4k-wke-B8pK":"3", "ydO2p2-64OmnPD-JT1-Xe9q":"6", "b9P9Pj-8o3jgFn-E5G-M6Wx":"3"} +0 {"swpp0q-vgNTgMG-U1h-GrtK":"5", "e0bNvm-ZuCo9v7-Gu1-9qsJ":"9", "CW2OBW-ET71l4k-wke-B8pK":"3", "ydO2p2-64OmnPD-JT1-Xe9q":"6", "b9P9Pj-8o3jgFn-E5G-M6Wx":"3"} +0 {"swpp0q-vgNTgMG-U1h-GrtK":"5", "e0bNvm-ZuCo9v7-Gu1-9qsJ":"9", "CW2OBW-ET71l4k-wke-B8pK":"3", "ydO2p2-64OmnPD-JT1-Xe9q":"6", "b9P9Pj-8o3jgFn-E5G-M6Wx":"3"} +1 {"nKWQnv-HolfdQc-tqs-Trnb":"0", "GEAmQi-bMHzvyG-MGd-GBjA":"5", "KtMT9e-jKcWd9t-9GG-8mG6":"3", "znKFYo-4abMgt6-k4A-L4Ki":"4", "IpXgBa-FGiyskO-R5R-T3CY":"6"} +1 {"nKWQnv-HolfdQc-tqs-Trnb":"0", "GEAmQi-bMHzvyG-MGd-GBjA":"5", "KtMT9e-jKcWd9t-9GG-8mG6":"3", "znKFYo-4abMgt6-k4A-L4Ki":"4", "IpXgBa-FGiyskO-R5R-T3CY":"6"} +1 {"nKWQnv-HolfdQc-tqs-Trnb":"0", "GEAmQi-bMHzvyG-MGd-GBjA":"5", "KtMT9e-jKcWd9t-9GG-8mG6":"3", "znKFYo-4abMgt6-k4A-L4Ki":"4", "IpXgBa-FGiyskO-R5R-T3CY":"6"} +1 {"nKWQnv-HolfdQc-tqs-Trnb":"0", "GEAmQi-bMHzvyG-MGd-GBjA":"5", "KtMT9e-jKcWd9t-9GG-8mG6":"3", "znKFYo-4abMgt6-k4A-L4Ki":"4", "IpXgBa-FGiyskO-R5R-T3CY":"6"} +1 {"nKWQnv-HolfdQc-tqs-Trnb":"0", "GEAmQi-bMHzvyG-MGd-GBjA":"5", "KtMT9e-jKcWd9t-9GG-8mG6":"3", "znKFYo-4abMgt6-k4A-L4Ki":"4", "IpXgBa-FGiyskO-R5R-T3CY":"6"} +1 {"nKWQnv-HolfdQc-tqs-Trnb":"0", "GEAmQi-bMHzvyG-MGd-GBjA":"5", "KtMT9e-jKcWd9t-9GG-8mG6":"3", "znKFYo-4abMgt6-k4A-L4Ki":"4", "IpXgBa-FGiyskO-R5R-T3CY":"6"} +10 {"kpjXle-CayOe2S-2W4-vqvG":"4", "VIWvQo-q2swzEQ-loT-PpQA":"9", "PnLRPK-AW1LRJj-QDh-WLfE":"5", "vGVTU8-SbgpatL-PXh-6DPp":"0", "YPH9gI-Uv5Ec0u-QHJ-5QKd":"3"} +10 {"kpjXle-CayOe2S-2W4-vqvG":"4", "VIWvQo-q2swzEQ-loT-PpQA":"9", "PnLRPK-AW1LRJj-QDh-WLfE":"5", "vGVTU8-SbgpatL-PXh-6DPp":"0", "YPH9gI-Uv5Ec0u-QHJ-5QKd":"3"} +10 {"kpjXle-CayOe2S-2W4-vqvG":"4", "VIWvQo-q2swzEQ-loT-PpQA":"9", "PnLRPK-AW1LRJj-QDh-WLfE":"5", "vGVTU8-SbgpatL-PXh-6DPp":"0", "YPH9gI-Uv5Ec0u-QHJ-5QKd":"3"} +10 {"kpjXle-CayOe2S-2W4-vqvG":"4", "VIWvQo-q2swzEQ-loT-PpQA":"9", "PnLRPK-AW1LRJj-QDh-WLfE":"5", "vGVTU8-SbgpatL-PXh-6DPp":"0", "YPH9gI-Uv5Ec0u-QHJ-5QKd":"3"} +10 {"kpjXle-CayOe2S-2W4-vqvG":"4", "VIWvQo-q2swzEQ-loT-PpQA":"9", "PnLRPK-AW1LRJj-QDh-WLfE":"5", "vGVTU8-SbgpatL-PXh-6DPp":"0", "YPH9gI-Uv5Ec0u-QHJ-5QKd":"3"} +11 {"Mqzc3X-DzSmJor-o9C-g73I":"4", "Xgdg1m-DZzCOqO-7pA-PKeS":"6", "JzTEl1-CwOaAmt-yV3-n2k0":"9", "pm0RBB-gdAl37a-8Pq-QLJV":"4", "oN1HDX-xCm8dRv-0wk-MJeu":"8"} +11 {"Mqzc3X-DzSmJor-o9C-g73I":"4", "Xgdg1m-DZzCOqO-7pA-PKeS":"6", "JzTEl1-CwOaAmt-yV3-n2k0":"9", "pm0RBB-gdAl37a-8Pq-QLJV":"4", "oN1HDX-xCm8dRv-0wk-MJeu":"8"} +11 {"Mqzc3X-DzSmJor-o9C-g73I":"4", "Xgdg1m-DZzCOqO-7pA-PKeS":"6", "JzTEl1-CwOaAmt-yV3-n2k0":"9", "pm0RBB-gdAl37a-8Pq-QLJV":"4", "oN1HDX-xCm8dRv-0wk-MJeu":"8"} +12 {"GFBqsx-TvhMATX-6ZB-0Zva":"2", "D6QtvK-8eTn7dE-CZn-h18O":"2", "tdKmN3-ealR73D-PO4-6nh8":"4", "veE9Ee-GBc3xZa-O1T-n8u3":"5", "0P9fws-Pb03WDb-lOu-l4Ww":"7"} +12 {"GFBqsx-TvhMATX-6ZB-0Zva":"2", "D6QtvK-8eTn7dE-CZn-h18O":"2", "tdKmN3-ealR73D-PO4-6nh8":"4", "veE9Ee-GBc3xZa-O1T-n8u3":"5", "0P9fws-Pb03WDb-lOu-l4Ww":"7"} +12 {"GFBqsx-TvhMATX-6ZB-0Zva":"2", "D6QtvK-8eTn7dE-CZn-h18O":"2", "tdKmN3-ealR73D-PO4-6nh8":"4", "veE9Ee-GBc3xZa-O1T-n8u3":"5", "0P9fws-Pb03WDb-lOu-l4Ww":"7"} +13 {"kIpyuK-FUQet1S-6JW-PJjH":"0", "KosGlG-X9weJpR-Olb-qNNY":"4", "x8HUd5-IzLbruK-ruY-HPIv":"8", "UaBaT4-ZKtEsWB-yJ7-ikwZ":"2", "tyiuhU-08ulAju-XWn-fsID":"6"} +13 {"kIpyuK-FUQet1S-6JW-PJjH":"0", "KosGlG-X9weJpR-Olb-qNNY":"4", "x8HUd5-IzLbruK-ruY-HPIv":"8", "UaBaT4-ZKtEsWB-yJ7-ikwZ":"2", "tyiuhU-08ulAju-XWn-fsID":"6"} +13 {"kIpyuK-FUQet1S-6JW-PJjH":"0", "KosGlG-X9weJpR-Olb-qNNY":"4", "x8HUd5-IzLbruK-ruY-HPIv":"8", "UaBaT4-ZKtEsWB-yJ7-ikwZ":"2", "tyiuhU-08ulAju-XWn-fsID":"6"} +14 {"Jfv2A1-Nh98daR-5eV-63xx":"9", "iqv9Dc-AovBuni-nRH-XnEZ":"1", "OQkgBo-duRmhW2-pTI-dSTS":"0", "pfWZRg-8lXC9OV-NPT-nHjl":"4", "Wso7dF-rmVTlDS-s9Q-MGtf":"0"} +14 {"Jfv2A1-Nh98daR-5eV-63xx":"9", "iqv9Dc-AovBuni-nRH-XnEZ":"1", "OQkgBo-duRmhW2-pTI-dSTS":"0", "pfWZRg-8lXC9OV-NPT-nHjl":"4", "Wso7dF-rmVTlDS-s9Q-MGtf":"0"} +14 {"Jfv2A1-Nh98daR-5eV-63xx":"9", "iqv9Dc-AovBuni-nRH-XnEZ":"1", "OQkgBo-duRmhW2-pTI-dSTS":"0", "pfWZRg-8lXC9OV-NPT-nHjl":"4", "Wso7dF-rmVTlDS-s9Q-MGtf":"0"} +14 {"Jfv2A1-Nh98daR-5eV-63xx":"9", "iqv9Dc-AovBuni-nRH-XnEZ":"1", "OQkgBo-duRmhW2-pTI-dSTS":"0", "pfWZRg-8lXC9OV-NPT-nHjl":"4", "Wso7dF-rmVTlDS-s9Q-MGtf":"0"} +14 {"Jfv2A1-Nh98daR-5eV-63xx":"9", "iqv9Dc-AovBuni-nRH-XnEZ":"1", "OQkgBo-duRmhW2-pTI-dSTS":"0", "pfWZRg-8lXC9OV-NPT-nHjl":"4", "Wso7dF-rmVTlDS-s9Q-MGtf":"0"} +14 {"Jfv2A1-Nh98daR-5eV-63xx":"9", "iqv9Dc-AovBuni-nRH-XnEZ":"1", "OQkgBo-duRmhW2-pTI-dSTS":"0", "pfWZRg-8lXC9OV-NPT-nHjl":"4", "Wso7dF-rmVTlDS-s9Q-MGtf":"0"} +14 {"Jfv2A1-Nh98daR-5eV-63xx":"9", "iqv9Dc-AovBuni-nRH-XnEZ":"1", "OQkgBo-duRmhW2-pTI-dSTS":"0", "pfWZRg-8lXC9OV-NPT-nHjl":"4", "Wso7dF-rmVTlDS-s9Q-MGtf":"0"} +14 {"Jfv2A1-Nh98daR-5eV-63xx":"9", "iqv9Dc-AovBuni-nRH-XnEZ":"1", "OQkgBo-duRmhW2-pTI-dSTS":"0", "pfWZRg-8lXC9OV-NPT-nHjl":"4", "Wso7dF-rmVTlDS-s9Q-MGtf":"0"} +14 {"Jfv2A1-Nh98daR-5eV-63xx":"9", "iqv9Dc-AovBuni-nRH-XnEZ":"1", "OQkgBo-duRmhW2-pTI-dSTS":"0", "pfWZRg-8lXC9OV-NPT-nHjl":"4", "Wso7dF-rmVTlDS-s9Q-MGtf":"0"} +15 {"ZZoIk6-FCvllrX-ynY-WH9b":"8", "Lo6mI2-FUuG0Kp-ALg-N9CN":"2", "lFhxJ0-iaZhTSc-XKg-Ats9":"9", "hiaVfD-qDapBgY-DBu-uCif":"1", "i5zkzH-UFvf6zO-ubG-bDEb":"9"} +15 {"ZZoIk6-FCvllrX-ynY-WH9b":"8", "Lo6mI2-FUuG0Kp-ALg-N9CN":"2", "lFhxJ0-iaZhTSc-XKg-Ats9":"9", "hiaVfD-qDapBgY-DBu-uCif":"1", "i5zkzH-UFvf6zO-ubG-bDEb":"9"} +15 {"ZZoIk6-FCvllrX-ynY-WH9b":"8", "Lo6mI2-FUuG0Kp-ALg-N9CN":"2", "lFhxJ0-iaZhTSc-XKg-Ats9":"9", "hiaVfD-qDapBgY-DBu-uCif":"1", "i5zkzH-UFvf6zO-ubG-bDEb":"9"} +15 {"ZZoIk6-FCvllrX-ynY-WH9b":"8", "Lo6mI2-FUuG0Kp-ALg-N9CN":"2", "lFhxJ0-iaZhTSc-XKg-Ats9":"9", "hiaVfD-qDapBgY-DBu-uCif":"1", "i5zkzH-UFvf6zO-ubG-bDEb":"9"} +16 {"6HGUM0-YSULhQm-tNd-WrUB":"6", "2iQyQQ-0Q79hSa-xas-mA1b":"2", "kHyrvC-FG8yKzY-bJi-FTGR":"5", "kCKYlJ-LlfcyEz-Zzc-pcuw":"2", "D97U6o-rNR0M8j-7FD-yB2i":"2"} +16 {"6HGUM0-YSULhQm-tNd-WrUB":"6", "2iQyQQ-0Q79hSa-xas-mA1b":"2", "kHyrvC-FG8yKzY-bJi-FTGR":"5", "kCKYlJ-LlfcyEz-Zzc-pcuw":"2", "D97U6o-rNR0M8j-7FD-yB2i":"2"} +16 {"6HGUM0-YSULhQm-tNd-WrUB":"6", "2iQyQQ-0Q79hSa-xas-mA1b":"2", "kHyrvC-FG8yKzY-bJi-FTGR":"5", "kCKYlJ-LlfcyEz-Zzc-pcuw":"2", "D97U6o-rNR0M8j-7FD-yB2i":"2"} +16 {"6HGUM0-YSULhQm-tNd-WrUB":"6", "2iQyQQ-0Q79hSa-xas-mA1b":"2", "kHyrvC-FG8yKzY-bJi-FTGR":"5", "kCKYlJ-LlfcyEz-Zzc-pcuw":"2", "D97U6o-rNR0M8j-7FD-yB2i":"2"} +16 {"6HGUM0-YSULhQm-tNd-WrUB":"6", "2iQyQQ-0Q79hSa-xas-mA1b":"2", "kHyrvC-FG8yKzY-bJi-FTGR":"5", "kCKYlJ-LlfcyEz-Zzc-pcuw":"2", "D97U6o-rNR0M8j-7FD-yB2i":"2"} +16 {"6HGUM0-YSULhQm-tNd-WrUB":"6", "2iQyQQ-0Q79hSa-xas-mA1b":"2", "kHyrvC-FG8yKzY-bJi-FTGR":"5", "kCKYlJ-LlfcyEz-Zzc-pcuw":"2", "D97U6o-rNR0M8j-7FD-yB2i":"2"} +16 {"6HGUM0-YSULhQm-tNd-WrUB":"6", "2iQyQQ-0Q79hSa-xas-mA1b":"2", "kHyrvC-FG8yKzY-bJi-FTGR":"5", "kCKYlJ-LlfcyEz-Zzc-pcuw":"2", "D97U6o-rNR0M8j-7FD-yB2i":"2"} +17 {"pvJ17T-e8sUqJE-UAM-3HJC":"1", "UqYyHo-6j72MWA-06u-ITXN":"0", "ALehFX-Jdmwn4V-i9x-VnTD":"6", "WRr0E1-5qGq3uV-MGG-8ugQ":"1", "JKKvYY-N1BCDLk-JBF-K7FN":"2"} +17 {"pvJ17T-e8sUqJE-UAM-3HJC":"1", "UqYyHo-6j72MWA-06u-ITXN":"0", "ALehFX-Jdmwn4V-i9x-VnTD":"6", "WRr0E1-5qGq3uV-MGG-8ugQ":"1", "JKKvYY-N1BCDLk-JBF-K7FN":"2"} +17 {"pvJ17T-e8sUqJE-UAM-3HJC":"1", "UqYyHo-6j72MWA-06u-ITXN":"0", "ALehFX-Jdmwn4V-i9x-VnTD":"6", "WRr0E1-5qGq3uV-MGG-8ugQ":"1", "JKKvYY-N1BCDLk-JBF-K7FN":"2"} +17 {"pvJ17T-e8sUqJE-UAM-3HJC":"1", "UqYyHo-6j72MWA-06u-ITXN":"0", "ALehFX-Jdmwn4V-i9x-VnTD":"6", "WRr0E1-5qGq3uV-MGG-8ugQ":"1", "JKKvYY-N1BCDLk-JBF-K7FN":"2"} +17 {"pvJ17T-e8sUqJE-UAM-3HJC":"1", "UqYyHo-6j72MWA-06u-ITXN":"0", "ALehFX-Jdmwn4V-i9x-VnTD":"6", "WRr0E1-5qGq3uV-MGG-8ugQ":"1", "JKKvYY-N1BCDLk-JBF-K7FN":"2"} +17 {"pvJ17T-e8sUqJE-UAM-3HJC":"1", "UqYyHo-6j72MWA-06u-ITXN":"0", "ALehFX-Jdmwn4V-i9x-VnTD":"6", "WRr0E1-5qGq3uV-MGG-8ugQ":"1", "JKKvYY-N1BCDLk-JBF-K7FN":"2"} +18 {"RSxqTF-B1xKGK9-9te-Z1xX":"0", "BcHNcY-Pwmjwi9-ZcZ-YE0Z":"8", "jLDmB5-YGodMV9-bTw-fa2T":"9", "u66Ef1-CwR2ipD-PME-fSP8":"1", "KDObx4-FHLBa7W-r44-z9tZ":"7"} +18 {"RSxqTF-B1xKGK9-9te-Z1xX":"0", "BcHNcY-Pwmjwi9-ZcZ-YE0Z":"8", "jLDmB5-YGodMV9-bTw-fa2T":"9", "u66Ef1-CwR2ipD-PME-fSP8":"1", "KDObx4-FHLBa7W-r44-z9tZ":"7"} +18 {"RSxqTF-B1xKGK9-9te-Z1xX":"0", "BcHNcY-Pwmjwi9-ZcZ-YE0Z":"8", "jLDmB5-YGodMV9-bTw-fa2T":"9", "u66Ef1-CwR2ipD-PME-fSP8":"1", "KDObx4-FHLBa7W-r44-z9tZ":"7"} +19 {"hBvx9O-RHmsr9n-bBE-h38U":"9", "tV1hKh-q5dhFb1-lfN-Mc3P":"0", "39ZYSM-aId3Vm9-Mjw-darG":"1", "v0P8jE-aSQ8EqB-Pld-3aSb":"7", "Pg5lQ0-cujZNmk-J9F-h9s4":"3"} +19 {"hBvx9O-RHmsr9n-bBE-h38U":"9", "tV1hKh-q5dhFb1-lfN-Mc3P":"0", "39ZYSM-aId3Vm9-Mjw-darG":"1", "v0P8jE-aSQ8EqB-Pld-3aSb":"7", "Pg5lQ0-cujZNmk-J9F-h9s4":"3"} +19 {"hBvx9O-RHmsr9n-bBE-h38U":"9", "tV1hKh-q5dhFb1-lfN-Mc3P":"0", "39ZYSM-aId3Vm9-Mjw-darG":"1", "v0P8jE-aSQ8EqB-Pld-3aSb":"7", "Pg5lQ0-cujZNmk-J9F-h9s4":"3"} +19 {"hBvx9O-RHmsr9n-bBE-h38U":"9", "tV1hKh-q5dhFb1-lfN-Mc3P":"0", "39ZYSM-aId3Vm9-Mjw-darG":"1", "v0P8jE-aSQ8EqB-Pld-3aSb":"7", "Pg5lQ0-cujZNmk-J9F-h9s4":"3"} +19 {"hBvx9O-RHmsr9n-bBE-h38U":"9", "tV1hKh-q5dhFb1-lfN-Mc3P":"0", "39ZYSM-aId3Vm9-Mjw-darG":"1", "v0P8jE-aSQ8EqB-Pld-3aSb":"7", "Pg5lQ0-cujZNmk-J9F-h9s4":"3"} +19 {"hBvx9O-RHmsr9n-bBE-h38U":"9", "tV1hKh-q5dhFb1-lfN-Mc3P":"0", "39ZYSM-aId3Vm9-Mjw-darG":"1", "v0P8jE-aSQ8EqB-Pld-3aSb":"7", "Pg5lQ0-cujZNmk-J9F-h9s4":"3"} +19 {"hBvx9O-RHmsr9n-bBE-h38U":"9", "tV1hKh-q5dhFb1-lfN-Mc3P":"0", "39ZYSM-aId3Vm9-Mjw-darG":"1", "v0P8jE-aSQ8EqB-Pld-3aSb":"7", "Pg5lQ0-cujZNmk-J9F-h9s4":"3"} +2 {"N1isOk-HqyCfZ3-Iqu-sHC4":"8", "zoEAUW-yYiiBxG-xER-4AVu":"0", "y6OK1G-wIHaG2A-oWB-WQel":"3", "oWys2y-FLQKf53-5mm-3BWr":"9", "Kkjpbh-vOvz3Hu-WdS-UVI6":"4"} +2 {"N1isOk-HqyCfZ3-Iqu-sHC4":"8", "zoEAUW-yYiiBxG-xER-4AVu":"0", "y6OK1G-wIHaG2A-oWB-WQel":"3", "oWys2y-FLQKf53-5mm-3BWr":"9", "Kkjpbh-vOvz3Hu-WdS-UVI6":"4"} +20 {"HJs9wu-EjXKZA0-AGv-Toi8":"4", "T7URbX-187t8gX-tYf-ks7L":"3", "f4i0Uf-oBOVCl5-1qx-n9Ss":"9", "sqsjvC-u9ge35C-KFD-86lp":"3", "n19oZV-8Ov2kKD-KWf-zb2s":"5"} +20 {"HJs9wu-EjXKZA0-AGv-Toi8":"4", "T7URbX-187t8gX-tYf-ks7L":"3", "f4i0Uf-oBOVCl5-1qx-n9Ss":"9", "sqsjvC-u9ge35C-KFD-86lp":"3", "n19oZV-8Ov2kKD-KWf-zb2s":"5"} +21 {"xwlOrk-MpiP6RS-Vm4-XKef":"3", "PTSQoh-9NkO5Sr-Bj9-DaSB":"3", "hC1kwZ-RkpFtX4-ALg-4nMp":"8", "CJwJkc-Q3Gn7BI-M2Z-jM3Q":"5", "MAug88-UBrKX5Q-W3w-ztwL":"2"} +21 {"xwlOrk-MpiP6RS-Vm4-XKef":"3", "PTSQoh-9NkO5Sr-Bj9-DaSB":"3", "hC1kwZ-RkpFtX4-ALg-4nMp":"8", "CJwJkc-Q3Gn7BI-M2Z-jM3Q":"5", "MAug88-UBrKX5Q-W3w-ztwL":"2"} +22 {"rY0E6a-ZcJhbVo-uWR-GmOY":"8", "Am9um1-LmHKPQy-pfo-wJcc":"4", "YMpSaO-IPhoMxN-p83-kc9V":"2", "rmWbeI-DqKZ4jH-NyE-7dW1":"2", "iiPxMz-2yRjDuq-dkc-Axcq":"5"} +22 {"rY0E6a-ZcJhbVo-uWR-GmOY":"8", "Am9um1-LmHKPQy-pfo-wJcc":"4", "YMpSaO-IPhoMxN-p83-kc9V":"2", "rmWbeI-DqKZ4jH-NyE-7dW1":"2", "iiPxMz-2yRjDuq-dkc-Axcq":"5"} +22 {"rY0E6a-ZcJhbVo-uWR-GmOY":"8", "Am9um1-LmHKPQy-pfo-wJcc":"4", "YMpSaO-IPhoMxN-p83-kc9V":"2", "rmWbeI-DqKZ4jH-NyE-7dW1":"2", "iiPxMz-2yRjDuq-dkc-Axcq":"5"} +22 {"rY0E6a-ZcJhbVo-uWR-GmOY":"8", "Am9um1-LmHKPQy-pfo-wJcc":"4", "YMpSaO-IPhoMxN-p83-kc9V":"2", "rmWbeI-DqKZ4jH-NyE-7dW1":"2", "iiPxMz-2yRjDuq-dkc-Axcq":"5"} +22 {"rY0E6a-ZcJhbVo-uWR-GmOY":"8", "Am9um1-LmHKPQy-pfo-wJcc":"4", "YMpSaO-IPhoMxN-p83-kc9V":"2", "rmWbeI-DqKZ4jH-NyE-7dW1":"2", "iiPxMz-2yRjDuq-dkc-Axcq":"5"} +22 {"rY0E6a-ZcJhbVo-uWR-GmOY":"8", "Am9um1-LmHKPQy-pfo-wJcc":"4", "YMpSaO-IPhoMxN-p83-kc9V":"2", "rmWbeI-DqKZ4jH-NyE-7dW1":"2", "iiPxMz-2yRjDuq-dkc-Axcq":"5"} +22 {"rY0E6a-ZcJhbVo-uWR-GmOY":"8", "Am9um1-LmHKPQy-pfo-wJcc":"4", "YMpSaO-IPhoMxN-p83-kc9V":"2", "rmWbeI-DqKZ4jH-NyE-7dW1":"2", "iiPxMz-2yRjDuq-dkc-Axcq":"5"} +22 {"rY0E6a-ZcJhbVo-uWR-GmOY":"8", "Am9um1-LmHKPQy-pfo-wJcc":"4", "YMpSaO-IPhoMxN-p83-kc9V":"2", "rmWbeI-DqKZ4jH-NyE-7dW1":"2", "iiPxMz-2yRjDuq-dkc-Axcq":"5"} +23 {"JGQBg2-5qtPR8C-5Re-twPm":"6", "Uj4xhW-KbWI4mC-72t-eepp":"3", "27tztk-g7ind7U-jKv-QuhC":"3", "iOwSq6-VqdFNrm-2nm-hY1n":"0", "bct8XY-dpPRzHE-k2n-kTH2":"0"} +23 {"JGQBg2-5qtPR8C-5Re-twPm":"6", "Uj4xhW-KbWI4mC-72t-eepp":"3", "27tztk-g7ind7U-jKv-QuhC":"3", "iOwSq6-VqdFNrm-2nm-hY1n":"0", "bct8XY-dpPRzHE-k2n-kTH2":"0"} +23 {"JGQBg2-5qtPR8C-5Re-twPm":"6", "Uj4xhW-KbWI4mC-72t-eepp":"3", "27tztk-g7ind7U-jKv-QuhC":"3", "iOwSq6-VqdFNrm-2nm-hY1n":"0", "bct8XY-dpPRzHE-k2n-kTH2":"0"} +23 {"JGQBg2-5qtPR8C-5Re-twPm":"6", "Uj4xhW-KbWI4mC-72t-eepp":"3", "27tztk-g7ind7U-jKv-QuhC":"3", "iOwSq6-VqdFNrm-2nm-hY1n":"0", "bct8XY-dpPRzHE-k2n-kTH2":"0"} +23 {"JGQBg2-5qtPR8C-5Re-twPm":"6", "Uj4xhW-KbWI4mC-72t-eepp":"3", "27tztk-g7ind7U-jKv-QuhC":"3", "iOwSq6-VqdFNrm-2nm-hY1n":"0", "bct8XY-dpPRzHE-k2n-kTH2":"0"} +23 {"JGQBg2-5qtPR8C-5Re-twPm":"6", "Uj4xhW-KbWI4mC-72t-eepp":"3", "27tztk-g7ind7U-jKv-QuhC":"3", "iOwSq6-VqdFNrm-2nm-hY1n":"0", "bct8XY-dpPRzHE-k2n-kTH2":"0"} +23 {"JGQBg2-5qtPR8C-5Re-twPm":"6", "Uj4xhW-KbWI4mC-72t-eepp":"3", "27tztk-g7ind7U-jKv-QuhC":"3", "iOwSq6-VqdFNrm-2nm-hY1n":"0", "bct8XY-dpPRzHE-k2n-kTH2":"0"} +24 {"Avnja3-OUO7GZp-Bbb-9WJ6":"5", "mLpE5W-beMQh4z-Z6P-jPjO":"9", "Rd3wR9-CHe1XzT-jcD-UBF3":"9", "gTx716-7et3HUP-jfk-oXWX":"1", "nEC8vl-y54mkM0-LKD-s18Q":"8"} +24 {"Avnja3-OUO7GZp-Bbb-9WJ6":"5", "mLpE5W-beMQh4z-Z6P-jPjO":"9", "Rd3wR9-CHe1XzT-jcD-UBF3":"9", "gTx716-7et3HUP-jfk-oXWX":"1", "nEC8vl-y54mkM0-LKD-s18Q":"8"} +24 {"Avnja3-OUO7GZp-Bbb-9WJ6":"5", "mLpE5W-beMQh4z-Z6P-jPjO":"9", "Rd3wR9-CHe1XzT-jcD-UBF3":"9", "gTx716-7et3HUP-jfk-oXWX":"1", "nEC8vl-y54mkM0-LKD-s18Q":"8"} +24 {"Avnja3-OUO7GZp-Bbb-9WJ6":"5", "mLpE5W-beMQh4z-Z6P-jPjO":"9", "Rd3wR9-CHe1XzT-jcD-UBF3":"9", "gTx716-7et3HUP-jfk-oXWX":"1", "nEC8vl-y54mkM0-LKD-s18Q":"8"} +24 {"Avnja3-OUO7GZp-Bbb-9WJ6":"5", "mLpE5W-beMQh4z-Z6P-jPjO":"9", "Rd3wR9-CHe1XzT-jcD-UBF3":"9", "gTx716-7et3HUP-jfk-oXWX":"1", "nEC8vl-y54mkM0-LKD-s18Q":"8"} +24 {"Avnja3-OUO7GZp-Bbb-9WJ6":"5", "mLpE5W-beMQh4z-Z6P-jPjO":"9", "Rd3wR9-CHe1XzT-jcD-UBF3":"9", "gTx716-7et3HUP-jfk-oXWX":"1", "nEC8vl-y54mkM0-LKD-s18Q":"8"} +3 {"sAAkg4-TsxcHvG-PU5-reih":"0", "gLZA8c-BegqvvI-c8X-k2sG":"0", "XsjVET-cVWzizh-IQL-oD9Z":"3", "mn5po7-sdVBNfT-D1h-J9cr":"1", "VhWPpg-MjnLQxg-fQW-HSEW":"5"} +3 {"sAAkg4-TsxcHvG-PU5-reih":"0", "gLZA8c-BegqvvI-c8X-k2sG":"0", "XsjVET-cVWzizh-IQL-oD9Z":"3", "mn5po7-sdVBNfT-D1h-J9cr":"1", "VhWPpg-MjnLQxg-fQW-HSEW":"5"} +3 {"sAAkg4-TsxcHvG-PU5-reih":"0", "gLZA8c-BegqvvI-c8X-k2sG":"0", "XsjVET-cVWzizh-IQL-oD9Z":"3", "mn5po7-sdVBNfT-D1h-J9cr":"1", "VhWPpg-MjnLQxg-fQW-HSEW":"5"} +4 {"GW98C8-Xv6AWLP-ivi-FLtW":"0", "Q635Ar-OMGGYWX-Dhg-7HgP":"8", "TbeKcq-4AUGF5H-DzO-KwWg":"6", "fKDLAp-tPmFPle-lF6-FGNZ":"1", "W5qTCQ-Lfqxf29-Z3J-9Yrz":"9"} +4 {"GW98C8-Xv6AWLP-ivi-FLtW":"0", "Q635Ar-OMGGYWX-Dhg-7HgP":"8", "TbeKcq-4AUGF5H-DzO-KwWg":"6", "fKDLAp-tPmFPle-lF6-FGNZ":"1", "W5qTCQ-Lfqxf29-Z3J-9Yrz":"9"} +5 {"iLOhUA-n05O2OY-LHB-qS3H":"8", "ATC5e8-MywwrP3-Vhc-QdxZ":"1", "fST2pb-uAaS3Mb-Pl6-9i1J":"0", "fppq1A-Q1WOvMj-Eod-xbTz":"3", "ebkDD2-T3jSVTg-zZS-Bqnh":"4"} +5 {"iLOhUA-n05O2OY-LHB-qS3H":"8", "ATC5e8-MywwrP3-Vhc-QdxZ":"1", "fST2pb-uAaS3Mb-Pl6-9i1J":"0", "fppq1A-Q1WOvMj-Eod-xbTz":"3", "ebkDD2-T3jSVTg-zZS-Bqnh":"4"} +5 {"iLOhUA-n05O2OY-LHB-qS3H":"8", "ATC5e8-MywwrP3-Vhc-QdxZ":"1", "fST2pb-uAaS3Mb-Pl6-9i1J":"0", "fppq1A-Q1WOvMj-Eod-xbTz":"3", "ebkDD2-T3jSVTg-zZS-Bqnh":"4"} +6 {"eSB1Yl-jDIrIHV-7WE-qyvM":"6", "4N8yoM-901A5FI-B9Q-3EpM":"9", "pBdMp1-nVbMzpB-WJj-dhDd":"4", "3jGdNP-NgyL5fW-9Ty-CLlY":"2", "5gaoHt-7MmTVLs-Jm0-KpaG":"2"} +6 {"eSB1Yl-jDIrIHV-7WE-qyvM":"6", "4N8yoM-901A5FI-B9Q-3EpM":"9", "pBdMp1-nVbMzpB-WJj-dhDd":"4", "3jGdNP-NgyL5fW-9Ty-CLlY":"2", "5gaoHt-7MmTVLs-Jm0-KpaG":"2"} +6 {"eSB1Yl-jDIrIHV-7WE-qyvM":"6", "4N8yoM-901A5FI-B9Q-3EpM":"9", "pBdMp1-nVbMzpB-WJj-dhDd":"4", "3jGdNP-NgyL5fW-9Ty-CLlY":"2", "5gaoHt-7MmTVLs-Jm0-KpaG":"2"} +7 {"yfDrEq-z2VBXi1-bSe-tgGW":"7", "zLkwFE-i6GiJut-yHW-e4wJ":"8", "pl1mqW-qSEYcv0-kTT-8Uuz":"3", "cEbTSo-CoT1upF-XsY-gafn":"8", "Shcq2l-iuYvkyo-ssN-okkW":"7"} +7 {"yfDrEq-z2VBXi1-bSe-tgGW":"7", "zLkwFE-i6GiJut-yHW-e4wJ":"8", "pl1mqW-qSEYcv0-kTT-8Uuz":"3", "cEbTSo-CoT1upF-XsY-gafn":"8", "Shcq2l-iuYvkyo-ssN-okkW":"7"} +7 {"yfDrEq-z2VBXi1-bSe-tgGW":"7", "zLkwFE-i6GiJut-yHW-e4wJ":"8", "pl1mqW-qSEYcv0-kTT-8Uuz":"3", "cEbTSo-CoT1upF-XsY-gafn":"8", "Shcq2l-iuYvkyo-ssN-okkW":"7"} +7 {"yfDrEq-z2VBXi1-bSe-tgGW":"7", "zLkwFE-i6GiJut-yHW-e4wJ":"8", "pl1mqW-qSEYcv0-kTT-8Uuz":"3", "cEbTSo-CoT1upF-XsY-gafn":"8", "Shcq2l-iuYvkyo-ssN-okkW":"7"} +8 {"lJFCyu-BhT7N4x-luz-9t85":"2", "wGuvM7-zJNjFUz-4z3-Y6Gf":"7", "F9KCc5-ZzE0IYN-c5t-VqUH":"8", "FgXjJi-l10Pijy-jtK-WgMf":"4", "uexWAB-xHkHMYL-vUi-4Umt":"9"} +8 {"lJFCyu-BhT7N4x-luz-9t85":"2", "wGuvM7-zJNjFUz-4z3-Y6Gf":"7", "F9KCc5-ZzE0IYN-c5t-VqUH":"8", "FgXjJi-l10Pijy-jtK-WgMf":"4", "uexWAB-xHkHMYL-vUi-4Umt":"9"} +8 {"lJFCyu-BhT7N4x-luz-9t85":"2", "wGuvM7-zJNjFUz-4z3-Y6Gf":"7", "F9KCc5-ZzE0IYN-c5t-VqUH":"8", "FgXjJi-l10Pijy-jtK-WgMf":"4", "uexWAB-xHkHMYL-vUi-4Umt":"9"} +8 {"lJFCyu-BhT7N4x-luz-9t85":"2", "wGuvM7-zJNjFUz-4z3-Y6Gf":"7", "F9KCc5-ZzE0IYN-c5t-VqUH":"8", "FgXjJi-l10Pijy-jtK-WgMf":"4", "uexWAB-xHkHMYL-vUi-4Umt":"9"} +8 {"lJFCyu-BhT7N4x-luz-9t85":"2", "wGuvM7-zJNjFUz-4z3-Y6Gf":"7", "F9KCc5-ZzE0IYN-c5t-VqUH":"8", "FgXjJi-l10Pijy-jtK-WgMf":"4", "uexWAB-xHkHMYL-vUi-4Umt":"9"} +8 {"lJFCyu-BhT7N4x-luz-9t85":"2", "wGuvM7-zJNjFUz-4z3-Y6Gf":"7", "F9KCc5-ZzE0IYN-c5t-VqUH":"8", "FgXjJi-l10Pijy-jtK-WgMf":"4", "uexWAB-xHkHMYL-vUi-4Umt":"9"} +9 {"s4d0hU-uWynWWe-eIf-ym5C":"9", "ANvepQ-oO7fMUr-qdl-XbUo":"4", "G16nau-7qY3fb1-UYL-oiNd":"5", "bVd4er-efUCDow-nzo-87fg":"2", "7I6zLI-hQTTlLe-sBr-oCTh":"7"} +9 {"s4d0hU-uWynWWe-eIf-ym5C":"9", "ANvepQ-oO7fMUr-qdl-XbUo":"4", "G16nau-7qY3fb1-UYL-oiNd":"5", "bVd4er-efUCDow-nzo-87fg":"2", "7I6zLI-hQTTlLe-sBr-oCTh":"7"} +9 {"s4d0hU-uWynWWe-eIf-ym5C":"9", "ANvepQ-oO7fMUr-qdl-XbUo":"4", "G16nau-7qY3fb1-UYL-oiNd":"5", "bVd4er-efUCDow-nzo-87fg":"2", "7I6zLI-hQTTlLe-sBr-oCTh":"7"} +9 {"s4d0hU-uWynWWe-eIf-ym5C":"9", "ANvepQ-oO7fMUr-qdl-XbUo":"4", "G16nau-7qY3fb1-UYL-oiNd":"5", "bVd4er-efUCDow-nzo-87fg":"2", "7I6zLI-hQTTlLe-sBr-oCTh":"7"} +9 {"s4d0hU-uWynWWe-eIf-ym5C":"9", "ANvepQ-oO7fMUr-qdl-XbUo":"4", "G16nau-7qY3fb1-UYL-oiNd":"5", "bVd4er-efUCDow-nzo-87fg":"2", "7I6zLI-hQTTlLe-sBr-oCTh":"7"} +9 {"s4d0hU-uWynWWe-eIf-ym5C":"9", "ANvepQ-oO7fMUr-qdl-XbUo":"4", "G16nau-7qY3fb1-UYL-oiNd":"5", "bVd4er-efUCDow-nzo-87fg":"2", "7I6zLI-hQTTlLe-sBr-oCTh":"7"} +9 {"s4d0hU-uWynWWe-eIf-ym5C":"9", "ANvepQ-oO7fMUr-qdl-XbUo":"4", "G16nau-7qY3fb1-UYL-oiNd":"5", "bVd4er-efUCDow-nzo-87fg":"2", "7I6zLI-hQTTlLe-sBr-oCTh":"7"} + diff --git a/regression-test/suites/query_p0/sql_functions/table_function/explode.groovy b/regression-test/suites/query_p0/sql_functions/table_function/explode.groovy index fabb847189ae05..982e27a96676f3 100644 --- a/regression-test/suites/query_p0/sql_functions/table_function/explode.groovy +++ b/regression-test/suites/query_p0/sql_functions/table_function/explode.groovy @@ -70,4 +70,71 @@ suite("explode") { sql "insert into baseall_explode_numbers values(1),(2),(3),(4),(5),(6),(7),(8),(9),(10),(11),(12),(13),(14),(15);" qt_test4 """select k3,e from baseall_explode_numbers as U lateral view explode_numbers(5) tmp1 as e order by k3,e;""" qt_test5 """select k3,e from baseall_explode_numbers as U lateral view explode_numbers(10) tmp1 as e order by k3,e;""" + + // test array nested array | map for explode + def testTable = "tam" + def dataFile = "am.json" + sql """ DROP TABLE IF EXISTS $testTable; """ + sql """ + CREATE TABLE `$testTable` ( + `id` bigint(20) NULL, + `arr_arr` array> NULL DEFAULT "[]", + `arr_map` array> NULL DEFAULT "[]" + ) ENGINE=OLAP + DUPLICATE KEY(`id`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`id`) BUCKETS 10 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "is_being_synced" = "false", + "storage_format" = "V2", + "light_schema_change" = "true", + "disable_auto_compaction" = "false", + "enable_single_replica_compaction" = "false" + ); + """ + + streamLoad { + table testTable + + // set http request header params + file dataFile // import json file + set 'format', 'json' // import format + set 'read_json_by_line', 'true' // read json by line + set 'strict_mode', 'true' + time 10000 // limit inflight 10s + + // if declared a check callback, the default check condition will ignore. + // So you must check all condition + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(25, json.NumberTotalRows) + assertEquals(25, json.NumberLoadedRows) + assertEquals(0, json.NumberFilteredRows) + assertTrue(json.LoadBytes > 0) + } + } + + sql "sync" + + // check result + order_qt_sql """ select id, eaa from $testTable lateral view explode(arr_arr) aa as eaa order by id; """ + order_qt_sql """ select id, eam from $testTable lateral view explode(arr_map) aa as eam order by id; """ + + def res_origin_am = sql "select array_size(arr_map) from $testTable where array_size(arr_map) > 0 order by id;" + def res_explode_am = sql "select count() from (select id, eam from $testTable lateral view explode(arr_map) aa as eam order by id) as t1 group by id order by id;" + for (int r = 0; r < res_origin_am.size(); ++ r) { + assertEquals(res_origin_am[r][0], res_explode_am[r][0]) + } + + def res_origin_size_aa = sql "select array_size(arr_arr) from $testTable where array_size(arr_arr) > 0 order by id;" + def res_explode_aa = sql "select count() from (select id, eaa from $testTable lateral view explode(arr_arr) aa as eaa order by id) as t1 group by id order by id;" + for (int r = 0; r < res_origin_size_aa.size(); ++ r) { + assertEquals(res_origin_size_aa[r][0], res_explode_aa[r][0]) + } } From 9a47e8fa737b22fccc34dd23bd85ab7edb952b69 Mon Sep 17 00:00:00 2001 From: shuke <37901441+shuke987@users.noreply.github.com> Date: Mon, 18 Sep 2023 16:56:01 +0800 Subject: [PATCH 10/12] [catalog lock](log) enable catalog lock log (#24530) --- .../src/main/java/org/apache/doris/catalog/Env.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 2c80226e3d2962..a0cdc7068f6e45 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -855,12 +855,10 @@ private boolean tryLock(boolean mustLock) { while (true) { try { if (!lock.tryLock(Config.catalog_try_lock_timeout_ms, TimeUnit.MILLISECONDS)) { - if (LOG.isDebugEnabled()) { - // to see which thread held this lock for long time. - Thread owner = lock.getOwner(); - if (owner != null) { - LOG.info("catalog lock is held by: {}", Util.dumpThread(owner, 10)); - } + // to see which thread held this lock for long time. + Thread owner = lock.getOwner(); + if (owner != null) { + LOG.info("catalog lock is held by: {}", Util.dumpThread(owner, 10)); } if (mustLock) { From dcabc06510e1e8151da7af2aef519622c831dc5f Mon Sep 17 00:00:00 2001 From: Dongyang Li Date: Mon, 18 Sep 2023 17:18:23 +0800 Subject: [PATCH 11/12] [fix](pipeline) update check-pr-if-need-run-build.sh (#24515) update check-pr-if-need-run-build.sh --- regression-test/pipeline/common/check-pr-if-need-run-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/pipeline/common/check-pr-if-need-run-build.sh b/regression-test/pipeline/common/check-pr-if-need-run-build.sh index 5f57cd55adc8e8..c33422f7cb74b1 100755 --- a/regression-test/pipeline/common/check-pr-if-need-run-build.sh +++ b/regression-test/pipeline/common/check-pr-if-need-run-build.sh @@ -81,7 +81,7 @@ https://github.com/apache/doris/pull/${PULL_NUMBER}/files all change files: } _only_modified_regression_conf() { - if [[ -n ${added_files} ]]; then echo "Not only modified regression conf, find added files" && return 1; fi + if [[ -n ${added_files} || -n ${removed_files} ]]; then echo "Not only modified regression conf, find added/removed files" && return 1; fi for f in ${modified_files}; do if [[ "${f}" == "regression-test/pipeline/p0/conf/regression-conf.groovy" ]] || [[ "${f}" == "regression-test/pipeline/p1/conf/regression-conf.groovy" ]]; then From 67e8951b72f3d48167734e4a060de40f23acb669 Mon Sep 17 00:00:00 2001 From: AKIRA <33112463+Kikyou1997@users.noreply.github.com> Date: Mon, 18 Sep 2023 18:27:10 +0900 Subject: [PATCH 12/12] [fix](stats) Fix analyze failed when there are thousands of partitions. (#24521) It's caused by we used same query id for multiple queries of same olap analyze task, but many structures related to query execution depends on query id. --- .../org/apache/doris/qe/StmtExecutor.java | 3 +++ .../doris/statistics/util/StatisticsUtil.java | 5 ----- .../suites/statistics/analyze_stats.groovy | 21 +++++++++++++++++++ 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index 114080036ed7a2..1a7a2e674da3e6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -2571,6 +2571,9 @@ public StatementBase setParsedStmt(StatementBase parsedStmt) { public List executeInternalQuery() { LOG.debug("INTERNAL QUERY: " + originStmt.toString()); + UUID uuid = UUID.randomUUID(); + TUniqueId queryId = new TUniqueId(uuid.getMostSignificantBits(), uuid.getLeastSignificantBits()); + context.setQueryId(queryId); try { List resultRows = new ArrayList<>(); try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java index 3d2d0b171882a5..1cb131099c5b4b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java @@ -73,7 +73,6 @@ import org.apache.doris.statistics.StatisticConstants; import org.apache.doris.system.Frontend; import org.apache.doris.system.SystemInfoService; -import org.apache.doris.thrift.TUniqueId; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; @@ -105,7 +104,6 @@ import java.util.Optional; import java.util.Set; import java.util.StringJoiner; -import java.util.UUID; import java.util.function.Function; import java.util.stream.Collectors; @@ -188,9 +186,6 @@ public static AutoCloseConnectContext buildConnectContext(boolean limitScan) { connectContext.setDatabase(FeConstants.INTERNAL_DB_NAME); connectContext.setQualifiedUser(UserIdentity.ROOT.getQualifiedUser()); connectContext.setCurrentUserIdentity(UserIdentity.ROOT); - UUID uuid = UUID.randomUUID(); - TUniqueId queryId = new TUniqueId(uuid.getMostSignificantBits(), uuid.getLeastSignificantBits()); - connectContext.setQueryId(queryId); connectContext.setStartTime(); connectContext.setCluster(SystemInfoService.DEFAULT_CLUSTER); return new AutoCloseConnectContext(connectContext); diff --git a/regression-test/suites/statistics/analyze_stats.groovy b/regression-test/suites/statistics/analyze_stats.groovy index bc9b42a1a5515e..4ee9cc2b2f30b8 100644 --- a/regression-test/suites/statistics/analyze_stats.groovy +++ b/regression-test/suites/statistics/analyze_stats.groovy @@ -1035,5 +1035,26 @@ PARTITION `p599` VALUES IN (599) SELECT * FROM analyze_test_with_schema_update; """ + sql """ + DROP TABLE IF EXISTS two_thousand_partition_table_test + """ + + sql """ + CREATE TABLE two_thousand_partition_table_test (col1 int(11451) not null) + DUPLICATE KEY(col1) + PARTITION BY RANGE(`col1`) + ( + from (0) to (1000001) INTERVAL 500 + ) + DISTRIBUTED BY HASH(col1) + BUCKETS 3 + PROPERTIES( + "replication_num"="1" + ); + """ + + sql """ + ANALYZE TABLE two_thousand_partition_table_test WITH SYNC; + """ }