From fa1e7198675ec0bfd1810acc12ec24d8c9a7cd94 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Tue, 30 Jan 2024 14:43:11 +0300 Subject: [PATCH 01/83] Check for Motions before scan if requesting Explicit Redistribute Motion Previously, `Explicit Redistribute Motion` was added if there were any motions in the subplan, ignoring any motions in `InitPlans`. This patch updates the logic: if an `Explicit Redistribute Motion` has no motions underneath it's subtree, then the row to update must originate from the same segment, and no `Explicit Redistribute Motion` is needed. If there are any motions, `Explicit Redistribute Motion` should be added only if there is a motion between the scan and the `ModifyTable` on the relation we are going to update. Even if previous Motions were performed on a leaf partition or inherited table, targetlists from Motions refer to relids of their parents. So, if relid has a superclass, we should check for inheritance instead of comparing Oids. --- src/backend/optimizer/plan/createplan.c | 151 +++++++++++++++++++++++- 1 file changed, 150 insertions(+), 1 deletion(-) diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c index fe03c8df7ce9..99fed6d7c06c 100644 --- a/src/backend/optimizer/plan/createplan.c +++ b/src/backend/optimizer/plan/createplan.c @@ -22,6 +22,7 @@ #include #include "catalog/pg_exttable.h" +#include "catalog/pg_inherits_fn.h" #include "access/skey.h" #include "access/sysattr.h" #include "catalog/pg_class.h" @@ -54,6 +55,7 @@ #include "utils/guc.h" #include "utils/lsyscache.h" #include "utils/uri.h" +#include "storage/lmgr.h" /* LockRelationOid(), UnlockRelationOid() */ #include "cdb/cdbllize.h" /* pull_up_Flow() */ #include "cdb/cdbmutate.h" @@ -63,6 +65,7 @@ #include "cdb/cdbsetop.h" #include "cdb/cdbsreh.h" #include "cdb/cdbvars.h" +#include "cdb/cdbpartition.h" /* rel_partition_get_master() etc. */ static Plan *create_subplan(PlannerInfo *root, Path *best_path); /* CDB */ @@ -170,6 +173,7 @@ static WorkTableScan *make_worktablescan(List *qptlist, List *qpqual, static BitmapAnd *make_bitmap_and(List *bitmapplans); static BitmapOr *make_bitmap_or(List *bitmapplans); static List *flatten_grouping_list(List *groupcls); +static bool can_elide_explicit_motion(Plan *plan, List *rtable, Oid relid); static void adjust_modifytable_flow(PlannerInfo *root, ModifyTable *node, List *is_split_updates); static Plan *prepare_sort_from_pathkeys(PlannerInfo *root, Plan *lefttree, List *pathkeys, @@ -6471,6 +6475,140 @@ make_modifytable(PlannerInfo *root, return node; } +/* + * can_elide_explicit_motion_recurse + * Recursive guts of can_elide_explicit_motion(). + * + * Check for motions by recursively looking at right subtree first. If it does + * not exist, go into left subtree. If we find a motion before the scan, we + * return false and exit from that subtree. A scan is always underneath the + * motion, so we won't encounter the scan if we exited the subtree. In that + * case, we can't elide Explicit Redistribute Motion. + * + * If we encounter the scan before any motions, then we can elide + * unneccessary Explicit Redistribute Motion. + */ +static bool +can_elide_explicit_motion_recurse(Plan *plan, List *rtable, Oid relid, + bool relid_is_subclass, Oid reltypeid) +{ + RangeTblEntry *target_rte; + + while (plan) + { + /* + * If this is a Motion, check Oids from the Motion's targetlist. + */ + if (IsA(plan, Motion)) + { + Motion *motion = (Motion *) plan; + + if ((motion->motionType == MOTIONTYPE_FIXED && motion->isBroadcast) || + motion->motionType == MOTIONTYPE_HASH) + { + TargetEntry *tle; + ListCell *lcr; + + foreach(lcr, plan->targetlist) + { + tle = (TargetEntry *) lfirst(lcr); + target_rte = rt_fetch(tle->resno, rtable); + + if (relid_is_subclass) + { + Oid target_reltypeid = get_rel_type_id(target_rte->relid); + + /* + * Does relid inherit from a table in targetlist? + */ + if (target_reltypeid != InvalidOid && reltypeid != InvalidOid && + typeInheritsFrom(reltypeid, target_reltypeid)) + /* There is a Motion on parent table before scan on + the child */ + return false; + } + + if (target_rte->relid == relid) + /* There is a Motion before scan */ + return false; + } + } + } + /* + * If this is a scan and it's scanrelid matches relid, we encountered a + * scan before any Motions. + */ + else + { + Scan *scan; + + switch (nodeTag(plan)) + { + case T_SeqScan: + case T_DynamicSeqScan: + case T_ExternalScan: + case T_IndexScan: + case T_DynamicIndexScan: + case T_IndexOnlyScan: + case T_BitmapIndexScan: + case T_DynamicBitmapIndexScan: + case T_BitmapHeapScan: + case T_DynamicBitmapHeapScan: + scan = (Scan *) plan; + target_rte = rt_fetch(scan->scanrelid, rtable); + + if (target_rte->relid == relid) + /* There wasn't any Motions before scan */ + return true; + break; + default: + break; + } + } + + /* + * If this plan has a right subtree, check it for Motions too. + */ + if (plan->righttree && + can_elide_explicit_motion_recurse(plan->righttree, rtable, relid, + relid_is_subclass, reltypeid)) + return true; + + plan = plan->lefttree; + } + + return false; +} + +/* + * can_elide_explicit_motion + * Check if there's any Redistribute or Broadcast Motions before scan in the + * same subtree for a relid. + */ +static bool +can_elide_explicit_motion(Plan *plan, List *rtable, Oid relid) +{ + bool relid_is_subclass; + Oid reltypeid; + + /* + * Even if previous Motions were performed on a leaf partition or inherited + * table, targetlists from Motions refer to relids of their parents. So, if + * relid has a superclass, we should check for type inheritance too. + */ + LockRelationOid(relid, AccessShareLock); + relid_is_subclass = has_superclass(relid); + UnlockRelationOid(relid, AccessShareLock); + + if (relid_is_subclass) + reltypeid = get_rel_type_id(relid); + else + reltypeid = InvalidOid; + + return can_elide_explicit_motion_recurse(plan, rtable, relid, + relid_is_subclass, reltypeid); +} + /* * Set the Flow in a ModifyTable and its children correctly. * @@ -6726,7 +6864,18 @@ adjust_modifytable_flow(PlannerInfo *root, ModifyTable *node, List *is_split_upd node->action_col_idxes = lappend_int(node->action_col_idxes, -1); node->ctid_col_idxes = lappend_int(node->ctid_col_idxes, -1); node->oid_col_idxes = lappend_int(node->oid_col_idxes, 0); - request_explicit_motion(subplan, rti, root->glob->finalrtable); + + /* + * If an Explicit Motion has no Motions underneath it, then + * the row to update must originate from the same segment, + * and no Motion is needed. + * + * We elide the motion even if there are Motions, as long as + * they are not between the scan on the target table and the + * ModifyTable. + */ + if (!can_elide_explicit_motion(subplan, root->parse->rtable, rte->relid)) + request_explicit_motion(subplan, rti, root->glob->finalrtable); } } else if (targetPolicyType == POLICYTYPE_ENTRY) From 51960cd906346d542aaba061bbd4c09c7efec253 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Tue, 30 Jan 2024 14:43:15 +0300 Subject: [PATCH 02/83] Update tests --- src/test/regress/expected/bfv_dml.out | 17 ++- src/test/regress/expected/gangsize.out | 2 - src/test/regress/expected/gp_unique_rowid.out | 56 +++++----- src/test/regress/expected/gporca.out | 21 ++-- .../regress/expected/partition_pruning.out | 37 +++---- src/test/regress/expected/qp_subquery.out | 42 ++++--- src/test/regress/expected/update_gp.out | 103 ++++++++---------- 7 files changed, 131 insertions(+), 147 deletions(-) diff --git a/src/test/regress/expected/bfv_dml.out b/src/test/regress/expected/bfv_dml.out index 444c9673c877..bb18be62e795 100644 --- a/src/test/regress/expected/bfv_dml.out +++ b/src/test/regress/expected/bfv_dml.out @@ -521,17 +521,16 @@ alter table bar set with(REORGANIZE=false) distributed randomly; analyze foo; analyze bar; explain delete from foo using bar; - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------ + QUERY PLAN +-------------------------------------------------------------------------------------------------- Delete on foo (cost=10000000000.00..10000000009.98 rows=34 width=16) - -> Explicit Redistribute Motion 3:3 (slice2; segments: 3) (cost=10000000000.00..10000000009.98 rows=34 width=16) - -> Nested Loop (cost=10000000000.00..10000000009.98 rows=34 width=16) - -> Seq Scan on foo (cost=0.00..3.10 rows=4 width=10) - -> Materialize (cost=0.00..3.65 rows=10 width=6) - -> Broadcast Motion 3:3 (slice1; segments: 3) (cost=0.00..3.50 rows=10 width=6) - -> Seq Scan on bar (cost=0.00..3.10 rows=4 width=6) + -> Nested Loop (cost=10000000000.00..10000000009.98 rows=34 width=16) + -> Seq Scan on foo (cost=0.00..3.10 rows=4 width=10) + -> Materialize (cost=0.00..3.65 rows=10 width=6) + -> Broadcast Motion 3:3 (slice1; segments: 3) (cost=0.00..3.50 rows=10 width=6) + -> Seq Scan on bar (cost=0.00..3.10 rows=4 width=6) Optimizer: Postgres query optimizer -(8 rows) +(7 rows) delete from foo using bar; drop table foo; diff --git a/src/test/regress/expected/gangsize.out b/src/test/regress/expected/gangsize.out index c72d8df7c6ff..4afce1ccb1ec 100644 --- a/src/test/regress/expected/gangsize.out +++ b/src/test/regress/expected/gangsize.out @@ -173,13 +173,11 @@ end; INFO: Distributed transaction command 'Distributed Commit (one-phase)' to PARTIAL contents: 0 1 update random_2_0 set a = 1 from hash_3_3_2 where hash_3_3_2.b = random_2_0.c; INFO: (slice 1) Dispatch command to ALL contents: 0 1 2 -INFO: (slice 2) Dispatch command to PARTIAL contents: 0 1 INFO: (slice 0) Dispatch command to PARTIAL contents: 0 1 INFO: Distributed transaction command 'Distributed Commit (one-phase)' to ALL contents: 0 1 2 begin; update random_2_0 set a = 1 from hash_3_3_2 where hash_3_3_2.b = random_2_0.c; INFO: (slice 1) Dispatch command to ALL contents: 0 1 2 -INFO: (slice 2) Dispatch command to PARTIAL contents: 0 1 INFO: (slice 0) Dispatch command to PARTIAL contents: 0 1 end; INFO: Distributed transaction command 'Distributed Commit (one-phase)' to ALL contents: 0 1 2 diff --git a/src/test/regress/expected/gp_unique_rowid.out b/src/test/regress/expected/gp_unique_rowid.out index ec782ff45645..f2bcbcf6c699 100644 --- a/src/test/regress/expected/gp_unique_rowid.out +++ b/src/test/regress/expected/gp_unique_rowid.out @@ -192,22 +192,21 @@ where e.x in select b from t2_12512 ) ; - QUERY PLAN ---------------------------------------------------------------------------- + QUERY PLAN +--------------------------------------------------------------------- Update on t_12512 - -> Explicit Redistribute Motion 3:3 (slice2; segments: 3) - -> HashAggregate - Group Key: t_12512.ctid, x.ctid - -> Hash Join - Hash Cond: (x.x = t2_12512.b) - -> Nested Loop - -> Seq Scan on t_12512 - -> Function Scan on x - -> Hash - -> Broadcast Motion 3:3 (slice1; segments: 3) - -> Seq Scan on t2_12512 + -> HashAggregate + Group Key: t_12512.ctid, x.ctid + -> Hash Join + Hash Cond: (x.x = t2_12512.b) + -> Nested Loop + -> Seq Scan on t_12512 + -> Function Scan on x + -> Hash + -> Broadcast Motion 3:3 (slice1; segments: 3) + -> Seq Scan on t2_12512 Optimizer: Postgres query optimizer -(13 rows) +(12 rows) update t_12512 set b = 1 from @@ -231,23 +230,22 @@ where e.x in select b from t2_12512 ) ; - QUERY PLAN ---------------------------------------------------------------------------- + QUERY PLAN +--------------------------------------------------------------------- Update on t_12512 - -> Explicit Redistribute Motion 3:3 (slice2; segments: 3) - -> HashAggregate - Group Key: t_12512.ctid, "*VALUES*".ctid - -> Hash Join - Hash Cond: ("*VALUES*".column1 = t2_12512.b) - -> Nested Loop - -> Seq Scan on t_12512 - -> Materialize - -> Values Scan on "*VALUES*" - -> Hash - -> Broadcast Motion 3:3 (slice1; segments: 3) - -> Seq Scan on t2_12512 + -> HashAggregate + Group Key: t_12512.ctid, "*VALUES*".ctid + -> Hash Join + Hash Cond: ("*VALUES*".column1 = t2_12512.b) + -> Nested Loop + -> Seq Scan on t_12512 + -> Materialize + -> Values Scan on "*VALUES*" + -> Hash + -> Broadcast Motion 3:3 (slice1; segments: 3) + -> Seq Scan on t2_12512 Optimizer: Postgres query optimizer -(14 rows) +(13 rows) update t_12512 set b = 1 from diff --git a/src/test/regress/expected/gporca.out b/src/test/regress/expected/gporca.out index b4ead0baf92c..26b7702057ea 100644 --- a/src/test/regress/expected/gporca.out +++ b/src/test/regress/expected/gporca.out @@ -11861,19 +11861,18 @@ select b, count(*) from gpexp_hash group by b order by b; (11 rows) explain update gpexp_rand set b=(select b from gpexp_hash where gpexp_rand.a = gpexp_hash.a); - QUERY PLAN ---------------------------------------------------------------------------------------------------------------- + QUERY PLAN +--------------------------------------------------------------------------------------------------------- Update on gpexp_rand (cost=0.00..133.75 rows=25 width=14) - -> Explicit Redistribute Motion 2:2 (slice2; segments: 2) (cost=0.00..133.75 rows=25 width=14) - -> Seq Scan on gpexp_rand (cost=0.00..133.75 rows=25 width=14) - SubPlan 1 (slice2; segments: 2) - -> Result (cost=0.00..2.63 rows=1 width=4) - Filter: (gpexp_rand.a = gpexp_hash.a) - -> Materialize (cost=0.00..2.63 rows=1 width=4) - -> Broadcast Motion 2:2 (slice1; segments: 2) (cost=0.00..2.62 rows=1 width=4) - -> Seq Scan on gpexp_hash (cost=0.00..2.62 rows=1 width=4) + -> Seq Scan on gpexp_rand (cost=0.00..133.75 rows=25 width=14) + SubPlan 1 (slice0; segments: 2) + -> Result (cost=0.00..2.63 rows=1 width=4) + Filter: (gpexp_rand.a = gpexp_hash.a) + -> Materialize (cost=0.00..2.63 rows=1 width=4) + -> Broadcast Motion 2:2 (slice1; segments: 2) (cost=0.00..2.62 rows=1 width=4) + -> Seq Scan on gpexp_hash (cost=0.00..2.62 rows=1 width=4) Optimizer: Postgres query optimizer -(10 rows) +(9 rows) update gpexp_rand set b=(select b from gpexp_hash where gpexp_rand.a = gpexp_hash.a); select b, count(*) from gpexp_rand group by b order by b; diff --git a/src/test/regress/expected/partition_pruning.out b/src/test/regress/expected/partition_pruning.out index d647a13c7360..dbdb4c75dfd4 100644 --- a/src/test/regress/expected/partition_pruning.out +++ b/src/test/regress/expected/partition_pruning.out @@ -3498,26 +3498,25 @@ from ( ) src where trg.key1 = src.key1 and trg.key1 = 2; - QUERY PLAN -------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------- Update on t_part1_1_prt_2 trg - -> Explicit Redistribute Motion 3:3 (slice3; segments: 3) - -> Nested Loop - -> Seq Scan on t_part1_1_prt_2 trg - Filter: (key1 = 2) - -> Materialize - -> Redistribute Motion 1:3 (slice2; segments: 1) - Hash Key: src.key1 - -> Subquery Scan on src - Filter: (src.key1 = 2) - -> WindowAgg - -> Gather Motion 3:1 (slice1; segments: 3) - -> Result - -> Append - -> Seq Scan on t_part1_1_prt_2 r - Filter: (key1 = 2) - Optimizer: Postgres query optimizer -(17 rows) + -> Nested Loop + -> Seq Scan on t_part1_1_prt_2 trg + Filter: (key1 = 2) + -> Materialize + -> Redistribute Motion 1:3 (slice2; segments: 1) + Hash Key: src.key1 + -> Subquery Scan on src + Filter: (src.key1 = 2) + -> WindowAgg + -> Gather Motion 3:1 (slice1; segments: 3) + -> Result + -> Append + -> Seq Scan on t_part1_1_prt_2 r + Filter: (key1 = 2) + Optimizer: Postgres query optimizer +(16 rows) DROP TABLE t_part1; -- Test that the dynamic partition pruning should not be performed if the partition's opclass and the diff --git a/src/test/regress/expected/qp_subquery.out b/src/test/regress/expected/qp_subquery.out index fc255ecd860d..b60668709932 100644 --- a/src/test/regress/expected/qp_subquery.out +++ b/src/test/regress/expected/qp_subquery.out @@ -1073,32 +1073,30 @@ create table TabDel4(a int not null, b int not null); insert into TabDel4 values(1,2); commit; explain delete from TabDel1 where TabDel1.a not in (select a from TabDel3); -- do not support this because we produce NLASJ - QUERY PLAN -------------------------------------------------------------------------------------------------------- - Delete on tabdel1 (cost=1.09..3.17 rows=2 width=16) - -> Explicit Redistribute Motion 3:3 (slice2; segments: 3) (cost=1.09..3.17 rows=2 width=10) - -> Hash Left Anti Semi (Not-In) Join (cost=1.09..3.17 rows=2 width=10) - Hash Cond: tabdel1.a = tabdel3.a - -> Seq Scan on tabdel1 (cost=0.00..2.03 rows=1 width=14) - -> Hash (cost=1.05..1.05 rows=1 width=4) - -> Broadcast Motion 3:3 (slice1; segments: 3) (cost=0.00..1.05 rows=1 width=4) - -> Seq Scan on tabdel3 (cost=0.00..1.01 rows=1 width=4) - Optimizer status: Postgres query optimizer -(9 rows) + QUERY PLAN +-------------------------------------------------------------------------------------------------- + Delete on tabdel1 (cost=1.09..4.17 rows=2 width=16) + -> Hash Left Anti Semi (Not-In) Join (cost=1.09..4.17 rows=2 width=16) + Hash Cond: (tabdel1.a = tabdel3.a) + -> Seq Scan on tabdel1 (cost=0.00..3.03 rows=1 width=14) + -> Hash (cost=1.05..1.05 rows=1 width=10) + -> Broadcast Motion 3:3 (slice1; segments: 3) (cost=0.00..1.05 rows=1 width=10) + -> Seq Scan on tabdel3 (cost=0.00..1.01 rows=1 width=10) + Optimizer: Postgres query optimizer +(8 rows) explain delete from TabDel2 where TabDel2.a not in (select a from TabDel4); -- support this - QUERY PLAN --------------------------------------------------------------------------------------------------------- + QUERY PLAN +-------------------------------------------------------------------------------------------------- Delete on tabdel2 (cost=1.09..4.17 rows=2 width=16) - -> Explicit Redistribute Motion 3:3 (slice2; segments: 3) (cost=1.09..4.17 rows=2 width=16) - -> Hash Left Anti Semi (Not-In) Join (cost=1.09..4.17 rows=2 width=16) - Hash Cond: (tabdel2.a = tabdel4.a) - -> Seq Scan on tabdel2 (cost=0.00..3.03 rows=1 width=14) - -> Hash (cost=1.05..1.05 rows=1 width=10) - -> Broadcast Motion 3:3 (slice1; segments: 3) (cost=0.00..1.05 rows=1 width=10) - -> Seq Scan on tabdel4 (cost=0.00..1.01 rows=1 width=10) + -> Hash Left Anti Semi (Not-In) Join (cost=1.09..4.17 rows=2 width=16) + Hash Cond: (tabdel2.a = tabdel4.a) + -> Seq Scan on tabdel2 (cost=0.00..3.03 rows=1 width=14) + -> Hash (cost=1.05..1.05 rows=1 width=10) + -> Broadcast Motion 3:3 (slice1; segments: 3) (cost=0.00..1.05 rows=1 width=10) + -> Seq Scan on tabdel4 (cost=0.00..1.01 rows=1 width=10) Optimizer: Postgres query optimizer -(9 rows) +(8 rows) delete from TabDel2 where TabDel2.a not in (select a from TabDel4); select * from TabDel2; diff --git a/src/test/regress/expected/update_gp.out b/src/test/regress/expected/update_gp.out index 2f8a4e309628..de2539cf45fb 100644 --- a/src/test/regress/expected/update_gp.out +++ b/src/test/regress/expected/update_gp.out @@ -82,8 +82,8 @@ NOTICE: merging column "a" with inherited definition NOTICE: merging column "b" with inherited definition insert into base_tbl select g, g from generate_series(1, 5) g; explain (costs off) update base_tbl set a=a+1; - QUERY PLAN ---------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------ Update on base_tbl -> Redistribute Motion 3:3 (slice1; segments: 3) Hash Key: ((base_tbl.a + 1)) @@ -93,10 +93,9 @@ explain (costs off) update base_tbl set a=a+1; Hash Key: ((child_a.a + 1)) -> Split -> Seq Scan on child_a - -> Explicit Redistribute Motion 3:3 (slice3; segments: 3) - -> Seq Scan on child_b + -> Seq Scan on child_b Optimizer: Postgres query optimizer -(12 rows) +(11 rows) update base_tbl set a = 5; -- @@ -623,26 +622,24 @@ insert into t2_13265 values (2, null, 2, 2); explain (verbose, costs off) update t1_13265 set b = 2 where (c, d) not in (select c, d from t2_13265 where a = 2); - QUERY PLAN -------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------------- Update on public.t1_13265 - -> Explicit Redistribute Motion 3:3 (slice2; segments: 3) + -> Nested Loop Left Anti Semi (Not-In) Join Output: t1_13265.a, 2, t1_13265.c, t1_13265.d, t1_13265.ctid, t1_13265.gp_segment_id, t2_13265.ctid - -> Nested Loop Left Anti Semi (Not-In) Join - Output: t1_13265.a, 2, t1_13265.c, t1_13265.d, t1_13265.ctid, t1_13265.gp_segment_id, t2_13265.ctid - Join Filter: ((t1_13265.c = t2_13265.c) AND (t1_13265.d = t2_13265.d)) - -> Seq Scan on public.t1_13265 - Output: t1_13265.a, t1_13265.c, t1_13265.d, t1_13265.ctid, t1_13265.gp_segment_id - -> Materialize + Join Filter: ((t1_13265.c = t2_13265.c) AND (t1_13265.d = t2_13265.d)) + -> Seq Scan on public.t1_13265 + Output: t1_13265.a, t1_13265.c, t1_13265.d, t1_13265.ctid, t1_13265.gp_segment_id + -> Materialize + Output: t2_13265.ctid, t2_13265.c, t2_13265.d, (2) + -> Broadcast Motion 1:3 (slice1; segments: 1) Output: t2_13265.ctid, t2_13265.c, t2_13265.d, (2) - -> Broadcast Motion 3:3 (slice1; segments: 3) - Output: t2_13265.ctid, t2_13265.c, t2_13265.d, (2) - -> Seq Scan on public.t2_13265 - Output: t2_13265.ctid, t2_13265.c, t2_13265.d, 2 - Filter: (t2_13265.a = 2) + -> Seq Scan on public.t2_13265 + Output: t2_13265.ctid, t2_13265.c, t2_13265.d, 2 + Filter: (t2_13265.a = 2) Optimizer: Postgres query optimizer Settings: optimizer=off -(17 rows) +(15 rows) update t1_13265 set b = 2 where (c, d) not in (select c, d from t2_13265 where a = 2); @@ -678,43 +675,39 @@ insert into from_table select i*1.5,i*2,i*3,'xx'||i,'yy'||i, i+1 from generate_s explain (costs off) update into_table set d=from_table.d, e=from_table.e, f=from_table.f from from_table where into_table.a=from_table.a and into_table.b=from_table.b and into_table.c=from_table.c; - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------ Update on into_table_1_prt_1 - -> Explicit Redistribute Motion 3:3 (slice2; segments: 3) - -> Hash Join - Hash Cond: ((into_table_1_prt_1.a = from_table.a) AND (into_table_1_prt_1.b = from_table.b) AND (into_table_1_prt_1.c = from_table.c)) - -> Seq Scan on into_table_1_prt_1 - -> Hash - -> Redistribute Motion 3:3 (slice1; segments: 3) - Hash Key: from_table.a, from_table.b, from_table.c - -> Seq Scan on from_table - -> Explicit Redistribute Motion 3:3 (slice4; segments: 3) - -> Hash Join - Hash Cond: ((into_table_1_prt_2.a = from_table.a) AND (into_table_1_prt_2.b = from_table.b) AND (into_table_1_prt_2.c = from_table.c)) - -> Seq Scan on into_table_1_prt_2 - -> Hash - -> Redistribute Motion 3:3 (slice3; segments: 3) - Hash Key: from_table.a, from_table.b, from_table.c - -> Seq Scan on from_table - -> Explicit Redistribute Motion 3:3 (slice6; segments: 3) - -> Hash Join - Hash Cond: ((into_table_1_prt_3.a = from_table.a) AND (into_table_1_prt_3.b = from_table.b) AND (into_table_1_prt_3.c = from_table.c)) - -> Seq Scan on into_table_1_prt_3 - -> Hash - -> Redistribute Motion 3:3 (slice5; segments: 3) - Hash Key: from_table.a, from_table.b, from_table.c - -> Seq Scan on from_table - -> Explicit Redistribute Motion 3:3 (slice8; segments: 3) - -> Hash Join - Hash Cond: ((into_table_1_prt_4.a = from_table.a) AND (into_table_1_prt_4.b = from_table.b) AND (into_table_1_prt_4.c = from_table.c)) - -> Seq Scan on into_table_1_prt_4 - -> Hash - -> Redistribute Motion 3:3 (slice7; segments: 3) - Hash Key: from_table.a, from_table.b, from_table.c - -> Seq Scan on from_table + -> Hash Join + Hash Cond: ((into_table_1_prt_1.a = from_table.a) AND (into_table_1_prt_1.b = from_table.b) AND (into_table_1_prt_1.c = from_table.c)) + -> Seq Scan on into_table_1_prt_1 + -> Hash + -> Redistribute Motion 3:3 (slice1; segments: 3) + Hash Key: from_table.a, from_table.b, from_table.c + -> Seq Scan on from_table + -> Hash Join + Hash Cond: ((into_table_1_prt_2.a = from_table.a) AND (into_table_1_prt_2.b = from_table.b) AND (into_table_1_prt_2.c = from_table.c)) + -> Seq Scan on into_table_1_prt_2 + -> Hash + -> Redistribute Motion 3:3 (slice2; segments: 3) + Hash Key: from_table.a, from_table.b, from_table.c + -> Seq Scan on from_table + -> Hash Join + Hash Cond: ((into_table_1_prt_3.a = from_table.a) AND (into_table_1_prt_3.b = from_table.b) AND (into_table_1_prt_3.c = from_table.c)) + -> Seq Scan on into_table_1_prt_3 + -> Hash + -> Redistribute Motion 3:3 (slice3; segments: 3) + Hash Key: from_table.a, from_table.b, from_table.c + -> Seq Scan on from_table + -> Hash Join + Hash Cond: ((into_table_1_prt_4.a = from_table.a) AND (into_table_1_prt_4.b = from_table.b) AND (into_table_1_prt_4.c = from_table.c)) + -> Seq Scan on into_table_1_prt_4 + -> Hash + -> Redistribute Motion 3:3 (slice4; segments: 3) + Hash Key: from_table.a, from_table.b, from_table.c + -> Seq Scan on from_table Optimizer: Postgres query optimizer -(34 rows) +(30 rows) -- start_matchsubs -- m/"into_table_1_prt_\d" to partition "into_table_1_prt_\d"/ From 4d5edb537f9078bff0d88d21b0131fea4ca98350 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Tue, 30 Jan 2024 14:43:19 +0300 Subject: [PATCH 03/83] Update tests for optimizer --- .../regress/expected/gangsize_optimizer.out | 2 - .../regress/expected/gporca_optimizer.out | 21 +++++------ .../regress/expected/update_gp_optimizer.out | 37 +++++++++---------- 3 files changed, 27 insertions(+), 33 deletions(-) diff --git a/src/test/regress/expected/gangsize_optimizer.out b/src/test/regress/expected/gangsize_optimizer.out index efe59dc3f6a1..fc664fe89260 100644 --- a/src/test/regress/expected/gangsize_optimizer.out +++ b/src/test/regress/expected/gangsize_optimizer.out @@ -171,13 +171,11 @@ end; INFO: Distributed transaction command 'Distributed Commit (one-phase)' to PARTIAL contents: 0 1 update random_2_0 set a = 1 from hash_3_3_2 where hash_3_3_2.b = random_2_0.c; INFO: (slice 1) Dispatch command to ALL contents: 0 1 2 -INFO: (slice 2) Dispatch command to PARTIAL contents: 0 1 INFO: (slice 0) Dispatch command to PARTIAL contents: 0 1 INFO: Distributed transaction command 'Distributed Commit (one-phase)' to ALL contents: 0 1 2 begin; update random_2_0 set a = 1 from hash_3_3_2 where hash_3_3_2.b = random_2_0.c; INFO: (slice 1) Dispatch command to ALL contents: 0 1 2 -INFO: (slice 2) Dispatch command to PARTIAL contents: 0 1 INFO: (slice 0) Dispatch command to PARTIAL contents: 0 1 end; INFO: Distributed transaction command 'Distributed Commit (one-phase)' to ALL contents: 0 1 2 diff --git a/src/test/regress/expected/gporca_optimizer.out b/src/test/regress/expected/gporca_optimizer.out index c362caeb1ed0..967f7ac8304b 100644 --- a/src/test/regress/expected/gporca_optimizer.out +++ b/src/test/regress/expected/gporca_optimizer.out @@ -11970,19 +11970,18 @@ DETAIL: Unknown error: Partially Distributed Data explain update gpexp_rand set b=(select b from gpexp_hash where gpexp_rand.a = gpexp_hash.a); INFO: GPORCA failed to produce a plan, falling back to planner DETAIL: Unknown error: Partially Distributed Data - QUERY PLAN ---------------------------------------------------------------------------------------------------------------- + QUERY PLAN +--------------------------------------------------------------------------------------------------------- Update on gpexp_rand (cost=0.00..133.75 rows=25 width=14) - -> Explicit Redistribute Motion 2:2 (slice2; segments: 2) (cost=0.00..133.75 rows=25 width=14) - -> Seq Scan on gpexp_rand (cost=0.00..133.75 rows=25 width=14) - SubPlan 1 (slice2; segments: 2) - -> Result (cost=0.00..2.63 rows=1 width=4) - Filter: (gpexp_rand.a = gpexp_hash.a) - -> Materialize (cost=0.00..2.63 rows=1 width=4) - -> Broadcast Motion 2:2 (slice1; segments: 2) (cost=0.00..2.62 rows=1 width=4) - -> Seq Scan on gpexp_hash (cost=0.00..2.62 rows=1 width=4) + -> Seq Scan on gpexp_rand (cost=0.00..133.75 rows=25 width=14) + SubPlan 1 (slice0; segments: 2) + -> Result (cost=0.00..2.63 rows=1 width=4) + Filter: (gpexp_rand.a = gpexp_hash.a) + -> Materialize (cost=0.00..2.63 rows=1 width=4) + -> Broadcast Motion 2:2 (slice1; segments: 2) (cost=0.00..2.62 rows=1 width=4) + -> Seq Scan on gpexp_hash (cost=0.00..2.62 rows=1 width=4) Optimizer: Postgres query optimizer -(10 rows) +(9 rows) update gpexp_rand set b=(select b from gpexp_hash where gpexp_rand.a = gpexp_hash.a); INFO: GPORCA failed to produce a plan, falling back to planner diff --git a/src/test/regress/expected/update_gp_optimizer.out b/src/test/regress/expected/update_gp_optimizer.out index 3edebee2b0e1..f14270c3e811 100644 --- a/src/test/regress/expected/update_gp_optimizer.out +++ b/src/test/regress/expected/update_gp_optimizer.out @@ -82,8 +82,8 @@ NOTICE: merging column "a" with inherited definition NOTICE: merging column "b" with inherited definition insert into base_tbl select g, g from generate_series(1, 5) g; explain (costs off) update base_tbl set a=a+1; - QUERY PLAN ---------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------ Update on base_tbl -> Redistribute Motion 3:3 (slice1; segments: 3) Hash Key: ((base_tbl.a + 1)) @@ -93,10 +93,9 @@ explain (costs off) update base_tbl set a=a+1; Hash Key: ((child_a.a + 1)) -> Split -> Seq Scan on child_a - -> Explicit Redistribute Motion 3:3 (slice3; segments: 3) - -> Seq Scan on child_b + -> Seq Scan on child_b Optimizer: Postgres query optimizer -(12 rows) +(11 rows) update base_tbl set a = 5; -- @@ -624,26 +623,24 @@ insert into t2_13265 values (2, null, 2, 2); explain (verbose, costs off) update t1_13265 set b = 2 where (c, d) not in (select c, d from t2_13265 where a = 2); - QUERY PLAN -------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------------- Update on public.t1_13265 - -> Explicit Redistribute Motion 3:3 (slice2; segments: 3) + -> Nested Loop Left Anti Semi (Not-In) Join Output: t1_13265.a, 2, t1_13265.c, t1_13265.d, t1_13265.ctid, t1_13265.gp_segment_id, t2_13265.ctid - -> Nested Loop Left Anti Semi (Not-In) Join - Output: t1_13265.a, 2, t1_13265.c, t1_13265.d, t1_13265.ctid, t1_13265.gp_segment_id, t2_13265.ctid - Join Filter: ((t1_13265.c = t2_13265.c) AND (t1_13265.d = t2_13265.d)) - -> Seq Scan on public.t1_13265 - Output: t1_13265.a, t1_13265.c, t1_13265.d, t1_13265.ctid, t1_13265.gp_segment_id - -> Materialize + Join Filter: ((t1_13265.c = t2_13265.c) AND (t1_13265.d = t2_13265.d)) + -> Seq Scan on public.t1_13265 + Output: t1_13265.a, t1_13265.c, t1_13265.d, t1_13265.ctid, t1_13265.gp_segment_id + -> Materialize + Output: t2_13265.ctid, t2_13265.c, t2_13265.d, (2) + -> Broadcast Motion 1:3 (slice1; segments: 1) Output: t2_13265.ctid, t2_13265.c, t2_13265.d, (2) - -> Broadcast Motion 3:3 (slice1; segments: 3) - Output: t2_13265.ctid, t2_13265.c, t2_13265.d, (2) - -> Seq Scan on public.t2_13265 - Output: t2_13265.ctid, t2_13265.c, t2_13265.d, 2 - Filter: (t2_13265.a = 2) + -> Seq Scan on public.t2_13265 + Output: t2_13265.ctid, t2_13265.c, t2_13265.d, 2 + Filter: (t2_13265.a = 2) Optimizer: Postgres query optimizer Settings: optimizer=on -(17 rows) +(15 rows) update t1_13265 set b = 2 where (c, d) not in (select c, d from t2_13265 where a = 2); From cbb6f6b54160520d0390fc9756d8e95a772c28fe Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Tue, 30 Jan 2024 14:43:20 +0300 Subject: [PATCH 04/83] Add original queries to tests --- src/test/regress/expected/update_gp.out | 72 +++++++++++++++++ .../regress/expected/update_gp_optimizer.out | 78 +++++++++++++++++++ src/test/regress/sql/update_gp.sql | 21 +++++ 3 files changed, 171 insertions(+) diff --git a/src/test/regress/expected/update_gp.out b/src/test/regress/expected/update_gp.out index de2539cf45fb..72b2442fc178 100644 --- a/src/test/regress/expected/update_gp.out +++ b/src/test/regress/expected/update_gp.out @@ -208,6 +208,78 @@ DROP TABLE keo2; DROP TABLE keo3; DROP TABLE keo4; DROP TABLE keo5; +-- Explicit Redistribute Motion shouled be added only if there is a motion +-- between the scan and the ModifyTable on the relation we are going to update. (test case not applicable to ORCA) +CREATE TABLE t1 (i int, j int) DISTRIBUTED BY (i); +CREATE TABLE t2 (i int) DISTRIBUTED BY (i); +CREATE TABLE t_strewn (i int) DISTRIBUTED RANDOMLY; +EXPLAIN (costs off) +UPDATE t1 SET j = t_strewn.i FROM t_strewn WHERE t_strewn.i = t1.i; + QUERY PLAN +------------------------------------------------------------ + Update on t1 + -> Hash Join + Hash Cond: (t_strewn.i = t1.i) + -> Redistribute Motion 3:3 (slice1; segments: 3) + Hash Key: t_strewn.i + -> Seq Scan on t_strewn + -> Hash + -> Seq Scan on t1 + Optimizer: Postgres query optimizer +(9 rows) + +EXPLAIN (costs off) +WITH CTE AS (DELETE FROM t1 RETURNING *) +SELECT count(*) AS a FROM t_strewn JOIN t2 USING (i) +WHERE 0 < ALL (SELECT i FROM cte); + QUERY PLAN +------------------------------------------------------------------------------ + Aggregate + -> Gather Motion 3:1 (slice3; segments: 3) + -> Aggregate + -> Result + One-Time Filter: (SubPlan 1) + -> Hash Join + Hash Cond: (t_strewn.i = t2.i) + -> Redistribute Motion 3:3 (slice1; segments: 3) + Hash Key: t_strewn.i + -> Seq Scan on t_strewn + -> Hash + -> Seq Scan on t2 + SubPlan 1 (slice3; segments: 3) + -> Materialize + -> Broadcast Motion 3:3 (slice2; segments: 3) + -> Subquery Scan on cte + -> Delete on t1 + -> Seq Scan on t1 + Optimizer: Postgres query optimizer +(19 rows) + +EXPLAIN (costs off, verbose) +DELETE FROM t_strewn WHERE t_strewn.i = (SELECT t2.i FROM t2 WHERE t_strewn.i = t2.i); + QUERY PLAN +----------------------------------------------------------------------- + Delete on public.t_strewn + -> Seq Scan on public.t_strewn + Output: t_strewn.ctid, t_strewn.gp_segment_id + Filter: (t_strewn.i = (SubPlan 1)) + SubPlan 1 (slice0; segments: 3) + -> Result + Output: t2.i + Filter: (t_strewn.i = t2.i) + -> Materialize + Output: t2.i, t2.i + -> Broadcast Motion 3:3 (slice1; segments: 3) + Output: t2.i, t2.i + -> Seq Scan on public.t2 + Output: t2.i, t2.i + Optimizer: Postgres query optimizer + Settings: optimizer=off +(16 rows) + +DROP TABLE t1; +DROP TABLE t2; +DROP TABLE t_strewn; -- -- text types. We should support the following updates. -- diff --git a/src/test/regress/expected/update_gp_optimizer.out b/src/test/regress/expected/update_gp_optimizer.out index f14270c3e811..d5557fd37685 100644 --- a/src/test/regress/expected/update_gp_optimizer.out +++ b/src/test/regress/expected/update_gp_optimizer.out @@ -213,6 +213,84 @@ DROP TABLE keo2; DROP TABLE keo3; DROP TABLE keo4; DROP TABLE keo5; +-- Explicit Redistribute Motion shouled be added only if there is a motion +-- between the scan and the ModifyTable on the relation we are going to update. (test case not applicable to ORCA) +CREATE TABLE t1 (i int, j int) DISTRIBUTED BY (i); +CREATE TABLE t2 (i int) DISTRIBUTED BY (i); +CREATE TABLE t_strewn (i int) DISTRIBUTED RANDOMLY; +EXPLAIN (costs off) +UPDATE t1 SET j = t_strewn.i FROM t_strewn WHERE t_strewn.i = t1.i; + QUERY PLAN +------------------------------------------------------------------------------ + Update + -> Redistribute Motion 3:3 (slice2; segments: 3) + Hash Key: t1.i + -> Split + -> Hash Join + Hash Cond: (t1.i = t_strewn.i) + -> Seq Scan on t1 + -> Hash + -> Redistribute Motion 3:3 (slice1; segments: 3) + Hash Key: t_strewn.i + -> Seq Scan on t_strewn + Optimizer: Pivotal Optimizer (GPORCA) +(12 rows) + +EXPLAIN (costs off) +WITH CTE AS (DELETE FROM t1 RETURNING *) +SELECT count(*) AS a FROM t_strewn JOIN t2 USING (i) +WHERE 0 < ALL (SELECT i FROM cte); + QUERY PLAN +------------------------------------------------------------------------------ + Aggregate + -> Gather Motion 3:1 (slice3; segments: 3) + -> Aggregate + -> Result + One-Time Filter: (SubPlan 1) + -> Hash Join + Hash Cond: (t_strewn.i = t2.i) + -> Redistribute Motion 3:3 (slice1; segments: 3) + Hash Key: t_strewn.i + -> Seq Scan on t_strewn + -> Hash + -> Seq Scan on t2 + SubPlan 1 (slice3; segments: 3) + -> Materialize + -> Broadcast Motion 3:3 (slice2; segments: 3) + -> Subquery Scan on cte + -> Delete on t1 + -> Seq Scan on t1 + Optimizer: Postgres query optimizer +(19 rows) + +EXPLAIN (costs off, verbose) +DELETE FROM t_strewn WHERE t_strewn.i = (SELECT t2.i FROM t2 WHERE t_strewn.i = t2.i); + QUERY PLAN +----------------------------------------------------------------------------- + Delete + Output: t_strewn.i, "outer".ColRef_0016, t_strewn.ctid + -> Result + Output: t_strewn.i, t_strewn.ctid, t_strewn.gp_segment_id, 0 + -> Seq Scan on public.t_strewn + Output: t_strewn.i, t_strewn.ctid, t_strewn.gp_segment_id + Filter: (t_strewn.i = (SubPlan 1)) + SubPlan 1 (slice0; segments: 3) + -> Result + Output: t2.i + Filter: (t_strewn.i = t2.i) + -> Materialize + Output: t2.i + -> Broadcast Motion 3:3 (slice1; segments: 3) + Output: t2.i + -> Seq Scan on public.t2 + Output: t2.i + Optimizer: Pivotal Optimizer (GPORCA) + Settings: optimizer=on +(19 rows) + +DROP TABLE t1; +DROP TABLE t2; +DROP TABLE t_strewn; -- -- text types. We should support the following updates. -- diff --git a/src/test/regress/sql/update_gp.sql b/src/test/regress/sql/update_gp.sql index f387738153d3..50c4599039ec 100644 --- a/src/test/regress/sql/update_gp.sql +++ b/src/test/regress/sql/update_gp.sql @@ -117,6 +117,27 @@ DROP TABLE keo3; DROP TABLE keo4; DROP TABLE keo5; +-- Explicit Redistribute Motion shouled be added only if there is a motion +-- between the scan and the ModifyTable on the relation we are going to update. (test case not applicable to ORCA) +CREATE TABLE t1 (i int, j int) DISTRIBUTED BY (i); +CREATE TABLE t2 (i int) DISTRIBUTED BY (i); +CREATE TABLE t_strewn (i int) DISTRIBUTED RANDOMLY; + +EXPLAIN (costs off) +UPDATE t1 SET j = t_strewn.i FROM t_strewn WHERE t_strewn.i = t1.i; + +EXPLAIN (costs off) +WITH CTE AS (DELETE FROM t1 RETURNING *) +SELECT count(*) AS a FROM t_strewn JOIN t2 USING (i) +WHERE 0 < ALL (SELECT i FROM cte); + +EXPLAIN (costs off, verbose) +DELETE FROM t_strewn WHERE t_strewn.i = (SELECT t2.i FROM t2 WHERE t_strewn.i = t2.i); + +DROP TABLE t1; +DROP TABLE t2; +DROP TABLE t_strewn; + -- -- text types. We should support the following updates. -- From 684ac6a2e65b877f01a61e6a8afc7c684a2238f4 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Tue, 30 Jan 2024 14:43:22 +0300 Subject: [PATCH 05/83] Add cases with inherited tables to tests --- src/test/regress/expected/update_gp.out | 153 ++++++++++++++++++ .../regress/expected/update_gp_optimizer.out | 111 +++++++++++++ src/test/regress/sql/update_gp.sql | 54 +++++++ 3 files changed, 318 insertions(+) diff --git a/src/test/regress/expected/update_gp.out b/src/test/regress/expected/update_gp.out index 72b2442fc178..65fd16bf0f52 100644 --- a/src/test/regress/expected/update_gp.out +++ b/src/test/regress/expected/update_gp.out @@ -280,6 +280,159 @@ DELETE FROM t_strewn WHERE t_strewn.i = (SELECT t2.i FROM t2 WHERE t_strewn.i = DROP TABLE t1; DROP TABLE t2; DROP TABLE t_strewn; +-- Explicit Redistribute Motion should not be mistakenly elided for inherited +-- tables. (test case not applicable to ORCA) +CREATE TABLE i (i int, j int) DISTRIBUTED BY (i); +INSERT INTO i SELECT + generate_series(1, 100), generate_series(1, 100) * 3; +CREATE TABLE foo (f1 serial, f2 text, f3 int) DISTRIBUTED RANDOMLY; +INSERT INTO foo (f2, f3) + VALUES ('first', 1), ('second', 2), ('third', 3); +CREATE TABLE foochild (fc int) INHERITS (foo); +NOTICE: table has parent, setting distribution columns to match parent table +INSERT INTO foochild + VALUES(123, 'child', 999, -123); +EXPLAIN (costs off, verbose) +DELETE FROM foo + USING i + WHERE foo.f1 = i.j; + QUERY PLAN +-------------------------------------------------------------------------------------------- + Delete on public.foo + -> Explicit Redistribute Motion 3:3 (slice2; segments: 3) + Output: foo.ctid, foo.gp_segment_id, i.ctid + -> Hash Join + Output: foo.ctid, foo.gp_segment_id, i.ctid + Hash Cond: (i.j = foo.f1) + -> Seq Scan on public.i + Output: i.ctid, i.j + -> Hash + Output: foo.ctid, foo.gp_segment_id, foo.f1 + -> Broadcast Motion 3:3 (slice1; segments: 3) + Output: foo.ctid, foo.gp_segment_id, foo.f1 + -> Seq Scan on public.foo + Output: foo.ctid, foo.gp_segment_id, foo.f1 + -> Explicit Redistribute Motion 3:3 (slice4; segments: 3) + Output: foochild.ctid, foochild.gp_segment_id, i.ctid + -> Hash Join + Output: foochild.ctid, foochild.gp_segment_id, i.ctid + Hash Cond: (i.j = foochild.f1) + -> Seq Scan on public.i + Output: i.ctid, i.j + -> Hash + Output: foochild.ctid, foochild.gp_segment_id, foochild.f1 + -> Broadcast Motion 3:3 (slice3; segments: 3) + Output: foochild.ctid, foochild.gp_segment_id, foochild.f1 + -> Seq Scan on public.foochild + Output: foochild.ctid, foochild.gp_segment_id, foochild.f1 + Optimizer: Postgres query optimizer + Settings: optimizer=off +(29 rows) + +DELETE FROM foo + USING i + WHERE foo.f1 = i.j; +DROP TABLE i; +DROP TABLE foochild; +DROP TABLE foo; +-- Explicit Redistribute Motion should not be mistakenly elided for partitioned +-- tables. (test case not applicable to ORCA) +CREATE TABLE t1 (a int, b int) DISTRIBUTED BY (a) +PARTITION BY + range(b) (start(1) end(101) every(50)); +NOTICE: CREATE TABLE will create partition "t1_1_prt_1" for table "t1" +NOTICE: CREATE TABLE will create partition "t1_1_prt_2" for table "t1" +CREATE TABLE t2 (a int, b int) DISTRIBUTED BY (b) +PARTITION BY + range(a) (start(1) end(101) every(25), default partition def); +NOTICE: CREATE TABLE will create partition "t2_1_prt_def" for table "t2" +NOTICE: CREATE TABLE will create partition "t2_1_prt_2" for table "t2" +NOTICE: CREATE TABLE will create partition "t2_1_prt_3" for table "t2" +NOTICE: CREATE TABLE will create partition "t2_1_prt_4" for table "t2" +NOTICE: CREATE TABLE will create partition "t2_1_prt_5" for table "t2" +INSERT INTO t1 SELECT + generate_series(1, 100) * 3, generate_series(1, 100); +INSERT INTO t2 SELECT + generate_series(1, 100), generate_series(1, 100) * 3; +INSERT INTO t2 VALUES + (generate_series(101, 111), NULL); +EXPLAIN (costs off, verbose) +DELETE FROM t1 USING t2 WHERE t1.a = t2.a; + QUERY PLAN +----------------------------------------------------------------------------------------------------------- + Delete on public.t1_1_prt_1 + -> Explicit Redistribute Motion 3:3 (slice2; segments: 3) + Output: t1_1_prt_1.ctid, t1_1_prt_1.gp_segment_id, t2_1_prt_def.ctid, t2_1_prt_def.tableoid + -> Hash Join + Output: t1_1_prt_1.ctid, t1_1_prt_1.gp_segment_id, t2_1_prt_def.ctid, t2_1_prt_def.tableoid + Hash Cond: (t2_1_prt_def.a = t1_1_prt_1.a) + -> Append + -> Result + Output: t2_1_prt_def.ctid, t2_1_prt_def.tableoid, t2_1_prt_def.a + One-Time Filter: PartSelected + -> Seq Scan on public.t2_1_prt_def + Output: t2_1_prt_def.ctid, t2_1_prt_def.tableoid, t2_1_prt_def.a + -> Result + Output: t2_1_prt_2.ctid, t2_1_prt_2.tableoid, t2_1_prt_2.a + One-Time Filter: PartSelected + -> Seq Scan on public.t2_1_prt_2 + Output: t2_1_prt_2.ctid, t2_1_prt_2.tableoid, t2_1_prt_2.a + -> Result + Output: t2_1_prt_3.ctid, t2_1_prt_3.tableoid, t2_1_prt_3.a + One-Time Filter: PartSelected + -> Seq Scan on public.t2_1_prt_3 + Output: t2_1_prt_3.ctid, t2_1_prt_3.tableoid, t2_1_prt_3.a + -> Result + Output: t2_1_prt_4.ctid, t2_1_prt_4.tableoid, t2_1_prt_4.a + One-Time Filter: PartSelected + -> Seq Scan on public.t2_1_prt_4 + Output: t2_1_prt_4.ctid, t2_1_prt_4.tableoid, t2_1_prt_4.a + -> Result + Output: t2_1_prt_5.ctid, t2_1_prt_5.tableoid, t2_1_prt_5.a + One-Time Filter: PartSelected + -> Seq Scan on public.t2_1_prt_5 + Output: t2_1_prt_5.ctid, t2_1_prt_5.tableoid, t2_1_prt_5.a + -> Hash + Output: t1_1_prt_1.ctid, t1_1_prt_1.gp_segment_id, t1_1_prt_1.a + -> Partition Selector for t2 (dynamic scan id: 2) + Output: t1_1_prt_1.ctid, t1_1_prt_1.gp_segment_id, t1_1_prt_1.a + Filter: t1_1_prt_1.a + -> Broadcast Motion 3:3 (slice1; segments: 3) + Output: t1_1_prt_1.ctid, t1_1_prt_1.gp_segment_id, t1_1_prt_1.a + -> Seq Scan on public.t1_1_prt_1 + Output: t1_1_prt_1.ctid, t1_1_prt_1.gp_segment_id, t1_1_prt_1.a + -> Explicit Redistribute Motion 3:3 (slice4; segments: 3) + Output: t1_1_prt_2.ctid, t1_1_prt_2.gp_segment_id, t2_1_prt_def.ctid, t2_1_prt_def.tableoid + -> Hash Join + Output: t1_1_prt_2.ctid, t1_1_prt_2.gp_segment_id, t2_1_prt_def.ctid, t2_1_prt_def.tableoid + Hash Cond: (t2_1_prt_def.a = t1_1_prt_2.a) + -> Append + -> Seq Scan on public.t2_1_prt_def + Output: t2_1_prt_def.ctid, t2_1_prt_def.tableoid, t2_1_prt_def.a + -> Seq Scan on public.t2_1_prt_2 + Output: t2_1_prt_2.ctid, t2_1_prt_2.tableoid, t2_1_prt_2.a + -> Seq Scan on public.t2_1_prt_3 + Output: t2_1_prt_3.ctid, t2_1_prt_3.tableoid, t2_1_prt_3.a + -> Seq Scan on public.t2_1_prt_4 + Output: t2_1_prt_4.ctid, t2_1_prt_4.tableoid, t2_1_prt_4.a + -> Seq Scan on public.t2_1_prt_5 + Output: t2_1_prt_5.ctid, t2_1_prt_5.tableoid, t2_1_prt_5.a + -> Hash + Output: t1_1_prt_2.ctid, t1_1_prt_2.gp_segment_id, t1_1_prt_2.a + -> Partition Selector for t2 (dynamic scan id: 2) + Output: t1_1_prt_2.ctid, t1_1_prt_2.gp_segment_id, t1_1_prt_2.a + Filter: t1_1_prt_2.a + -> Broadcast Motion 3:3 (slice3; segments: 3) + Output: t1_1_prt_2.ctid, t1_1_prt_2.gp_segment_id, t1_1_prt_2.a + -> Seq Scan on public.t1_1_prt_2 + Output: t1_1_prt_2.ctid, t1_1_prt_2.gp_segment_id, t1_1_prt_2.a + Optimizer: Postgres query optimizer + Settings: optimizer=off +(68 rows) + +DELETE FROM t1 USING t2 WHERE t1.a = t2.a; +DROP TABLE t1; +DROP TABLE t2; -- -- text types. We should support the following updates. -- diff --git a/src/test/regress/expected/update_gp_optimizer.out b/src/test/regress/expected/update_gp_optimizer.out index d5557fd37685..c4cf892ddea6 100644 --- a/src/test/regress/expected/update_gp_optimizer.out +++ b/src/test/regress/expected/update_gp_optimizer.out @@ -291,6 +291,117 @@ DELETE FROM t_strewn WHERE t_strewn.i = (SELECT t2.i FROM t2 WHERE t_strewn.i = DROP TABLE t1; DROP TABLE t2; DROP TABLE t_strewn; +-- Explicit Redistribute Motion should not be mistakenly elided for inherited +-- tables. (test case not applicable to ORCA) +CREATE TABLE i (i int, j int) DISTRIBUTED BY (i); +INSERT INTO i SELECT + generate_series(1, 100), generate_series(1, 100) * 3; +CREATE TABLE foo (f1 serial, f2 text, f3 int) DISTRIBUTED RANDOMLY; +INSERT INTO foo (f2, f3) + VALUES ('first', 1), ('second', 2), ('third', 3); +CREATE TABLE foochild (fc int) INHERITS (foo); +NOTICE: table has parent, setting distribution columns to match parent table +INSERT INTO foochild + VALUES(123, 'child', 999, -123); +EXPLAIN (costs off, verbose) +DELETE FROM foo + USING i + WHERE foo.f1 = i.j; + QUERY PLAN +-------------------------------------------------------------------------------------------- + Delete on public.foo + -> Explicit Redistribute Motion 3:3 (slice2; segments: 3) + Output: foo.ctid, foo.gp_segment_id, i.ctid + -> Hash Join + Output: foo.ctid, foo.gp_segment_id, i.ctid + Hash Cond: (i.j = foo.f1) + -> Seq Scan on public.i + Output: i.ctid, i.j + -> Hash + Output: foo.ctid, foo.gp_segment_id, foo.f1 + -> Broadcast Motion 3:3 (slice1; segments: 3) + Output: foo.ctid, foo.gp_segment_id, foo.f1 + -> Seq Scan on public.foo + Output: foo.ctid, foo.gp_segment_id, foo.f1 + -> Explicit Redistribute Motion 3:3 (slice4; segments: 3) + Output: foochild.ctid, foochild.gp_segment_id, i.ctid + -> Hash Join + Output: foochild.ctid, foochild.gp_segment_id, i.ctid + Hash Cond: (i.j = foochild.f1) + -> Seq Scan on public.i + Output: i.ctid, i.j + -> Hash + Output: foochild.ctid, foochild.gp_segment_id, foochild.f1 + -> Broadcast Motion 3:3 (slice3; segments: 3) + Output: foochild.ctid, foochild.gp_segment_id, foochild.f1 + -> Seq Scan on public.foochild + Output: foochild.ctid, foochild.gp_segment_id, foochild.f1 + Optimizer: Postgres query optimizer + Settings: optimizer=on +(29 rows) + +DELETE FROM foo + USING i + WHERE foo.f1 = i.j; +DROP TABLE i; +DROP TABLE foochild; +DROP TABLE foo; +-- Explicit Redistribute Motion should not be mistakenly elided for partitioned +-- tables. (test case not applicable to ORCA) +CREATE TABLE t1 (a int, b int) DISTRIBUTED BY (a) +PARTITION BY + range(b) (start(1) end(101) every(50)); +NOTICE: CREATE TABLE will create partition "t1_1_prt_1" for table "t1" +NOTICE: CREATE TABLE will create partition "t1_1_prt_2" for table "t1" +CREATE TABLE t2 (a int, b int) DISTRIBUTED BY (b) +PARTITION BY + range(a) (start(1) end(101) every(25), default partition def); +NOTICE: CREATE TABLE will create partition "t2_1_prt_def" for table "t2" +NOTICE: CREATE TABLE will create partition "t2_1_prt_2" for table "t2" +NOTICE: CREATE TABLE will create partition "t2_1_prt_3" for table "t2" +NOTICE: CREATE TABLE will create partition "t2_1_prt_4" for table "t2" +NOTICE: CREATE TABLE will create partition "t2_1_prt_5" for table "t2" +INSERT INTO t1 SELECT + generate_series(1, 100) * 3, generate_series(1, 100); +INSERT INTO t2 SELECT + generate_series(1, 100), generate_series(1, 100) * 3; +INSERT INTO t2 VALUES + (generate_series(101, 111), NULL); +EXPLAIN (costs off, verbose) +DELETE FROM t1 USING t2 WHERE t1.a = t2.a; + QUERY PLAN +---------------------------------------------------------------------------------------- + Delete + Output: t1.a, t1.b, "outer".ColRef_0018, t1.ctid + -> Result + Output: t1.a, t1.b, t1.ctid, t1.gp_segment_id, 0 + -> Hash Join + Output: t1.a, t1.b, t1.ctid, t1.gp_segment_id + Hash Cond: (t1.a = t2.a) + -> Sequence + Output: t1.a, t1.b, t1.ctid, t1.gp_segment_id + -> Partition Selector for t1 (dynamic scan id: 1) + Partitions selected: 2 (out of 2) + -> Dynamic Seq Scan on public.t1 (dynamic scan id: 1) + Output: t1.a, t1.b, t1.ctid, t1.gp_segment_id + -> Hash + Output: t2.a + -> Redistribute Motion 3:3 (slice1; segments: 3) + Output: t2.a + Hash Key: t2.a + -> Sequence + Output: t2.a + -> Partition Selector for t2 (dynamic scan id: 2) + Partitions selected: 5 (out of 5) + -> Dynamic Seq Scan on public.t2 (dynamic scan id: 2) + Output: t2.a + Optimizer: Pivotal Optimizer (GPORCA) + Settings: optimizer=on +(26 rows) + +DELETE FROM t1 USING t2 WHERE t1.a = t2.a; +DROP TABLE t1; +DROP TABLE t2; -- -- text types. We should support the following updates. -- diff --git a/src/test/regress/sql/update_gp.sql b/src/test/regress/sql/update_gp.sql index 50c4599039ec..c28463d7abb7 100644 --- a/src/test/regress/sql/update_gp.sql +++ b/src/test/regress/sql/update_gp.sql @@ -138,6 +138,60 @@ DROP TABLE t1; DROP TABLE t2; DROP TABLE t_strewn; +-- Explicit Redistribute Motion should not be mistakenly elided for inherited +-- tables. (test case not applicable to ORCA) +CREATE TABLE i (i int, j int) DISTRIBUTED BY (i); +INSERT INTO i SELECT + generate_series(1, 100), generate_series(1, 100) * 3; + +CREATE TABLE foo (f1 serial, f2 text, f3 int) DISTRIBUTED RANDOMLY; + +INSERT INTO foo (f2, f3) + VALUES ('first', 1), ('second', 2), ('third', 3); + +CREATE TABLE foochild (fc int) INHERITS (foo); + +INSERT INTO foochild + VALUES(123, 'child', 999, -123); + +EXPLAIN (costs off, verbose) +DELETE FROM foo + USING i + WHERE foo.f1 = i.j; + +DELETE FROM foo + USING i + WHERE foo.f1 = i.j; + +DROP TABLE i; +DROP TABLE foochild; +DROP TABLE foo; + +-- Explicit Redistribute Motion should not be mistakenly elided for partitioned +-- tables. (test case not applicable to ORCA) +CREATE TABLE t1 (a int, b int) DISTRIBUTED BY (a) +PARTITION BY + range(b) (start(1) end(101) every(50)); + +CREATE TABLE t2 (a int, b int) DISTRIBUTED BY (b) +PARTITION BY + range(a) (start(1) end(101) every(25), default partition def); + +INSERT INTO t1 SELECT + generate_series(1, 100) * 3, generate_series(1, 100); +INSERT INTO t2 SELECT + generate_series(1, 100), generate_series(1, 100) * 3; +INSERT INTO t2 VALUES + (generate_series(101, 111), NULL); + +EXPLAIN (costs off, verbose) +DELETE FROM t1 USING t2 WHERE t1.a = t2.a; + +DELETE FROM t1 USING t2 WHERE t1.a = t2.a; + +DROP TABLE t1; +DROP TABLE t2; + -- -- text types. We should support the following updates. -- From da58c4349aa3435a8273c18527bddbf6201b3081 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Tue, 30 Jan 2024 14:43:23 +0300 Subject: [PATCH 06/83] Clean up --- src/backend/optimizer/plan/createplan.c | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c index 99fed6d7c06c..9268c1ee931d 100644 --- a/src/backend/optimizer/plan/createplan.c +++ b/src/backend/optimizer/plan/createplan.c @@ -6492,8 +6492,6 @@ static bool can_elide_explicit_motion_recurse(Plan *plan, List *rtable, Oid relid, bool relid_is_subclass, Oid reltypeid) { - RangeTblEntry *target_rte; - while (plan) { /* @@ -6506,13 +6504,12 @@ can_elide_explicit_motion_recurse(Plan *plan, List *rtable, Oid relid, if ((motion->motionType == MOTIONTYPE_FIXED && motion->isBroadcast) || motion->motionType == MOTIONTYPE_HASH) { - TargetEntry *tle; ListCell *lcr; foreach(lcr, plan->targetlist) { - tle = (TargetEntry *) lfirst(lcr); - target_rte = rt_fetch(tle->resno, rtable); + TargetEntry *tle = (TargetEntry *) lfirst(lcr); + RangeTblEntry *target_rte = rt_fetch(tle->resno, rtable); if (relid_is_subclass) { @@ -6540,8 +6537,6 @@ can_elide_explicit_motion_recurse(Plan *plan, List *rtable, Oid relid, */ else { - Scan *scan; - switch (nodeTag(plan)) { case T_SeqScan: @@ -6553,14 +6548,14 @@ can_elide_explicit_motion_recurse(Plan *plan, List *rtable, Oid relid, case T_BitmapIndexScan: case T_DynamicBitmapIndexScan: case T_BitmapHeapScan: - case T_DynamicBitmapHeapScan: - scan = (Scan *) plan; - target_rte = rt_fetch(scan->scanrelid, rtable); + case T_DynamicBitmapHeapScan: { + Scan *scan = (Scan *) plan; + RangeTblEntry *target_rte = rt_fetch(scan->scanrelid, rtable); if (target_rte->relid == relid) /* There wasn't any Motions before scan */ return true; - break; + } break; default: break; } @@ -6600,10 +6595,7 @@ can_elide_explicit_motion(Plan *plan, List *rtable, Oid relid) relid_is_subclass = has_superclass(relid); UnlockRelationOid(relid, AccessShareLock); - if (relid_is_subclass) - reltypeid = get_rel_type_id(relid); - else - reltypeid = InvalidOid; + reltypeid = relid_is_subclass ? get_rel_type_id(relid) : InvalidOid; return can_elide_explicit_motion_recurse(plan, rtable, relid, relid_is_subclass, reltypeid); From 208903bc4f6ed327d592fc2a0d5bd15cbacb8a41 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Tue, 30 Jan 2024 14:43:24 +0300 Subject: [PATCH 07/83] Apply remarks --- src/backend/optimizer/plan/createplan.c | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c index 9268c1ee931d..a0c372943926 100644 --- a/src/backend/optimizer/plan/createplan.c +++ b/src/backend/optimizer/plan/createplan.c @@ -6518,16 +6518,20 @@ can_elide_explicit_motion_recurse(Plan *plan, List *rtable, Oid relid, /* * Does relid inherit from a table in targetlist? */ - if (target_reltypeid != InvalidOid && reltypeid != InvalidOid && + if (OidIsValid(target_reltypeid) && OidIsValid(reltypeid) && typeInheritsFrom(reltypeid, target_reltypeid)) + { /* There is a Motion on parent table before scan on the child */ return false; + } } if (target_rte->relid == relid) + { /* There is a Motion before scan */ return false; + } } } } @@ -6548,14 +6552,26 @@ can_elide_explicit_motion_recurse(Plan *plan, List *rtable, Oid relid, case T_BitmapIndexScan: case T_DynamicBitmapIndexScan: case T_BitmapHeapScan: - case T_DynamicBitmapHeapScan: { + case T_DynamicBitmapHeapScan: + case T_TidScan: + case T_SubqueryScan: + case T_FunctionScan: + case T_TableFunctionScan: + case T_ValuesScan: + case T_CteScan: + case T_WorkTableScan: + case T_ForeignScan: + { Scan *scan = (Scan *) plan; RangeTblEntry *target_rte = rt_fetch(scan->scanrelid, rtable); if (target_rte->relid == relid) + { /* There wasn't any Motions before scan */ return true; - } break; + } + break; + } default: break; } From 848cc16b42cf385e0b86e91ff6ba73333648d313 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Tue, 30 Jan 2024 14:43:25 +0300 Subject: [PATCH 08/83] Run pgindent --- src/backend/optimizer/plan/createplan.c | 56 +++++++++++++------------ 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c index a0c372943926..bbe631792ec7 100644 --- a/src/backend/optimizer/plan/createplan.c +++ b/src/backend/optimizer/plan/createplan.c @@ -6499,12 +6499,12 @@ can_elide_explicit_motion_recurse(Plan *plan, List *rtable, Oid relid, */ if (IsA(plan, Motion)) { - Motion *motion = (Motion *) plan; + Motion *motion = (Motion *) plan; if ((motion->motionType == MOTIONTYPE_FIXED && motion->isBroadcast) || motion->motionType == MOTIONTYPE_HASH) { - ListCell *lcr; + ListCell *lcr; foreach(lcr, plan->targetlist) { @@ -6513,7 +6513,7 @@ can_elide_explicit_motion_recurse(Plan *plan, List *rtable, Oid relid, if (relid_is_subclass) { - Oid target_reltypeid = get_rel_type_id(target_rte->relid); + Oid target_reltypeid = get_rel_type_id(target_rte->relid); /* * Does relid inherit from a table in targetlist? @@ -6521,8 +6521,10 @@ can_elide_explicit_motion_recurse(Plan *plan, List *rtable, Oid relid, if (OidIsValid(target_reltypeid) && OidIsValid(reltypeid) && typeInheritsFrom(reltypeid, target_reltypeid)) { - /* There is a Motion on parent table before scan on - the child */ + /* + * There is a Motion on parent table before scan + * on the child + */ return false; } } @@ -6535,9 +6537,10 @@ can_elide_explicit_motion_recurse(Plan *plan, List *rtable, Oid relid, } } } + /* - * If this is a scan and it's scanrelid matches relid, we encountered a - * scan before any Motions. + * If this is a scan and it's scanrelid matches relid, we encountered + * a scan before any Motions. */ else { @@ -6561,17 +6564,17 @@ can_elide_explicit_motion_recurse(Plan *plan, List *rtable, Oid relid, case T_CteScan: case T_WorkTableScan: case T_ForeignScan: - { - Scan *scan = (Scan *) plan; - RangeTblEntry *target_rte = rt_fetch(scan->scanrelid, rtable); - - if (target_rte->relid == relid) { - /* There wasn't any Motions before scan */ - return true; + Scan *scan = (Scan *) plan; + RangeTblEntry *target_rte = rt_fetch(scan->scanrelid, rtable); + + if (target_rte->relid == relid) + { + /* There wasn't any Motions before scan */ + return true; + } } break; - } default: break; } @@ -6599,13 +6602,14 @@ can_elide_explicit_motion_recurse(Plan *plan, List *rtable, Oid relid, static bool can_elide_explicit_motion(Plan *plan, List *rtable, Oid relid) { - bool relid_is_subclass; - Oid reltypeid; + bool relid_is_subclass; + Oid reltypeid; /* - * Even if previous Motions were performed on a leaf partition or inherited - * table, targetlists from Motions refer to relids of their parents. So, if - * relid has a superclass, we should check for type inheritance too. + * Even if previous Motions were performed on a leaf partition or + * inherited table, targetlists from Motions refer to relids of their + * parents. So, if relid has a superclass, we should check for type + * inheritance too. */ LockRelationOid(relid, AccessShareLock); relid_is_subclass = has_superclass(relid); @@ -6874,13 +6878,13 @@ adjust_modifytable_flow(PlannerInfo *root, ModifyTable *node, List *is_split_upd node->oid_col_idxes = lappend_int(node->oid_col_idxes, 0); /* - * If an Explicit Motion has no Motions underneath it, then - * the row to update must originate from the same segment, - * and no Motion is needed. + * If an Explicit Motion has no Motions underneath it, + * then the row to update must originate from the same + * segment, and no Motion is needed. * - * We elide the motion even if there are Motions, as long as - * they are not between the scan on the target table and the - * ModifyTable. + * We elide the motion even if there are Motions, as long + * as they are not between the scan on the target table + * and the ModifyTable. */ if (!can_elide_explicit_motion(subplan, root->parse->rtable, rte->relid)) request_explicit_motion(subplan, rti, root->glob->finalrtable); From b24fea9bc4ea17d1be94ebfff5f471966d04fdb2 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Tue, 30 Jan 2024 14:43:26 +0300 Subject: [PATCH 09/83] Look into hashExprs for Redistribute Motion While checking the assumption that the right subtree is more frequent, I found out that target list of `Redistribute Motion` may contain columns which are not used for distribution. Behavior was correct only because the right subtree is checked first. In cases when the scan would be in the left subtree, this is incorrect, and `hashExprs` need to be checked instead. Checking `hashExprs` may contain different structures that inherit `Expr` node. The structure we need is of type `Var`, because it's the only child of `Expr` node that we can get relation ID from. --- src/backend/optimizer/plan/createplan.c | 92 ++++++++++++++++++++----- 1 file changed, 74 insertions(+), 18 deletions(-) diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c index bbe631792ec7..bcbc5cc6fc10 100644 --- a/src/backend/optimizer/plan/createplan.c +++ b/src/backend/optimizer/plan/createplan.c @@ -6490,30 +6490,30 @@ make_modifytable(PlannerInfo *root, */ static bool can_elide_explicit_motion_recurse(Plan *plan, List *rtable, Oid relid, - bool relid_is_subclass, Oid reltypeid) + Oid reltypeid, bool *early_exit) { - while (plan) + while (plan && !(*early_exit)) { - /* - * If this is a Motion, check Oids from the Motion's targetlist. - */ if (IsA(plan, Motion)) { - Motion *motion = (Motion *) plan; + Motion *motion = (Motion *) plan; - if ((motion->motionType == MOTIONTYPE_FIXED && motion->isBroadcast) || - motion->motionType == MOTIONTYPE_HASH) + /* + * If this is a Broadcast Motion, check Oids from the Motion's + * targetlist. + */ + if (motion->motionType == MOTIONTYPE_FIXED && motion->isBroadcast) { - ListCell *lcr; + ListCell *lcr; foreach(lcr, plan->targetlist) { TargetEntry *tle = (TargetEntry *) lfirst(lcr); RangeTblEntry *target_rte = rt_fetch(tle->resno, rtable); - if (relid_is_subclass) + if (OidIsValid(reltypeid)) { - Oid target_reltypeid = get_rel_type_id(target_rte->relid); + Oid target_reltypeid = get_rel_type_id(target_rte->relid); /* * Does relid inherit from a table in targetlist? @@ -6536,8 +6536,62 @@ can_elide_explicit_motion_recurse(Plan *plan, List *rtable, Oid relid, } } } - } + /* + * If this is a Redistribute Motion, get relation ID from Motion's + * hashExprs. We can only get relation ID out of Var nodes, so we + * bail out early when we encounter any other expression node. + */ + else if (motion->motionType == MOTIONTYPE_HASH) + { + ListCell *lcr; + + foreach(lcr, motion->hashExprs) + { + Node *expr = (Node *) lfirst(lcr); + Oid expr_relid = InvalidOid; + + if (IsA(expr, Var)) + { + Var *var = (Var *) expr; + RangeTblEntry *rte = rt_fetch(var->varno, rtable); + expr_relid = rte->relid; + } + else + { + /* + * We cant't get relation ID here. Bail out before we + * hit a scan. + */ + *early_exit = true; + return false; + } + + if (OidIsValid(reltypeid)) + { + Oid expr_reltypeid = get_rel_type_id(expr_relid); + /* + * Does relid inherit from this table? + */ + if (OidIsValid(expr_reltypeid) && OidIsValid(reltypeid) && + typeInheritsFrom(reltypeid, expr_reltypeid)) + { + /* + * There is a Motion on parent table before scan + * on the child + */ + return false; + } + } + + if (relid == expr_relid) + { + /* There is a Motion before scan */ + return false; + } + } + } + } /* * If this is a scan and it's scanrelid matches relid, we encountered * a scan before any Motions. @@ -6565,7 +6619,7 @@ can_elide_explicit_motion_recurse(Plan *plan, List *rtable, Oid relid, case T_WorkTableScan: case T_ForeignScan: { - Scan *scan = (Scan *) plan; + Scan *scan = (Scan *) plan; RangeTblEntry *target_rte = rt_fetch(scan->scanrelid, rtable); if (target_rte->relid == relid) @@ -6585,7 +6639,7 @@ can_elide_explicit_motion_recurse(Plan *plan, List *rtable, Oid relid, */ if (plan->righttree && can_elide_explicit_motion_recurse(plan->righttree, rtable, relid, - relid_is_subclass, reltypeid)) + reltypeid, early_exit)) return true; plan = plan->lefttree; @@ -6602,8 +6656,10 @@ can_elide_explicit_motion_recurse(Plan *plan, List *rtable, Oid relid, static bool can_elide_explicit_motion(Plan *plan, List *rtable, Oid relid) { - bool relid_is_subclass; - Oid reltypeid; + bool relid_is_subclass; + Oid reltypeid; + + bool early_exit = false; /* * Even if previous Motions were performed on a leaf partition or @@ -6617,8 +6673,8 @@ can_elide_explicit_motion(Plan *plan, List *rtable, Oid relid) reltypeid = relid_is_subclass ? get_rel_type_id(relid) : InvalidOid; - return can_elide_explicit_motion_recurse(plan, rtable, relid, - relid_is_subclass, reltypeid); + return can_elide_explicit_motion_recurse(plan, rtable, relid, reltypeid, + &early_exit); } /* From 21448723773c627691124dc64b09efc8283db799 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Tue, 30 Jan 2024 14:43:48 +0300 Subject: [PATCH 10/83] Move the logic to cdbmutate.c --- src/backend/cdb/cdbmutate.c | 317 ++++++++++++++++++++---- src/backend/optimizer/plan/createplan.c | 218 +--------------- 2 files changed, 270 insertions(+), 265 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index 67d3535a1011..f2a86bb3a0c1 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -46,6 +46,7 @@ #include "catalog/gp_policy.h" #include "catalog/pg_operator.h" #include "catalog/pg_type.h" +#include "catalog/pg_inherits_fn.h" /* typeInheritsFrom() */ #include "catalog/pg_proc.h" #include "catalog/pg_trigger.h" @@ -63,6 +64,7 @@ #include "cdb/cdbtargeteddispatch.h" #include "executor/executor.h" +#include "storage/lmgr.h" /* LockRelationOid(), UnlockRelationOid() */ /* * An ApplyMotionState holds state for the recursive apply_motion_mutator(). @@ -75,8 +77,15 @@ typedef struct ApplyMotionState * plan_tree_walker/mutator */ int nextMotionID; int sliceDepth; - bool containMotionNodes; HTAB *planid_subplans; /* hash table for InitPlanItem */ + + /* Context for ModifyTable to elide Explicit Redistribute Motion */ + List *mtResultRelids; /* Oid list of relations to be modified */ + List *mtResultReltypeids; /* Oid list of relation's types. If a + * relation isn't a subclass, it's + * InvalidOid */ + bool mtNeedExplicitMotion; + bool mtIsChecking; } ApplyMotionState; typedef struct InitPlanItem @@ -404,7 +413,12 @@ apply_motion(PlannerInfo *root, Plan *plan, Query *query) * plan */ state.nextMotionID = 1; /* Start at 1 so zero will mean "unassigned". */ state.sliceDepth = 0; - state.containMotionNodes = false; + + state.mtResultRelids = NIL; + state.mtResultReltypeids = NIL; + state.mtNeedExplicitMotion = false; + state.mtIsChecking = false; + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(int); ctl.entrysize = sizeof(InitPlanItem); @@ -729,7 +743,6 @@ apply_motion(PlannerInfo *root, Plan *plan, Query *query) return result; } - /* * Function apply_motion_mutator() is the workhorse for apply_motion(). */ @@ -751,43 +764,265 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) if (node == NULL) return NULL; - /* An expression node might have subtrees containing plans to be mutated. */ - if (!is_plan_node(node)) + /* + * For UPDATE/DELETE, we check if there's any Redistribute or Broadcast + * Motions before scan in the same subtree for the table we're going to + * modify. If we encounter the scan before any motions, then we can elide + * unneccessary Explicit Redistribute Motion. + */ + if (IsA(node, ModifyTable)) { - /* - * The containMotionNodes flag keeps track of whether there are any - * Motion nodes, ignoring any in InitPlans. So if we recurse into an - * InitPlan, save and restore the flag. - */ - if (IsA(node, SubPlan) &&((SubPlan *) node)->is_initplan) + ModifyTable *mt = (ModifyTable *) node; + + if (mt->operation == CMD_UPDATE || mt->operation == CMD_DELETE) + { + ListCell *lcr; + + context->mtResultRelids = NIL; + context->mtResultReltypeids = NIL; + + /* + * Make lists of resulting relations' Oids + */ + foreach(lcr, mt->resultRelations) + { + Oid reltypeid; + + Index rti = lfirst_int(lcr); + RangeTblEntry *rte = rt_fetch(rti, root->glob->finalrtable); + bool is_subclass = false; + + LockRelationOid(rte->relid, AccessShareLock); + is_subclass = has_superclass(rte->relid); + UnlockRelationOid(rte->relid, AccessShareLock); + + reltypeid = is_subclass ? get_rel_type_id(rte->relid) : InvalidOid; + + context->mtResultRelids = lappend_oid(context->mtResultRelids, + rte->relid); + context->mtResultReltypeids = lappend_oid(context->mtResultReltypeids, + reltypeid); + } + + context->mtIsChecking = true; + } + } + + /* + * Elide Explicit Redistribute Motion if there's no motions between the scan + * and the ModifyTable that affect the relation we are going to update. + */ + if (context->mtIsChecking) + { + + if (IsA(node, Motion)) { - bool found; - bool saveContainMotionNodes = context->containMotionNodes; - int saveSliceDepth = context->sliceDepth; - SubPlan *subplan = (SubPlan *) node; + Motion *motion = (Motion *) node; + List *rtable = root->glob->finalrtable; + /* - * If the init-plan refered by `subplan` has been visited, we should - * not re-visit the subplan, or the motions under the init-plan are - * re-counted. + * If this is a Redistribute Motion, get relation ID from Motion's + * hashExprs. We can only get relation ID out of Var nodes, so we + * bail out early when we encounter any other expression node. */ - hash_search(context->planid_subplans, &subplan->plan_id, - HASH_FIND, &found); - if (found) - return node; + if (motion->motionType == MOTIONTYPE_HASH) + { + ListCell *expr_lce; + + foreach (expr_lce, motion->hashExprs) + { + ListCell *mt_lcr; + ListCell *mt_lct; + + Node *expr = (Node *) lfirst(expr_lce); + Oid expr_relid = InvalidOid; + + if (!context->mtIsChecking) + break; + + if (IsA(expr, Var)) + { + RangeTblEntry *expr_rte; + Var *var = (Var *) expr; + + if (var->varnoold > 0 && + var->varnoold <= list_length(rtable)) + { + expr_rte = rt_fetch(var->varnoold, rtable); + expr_relid = expr_rte->relid; + } + } + + if (!OidIsValid(expr_relid)) + { + /* We can't get relation ID easily */ + context->mtIsChecking = false; + context->mtNeedExplicitMotion = true; + break; + } + + forboth(mt_lcr, context->mtResultRelids, mt_lct, context->mtResultReltypeids) + { + Oid mt_relid = lfirst_oid(mt_lcr); + Oid mt_reltypeid = lfirst_oid(mt_lct); + + if (OidIsValid(mt_reltypeid)) + { + Oid expr_reltypeid = get_rel_type_id(expr_relid); + + if (OidIsValid(expr_reltypeid) && + typeInheritsFrom(mt_reltypeid, expr_reltypeid)) + { + /* + * There is a Motion on parent table before scan + * on the child + */ + context->mtIsChecking = false; + context->mtNeedExplicitMotion = true; + break; + } + } + + if (expr_relid == mt_relid) + { + /* There is a Motion before scan */ + context->mtIsChecking = false; + context->mtNeedExplicitMotion = true; + break; + } + } + } + } + /* + * If this is a Broadcast Motion, check Oids from the Motion's + * targetlist. + */ + else if (motion->motionType == MOTIONTYPE_FIXED && motion->isBroadcast) + { + ListCell *target_lcr; + + foreach(target_lcr, motion->plan.targetlist) + { + ListCell *mt_lcr; + ListCell *mt_lct; + + TargetEntry *target_tle = (TargetEntry *) lfirst(target_lcr); + Oid expr_relid = InvalidOid; + + if (!context->mtIsChecking) + break; + + if (IsA(target_tle->expr, Var)) + { + RangeTblEntry *expr_rte; + Var *var = (Var *) target_tle->expr; + + if (var->varnoold > 0 && + var->varnoold <= list_length(rtable)) + { + expr_rte = rt_fetch(var->varnoold, rtable); + expr_relid = expr_rte->relid; + } + } + + if (!OidIsValid(expr_relid)) + { + /* We can't get relation ID easily */ + context->mtIsChecking = false; + context->mtNeedExplicitMotion = true; + break; + } - /* reset sliceDepth for each init plan */ - context->sliceDepth = 0; - node = plan_tree_mutator(node, apply_motion_mutator, context); + forboth(mt_lcr, context->mtResultRelids, mt_lct, context->mtResultReltypeids) + { + Oid mt_relid = lfirst_oid(mt_lcr); + Oid mt_reltypeid = lfirst_oid(mt_lct); - context->containMotionNodes = saveContainMotionNodes; - context->sliceDepth = saveSliceDepth; + if (OidIsValid(mt_reltypeid)) + { + Oid target_reltypeid = get_rel_type_id(expr_relid); + + if (OidIsValid(target_reltypeid) && + typeInheritsFrom(mt_reltypeid, target_reltypeid)) + { + /* + * There is a Motion on parent table before scan + * on the child + */ + context->mtIsChecking = false; + context->mtNeedExplicitMotion = true; + break; + } + } - return node; + if (expr_relid == mt_relid) + { + /* There is a Motion before scan */ + context->mtIsChecking = false; + context->mtNeedExplicitMotion = true; + break; + } + } + } + } } + /* + * If this is a scan and it's scanrelid matches ModifyTable's relid, we + * encountered a scan before any Motions. + */ else - return plan_tree_mutator(node, apply_motion_mutator, context); + { + switch (nodeTag(node)) + { + case T_SeqScan: + case T_DynamicSeqScan: + case T_ExternalScan: + case T_IndexScan: + case T_DynamicIndexScan: + case T_IndexOnlyScan: + case T_BitmapIndexScan: + case T_DynamicBitmapIndexScan: + case T_BitmapHeapScan: + case T_DynamicBitmapHeapScan: + case T_TidScan: + case T_SubqueryScan: + case T_FunctionScan: + case T_TableFunctionScan: + case T_ValuesScan: + case T_CteScan: + case T_WorkTableScan: + case T_ForeignScan: + { + ListCell *mt_lcr; + + Scan *scan = (Scan *) node; + List *rtable = root->glob->finalrtable; + RangeTblEntry *target_rte = rt_fetch(scan->scanrelid, rtable); + + foreach(mt_lcr, context->mtResultRelids) + { + Oid mt_relid = lfirst_oid(mt_lcr); + + if (target_rte->relid == mt_relid) + { + /* There wasn't any Motions before scan */ + context->mtIsChecking = false; + context->mtNeedExplicitMotion = false; + break; + } + } + } + break; + default: + break; + } + } } + /* An expression node might have subtrees containing plans to be mutated. */ + if (!is_plan_node(node)) + return plan_tree_mutator(node, apply_motion_mutator, context); + plan = (Plan *) node; flow = plan->flow; @@ -936,20 +1171,15 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) break; case MOVEMENT_EXPLICIT: - /* * add an ExplicitRedistribute motion node only if child plan - * nodes have a motion node + * nodes have a motion node between ModifyTable and the scan */ - if (context->containMotionNodes) + if (context->mtNeedExplicitMotion) { - /* - * motion node in child nodes: add a ExplicitRedistribute - * motion - */ newnode = (Node *) make_explicit_motion(plan, flow->segidColIdx, - true /* useExecutorVarFormat */ + true /* useExecutorVarFormat */ ); } else @@ -962,6 +1192,8 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) flow->flow_before_req_move = NULL; } + /* Continue checking in case another Explicit Motion is needed */ + context->mtIsChecking = true; break; case MOVEMENT_NONE: @@ -1009,17 +1241,6 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) plan->nMotionNodes = context->nextMotionID - saveNextMotionID; plan->nInitPlans = hash_get_num_entries(context->planid_subplans) - saveNumInitPlans; - /* - * Remember if this was a Motion node. This is used at the top of the - * tree, with MOVEMENT_EXPLICIT, to avoid adding an explicit motion, if - * there were no Motion in the subtree. Note that this does not take - * InitPlans containing Motion nodes into account. InitPlans are executed - * as a separate step before the main plan, and hence any Motion nodes in - * them don't need to affect the way the main plan is executed. - */ - if (IsA(newnode, Motion)) - context->containMotionNodes = true; - return newnode; } /* apply_motion_mutator */ diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c index bcbc5cc6fc10..d7f3a7d01721 100644 --- a/src/backend/optimizer/plan/createplan.c +++ b/src/backend/optimizer/plan/createplan.c @@ -22,7 +22,6 @@ #include #include "catalog/pg_exttable.h" -#include "catalog/pg_inherits_fn.h" #include "access/skey.h" #include "access/sysattr.h" #include "catalog/pg_class.h" @@ -55,7 +54,6 @@ #include "utils/guc.h" #include "utils/lsyscache.h" #include "utils/uri.h" -#include "storage/lmgr.h" /* LockRelationOid(), UnlockRelationOid() */ #include "cdb/cdbllize.h" /* pull_up_Flow() */ #include "cdb/cdbmutate.h" @@ -173,7 +171,6 @@ static WorkTableScan *make_worktablescan(List *qptlist, List *qpqual, static BitmapAnd *make_bitmap_and(List *bitmapplans); static BitmapOr *make_bitmap_or(List *bitmapplans); static List *flatten_grouping_list(List *groupcls); -static bool can_elide_explicit_motion(Plan *plan, List *rtable, Oid relid); static void adjust_modifytable_flow(PlannerInfo *root, ModifyTable *node, List *is_split_updates); static Plan *prepare_sort_from_pathkeys(PlannerInfo *root, Plan *lefttree, List *pathkeys, @@ -6475,208 +6472,6 @@ make_modifytable(PlannerInfo *root, return node; } -/* - * can_elide_explicit_motion_recurse - * Recursive guts of can_elide_explicit_motion(). - * - * Check for motions by recursively looking at right subtree first. If it does - * not exist, go into left subtree. If we find a motion before the scan, we - * return false and exit from that subtree. A scan is always underneath the - * motion, so we won't encounter the scan if we exited the subtree. In that - * case, we can't elide Explicit Redistribute Motion. - * - * If we encounter the scan before any motions, then we can elide - * unneccessary Explicit Redistribute Motion. - */ -static bool -can_elide_explicit_motion_recurse(Plan *plan, List *rtable, Oid relid, - Oid reltypeid, bool *early_exit) -{ - while (plan && !(*early_exit)) - { - if (IsA(plan, Motion)) - { - Motion *motion = (Motion *) plan; - - /* - * If this is a Broadcast Motion, check Oids from the Motion's - * targetlist. - */ - if (motion->motionType == MOTIONTYPE_FIXED && motion->isBroadcast) - { - ListCell *lcr; - - foreach(lcr, plan->targetlist) - { - TargetEntry *tle = (TargetEntry *) lfirst(lcr); - RangeTblEntry *target_rte = rt_fetch(tle->resno, rtable); - - if (OidIsValid(reltypeid)) - { - Oid target_reltypeid = get_rel_type_id(target_rte->relid); - - /* - * Does relid inherit from a table in targetlist? - */ - if (OidIsValid(target_reltypeid) && OidIsValid(reltypeid) && - typeInheritsFrom(reltypeid, target_reltypeid)) - { - /* - * There is a Motion on parent table before scan - * on the child - */ - return false; - } - } - - if (target_rte->relid == relid) - { - /* There is a Motion before scan */ - return false; - } - } - } - /* - * If this is a Redistribute Motion, get relation ID from Motion's - * hashExprs. We can only get relation ID out of Var nodes, so we - * bail out early when we encounter any other expression node. - */ - else if (motion->motionType == MOTIONTYPE_HASH) - { - ListCell *lcr; - - foreach(lcr, motion->hashExprs) - { - Node *expr = (Node *) lfirst(lcr); - Oid expr_relid = InvalidOid; - - if (IsA(expr, Var)) - { - Var *var = (Var *) expr; - RangeTblEntry *rte = rt_fetch(var->varno, rtable); - expr_relid = rte->relid; - } - else - { - /* - * We cant't get relation ID here. Bail out before we - * hit a scan. - */ - *early_exit = true; - return false; - } - - if (OidIsValid(reltypeid)) - { - Oid expr_reltypeid = get_rel_type_id(expr_relid); - - /* - * Does relid inherit from this table? - */ - if (OidIsValid(expr_reltypeid) && OidIsValid(reltypeid) && - typeInheritsFrom(reltypeid, expr_reltypeid)) - { - /* - * There is a Motion on parent table before scan - * on the child - */ - return false; - } - } - - if (relid == expr_relid) - { - /* There is a Motion before scan */ - return false; - } - } - } - } - /* - * If this is a scan and it's scanrelid matches relid, we encountered - * a scan before any Motions. - */ - else - { - switch (nodeTag(plan)) - { - case T_SeqScan: - case T_DynamicSeqScan: - case T_ExternalScan: - case T_IndexScan: - case T_DynamicIndexScan: - case T_IndexOnlyScan: - case T_BitmapIndexScan: - case T_DynamicBitmapIndexScan: - case T_BitmapHeapScan: - case T_DynamicBitmapHeapScan: - case T_TidScan: - case T_SubqueryScan: - case T_FunctionScan: - case T_TableFunctionScan: - case T_ValuesScan: - case T_CteScan: - case T_WorkTableScan: - case T_ForeignScan: - { - Scan *scan = (Scan *) plan; - RangeTblEntry *target_rte = rt_fetch(scan->scanrelid, rtable); - - if (target_rte->relid == relid) - { - /* There wasn't any Motions before scan */ - return true; - } - } - break; - default: - break; - } - } - - /* - * If this plan has a right subtree, check it for Motions too. - */ - if (plan->righttree && - can_elide_explicit_motion_recurse(plan->righttree, rtable, relid, - reltypeid, early_exit)) - return true; - - plan = plan->lefttree; - } - - return false; -} - -/* - * can_elide_explicit_motion - * Check if there's any Redistribute or Broadcast Motions before scan in the - * same subtree for a relid. - */ -static bool -can_elide_explicit_motion(Plan *plan, List *rtable, Oid relid) -{ - bool relid_is_subclass; - Oid reltypeid; - - bool early_exit = false; - - /* - * Even if previous Motions were performed on a leaf partition or - * inherited table, targetlists from Motions refer to relids of their - * parents. So, if relid has a superclass, we should check for type - * inheritance too. - */ - LockRelationOid(relid, AccessShareLock); - relid_is_subclass = has_superclass(relid); - UnlockRelationOid(relid, AccessShareLock); - - reltypeid = relid_is_subclass ? get_rel_type_id(relid) : InvalidOid; - - return can_elide_explicit_motion_recurse(plan, rtable, relid, reltypeid, - &early_exit); -} - /* * Set the Flow in a ModifyTable and its children correctly. * @@ -6932,18 +6727,7 @@ adjust_modifytable_flow(PlannerInfo *root, ModifyTable *node, List *is_split_upd node->action_col_idxes = lappend_int(node->action_col_idxes, -1); node->ctid_col_idxes = lappend_int(node->ctid_col_idxes, -1); node->oid_col_idxes = lappend_int(node->oid_col_idxes, 0); - - /* - * If an Explicit Motion has no Motions underneath it, - * then the row to update must originate from the same - * segment, and no Motion is needed. - * - * We elide the motion even if there are Motions, as long - * as they are not between the scan on the target table - * and the ModifyTable. - */ - if (!can_elide_explicit_motion(subplan, root->parse->rtable, rte->relid)) - request_explicit_motion(subplan, rti, root->glob->finalrtable); + request_explicit_motion(subplan, rti, root->glob->finalrtable); } } else if (targetPolicyType == POLICYTYPE_ENTRY) From 671b4392cae82975d41b8c7e67e5fd3e64d9ed75 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Tue, 30 Jan 2024 14:43:48 +0300 Subject: [PATCH 11/83] Change comments, since varnoold is used --- src/include/nodes/primnodes.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h index caed77f1c29d..5c5d2f23d6a8 100644 --- a/src/include/nodes/primnodes.h +++ b/src/include/nodes/primnodes.h @@ -154,8 +154,8 @@ typedef struct Expr * subplans; for example, in a join node varno becomes INNER_VAR or OUTER_VAR * and varattno becomes the index of the proper element of that subplan's * target list. But varnoold/varoattno continue to hold the original values. - * The code doesn't really need varnoold/varoattno, but they are very useful - * for debugging and interpreting completed plans, so we keep them around. + * They are very useful for debugging and interpreting completed plans, so we + * keep them around. */ #define INNER_VAR 65000 /* reference to inner subplan */ #define OUTER_VAR 65001 /* reference to outer subplan */ @@ -180,7 +180,7 @@ typedef struct Var Index varlevelsup; /* for subquery variables referencing outer * relations; 0 in a normal var, >0 means N * levels up */ - Index varnoold; /* original value of varno, for debugging */ + Index varnoold; /* original value of varno */ AttrNumber varoattno; /* original value of varattno */ int location; /* token location, or -1 if unknown */ } Var; From 7c191710da4c1835571975582067451e8e2a3e05 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Tue, 30 Jan 2024 14:43:48 +0300 Subject: [PATCH 12/83] Run pgindent --- src/backend/cdb/cdbmutate.c | 97 +++++++++++++++++++------------------ 1 file changed, 50 insertions(+), 47 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index f2a86bb3a0c1..91c6cf2791f1 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -80,10 +80,10 @@ typedef struct ApplyMotionState HTAB *planid_subplans; /* hash table for InitPlanItem */ /* Context for ModifyTable to elide Explicit Redistribute Motion */ - List *mtResultRelids; /* Oid list of relations to be modified */ - List *mtResultReltypeids; /* Oid list of relation's types. If a - * relation isn't a subclass, it's - * InvalidOid */ + List *mtResultRelids; /* Oid list of relations to be modified */ + List *mtResultReltypeids; /* Oid list of relation's types. If a + * relation isn't a subclass, it's + * InvalidOid */ bool mtNeedExplicitMotion; bool mtIsChecking; } ApplyMotionState; @@ -773,29 +773,29 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) if (IsA(node, ModifyTable)) { ModifyTable *mt = (ModifyTable *) node; - + if (mt->operation == CMD_UPDATE || mt->operation == CMD_DELETE) { - ListCell *lcr; + ListCell *lcr; context->mtResultRelids = NIL; context->mtResultReltypeids = NIL; /* - * Make lists of resulting relations' Oids - */ + * Make lists of resulting relations' Oids + */ foreach(lcr, mt->resultRelations) { - Oid reltypeid; + Oid reltypeid; - Index rti = lfirst_int(lcr); + Index rti = lfirst_int(lcr); RangeTblEntry *rte = rt_fetch(rti, root->glob->finalrtable); - bool is_subclass = false; + bool is_subclass = false; LockRelationOid(rte->relid, AccessShareLock); is_subclass = has_superclass(rte->relid); UnlockRelationOid(rte->relid, AccessShareLock); - + reltypeid = is_subclass ? get_rel_type_id(rte->relid) : InvalidOid; context->mtResultRelids = lappend_oid(context->mtResultRelids, @@ -809,16 +809,17 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) } /* - * Elide Explicit Redistribute Motion if there's no motions between the scan - * and the ModifyTable that affect the relation we are going to update. + * Elide Explicit Redistribute Motion if there's no motions between the + * scan and the ModifyTable that affect the relation we are going to + * update. */ if (context->mtIsChecking) { if (IsA(node, Motion)) { - Motion *motion = (Motion *) node; - List *rtable = root->glob->finalrtable; + Motion *motion = (Motion *) node; + List *rtable = root->glob->finalrtable; /* * If this is a Redistribute Motion, get relation ID from Motion's @@ -827,15 +828,15 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) */ if (motion->motionType == MOTIONTYPE_HASH) { - ListCell *expr_lce; + ListCell *expr_lce; - foreach (expr_lce, motion->hashExprs) + foreach(expr_lce, motion->hashExprs) { - ListCell *mt_lcr; - ListCell *mt_lct; + ListCell *mt_lcr; + ListCell *mt_lct; - Node *expr = (Node *) lfirst(expr_lce); - Oid expr_relid = InvalidOid; + Node *expr = (Node *) lfirst(expr_lce); + Oid expr_relid = InvalidOid; if (!context->mtIsChecking) break; @@ -843,7 +844,7 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) if (IsA(expr, Var)) { RangeTblEntry *expr_rte; - Var *var = (Var *) expr; + Var *var = (Var *) expr; if (var->varnoold > 0 && var->varnoold <= list_length(rtable)) @@ -863,19 +864,19 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) forboth(mt_lcr, context->mtResultRelids, mt_lct, context->mtResultReltypeids) { - Oid mt_relid = lfirst_oid(mt_lcr); - Oid mt_reltypeid = lfirst_oid(mt_lct); + Oid mt_relid = lfirst_oid(mt_lcr); + Oid mt_reltypeid = lfirst_oid(mt_lct); if (OidIsValid(mt_reltypeid)) { - Oid expr_reltypeid = get_rel_type_id(expr_relid); + Oid expr_reltypeid = get_rel_type_id(expr_relid); if (OidIsValid(expr_reltypeid) && - typeInheritsFrom(mt_reltypeid, expr_reltypeid)) + typeInheritsFrom(mt_reltypeid, expr_reltypeid)) { /* - * There is a Motion on parent table before scan - * on the child + * There is a Motion on parent table before + * scan on the child */ context->mtIsChecking = false; context->mtNeedExplicitMotion = true; @@ -893,21 +894,22 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) } } } + /* * If this is a Broadcast Motion, check Oids from the Motion's * targetlist. */ else if (motion->motionType == MOTIONTYPE_FIXED && motion->isBroadcast) { - ListCell *target_lcr; + ListCell *target_lcr; foreach(target_lcr, motion->plan.targetlist) { - ListCell *mt_lcr; - ListCell *mt_lct; + ListCell *mt_lcr; + ListCell *mt_lct; TargetEntry *target_tle = (TargetEntry *) lfirst(target_lcr); - Oid expr_relid = InvalidOid; + Oid expr_relid = InvalidOid; if (!context->mtIsChecking) break; @@ -915,9 +917,9 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) if (IsA(target_tle->expr, Var)) { RangeTblEntry *expr_rte; - Var *var = (Var *) target_tle->expr; + Var *var = (Var *) target_tle->expr; - if (var->varnoold > 0 && + if (var->varnoold > 0 && var->varnoold <= list_length(rtable)) { expr_rte = rt_fetch(var->varnoold, rtable); @@ -935,19 +937,19 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) forboth(mt_lcr, context->mtResultRelids, mt_lct, context->mtResultReltypeids) { - Oid mt_relid = lfirst_oid(mt_lcr); - Oid mt_reltypeid = lfirst_oid(mt_lct); + Oid mt_relid = lfirst_oid(mt_lcr); + Oid mt_reltypeid = lfirst_oid(mt_lct); if (OidIsValid(mt_reltypeid)) { - Oid target_reltypeid = get_rel_type_id(expr_relid); + Oid target_reltypeid = get_rel_type_id(expr_relid); if (OidIsValid(target_reltypeid) && - typeInheritsFrom(mt_reltypeid, target_reltypeid)) + typeInheritsFrom(mt_reltypeid, target_reltypeid)) { /* - * There is a Motion on parent table before scan - * on the child + * There is a Motion on parent table before + * scan on the child */ context->mtIsChecking = false; context->mtNeedExplicitMotion = true; @@ -966,9 +968,10 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) } } } + /* - * If this is a scan and it's scanrelid matches ModifyTable's relid, we - * encountered a scan before any Motions. + * If this is a scan and it's scanrelid matches ModifyTable's relid, + * we encountered a scan before any Motions. */ else { @@ -993,15 +996,15 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) case T_WorkTableScan: case T_ForeignScan: { - ListCell *mt_lcr; + ListCell *mt_lcr; - Scan *scan = (Scan *) node; - List *rtable = root->glob->finalrtable; + Scan *scan = (Scan *) node; + List *rtable = root->glob->finalrtable; RangeTblEntry *target_rte = rt_fetch(scan->scanrelid, rtable); foreach(mt_lcr, context->mtResultRelids) { - Oid mt_relid = lfirst_oid(mt_lcr); + Oid mt_relid = lfirst_oid(mt_lcr); if (target_rte->relid == mt_relid) { From e61fbf3be397c82dbf2eae31bf6c6cdcf9c63cf9 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Tue, 30 Jan 2024 14:43:48 +0300 Subject: [PATCH 13/83] Bring back resetting slice depth for InitPlan nodes --- src/backend/cdb/cdbmutate.c | 37 +++++++++++++++++++++---- src/backend/optimizer/plan/createplan.c | 1 - 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index 91c6cf2791f1..f9610ce54be8 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -743,6 +743,7 @@ apply_motion(PlannerInfo *root, Plan *plan, Query *query) return result; } + /* * Function apply_motion_mutator() is the workhorse for apply_motion(). */ @@ -764,6 +765,36 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) if (node == NULL) return NULL; + /* An expression node might have subtrees containing plans to be mutated. */ + if (!is_plan_node(node)) + { + if (IsA(node, SubPlan) &&((SubPlan *) node)->is_initplan) + { + bool found; + int saveSliceDepth = context->sliceDepth; + SubPlan *subplan = (SubPlan *) node; + /* + * If the init-plan refered by `subplan` has been visited, we should + * not re-visit the subplan, or the motions under the init-plan are + * re-counted. + */ + hash_search(context->planid_subplans, &subplan->plan_id, + HASH_FIND, &found); + if (found) + return node; + + /* reset sliceDepth for each init plan */ + context->sliceDepth = 0; + node = plan_tree_mutator(node, apply_motion_mutator, context); + + context->sliceDepth = saveSliceDepth; + + return node; + } + else + return plan_tree_mutator(node, apply_motion_mutator, context); + } + /* * For UPDATE/DELETE, we check if there's any Redistribute or Broadcast * Motions before scan in the same subtree for the table we're going to @@ -1022,10 +1053,6 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) } } - /* An expression node might have subtrees containing plans to be mutated. */ - if (!is_plan_node(node)) - return plan_tree_mutator(node, apply_motion_mutator, context); - plan = (Plan *) node; flow = plan->flow; @@ -1182,7 +1209,7 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) { newnode = (Node *) make_explicit_motion(plan, flow->segidColIdx, - true /* useExecutorVarFormat */ + true /* useExecutorVarFormat */ ); } else diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c index d7f3a7d01721..fe03c8df7ce9 100644 --- a/src/backend/optimizer/plan/createplan.c +++ b/src/backend/optimizer/plan/createplan.c @@ -63,7 +63,6 @@ #include "cdb/cdbsetop.h" #include "cdb/cdbsreh.h" #include "cdb/cdbvars.h" -#include "cdb/cdbpartition.h" /* rel_partition_get_master() etc. */ static Plan *create_subplan(PlannerInfo *root, Path *best_path); /* CDB */ From e0e839c67df7b6a02e1b96ab39190414c6a1438c Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Tue, 30 Jan 2024 14:43:48 +0300 Subject: [PATCH 14/83] Save state if there is several ModifyTable nodes --- src/backend/cdb/cdbmutate.c | 94 ++++++++++++++++++++++--------------- 1 file changed, 55 insertions(+), 39 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index f9610ce54be8..d3cd1c569025 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -66,6 +66,17 @@ #include "executor/executor.h" #include "storage/lmgr.h" /* LockRelationOid(), UnlockRelationOid() */ +typedef struct ModifyTableMotionState +{ + List *resultRelids; /* Oid list of relations to be + * modified */ + List *resultReltypeids; /* Oid list of relation's types. If a + * relation isn't a subclass, it's + * InvalidOid */ + bool needExplicitMotion; + bool isChecking; +} ModifyTableMotionState; + /* * An ApplyMotionState holds state for the recursive apply_motion_mutator(). * It is externalized here to make it shareable by helper code in other @@ -80,12 +91,7 @@ typedef struct ApplyMotionState HTAB *planid_subplans; /* hash table for InitPlanItem */ /* Context for ModifyTable to elide Explicit Redistribute Motion */ - List *mtResultRelids; /* Oid list of relations to be modified */ - List *mtResultReltypeids; /* Oid list of relation's types. If a - * relation isn't a subclass, it's - * InvalidOid */ - bool mtNeedExplicitMotion; - bool mtIsChecking; + ModifyTableMotionState mt; } ApplyMotionState; typedef struct InitPlanItem @@ -414,10 +420,10 @@ apply_motion(PlannerInfo *root, Plan *plan, Query *query) state.nextMotionID = 1; /* Start at 1 so zero will mean "unassigned". */ state.sliceDepth = 0; - state.mtResultRelids = NIL; - state.mtResultReltypeids = NIL; - state.mtNeedExplicitMotion = false; - state.mtIsChecking = false; + state.mt.resultRelids = NIL; + state.mt.resultReltypeids = NIL; + state.mt.needExplicitMotion = false; + state.mt.isChecking = false; memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(int); @@ -756,6 +762,7 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) int saveNextMotionID; int saveNumInitPlans; int saveSliceDepth; + ModifyTableMotionState save_mt; #ifdef USE_ASSERT_CHECKING PlannerInfo *root = (PlannerInfo *) context->base.node; @@ -809,8 +816,13 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) { ListCell *lcr; - context->mtResultRelids = NIL; - context->mtResultReltypeids = NIL; + /* If there was previous ModifyTable node, save it's state. We can't + * keep just a single boolean value, since there may be several + * ModifyTable nodes in the tree. So we check them separately. */ + save_mt = context->mt; + + context->mt.resultRelids = NIL; + context->mt.resultReltypeids = NIL; /* * Make lists of resulting relations' Oids @@ -829,13 +841,14 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) reltypeid = is_subclass ? get_rel_type_id(rte->relid) : InvalidOid; - context->mtResultRelids = lappend_oid(context->mtResultRelids, - rte->relid); - context->mtResultReltypeids = lappend_oid(context->mtResultReltypeids, - reltypeid); + context->mt.resultRelids = lappend_oid(context->mt.resultRelids, + rte->relid); + context->mt.resultReltypeids = lappend_oid(context->mt.resultReltypeids, + reltypeid); } - context->mtIsChecking = true; + context->mt.isChecking = true; + context->mt.needExplicitMotion = false; } } @@ -844,7 +857,7 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) * scan and the ModifyTable that affect the relation we are going to * update. */ - if (context->mtIsChecking) + if (context->mt.isChecking) { if (IsA(node, Motion)) @@ -869,7 +882,7 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) Node *expr = (Node *) lfirst(expr_lce); Oid expr_relid = InvalidOid; - if (!context->mtIsChecking) + if (!context->mt.isChecking) break; if (IsA(expr, Var)) @@ -888,12 +901,12 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) if (!OidIsValid(expr_relid)) { /* We can't get relation ID easily */ - context->mtIsChecking = false; - context->mtNeedExplicitMotion = true; + context->mt.isChecking = false; + context->mt.needExplicitMotion = true; break; } - forboth(mt_lcr, context->mtResultRelids, mt_lct, context->mtResultReltypeids) + forboth(mt_lcr, context->mt.resultRelids, mt_lct, context->mt.resultReltypeids) { Oid mt_relid = lfirst_oid(mt_lcr); Oid mt_reltypeid = lfirst_oid(mt_lct); @@ -909,8 +922,8 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) * There is a Motion on parent table before * scan on the child */ - context->mtIsChecking = false; - context->mtNeedExplicitMotion = true; + context->mt.isChecking = false; + context->mt.needExplicitMotion = true; break; } } @@ -918,8 +931,8 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) if (expr_relid == mt_relid) { /* There is a Motion before scan */ - context->mtIsChecking = false; - context->mtNeedExplicitMotion = true; + context->mt.isChecking = false; + context->mt.needExplicitMotion = true; break; } } @@ -942,7 +955,7 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) TargetEntry *target_tle = (TargetEntry *) lfirst(target_lcr); Oid expr_relid = InvalidOid; - if (!context->mtIsChecking) + if (!context->mt.isChecking) break; if (IsA(target_tle->expr, Var)) @@ -961,12 +974,12 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) if (!OidIsValid(expr_relid)) { /* We can't get relation ID easily */ - context->mtIsChecking = false; - context->mtNeedExplicitMotion = true; + context->mt.isChecking = false; + context->mt.needExplicitMotion = true; break; } - forboth(mt_lcr, context->mtResultRelids, mt_lct, context->mtResultReltypeids) + forboth(mt_lcr, context->mt.resultRelids, mt_lct, context->mt.resultReltypeids) { Oid mt_relid = lfirst_oid(mt_lcr); Oid mt_reltypeid = lfirst_oid(mt_lct); @@ -982,8 +995,8 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) * There is a Motion on parent table before * scan on the child */ - context->mtIsChecking = false; - context->mtNeedExplicitMotion = true; + context->mt.isChecking = false; + context->mt.needExplicitMotion = true; break; } } @@ -991,8 +1004,8 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) if (expr_relid == mt_relid) { /* There is a Motion before scan */ - context->mtIsChecking = false; - context->mtNeedExplicitMotion = true; + context->mt.isChecking = false; + context->mt.needExplicitMotion = true; break; } } @@ -1033,15 +1046,15 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) List *rtable = root->glob->finalrtable; RangeTblEntry *target_rte = rt_fetch(scan->scanrelid, rtable); - foreach(mt_lcr, context->mtResultRelids) + foreach(mt_lcr, context->mt.resultRelids) { Oid mt_relid = lfirst_oid(mt_lcr); if (target_rte->relid == mt_relid) { /* There wasn't any Motions before scan */ - context->mtIsChecking = false; - context->mtNeedExplicitMotion = false; + context->mt.isChecking = false; + context->mt.needExplicitMotion = false; break; } } @@ -1205,7 +1218,7 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) * add an ExplicitRedistribute motion node only if child plan * nodes have a motion node between ModifyTable and the scan */ - if (context->mtNeedExplicitMotion) + if (context->mt.needExplicitMotion) { newnode = (Node *) make_explicit_motion(plan, flow->segidColIdx, @@ -1223,7 +1236,7 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) } /* Continue checking in case another Explicit Motion is needed */ - context->mtIsChecking = true; + context->mt.isChecking = true; break; case MOVEMENT_NONE: @@ -1271,6 +1284,9 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) plan->nMotionNodes = context->nextMotionID - saveNextMotionID; plan->nInitPlans = hash_get_num_entries(context->planid_subplans) - saveNumInitPlans; + if(IsA(node, ModifyTable)) + context->mt = save_mt; + return newnode; } /* apply_motion_mutator */ From aad110bb53ca28d02ff48d6a8fca470d49e23da5 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Tue, 30 Jan 2024 14:43:48 +0300 Subject: [PATCH 15/83] Fix case when there are multiple subtrees with scan on the same table --- src/backend/cdb/cdbmutate.c | 297 ++++++++++++++++-------------------- 1 file changed, 132 insertions(+), 165 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index d3cd1c569025..74fb4195c627 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -66,14 +66,12 @@ #include "executor/executor.h" #include "storage/lmgr.h" /* LockRelationOid(), UnlockRelationOid() */ + typedef struct ModifyTableMotionState { List *resultRelids; /* Oid list of relations to be * modified */ - List *resultReltypeids; /* Oid list of relation's types. If a - * relation isn't a subclass, it's - * InvalidOid */ - bool needExplicitMotion; + List *needExplicitMotion; /* Boolean list matching resultRelids */ bool isChecking; } ModifyTableMotionState; @@ -421,8 +419,7 @@ apply_motion(PlannerInfo *root, Plan *plan, Query *query) state.sliceDepth = 0; state.mt.resultRelids = NIL; - state.mt.resultReltypeids = NIL; - state.mt.needExplicitMotion = false; + state.mt.needExplicitMotion = NIL; state.mt.isChecking = false; memset(&ctl, 0, sizeof(ctl)); @@ -750,6 +747,35 @@ apply_motion(PlannerInfo *root, Plan *plan, Query *query) } +/* + * Try to retrieve relation ID from this Expr node. + * + * This is used to look inside Motion's targetlists to compare with IDs from + * ModifyTable. We look at varnoold, because varno may be already set to + * INNER_VAR/OUTER_VAR. + * + * Result is InvalidOid if we couldn't get relation ID. + */ +static Oid +get_relid_from_expr(Expr *expr, List *rtable) +{ + if (IsA(expr, Var)) + { + Var *var = (Var *) expr; + + if (var->varnoold > 0 && + var->varnoold <= list_length(rtable)) + { + RangeTblEntry *expr_rte = rt_fetch(var->varnoold, rtable); + return expr_rte->relid; + } + } + + /* We can't get relation ID easily */ + return InvalidOid; +} + + /* * Function apply_motion_mutator() is the workhorse for apply_motion(). */ @@ -816,39 +842,36 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) { ListCell *lcr; - /* If there was previous ModifyTable node, save it's state. We can't - * keep just a single boolean value, since there may be several - * ModifyTable nodes in the tree. So we check them separately. */ + /* + * If there was another ModifyTable node before, save the previous + * state. + */ save_mt = context->mt; context->mt.resultRelids = NIL; - context->mt.resultReltypeids = NIL; + context->mt.needExplicitMotion = NIL; /* - * Make lists of resulting relations' Oids + * When UPDATE/DELETE occurs on a partitioned table, or a table that + * is a part of inheritance tree, ModifyTable node will have more + * than one relation in resultRelations. + * + * We make a list of resulting relations' Oids to compare them + * later, and a list of booleans of the same length to remember + * what we found. */ foreach(lcr, mt->resultRelations) { - Oid reltypeid; - Index rti = lfirst_int(lcr); RangeTblEntry *rte = rt_fetch(rti, root->glob->finalrtable); - bool is_subclass = false; - - LockRelationOid(rte->relid, AccessShareLock); - is_subclass = has_superclass(rte->relid); - UnlockRelationOid(rte->relid, AccessShareLock); - - reltypeid = is_subclass ? get_rel_type_id(rte->relid) : InvalidOid; context->mt.resultRelids = lappend_oid(context->mt.resultRelids, rte->relid); - context->mt.resultReltypeids = lappend_oid(context->mt.resultReltypeids, - reltypeid); + context->mt.needExplicitMotion = lappend_int(context->mt.needExplicitMotion, + false); } context->mt.isChecking = true; - context->mt.needExplicitMotion = false; } } @@ -866,91 +889,17 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) List *rtable = root->glob->finalrtable; /* - * If this is a Redistribute Motion, get relation ID from Motion's - * hashExprs. We can only get relation ID out of Var nodes, so we - * bail out early when we encounter any other expression node. - */ - if (motion->motionType == MOTIONTYPE_HASH) - { - ListCell *expr_lce; - - foreach(expr_lce, motion->hashExprs) - { - ListCell *mt_lcr; - ListCell *mt_lct; - - Node *expr = (Node *) lfirst(expr_lce); - Oid expr_relid = InvalidOid; - - if (!context->mt.isChecking) - break; - - if (IsA(expr, Var)) - { - RangeTblEntry *expr_rte; - Var *var = (Var *) expr; - - if (var->varnoold > 0 && - var->varnoold <= list_length(rtable)) - { - expr_rte = rt_fetch(var->varnoold, rtable); - expr_relid = expr_rte->relid; - } - } - - if (!OidIsValid(expr_relid)) - { - /* We can't get relation ID easily */ - context->mt.isChecking = false; - context->mt.needExplicitMotion = true; - break; - } - - forboth(mt_lcr, context->mt.resultRelids, mt_lct, context->mt.resultReltypeids) - { - Oid mt_relid = lfirst_oid(mt_lcr); - Oid mt_reltypeid = lfirst_oid(mt_lct); - - if (OidIsValid(mt_reltypeid)) - { - Oid expr_reltypeid = get_rel_type_id(expr_relid); - - if (OidIsValid(expr_reltypeid) && - typeInheritsFrom(mt_reltypeid, expr_reltypeid)) - { - /* - * There is a Motion on parent table before - * scan on the child - */ - context->mt.isChecking = false; - context->mt.needExplicitMotion = true; - break; - } - } - - if (expr_relid == mt_relid) - { - /* There is a Motion before scan */ - context->mt.isChecking = false; - context->mt.needExplicitMotion = true; - break; - } - } - } - } - - /* - * If this is a Broadcast Motion, check Oids from the Motion's - * targetlist. + * Check Oids from the Motion's targetlist. */ - else if (motion->motionType == MOTIONTYPE_FIXED && motion->isBroadcast) + if (motion->motionType == MOTIONTYPE_HASH || + (motion->motionType == MOTIONTYPE_FIXED && motion->isBroadcast)) { ListCell *target_lcr; foreach(target_lcr, motion->plan.targetlist) { ListCell *mt_lcr; - ListCell *mt_lct; + ListCell *mt_lcm; TargetEntry *target_tle = (TargetEntry *) lfirst(target_lcr); Oid expr_relid = InvalidOid; @@ -958,55 +907,20 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) if (!context->mt.isChecking) break; - if (IsA(target_tle->expr, Var)) - { - RangeTblEntry *expr_rte; - Var *var = (Var *) target_tle->expr; - - if (var->varnoold > 0 && - var->varnoold <= list_length(rtable)) - { - expr_rte = rt_fetch(var->varnoold, rtable); - expr_relid = expr_rte->relid; - } - } + expr_relid = get_relid_from_expr(target_tle->expr, rtable); if (!OidIsValid(expr_relid)) - { - /* We can't get relation ID easily */ - context->mt.isChecking = false; - context->mt.needExplicitMotion = true; - break; - } + continue; - forboth(mt_lcr, context->mt.resultRelids, mt_lct, context->mt.resultReltypeids) + forboth(mt_lcr, context->mt.resultRelids, mt_lcm, context->mt.needExplicitMotion) { Oid mt_relid = lfirst_oid(mt_lcr); - Oid mt_reltypeid = lfirst_oid(mt_lct); - - if (OidIsValid(mt_reltypeid)) - { - Oid target_reltypeid = get_rel_type_id(expr_relid); - - if (OidIsValid(target_reltypeid) && - typeInheritsFrom(mt_reltypeid, target_reltypeid)) - { - /* - * There is a Motion on parent table before - * scan on the child - */ - context->mt.isChecking = false; - context->mt.needExplicitMotion = true; - break; - } - } if (expr_relid == mt_relid) { - /* There is a Motion before scan */ + /* There is a Motion before scan. */ + lfirst_int(mt_lcm) = true; context->mt.isChecking = false; - context->mt.needExplicitMotion = true; - break; } } } @@ -1041,21 +955,25 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) case T_ForeignScan: { ListCell *mt_lcr; + ListCell *mt_lcm; Scan *scan = (Scan *) node; List *rtable = root->glob->finalrtable; RangeTblEntry *target_rte = rt_fetch(scan->scanrelid, rtable); - foreach(mt_lcr, context->mt.resultRelids) + forboth(mt_lcr, context->mt.resultRelids, mt_lcm, context->mt.needExplicitMotion) { Oid mt_relid = lfirst_oid(mt_lcr); if (target_rte->relid == mt_relid) { - /* There wasn't any Motions before scan */ - context->mt.isChecking = false; - context->mt.needExplicitMotion = false; - break; + /* + * There is a scan before Motions. + * + * We don't stop checking because there might + * be motions in other subtrees. + */ + lfirst_oid(mt_lcm) = false; } } } @@ -1214,29 +1132,74 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) break; case MOVEMENT_EXPLICIT: - /* - * add an ExplicitRedistribute motion node only if child plan - * nodes have a motion node between ModifyTable and the scan - */ - if (context->mt.needExplicitMotion) - { - newnode = (Node *) make_explicit_motion(plan, - flow->segidColIdx, - true /* useExecutorVarFormat */ - ); - } - else { + ListCell *target_lcr; + + bool finished = false; + bool need_motion = true; + List *rtable = root->glob->finalrtable; + /* - * no motion nodes in child plan nodes - no need for - * ExplicitRedistribute: restore flow + * Search this motion's targetlist for matching relations in + * ModifyTable context. On match, retrieve boolean value we set + * when checking for motions and scans. */ - flow->req_move = MOVEMENT_NONE; - flow->flow_before_req_move = NULL; - } + foreach(target_lcr, plan->targetlist) + { + ListCell *mt_lcr; + ListCell *mt_lcm; - /* Continue checking in case another Explicit Motion is needed */ - context->mt.isChecking = true; + TargetEntry *target_tle = (TargetEntry *) lfirst(target_lcr); + Oid expr_relid = InvalidOid; + + if (finished) + break; + + expr_relid = get_relid_from_expr(target_tle->expr, rtable); + + if (!OidIsValid(expr_relid)) + { + /* Unknown relation ID, expect Explicit Motion */ + need_motion = true; + continue; + } + + forboth(mt_lcr, context->mt.resultRelids, mt_lcm, context->mt.needExplicitMotion) + { + Oid mt_relid = lfirst_oid(mt_lcr); + + if (expr_relid == mt_relid) + { + need_motion = (bool) lfirst_int(mt_lcm); + finished = true; + break; + } + } + } + + if (need_motion) + { + newnode = (Node *) make_explicit_motion(plan, + flow->segidColIdx, + true /* useExecutorVarFormat */ + ); + } + else + { + /* + * Restore flow if Explicit Redistribute Motion is not + * needed + */ + flow->req_move = MOVEMENT_NONE; + flow->flow_before_req_move = NULL; + } + + /* + * Continue checking in case of another Explicit Redistribute + * Motion + */ + context->mt.isChecking = true; + } break; case MOVEMENT_NONE: @@ -1284,6 +1247,10 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) plan->nMotionNodes = context->nextMotionID - saveNextMotionID; plan->nInitPlans = hash_get_num_entries(context->planid_subplans) - saveNumInitPlans; + /* + * Restore previous ModifyTable state, in case this wasn't the first + * one. + */ if(IsA(node, ModifyTable)) context->mt = save_mt; From b7757288c825be940ac42ae6d1d582784536c3c7 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Tue, 30 Jan 2024 14:43:48 +0300 Subject: [PATCH 16/83] Clean up --- src/backend/cdb/cdbmutate.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index 74fb4195c627..9340944c12e2 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -46,7 +46,6 @@ #include "catalog/gp_policy.h" #include "catalog/pg_operator.h" #include "catalog/pg_type.h" -#include "catalog/pg_inherits_fn.h" /* typeInheritsFrom() */ #include "catalog/pg_proc.h" #include "catalog/pg_trigger.h" @@ -64,7 +63,6 @@ #include "cdb/cdbtargeteddispatch.h" #include "executor/executor.h" -#include "storage/lmgr.h" /* LockRelationOid(), UnlockRelationOid() */ typedef struct ModifyTableMotionState @@ -904,9 +902,6 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) TargetEntry *target_tle = (TargetEntry *) lfirst(target_lcr); Oid expr_relid = InvalidOid; - if (!context->mt.isChecking) - break; - expr_relid = get_relid_from_expr(target_tle->expr, rtable); if (!OidIsValid(expr_relid)) @@ -1170,8 +1165,8 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) if (expr_relid == mt_relid) { - need_motion = (bool) lfirst_int(mt_lcm); finished = true; + need_motion = (bool) lfirst_int(mt_lcm); break; } } From 8e69fae579a0a550b3a006ae0a258d2cdc7b41c8 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Tue, 30 Jan 2024 14:43:49 +0300 Subject: [PATCH 17/83] Remove extra checks --- src/backend/cdb/cdbmutate.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index 9340944c12e2..a73d6cc4b43a 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -900,9 +900,7 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) ListCell *mt_lcm; TargetEntry *target_tle = (TargetEntry *) lfirst(target_lcr); - Oid expr_relid = InvalidOid; - - expr_relid = get_relid_from_expr(target_tle->expr, rtable); + Oid expr_relid = get_relid_from_expr(target_tle->expr, rtable); if (!OidIsValid(expr_relid)) continue; @@ -1131,7 +1129,7 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) ListCell *target_lcr; bool finished = false; - bool need_motion = true; + bool need_motion = false; List *rtable = root->glob->finalrtable; /* @@ -1153,11 +1151,7 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) expr_relid = get_relid_from_expr(target_tle->expr, rtable); if (!OidIsValid(expr_relid)) - { - /* Unknown relation ID, expect Explicit Motion */ - need_motion = true; continue; - } forboth(mt_lcr, context->mt.resultRelids, mt_lcm, context->mt.needExplicitMotion) { From 9ebcff55252bd2a1974677054903963a099f9cb3 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Tue, 30 Jan 2024 14:43:49 +0300 Subject: [PATCH 18/83] Fix formatting --- src/backend/cdb/cdbmutate.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index a73d6cc4b43a..7086251dbed8 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -1240,7 +1240,7 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) * Restore previous ModifyTable state, in case this wasn't the first * one. */ - if(IsA(node, ModifyTable)) + if (IsA(node, ModifyTable)) context->mt = save_mt; return newnode; From 951ece2af22bab688d0b96ed4bec8875497973ab Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Tue, 30 Jan 2024 14:43:49 +0300 Subject: [PATCH 19/83] Fix isChecking being enabled for INSERT ModifyTable after first Explicit Motion --- src/backend/cdb/cdbmutate.c | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index 7086251dbed8..a3a109735a25 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -70,7 +70,8 @@ typedef struct ModifyTableMotionState List *resultRelids; /* Oid list of relations to be * modified */ List *needExplicitMotion; /* Boolean list matching resultRelids */ - bool isChecking; + bool enabled; /* Did we encounter UPDATE/DELETE? */ + bool isChecking; /* Are we still looking for motions? */ } ModifyTableMotionState; /* @@ -418,6 +419,7 @@ apply_motion(PlannerInfo *root, Plan *plan, Query *query) state.mt.resultRelids = NIL; state.mt.needExplicitMotion = NIL; + state.mt.enabled = false; state.mt.isChecking = false; memset(&ctl, 0, sizeof(ctl)); @@ -869,6 +871,7 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) false); } + context->mt.enabled = true; context->mt.isChecking = true; } } @@ -1128,14 +1131,21 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) { ListCell *target_lcr; - bool finished = false; bool need_motion = false; + bool finished = false; List *rtable = root->glob->finalrtable; /* - * Search this motion's targetlist for matching relations in - * ModifyTable context. On match, retrieve boolean value we set - * when checking for motions and scans. + * If this isn't UPDATE/DELETE, we don't need Explicit + * Redistribute Motion + */ + if (!context->mt.enabled) + break; + + /* + * Search this motion's targetlist for matching relations + * in ModifyTable context. On match, retrieve boolean value + * we set when checking for motions and scans. */ foreach(target_lcr, plan->targetlist) { @@ -1176,17 +1186,17 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) else { /* - * Restore flow if Explicit Redistribute Motion is not - * needed - */ + * Restore flow if Explicit Redistribute Motion is not + * needed + */ flow->req_move = MOVEMENT_NONE; flow->flow_before_req_move = NULL; } /* - * Continue checking in case of another Explicit Redistribute - * Motion - */ + * Continue checking in case of another Explicit Redistribute + * Motion + */ context->mt.isChecking = true; } break; From 049d111c1d11f12c42918f2b5b3dcc1decc9efa1 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Tue, 30 Jan 2024 14:43:49 +0300 Subject: [PATCH 20/83] Revert "Fix isChecking being enabled for INSERT ModifyTable after first Explicit Motion" This reverts commit 8cfe09cb4543fb4e3bd16a511ab1f28902bb117f. Explicit Redidistribute Motion is not requested unless IT IS UPDATE/DELETE --- src/backend/cdb/cdbmutate.c | 32 +++++++++++--------------------- 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index a3a109735a25..7086251dbed8 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -70,8 +70,7 @@ typedef struct ModifyTableMotionState List *resultRelids; /* Oid list of relations to be * modified */ List *needExplicitMotion; /* Boolean list matching resultRelids */ - bool enabled; /* Did we encounter UPDATE/DELETE? */ - bool isChecking; /* Are we still looking for motions? */ + bool isChecking; } ModifyTableMotionState; /* @@ -419,7 +418,6 @@ apply_motion(PlannerInfo *root, Plan *plan, Query *query) state.mt.resultRelids = NIL; state.mt.needExplicitMotion = NIL; - state.mt.enabled = false; state.mt.isChecking = false; memset(&ctl, 0, sizeof(ctl)); @@ -871,7 +869,6 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) false); } - context->mt.enabled = true; context->mt.isChecking = true; } } @@ -1131,21 +1128,14 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) { ListCell *target_lcr; - bool need_motion = false; bool finished = false; + bool need_motion = false; List *rtable = root->glob->finalrtable; /* - * If this isn't UPDATE/DELETE, we don't need Explicit - * Redistribute Motion - */ - if (!context->mt.enabled) - break; - - /* - * Search this motion's targetlist for matching relations - * in ModifyTable context. On match, retrieve boolean value - * we set when checking for motions and scans. + * Search this motion's targetlist for matching relations in + * ModifyTable context. On match, retrieve boolean value we set + * when checking for motions and scans. */ foreach(target_lcr, plan->targetlist) { @@ -1186,17 +1176,17 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) else { /* - * Restore flow if Explicit Redistribute Motion is not - * needed - */ + * Restore flow if Explicit Redistribute Motion is not + * needed + */ flow->req_move = MOVEMENT_NONE; flow->flow_before_req_move = NULL; } /* - * Continue checking in case of another Explicit Redistribute - * Motion - */ + * Continue checking in case of another Explicit Redistribute + * Motion + */ context->mt.isChecking = true; } break; From db1b2f4e836cb567ec461393464639e14eec9772 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Tue, 30 Jan 2024 14:43:49 +0300 Subject: [PATCH 21/83] Restore state only for UPDATE/DELETE --- src/backend/cdb/cdbmutate.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index 7086251dbed8..372a7a8f9c41 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -1241,7 +1241,12 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) * one. */ if (IsA(node, ModifyTable)) - context->mt = save_mt; + { + ModifyTable *mt = (ModifyTable *) node; + + if (mt->operation == CMD_UPDATE || mt->operation == CMD_DELETE) + context->mt = save_mt; + } return newnode; } /* apply_motion_mutator */ From ec27689b6846b51c80bf86691b0bda6629c628cc Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Tue, 30 Jan 2024 14:43:49 +0300 Subject: [PATCH 22/83] Simplify the logic and don't look into motions' targetlist --- src/backend/cdb/cdbmutate.c | 201 ++++++++++++------------------------ 1 file changed, 68 insertions(+), 133 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index 372a7a8f9c41..79b59f4dae5f 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -69,7 +69,8 @@ typedef struct ModifyTableMotionState { List *resultRelids; /* Oid list of relations to be * modified */ - List *needExplicitMotion; /* Boolean list matching resultRelids */ + bool needExplicitMotion; + int nMotionsAbove; bool isChecking; } ModifyTableMotionState; @@ -417,7 +418,8 @@ apply_motion(PlannerInfo *root, Plan *plan, Query *query) state.sliceDepth = 0; state.mt.resultRelids = NIL; - state.mt.needExplicitMotion = NIL; + state.mt.needExplicitMotion = false; + state.mt.nMotionsAbove = 0; state.mt.isChecking = false; memset(&ctl, 0, sizeof(ctl)); @@ -745,35 +747,6 @@ apply_motion(PlannerInfo *root, Plan *plan, Query *query) } -/* - * Try to retrieve relation ID from this Expr node. - * - * This is used to look inside Motion's targetlists to compare with IDs from - * ModifyTable. We look at varnoold, because varno may be already set to - * INNER_VAR/OUTER_VAR. - * - * Result is InvalidOid if we couldn't get relation ID. - */ -static Oid -get_relid_from_expr(Expr *expr, List *rtable) -{ - if (IsA(expr, Var)) - { - Var *var = (Var *) expr; - - if (var->varnoold > 0 && - var->varnoold <= list_length(rtable)) - { - RangeTblEntry *expr_rte = rt_fetch(var->varnoold, rtable); - return expr_rte->relid; - } - } - - /* We can't get relation ID easily */ - return InvalidOid; -} - - /* * Function apply_motion_mutator() is the workhorse for apply_motion(). */ @@ -847,7 +820,8 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) save_mt = context->mt; context->mt.resultRelids = NIL; - context->mt.needExplicitMotion = NIL; + context->mt.needExplicitMotion = false; + context->mt.nMotionsAbove = 0; /* * When UPDATE/DELETE occurs on a partitioned table, or a table that @@ -855,8 +829,7 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) * than one relation in resultRelations. * * We make a list of resulting relations' Oids to compare them - * later, and a list of booleans of the same length to remember - * what we found. + * later. */ foreach(lcr, mt->resultRelations) { @@ -865,8 +838,6 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) context->mt.resultRelids = lappend_oid(context->mt.resultRelids, rte->relid); - context->mt.needExplicitMotion = lappend_int(context->mt.needExplicitMotion, - false); } context->mt.isChecking = true; @@ -875,48 +846,23 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) /* * Elide Explicit Redistribute Motion if there's no motions between the - * scan and the ModifyTable that affect the relation we are going to - * update. + * scan and the ModifyTable. */ if (context->mt.isChecking) { + /* + * Remember if we are descending into a Redistribute or Broadcast + * Motion node. + */ if (IsA(node, Motion)) { Motion *motion = (Motion *) node; - List *rtable = root->glob->finalrtable; - /* - * Check Oids from the Motion's targetlist. - */ if (motion->motionType == MOTIONTYPE_HASH || (motion->motionType == MOTIONTYPE_FIXED && motion->isBroadcast)) { - ListCell *target_lcr; - - foreach(target_lcr, motion->plan.targetlist) - { - ListCell *mt_lcr; - ListCell *mt_lcm; - - TargetEntry *target_tle = (TargetEntry *) lfirst(target_lcr); - Oid expr_relid = get_relid_from_expr(target_tle->expr, rtable); - - if (!OidIsValid(expr_relid)) - continue; - - forboth(mt_lcr, context->mt.resultRelids, mt_lcm, context->mt.needExplicitMotion) - { - Oid mt_relid = lfirst_oid(mt_lcr); - - if (expr_relid == mt_relid) - { - /* There is a Motion before scan. */ - lfirst_int(mt_lcm) = true; - context->mt.isChecking = false; - } - } - } + context->mt.nMotionsAbove += 1; } } @@ -948,25 +894,37 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) case T_ForeignScan: { ListCell *mt_lcr; - ListCell *mt_lcm; Scan *scan = (Scan *) node; List *rtable = root->glob->finalrtable; RangeTblEntry *target_rte = rt_fetch(scan->scanrelid, rtable); - forboth(mt_lcr, context->mt.resultRelids, mt_lcm, context->mt.needExplicitMotion) + foreach(mt_lcr, context->mt.resultRelids) { Oid mt_relid = lfirst_oid(mt_lcr); if (target_rte->relid == mt_relid) { - /* - * There is a scan before Motions. - * - * We don't stop checking because there might - * be motions in other subtrees. - */ - lfirst_oid(mt_lcm) = false; + if (context->mt.nMotionsAbove > 0) + { + /* + * There are motions above. We don't need + * to check other nodes in this subtree + * anymore. + */ + context->mt.needExplicitMotion = true; + context->mt.isChecking = false; + } + else + { + /* + * There aren't any motions above, but + * there might be some underneath. + */ + context->mt.needExplicitMotion = false; + } + + break; } } } @@ -1125,70 +1083,30 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) break; case MOVEMENT_EXPLICIT: + /* + * Were there any motions above the scan? + */ + if (context->mt.needExplicitMotion) { - ListCell *target_lcr; - - bool finished = false; - bool need_motion = false; - List *rtable = root->glob->finalrtable; - - /* - * Search this motion's targetlist for matching relations in - * ModifyTable context. On match, retrieve boolean value we set - * when checking for motions and scans. - */ - foreach(target_lcr, plan->targetlist) - { - ListCell *mt_lcr; - ListCell *mt_lcm; - - TargetEntry *target_tle = (TargetEntry *) lfirst(target_lcr); - Oid expr_relid = InvalidOid; - - if (finished) - break; - - expr_relid = get_relid_from_expr(target_tle->expr, rtable); - - if (!OidIsValid(expr_relid)) - continue; - - forboth(mt_lcr, context->mt.resultRelids, mt_lcm, context->mt.needExplicitMotion) - { - Oid mt_relid = lfirst_oid(mt_lcr); - - if (expr_relid == mt_relid) - { - finished = true; - need_motion = (bool) lfirst_int(mt_lcm); - break; - } - } - } - - if (need_motion) - { - newnode = (Node *) make_explicit_motion(plan, - flow->segidColIdx, - true /* useExecutorVarFormat */ - ); - } - else - { - /* - * Restore flow if Explicit Redistribute Motion is not - * needed - */ - flow->req_move = MOVEMENT_NONE; - flow->flow_before_req_move = NULL; - } + newnode = (Node *) make_explicit_motion(plan, + flow->segidColIdx, + true /* useExecutorVarFormat */ + ); /* * Continue checking in case of another Explicit Redistribute - * Motion + * Motion is needed. */ context->mt.isChecking = true; } + else + { + /* + * Restore flow if Explicit Redistribute Motion is not needed + */ + flow->req_move = MOVEMENT_NONE; + flow->flow_before_req_move = NULL; + } break; case MOVEMENT_NONE: @@ -1248,6 +1166,23 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) context->mt = save_mt; } + if (context->mt.isChecking) + { + if (IsA(node, Motion)) + { + Motion *motion = (Motion *) node; + + if (motion->motionType == MOTIONTYPE_HASH || + (motion->motionType == MOTIONTYPE_FIXED && motion->isBroadcast)) + { + /* + * We're going out of this motion node. + */ + context->mt.nMotionsAbove -= 1; + } + } + } + return newnode; } /* apply_motion_mutator */ From 894ffd3c397747dbf6d828fb5e3073fa80ad7ea5 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Tue, 30 Jan 2024 14:43:49 +0300 Subject: [PATCH 23/83] Fix a test --- .../regress/expected/partition_pruning.out | 37 ++++++++++--------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/src/test/regress/expected/partition_pruning.out b/src/test/regress/expected/partition_pruning.out index dbdb4c75dfd4..d84cda71b13a 100644 --- a/src/test/regress/expected/partition_pruning.out +++ b/src/test/regress/expected/partition_pruning.out @@ -3498,25 +3498,26 @@ from ( ) src where trg.key1 = src.key1 and trg.key1 = 2; - QUERY PLAN -------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------- Update on t_part1_1_prt_2 trg - -> Nested Loop - -> Seq Scan on t_part1_1_prt_2 trg - Filter: (key1 = 2) - -> Materialize - -> Redistribute Motion 1:3 (slice2; segments: 1) - Hash Key: src.key1 - -> Subquery Scan on src - Filter: (src.key1 = 2) - -> WindowAgg - -> Gather Motion 3:1 (slice1; segments: 3) - -> Result - -> Append - -> Seq Scan on t_part1_1_prt_2 r - Filter: (key1 = 2) - Optimizer: Postgres query optimizer -(16 rows) + -> Explicit Redistribute Motion 3:3 (slice3; segments: 3) + -> Nested Loop + -> Seq Scan on t_part1_1_prt_2 trg + Filter: (key1 = 2) + -> Materialize + -> Redistribute Motion 1:3 (slice2; segments: 1) + Hash Key: src.key1 + -> Subquery Scan on src + Filter: (src.key1 = 2) + -> WindowAgg + -> Gather Motion 3:1 (slice1; segments: 3) + -> Result + -> Append + -> Seq Scan on t_part1_1_prt_2 r + Filter: (key1 = 2) + Optimizer: Postgres query optimizer +(17 rows) DROP TABLE t_part1; -- Test that the dynamic partition pruning should not be performed if the partition's opclass and the From 7d9eea92d1ea41d3bef232a5cad12f083a24903e Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Tue, 30 Jan 2024 14:43:49 +0300 Subject: [PATCH 24/83] Don't save previous context (we can't have two ModifyTable nodes in the same plan) --- src/backend/cdb/cdbmutate.c | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index 79b59f4dae5f..4667becc9acc 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -759,7 +759,6 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) int saveNextMotionID; int saveNumInitPlans; int saveSliceDepth; - ModifyTableMotionState save_mt; #ifdef USE_ASSERT_CHECKING PlannerInfo *root = (PlannerInfo *) context->base.node; @@ -813,12 +812,6 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) { ListCell *lcr; - /* - * If there was another ModifyTable node before, save the previous - * state. - */ - save_mt = context->mt; - context->mt.resultRelids = NIL; context->mt.needExplicitMotion = false; context->mt.nMotionsAbove = 0; @@ -1154,18 +1147,6 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) plan->nMotionNodes = context->nextMotionID - saveNextMotionID; plan->nInitPlans = hash_get_num_entries(context->planid_subplans) - saveNumInitPlans; - /* - * Restore previous ModifyTable state, in case this wasn't the first - * one. - */ - if (IsA(node, ModifyTable)) - { - ModifyTable *mt = (ModifyTable *) node; - - if (mt->operation == CMD_UPDATE || mt->operation == CMD_DELETE) - context->mt = save_mt; - } - if (context->mt.isChecking) { if (IsA(node, Motion)) From 3531176c8f131c09253c5caba6afdd7e8c08a2dd Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Tue, 30 Jan 2024 14:43:50 +0300 Subject: [PATCH 25/83] Revert "Change comments, since varnoold is used" This reverts commit 955356e3065236a552953bdc6958940d34c28966. --- src/include/nodes/primnodes.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h index 5c5d2f23d6a8..caed77f1c29d 100644 --- a/src/include/nodes/primnodes.h +++ b/src/include/nodes/primnodes.h @@ -154,8 +154,8 @@ typedef struct Expr * subplans; for example, in a join node varno becomes INNER_VAR or OUTER_VAR * and varattno becomes the index of the proper element of that subplan's * target list. But varnoold/varoattno continue to hold the original values. - * They are very useful for debugging and interpreting completed plans, so we - * keep them around. + * The code doesn't really need varnoold/varoattno, but they are very useful + * for debugging and interpreting completed plans, so we keep them around. */ #define INNER_VAR 65000 /* reference to inner subplan */ #define OUTER_VAR 65001 /* reference to outer subplan */ @@ -180,7 +180,7 @@ typedef struct Var Index varlevelsup; /* for subquery variables referencing outer * relations; 0 in a normal var, >0 means N * levels up */ - Index varnoold; /* original value of varno */ + Index varnoold; /* original value of varno, for debugging */ AttrNumber varoattno; /* original value of varattno */ int location; /* token location, or -1 if unknown */ } Var; From 02256573aaf116fe898e7ccaf2bf2b2d95af81b2 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Tue, 30 Jan 2024 14:43:50 +0300 Subject: [PATCH 26/83] Describe ModifyTableState members --- src/backend/cdb/cdbmutate.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index 4667becc9acc..705b25829eee 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -70,8 +70,11 @@ typedef struct ModifyTableMotionState List *resultRelids; /* Oid list of relations to be * modified */ bool needExplicitMotion; - int nMotionsAbove; - bool isChecking; + int nMotionsAbove; /* Number of Redistribute/Broadcast + * motions above the current node */ + bool isChecking; /* True if we encountered ModifyTable + * node with UPDATE/DELETE and we plan + * to insert Explicit Motions */ } ModifyTableMotionState; /* From 52810224497c829442adbe060f088af32965086e Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Tue, 30 Jan 2024 14:43:50 +0300 Subject: [PATCH 27/83] Change null assignments to asserts --- src/backend/cdb/cdbmutate.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index 705b25829eee..0528cc6f037f 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -815,9 +815,13 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) { ListCell *lcr; - context->mt.resultRelids = NIL; - context->mt.needExplicitMotion = false; - context->mt.nMotionsAbove = 0; + /* + * Sanity check, since we don't allow multiple ModifyTable nodes + * in the same plan. + */ + Assert(context->mt.resultRelids == NULL); + Assert(context->mt.nMotionsAbove == 0); + Assert(!context->mt.needExplicitMotion); /* * When UPDATE/DELETE occurs on a partitioned table, or a table that From 7efdd12d7a92b1dc004ce3b7fb217fbf084c2164 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Tue, 30 Jan 2024 14:43:50 +0300 Subject: [PATCH 28/83] Change List to Bitmapset --- src/backend/cdb/cdbmutate.c | 56 ++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 32 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index 0528cc6f037f..1e06246c1185 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -67,8 +67,7 @@ typedef struct ModifyTableMotionState { - List *resultRelids; /* Oid list of relations to be - * modified */ + Bitmapset *resultReloids; /* Oids of relations to be modified */ bool needExplicitMotion; int nMotionsAbove; /* Number of Redistribute/Broadcast * motions above the current node */ @@ -420,7 +419,7 @@ apply_motion(PlannerInfo *root, Plan *plan, Query *query) state.nextMotionID = 1; /* Start at 1 so zero will mean "unassigned". */ state.sliceDepth = 0; - state.mt.resultRelids = NIL; + state.mt.resultReloids = NULL; state.mt.needExplicitMotion = false; state.mt.nMotionsAbove = 0; state.mt.isChecking = false; @@ -819,7 +818,7 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) * Sanity check, since we don't allow multiple ModifyTable nodes * in the same plan. */ - Assert(context->mt.resultRelids == NULL); + Assert(context->mt.resultReloids == NULL); Assert(context->mt.nMotionsAbove == 0); Assert(!context->mt.needExplicitMotion); @@ -836,8 +835,8 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) Index rti = lfirst_int(lcr); RangeTblEntry *rte = rt_fetch(rti, root->glob->finalrtable); - context->mt.resultRelids = lappend_oid(context->mt.resultRelids, - rte->relid); + context->mt.resultReloids = bms_add_member(context->mt.resultReloids, + rte->relid); } context->mt.isChecking = true; @@ -893,39 +892,32 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) case T_WorkTableScan: case T_ForeignScan: { - ListCell *mt_lcr; - Scan *scan = (Scan *) node; List *rtable = root->glob->finalrtable; RangeTblEntry *target_rte = rt_fetch(scan->scanrelid, rtable); - foreach(mt_lcr, context->mt.resultRelids) + if (bms_is_member(target_rte->relid, context->mt.resultReloids)) { - Oid mt_relid = lfirst_oid(mt_lcr); - - if (target_rte->relid == mt_relid) + if (context->mt.nMotionsAbove > 0) { - if (context->mt.nMotionsAbove > 0) - { - /* - * There are motions above. We don't need - * to check other nodes in this subtree - * anymore. - */ - context->mt.needExplicitMotion = true; - context->mt.isChecking = false; - } - else - { - /* - * There aren't any motions above, but - * there might be some underneath. - */ - context->mt.needExplicitMotion = false; - } - - break; + /* + * There are motions above. We don't need + * to check other nodes in this subtree + * anymore. + */ + context->mt.needExplicitMotion = true; + context->mt.isChecking = false; } + else + { + /* + * There aren't any motions above, but + * there might be some underneath. + */ + context->mt.needExplicitMotion = false; + } + + break; } } break; From 4f609c4ae9a78b1559e5428272ef820bb2bf7438 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Tue, 30 Jan 2024 14:43:50 +0300 Subject: [PATCH 29/83] Use getrelid macro --- src/backend/cdb/cdbmutate.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index 1e06246c1185..96dd0e1c940e 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -833,10 +833,10 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) foreach(lcr, mt->resultRelations) { Index rti = lfirst_int(lcr); - RangeTblEntry *rte = rt_fetch(rti, root->glob->finalrtable); + Oid relid = getrelid(rti, root->glob->finalrtable); context->mt.resultReloids = bms_add_member(context->mt.resultReloids, - rte->relid); + relid); } context->mt.isChecking = true; @@ -894,9 +894,9 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) { Scan *scan = (Scan *) node; List *rtable = root->glob->finalrtable; - RangeTblEntry *target_rte = rt_fetch(scan->scanrelid, rtable); + Oid scan_reloid = getrelid(scan->scanrelid, rtable); - if (bms_is_member(target_rte->relid, context->mt.resultReloids)) + if (bms_is_member(scan_reloid, context->mt.resultReloids)) { if (context->mt.nMotionsAbove > 0) { From fe9905b4b355d5cb51c785ec0f5807ac9fd2c6cb Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Tue, 30 Jan 2024 14:43:50 +0300 Subject: [PATCH 30/83] Continue checking only when we reach the Explicit Redistribute Motion --- src/backend/cdb/cdbmutate.c | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index 96dd0e1c940e..4058273c532f 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -900,23 +900,19 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) { if (context->mt.nMotionsAbove > 0) { - /* - * There are motions above. We don't need - * to check other nodes in this subtree - * anymore. - */ + /* There are motions above */ context->mt.needExplicitMotion = true; - context->mt.isChecking = false; } else { - /* - * There aren't any motions above, but - * there might be some underneath. - */ + /* There aren't any motions above */ context->mt.needExplicitMotion = false; } - + /* + * We don't need to check other nodes in this + * subtree anymore. + */ + context->mt.isChecking = false; break; } } @@ -1084,12 +1080,6 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) flow->segidColIdx, true /* useExecutorVarFormat */ ); - - /* - * Continue checking in case of another Explicit Redistribute - * Motion is needed. - */ - context->mt.isChecking = true; } else { @@ -1099,6 +1089,12 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) flow->req_move = MOVEMENT_NONE; flow->flow_before_req_move = NULL; } + + /* + * Continue checking in case of another Explicit Redistribute + * Motion is needed. + */ + context->mt.isChecking = true; break; case MOVEMENT_NONE: From 98132ebb75119972941275a2b511cacf9e2e992f Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Tue, 30 Jan 2024 14:43:50 +0300 Subject: [PATCH 31/83] Revert "Fix a test" This reverts commit e5fbce1551b11949335a68a1f0309a536e5c10dc. Explicit Redistribute Motion here is unnecessary. --- .../regress/expected/partition_pruning.out | 37 +++++++++---------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/src/test/regress/expected/partition_pruning.out b/src/test/regress/expected/partition_pruning.out index d84cda71b13a..dbdb4c75dfd4 100644 --- a/src/test/regress/expected/partition_pruning.out +++ b/src/test/regress/expected/partition_pruning.out @@ -3498,26 +3498,25 @@ from ( ) src where trg.key1 = src.key1 and trg.key1 = 2; - QUERY PLAN -------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------- Update on t_part1_1_prt_2 trg - -> Explicit Redistribute Motion 3:3 (slice3; segments: 3) - -> Nested Loop - -> Seq Scan on t_part1_1_prt_2 trg - Filter: (key1 = 2) - -> Materialize - -> Redistribute Motion 1:3 (slice2; segments: 1) - Hash Key: src.key1 - -> Subquery Scan on src - Filter: (src.key1 = 2) - -> WindowAgg - -> Gather Motion 3:1 (slice1; segments: 3) - -> Result - -> Append - -> Seq Scan on t_part1_1_prt_2 r - Filter: (key1 = 2) - Optimizer: Postgres query optimizer -(17 rows) + -> Nested Loop + -> Seq Scan on t_part1_1_prt_2 trg + Filter: (key1 = 2) + -> Materialize + -> Redistribute Motion 1:3 (slice2; segments: 1) + Hash Key: src.key1 + -> Subquery Scan on src + Filter: (src.key1 = 2) + -> WindowAgg + -> Gather Motion 3:1 (slice1; segments: 3) + -> Result + -> Append + -> Seq Scan on t_part1_1_prt_2 r + Filter: (key1 = 2) + Optimizer: Postgres query optimizer +(16 rows) DROP TABLE t_part1; -- Test that the dynamic partition pruning should not be performed if the partition's opclass and the From d5f68e185b1765646b4d9e32ba5ee3314db659e6 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Tue, 30 Jan 2024 14:43:50 +0300 Subject: [PATCH 32/83] Ignore motions in InitPlans --- src/backend/cdb/cdbmutate.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index 4058273c532f..32a8c578d36a 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -773,9 +773,15 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) /* An expression node might have subtrees containing plans to be mutated. */ if (!is_plan_node(node)) { + /* + * mt.isChecking is true whether we are checking for motions underneath + * to add Explicit Reditribute Motion, ignoring any in InitPlans. So if + * we recurse into an InitPlan, save it and temporarily set it to false. + */ if (IsA(node, SubPlan) &&((SubPlan *) node)->is_initplan) { bool found; + bool saveMtIsChecking = context->mt.isChecking; int saveSliceDepth = context->sliceDepth; SubPlan *subplan = (SubPlan *) node; /* @@ -790,8 +796,10 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) /* reset sliceDepth for each init plan */ context->sliceDepth = 0; + context->mt.isChecking = false; node = plan_tree_mutator(node, apply_motion_mutator, context); + context->mt.isChecking = saveMtIsChecking; context->sliceDepth = saveSliceDepth; return node; From 31f4f3e96c44d66f989fe40cf448c36bc493579e Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Tue, 30 Jan 2024 14:43:51 +0300 Subject: [PATCH 33/83] Fix comment in test --- src/test/regress/expected/update_gp.out | 5 +++-- src/test/regress/expected/update_gp_optimizer.out | 5 +++-- src/test/regress/sql/update_gp.sql | 5 +++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/test/regress/expected/update_gp.out b/src/test/regress/expected/update_gp.out index 65fd16bf0f52..355890751ac7 100644 --- a/src/test/regress/expected/update_gp.out +++ b/src/test/regress/expected/update_gp.out @@ -208,8 +208,9 @@ DROP TABLE keo2; DROP TABLE keo3; DROP TABLE keo4; DROP TABLE keo5; --- Explicit Redistribute Motion shouled be added only if there is a motion --- between the scan and the ModifyTable on the relation we are going to update. (test case not applicable to ORCA) +-- Explicit Redistribute Motion should be added only if there is a motion +-- between the scan and the ModifyTable on the relation we are going to update. +-- (test case not applicable to ORCA) CREATE TABLE t1 (i int, j int) DISTRIBUTED BY (i); CREATE TABLE t2 (i int) DISTRIBUTED BY (i); CREATE TABLE t_strewn (i int) DISTRIBUTED RANDOMLY; diff --git a/src/test/regress/expected/update_gp_optimizer.out b/src/test/regress/expected/update_gp_optimizer.out index c4cf892ddea6..c51bf19f63fd 100644 --- a/src/test/regress/expected/update_gp_optimizer.out +++ b/src/test/regress/expected/update_gp_optimizer.out @@ -213,8 +213,9 @@ DROP TABLE keo2; DROP TABLE keo3; DROP TABLE keo4; DROP TABLE keo5; --- Explicit Redistribute Motion shouled be added only if there is a motion --- between the scan and the ModifyTable on the relation we are going to update. (test case not applicable to ORCA) +-- Explicit Redistribute Motion should be added only if there is a motion +-- between the scan and the ModifyTable on the relation we are going to update. +-- (test case not applicable to ORCA) CREATE TABLE t1 (i int, j int) DISTRIBUTED BY (i); CREATE TABLE t2 (i int) DISTRIBUTED BY (i); CREATE TABLE t_strewn (i int) DISTRIBUTED RANDOMLY; diff --git a/src/test/regress/sql/update_gp.sql b/src/test/regress/sql/update_gp.sql index c28463d7abb7..53d7f25b2a38 100644 --- a/src/test/regress/sql/update_gp.sql +++ b/src/test/regress/sql/update_gp.sql @@ -117,8 +117,9 @@ DROP TABLE keo3; DROP TABLE keo4; DROP TABLE keo5; --- Explicit Redistribute Motion shouled be added only if there is a motion --- between the scan and the ModifyTable on the relation we are going to update. (test case not applicable to ORCA) +-- Explicit Redistribute Motion should be added only if there is a motion +-- between the scan and the ModifyTable on the relation we are going to update. +-- (test case not applicable to ORCA) CREATE TABLE t1 (i int, j int) DISTRIBUTED BY (i); CREATE TABLE t2 (i int) DISTRIBUTED BY (i); CREATE TABLE t_strewn (i int) DISTRIBUTED RANDOMLY; From 5a53ecde8588fb920d8e4b9031147501b431ce17 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Tue, 30 Jan 2024 14:43:51 +0300 Subject: [PATCH 34/83] Add a test that requies Explicit Redistribute Motion --- src/test/regress/expected/update_gp.out | 21 +++++++++++++++++++ .../regress/expected/update_gp_optimizer.out | 18 ++++++++++++++++ src/test/regress/sql/update_gp.sql | 6 ++++++ 3 files changed, 45 insertions(+) diff --git a/src/test/regress/expected/update_gp.out b/src/test/regress/expected/update_gp.out index 355890751ac7..26670b1bf65a 100644 --- a/src/test/regress/expected/update_gp.out +++ b/src/test/regress/expected/update_gp.out @@ -214,6 +214,7 @@ DROP TABLE keo5; CREATE TABLE t1 (i int, j int) DISTRIBUTED BY (i); CREATE TABLE t2 (i int) DISTRIBUTED BY (i); CREATE TABLE t_strewn (i int) DISTRIBUTED RANDOMLY; +CREATE TABLE t_strewn2 (i int) DISTRIBUTED RANDOMLY; EXPLAIN (costs off) UPDATE t1 SET j = t_strewn.i FROM t_strewn WHERE t_strewn.i = t1.i; QUERY PLAN @@ -278,9 +279,29 @@ DELETE FROM t_strewn WHERE t_strewn.i = (SELECT t2.i FROM t2 WHERE t_strewn.i = Settings: optimizer=off (16 rows) +EXPLAIN (costs off) +UPDATE t_strewn SET i = t_strewn2.i +FROM t_strewn2 WHERE t_strewn.i = t_strewn2.i; + QUERY PLAN +------------------------------------------------------------------------ + Update on t_strewn + -> Explicit Redistribute Motion 3:3 (slice3; segments: 3) + -> Hash Join + Hash Cond: (t_strewn.i = t_strewn2.i) + -> Redistribute Motion 3:3 (slice1; segments: 3) + Hash Key: t_strewn.i + -> Seq Scan on t_strewn + -> Hash + -> Redistribute Motion 3:3 (slice2; segments: 3) + Hash Key: t_strewn2.i + -> Seq Scan on t_strewn2 + Optimizer: Postgres query optimizer +(12 rows) + DROP TABLE t1; DROP TABLE t2; DROP TABLE t_strewn; +DROP TABLE t_strewn2; -- Explicit Redistribute Motion should not be mistakenly elided for inherited -- tables. (test case not applicable to ORCA) CREATE TABLE i (i int, j int) DISTRIBUTED BY (i); diff --git a/src/test/regress/expected/update_gp_optimizer.out b/src/test/regress/expected/update_gp_optimizer.out index c51bf19f63fd..cc86a9851291 100644 --- a/src/test/regress/expected/update_gp_optimizer.out +++ b/src/test/regress/expected/update_gp_optimizer.out @@ -219,6 +219,7 @@ DROP TABLE keo5; CREATE TABLE t1 (i int, j int) DISTRIBUTED BY (i); CREATE TABLE t2 (i int) DISTRIBUTED BY (i); CREATE TABLE t_strewn (i int) DISTRIBUTED RANDOMLY; +CREATE TABLE t_strewn2 (i int) DISTRIBUTED RANDOMLY; EXPLAIN (costs off) UPDATE t1 SET j = t_strewn.i FROM t_strewn WHERE t_strewn.i = t1.i; QUERY PLAN @@ -289,9 +290,26 @@ DELETE FROM t_strewn WHERE t_strewn.i = (SELECT t2.i FROM t2 WHERE t_strewn.i = Settings: optimizer=on (19 rows) +EXPLAIN (costs off) +UPDATE t_strewn SET i = t_strewn2.i +FROM t_strewn2 WHERE t_strewn.i = t_strewn2.i; + QUERY PLAN +--------------------------------------------------------------------- + Update + -> Split + -> Hash Join + Hash Cond: (t_strewn.i = t_strewn2.i) + -> Seq Scan on t_strewn + -> Hash + -> Broadcast Motion 3:3 (slice1; segments: 3) + -> Seq Scan on t_strewn2 + Optimizer: Pivotal Optimizer (GPORCA) +(9 rows) + DROP TABLE t1; DROP TABLE t2; DROP TABLE t_strewn; +DROP TABLE t_strewn2; -- Explicit Redistribute Motion should not be mistakenly elided for inherited -- tables. (test case not applicable to ORCA) CREATE TABLE i (i int, j int) DISTRIBUTED BY (i); diff --git a/src/test/regress/sql/update_gp.sql b/src/test/regress/sql/update_gp.sql index 53d7f25b2a38..df8a3bd3115b 100644 --- a/src/test/regress/sql/update_gp.sql +++ b/src/test/regress/sql/update_gp.sql @@ -123,6 +123,7 @@ DROP TABLE keo5; CREATE TABLE t1 (i int, j int) DISTRIBUTED BY (i); CREATE TABLE t2 (i int) DISTRIBUTED BY (i); CREATE TABLE t_strewn (i int) DISTRIBUTED RANDOMLY; +CREATE TABLE t_strewn2 (i int) DISTRIBUTED RANDOMLY; EXPLAIN (costs off) UPDATE t1 SET j = t_strewn.i FROM t_strewn WHERE t_strewn.i = t1.i; @@ -135,9 +136,14 @@ WHERE 0 < ALL (SELECT i FROM cte); EXPLAIN (costs off, verbose) DELETE FROM t_strewn WHERE t_strewn.i = (SELECT t2.i FROM t2 WHERE t_strewn.i = t2.i); +EXPLAIN (costs off) +UPDATE t_strewn SET i = t_strewn2.i +FROM t_strewn2 WHERE t_strewn.i = t_strewn2.i; + DROP TABLE t1; DROP TABLE t2; DROP TABLE t_strewn; +DROP TABLE t_strewn2; -- Explicit Redistribute Motion should not be mistakenly elided for inherited -- tables. (test case not applicable to ORCA) From bf60efae4cec145a8cfad6e91f124572f21b07ed Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Tue, 30 Jan 2024 14:43:51 +0300 Subject: [PATCH 35/83] Add a test for motions in InitPlans --- src/test/regress/expected/update_gp.out | 30 ++++++++++++++++++ .../regress/expected/update_gp_optimizer.out | 31 +++++++++++++++++++ src/test/regress/sql/update_gp.sql | 12 +++++++ 3 files changed, 73 insertions(+) diff --git a/src/test/regress/expected/update_gp.out b/src/test/regress/expected/update_gp.out index 26670b1bf65a..fe7d92aacde9 100644 --- a/src/test/regress/expected/update_gp.out +++ b/src/test/regress/expected/update_gp.out @@ -302,6 +302,36 @@ DROP TABLE t1; DROP TABLE t2; DROP TABLE t_strewn; DROP TABLE t_strewn2; +-- Explicit Redistribute Motion should not be added if there are motions in Init +-- Plans but not in the main plan. (test case not applicable to ORCA) +CREATE TABLE t1 (a int, b int) DISTRIBUTED RANDOMLY; +CREATE TABLE t2 (a int, b int) DISTRIBUTED RANDOMLY; +EXPLAIN (costs off) WITH cte AS ( + SELECT count(*) AS c FROM t1, t2 WHERE t1.b = t2.b +) UPDATE t2 SET b = 10 WHERE a = (SELECT * FROM cte); + QUERY PLAN +-------------------------------------------------------------------------------------- + Update on t2 + InitPlan 1 (returns $0) (slice4) + -> Aggregate + -> Gather Motion 3:1 (slice3; segments: 3) + -> Aggregate + -> Hash Join + Hash Cond: (t1.b = t2_1.b) + -> Redistribute Motion 3:3 (slice1; segments: 3) + Hash Key: t1.b + -> Seq Scan on t1 + -> Hash + -> Redistribute Motion 3:3 (slice2; segments: 3) + Hash Key: t2_1.b + -> Seq Scan on t2 t2_1 + -> Seq Scan on t2 + Filter: (a = $0) + Optimizer: Postgres query optimizer +(17 rows) + +DROP TABLE t1; +DROP TABLE t2; -- Explicit Redistribute Motion should not be mistakenly elided for inherited -- tables. (test case not applicable to ORCA) CREATE TABLE i (i int, j int) DISTRIBUTED BY (i); diff --git a/src/test/regress/expected/update_gp_optimizer.out b/src/test/regress/expected/update_gp_optimizer.out index cc86a9851291..538d8a7f8437 100644 --- a/src/test/regress/expected/update_gp_optimizer.out +++ b/src/test/regress/expected/update_gp_optimizer.out @@ -310,6 +310,37 @@ DROP TABLE t1; DROP TABLE t2; DROP TABLE t_strewn; DROP TABLE t_strewn2; +-- Explicit Redistribute Motion should not be added if there are motions in Init +-- Plans but not in the main plan. (test case not applicable to ORCA) +CREATE TABLE t1 (a int, b int) DISTRIBUTED RANDOMLY; +CREATE TABLE t2 (a int, b int) DISTRIBUTED RANDOMLY; +EXPLAIN (costs off) WITH cte AS ( + SELECT count(*) AS c FROM t1, t2 WHERE t1.b = t2.b +) UPDATE t2 SET b = 10 WHERE a = (SELECT * FROM cte); + QUERY PLAN +--------------------------------------------------------------------------------------------------------- + Update + -> Explicit Redistribute Motion 1:3 (slice4; segments: 1) + -> Split + -> Result + -> Hash Join + Hash Cond: ((t2.a)::bigint = (count())) + -> Gather Motion 3:1 (slice1; segments: 3) + -> Seq Scan on t2 + -> Hash + -> Aggregate + -> Gather Motion 3:1 (slice3; segments: 3) + -> Hash Join + Hash Cond: (t1.b = t2_1.b) + -> Seq Scan on t1 + -> Hash + -> Broadcast Motion 3:3 (slice2; segments: 3) + -> Seq Scan on t2 t2_1 + Optimizer: Pivotal Optimizer (GPORCA) +(18 rows) + +DROP TABLE t1; +DROP TABLE t2; -- Explicit Redistribute Motion should not be mistakenly elided for inherited -- tables. (test case not applicable to ORCA) CREATE TABLE i (i int, j int) DISTRIBUTED BY (i); diff --git a/src/test/regress/sql/update_gp.sql b/src/test/regress/sql/update_gp.sql index df8a3bd3115b..191ff90e69a0 100644 --- a/src/test/regress/sql/update_gp.sql +++ b/src/test/regress/sql/update_gp.sql @@ -145,6 +145,18 @@ DROP TABLE t2; DROP TABLE t_strewn; DROP TABLE t_strewn2; +-- Explicit Redistribute Motion should not be added if there are motions in Init +-- Plans but not in the main plan. (test case not applicable to ORCA) +CREATE TABLE t1 (a int, b int) DISTRIBUTED RANDOMLY; +CREATE TABLE t2 (a int, b int) DISTRIBUTED RANDOMLY; + +EXPLAIN (costs off) WITH cte AS ( + SELECT count(*) AS c FROM t1, t2 WHERE t1.b = t2.b +) UPDATE t2 SET b = 10 WHERE a = (SELECT * FROM cte); + +DROP TABLE t1; +DROP TABLE t2; + -- Explicit Redistribute Motion should not be mistakenly elided for inherited -- tables. (test case not applicable to ORCA) CREATE TABLE i (i int, j int) DISTRIBUTED BY (i); From a60385fbfa1bdfc69d7b87cc5c1b2eae69c508f6 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Tue, 30 Jan 2024 14:43:51 +0300 Subject: [PATCH 36/83] Squash if condition --- src/backend/cdb/cdbmutate.c | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index 32a8c578d36a..00a6a59a4651 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -906,16 +906,12 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) if (bms_is_member(scan_reloid, context->mt.resultReloids)) { - if (context->mt.nMotionsAbove > 0) - { - /* There are motions above */ - context->mt.needExplicitMotion = true; - } - else - { - /* There aren't any motions above */ - context->mt.needExplicitMotion = false; - } + /* + * We need Explicit Redistribute Motion only if + * there were any motions above. + */ + context->mt.needExplicitMotion = context->mt.nMotionsAbove > 0; + /* * We don't need to check other nodes in this * subtree anymore. From 8d91f93414711658ce95ec28277d20eae6cf04fb Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Tue, 30 Jan 2024 14:43:51 +0300 Subject: [PATCH 37/83] Add a comment --- src/backend/cdb/cdbmutate.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index 00a6a59a4651..52da004252e8 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -1095,8 +1095,10 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) } /* - * Continue checking in case of another Explicit Redistribute - * Motion is needed. + * If we're here, it means we are directly under the ModifyTable + * node. We are about to go to out of the recursion and go into + * other subtree. So continue checking in case of another Explicit + * Redistribute Motion is needed. */ context->mt.isChecking = true; break; From c0d72696b8570c028955fbce1896afb8048f57c4 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Tue, 30 Jan 2024 14:43:51 +0300 Subject: [PATCH 38/83] Squash if condition --- src/backend/cdb/cdbmutate.c | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index 52da004252e8..fbee3105b390 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -1148,20 +1148,17 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) plan->nMotionNodes = context->nextMotionID - saveNextMotionID; plan->nInitPlans = hash_get_num_entries(context->planid_subplans) - saveNumInitPlans; - if (context->mt.isChecking) + if (context->mt.isChecking && IsA(node, Motion)) { - if (IsA(node, Motion)) - { - Motion *motion = (Motion *) node; + Motion *motion = (Motion *) node; - if (motion->motionType == MOTIONTYPE_HASH || - (motion->motionType == MOTIONTYPE_FIXED && motion->isBroadcast)) - { - /* - * We're going out of this motion node. - */ - context->mt.nMotionsAbove -= 1; - } + if (motion->motionType == MOTIONTYPE_HASH || + (motion->motionType == MOTIONTYPE_FIXED && motion->isBroadcast)) + { + /* + * We're going out of this motion node. + */ + context->mt.nMotionsAbove -= 1; } } From a99c4b16fccea38460afde7dc28d94641d87afb1 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Tue, 30 Jan 2024 14:43:51 +0300 Subject: [PATCH 39/83] Fix nMotionsAbove not decrementing when we already found motions --- src/backend/cdb/cdbmutate.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index fbee3105b390..1e90b041e2c0 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -71,9 +71,10 @@ typedef struct ModifyTableMotionState bool needExplicitMotion; int nMotionsAbove; /* Number of Redistribute/Broadcast * motions above the current node */ - bool isChecking; /* True if we encountered ModifyTable - * node with UPDATE/DELETE and we plan - * to insert Explicit Motions */ + bool isChecking; /* True if we are currently checking + * the plan for motions */ + bool isEnabled; /* True if we encountered ModifyTable + * node with UPDATE/DELETE */ } ModifyTableMotionState; /* @@ -422,6 +423,7 @@ apply_motion(PlannerInfo *root, Plan *plan, Query *query) state.mt.resultReloids = NULL; state.mt.needExplicitMotion = false; state.mt.nMotionsAbove = 0; + state.mt.isEnabled = false; state.mt.isChecking = false; memset(&ctl, 0, sizeof(ctl)); @@ -828,6 +830,7 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) */ Assert(context->mt.resultReloids == NULL); Assert(context->mt.nMotionsAbove == 0); + Assert(!context->mt.isEnabled); Assert(!context->mt.needExplicitMotion); /* @@ -847,6 +850,7 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) relid); } + context->mt.isEnabled = true; context->mt.isChecking = true; } } @@ -1148,7 +1152,7 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) plan->nMotionNodes = context->nextMotionID - saveNextMotionID; plan->nInitPlans = hash_get_num_entries(context->planid_subplans) - saveNumInitPlans; - if (context->mt.isChecking && IsA(node, Motion)) + if (context->mt.isEnabled && IsA(node, Motion)) { Motion *motion = (Motion *) node; From 43b562b873e255756f74106f8571328685d659d1 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Tue, 30 Jan 2024 14:43:51 +0300 Subject: [PATCH 40/83] Revert "Fix nMotionsAbove not decrementing when we already found motions" This reverts commit 99b32b35be61cb60eddf06d3d81d85cccdd913e3. --- src/backend/cdb/cdbmutate.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index 1e90b041e2c0..fbee3105b390 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -71,10 +71,9 @@ typedef struct ModifyTableMotionState bool needExplicitMotion; int nMotionsAbove; /* Number of Redistribute/Broadcast * motions above the current node */ - bool isChecking; /* True if we are currently checking - * the plan for motions */ - bool isEnabled; /* True if we encountered ModifyTable - * node with UPDATE/DELETE */ + bool isChecking; /* True if we encountered ModifyTable + * node with UPDATE/DELETE and we plan + * to insert Explicit Motions */ } ModifyTableMotionState; /* @@ -423,7 +422,6 @@ apply_motion(PlannerInfo *root, Plan *plan, Query *query) state.mt.resultReloids = NULL; state.mt.needExplicitMotion = false; state.mt.nMotionsAbove = 0; - state.mt.isEnabled = false; state.mt.isChecking = false; memset(&ctl, 0, sizeof(ctl)); @@ -830,7 +828,6 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) */ Assert(context->mt.resultReloids == NULL); Assert(context->mt.nMotionsAbove == 0); - Assert(!context->mt.isEnabled); Assert(!context->mt.needExplicitMotion); /* @@ -850,7 +847,6 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) relid); } - context->mt.isEnabled = true; context->mt.isChecking = true; } } @@ -1152,7 +1148,7 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) plan->nMotionNodes = context->nextMotionID - saveNextMotionID; plan->nInitPlans = hash_get_num_entries(context->planid_subplans) - saveNumInitPlans; - if (context->mt.isEnabled && IsA(node, Motion)) + if (context->mt.isChecking && IsA(node, Motion)) { Motion *motion = (Motion *) node; From 2f170f895c8025a22e6b57f83cb905ee9add407e Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Tue, 30 Jan 2024 14:43:52 +0300 Subject: [PATCH 41/83] Reset nMotionsAbove and needExplicitMotion when we are exiting recursion --- src/backend/cdb/cdbmutate.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index fbee3105b390..c8414ea77a7e 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -1097,10 +1097,12 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) /* * If we're here, it means we are directly under the ModifyTable * node. We are about to go to out of the recursion and go into - * other subtree. So continue checking in case of another Explicit - * Redistribute Motion is needed. + * other subtree. So reset the state and continue checking in case + * of another Explicit Redistribute Motion is needed. */ context->mt.isChecking = true; + context->mt.needExplicitMotion = false; + context->mt.nMotionsAbove = 0; break; case MOVEMENT_NONE: From 44e7492fab0166d380544be3c29acd5b8ff0c1f7 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Tue, 30 Jan 2024 14:43:52 +0300 Subject: [PATCH 42/83] Check if scan retrieves ctid --- src/backend/cdb/cdbmutate.c | 52 ++++++++++++++++++++++++++++++++----- 1 file changed, 45 insertions(+), 7 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index c8414ea77a7e..87a73f65ccda 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -900,24 +900,62 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) case T_WorkTableScan: case T_ForeignScan: { + ListCell *lcr; + Scan *scan = (Scan *) node; List *rtable = root->glob->finalrtable; Oid scan_reloid = getrelid(scan->scanrelid, rtable); if (bms_is_member(scan_reloid, context->mt.resultReloids)) { + bool has_ctid = false; + /* - * We need Explicit Redistribute Motion only if - * there were any motions above. + * We need to make sure that this scan retrieves + * ctid from the relation, because citd is required + * for DELETE/UPDATE to find modified tuples. If + * this scan's targetlist does not retrive ctid, it + * does not affect the result of UPDATE/DELETE. */ - context->mt.needExplicitMotion = context->mt.nMotionsAbove > 0; + foreach(lcr, scan->plan.targetlist) + { + TargetEntry *scan_tle = lfirst(lcr); + + if (IsA(scan_tle->expr, Var)) + { + Var *var = (Var *) scan_tle->expr; + + if (var->varoattno == SelfItemPointerAttributeNumber) + { + has_ctid = true; + break; + } + } + } + + if (!has_ctid) + break; /* - * We don't need to check other nodes in this - * subtree anymore. + * We need Explicit Redistribute Motion only if + * there were any motions above. */ - context->mt.isChecking = false; - break; + if (context->mt.nMotionsAbove > 0) + { + context->mt.needExplicitMotion = true; + } + else + { + context->mt.needExplicitMotion = false; + /* + * This is a scan without any Motions above. + * We don't care if there are more scans of + * this table with any motions above, as the + * unaltered ctid we need for ModifyTable is + * already available. + */ + context->mt.isChecking = false; + } } } break; From 7eb73d565541193b5d7f34a572c75522c1dbad49 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Tue, 30 Jan 2024 14:43:52 +0300 Subject: [PATCH 43/83] Add missing assert --- src/backend/cdb/cdbmutate.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index 87a73f65ccda..01d5ebc479b0 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -829,6 +829,7 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) Assert(context->mt.resultReloids == NULL); Assert(context->mt.nMotionsAbove == 0); Assert(!context->mt.needExplicitMotion); + Assert(!context->mt.isChecking); /* * When UPDATE/DELETE occurs on a partitioned table, or a table that From b6f50494e2dcdf523e5da13589d749a6043e3c84 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Tue, 30 Jan 2024 14:43:52 +0300 Subject: [PATCH 44/83] Revert "Check if scan retrieves ctid" This reverts commit e7a5544420e4b28e19bfd7727a7215c6efa06dbf. --- src/backend/cdb/cdbmutate.c | 52 +++++-------------------------------- 1 file changed, 7 insertions(+), 45 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index 01d5ebc479b0..f289445193b6 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -901,62 +901,24 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) case T_WorkTableScan: case T_ForeignScan: { - ListCell *lcr; - Scan *scan = (Scan *) node; List *rtable = root->glob->finalrtable; Oid scan_reloid = getrelid(scan->scanrelid, rtable); if (bms_is_member(scan_reloid, context->mt.resultReloids)) { - bool has_ctid = false; - /* - * We need to make sure that this scan retrieves - * ctid from the relation, because citd is required - * for DELETE/UPDATE to find modified tuples. If - * this scan's targetlist does not retrive ctid, it - * does not affect the result of UPDATE/DELETE. + * We need Explicit Redistribute Motion only if + * there were any motions above. */ - foreach(lcr, scan->plan.targetlist) - { - TargetEntry *scan_tle = lfirst(lcr); - - if (IsA(scan_tle->expr, Var)) - { - Var *var = (Var *) scan_tle->expr; - - if (var->varoattno == SelfItemPointerAttributeNumber) - { - has_ctid = true; - break; - } - } - } - - if (!has_ctid) - break; + context->mt.needExplicitMotion = context->mt.nMotionsAbove > 0; /* - * We need Explicit Redistribute Motion only if - * there were any motions above. + * We don't need to check other nodes in this + * subtree anymore. */ - if (context->mt.nMotionsAbove > 0) - { - context->mt.needExplicitMotion = true; - } - else - { - context->mt.needExplicitMotion = false; - /* - * This is a scan without any Motions above. - * We don't care if there are more scans of - * this table with any motions above, as the - * unaltered ctid we need for ModifyTable is - * already available. - */ - context->mt.isChecking = false; - } + context->mt.isChecking = false; + break; } } break; From b0bebc8698ebedf8ca96d39e3fbd7e864cfc2b1f Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Tue, 30 Jan 2024 14:43:52 +0300 Subject: [PATCH 45/83] Check for matching RTIs instead of relation IDs --- src/backend/cdb/cdbmutate.c | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index f289445193b6..ce50e496988c 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -67,7 +67,8 @@ typedef struct ModifyTableMotionState { - Bitmapset *resultReloids; /* Oids of relations to be modified */ + Bitmapset *resultRtis; /* Indexes into rtable for relations to + * be modified */ bool needExplicitMotion; int nMotionsAbove; /* Number of Redistribute/Broadcast * motions above the current node */ @@ -419,7 +420,7 @@ apply_motion(PlannerInfo *root, Plan *plan, Query *query) state.nextMotionID = 1; /* Start at 1 so zero will mean "unassigned". */ state.sliceDepth = 0; - state.mt.resultReloids = NULL; + state.mt.resultRtis = NULL; state.mt.needExplicitMotion = false; state.mt.nMotionsAbove = 0; state.mt.isChecking = false; @@ -826,7 +827,7 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) * Sanity check, since we don't allow multiple ModifyTable nodes * in the same plan. */ - Assert(context->mt.resultReloids == NULL); + Assert(context->mt.resultRtis == NULL); Assert(context->mt.nMotionsAbove == 0); Assert(!context->mt.needExplicitMotion); Assert(!context->mt.isChecking); @@ -836,16 +837,15 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) * is a part of inheritance tree, ModifyTable node will have more * than one relation in resultRelations. * - * We make a list of resulting relations' Oids to compare them + * We make a list of resulting relations' indexes to compare them * later. */ foreach(lcr, mt->resultRelations) { Index rti = lfirst_int(lcr); - Oid relid = getrelid(rti, root->glob->finalrtable); - context->mt.resultReloids = bms_add_member(context->mt.resultReloids, - relid); + context->mt.resultRtis = bms_add_member(context->mt.resultRtis, + rti); } context->mt.isChecking = true; @@ -902,10 +902,8 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) case T_ForeignScan: { Scan *scan = (Scan *) node; - List *rtable = root->glob->finalrtable; - Oid scan_reloid = getrelid(scan->scanrelid, rtable); - if (bms_is_member(scan_reloid, context->mt.resultReloids)) + if (bms_is_member(scan->scanrelid, context->mt.resultRtis)) { /* * We need Explicit Redistribute Motion only if From 38beaa2eb0d262ef4688af2644f5996d74009ce5 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Fri, 9 Feb 2024 14:45:42 +0300 Subject: [PATCH 46/83] Get rid of rti variable --- src/backend/cdb/cdbmutate.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index ce50e496988c..ac32357fdd0f 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -842,10 +842,8 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) */ foreach(lcr, mt->resultRelations) { - Index rti = lfirst_int(lcr); - context->mt.resultRtis = bms_add_member(context->mt.resultRtis, - rti); + lfirst_int(lcr)); } context->mt.isChecking = true; From b8c919ceda85c6d2b9357cd16664a9aeae79c705 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Fri, 9 Feb 2024 14:47:14 +0300 Subject: [PATCH 47/83] Add tests for partitions and swapped trees --- src/test/regress/expected/update_gp.out | 114 ++++++++++++++++++ .../regress/expected/update_gp_optimizer.out | 106 ++++++++++++++++ src/test/regress/sql/update_gp.sql | 54 +++++++++ 3 files changed, 274 insertions(+) diff --git a/src/test/regress/expected/update_gp.out b/src/test/regress/expected/update_gp.out index fe7d92aacde9..ec64995cddb4 100644 --- a/src/test/regress/expected/update_gp.out +++ b/src/test/regress/expected/update_gp.out @@ -485,6 +485,120 @@ DELETE FROM t1 USING t2 WHERE t1.a = t2.a; DELETE FROM t1 USING t2 WHERE t1.a = t2.a; DROP TABLE t1; DROP TABLE t2; +-- Explicit Redistribute Motion should not be elided if we encounter a scan on +-- the same table that we are going to update, but with different range table +-- index. +CREATE TABLE t(a int, b int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +CREATE TABLE d(a int, b int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +INSERT INTO t SELECT a, a FROM generate_series(1, 100) a; +INSERT INTO d SELECT a, a FROM generate_series(1, 100) a; +EXPLAIN (costs off) update d trg +SET b = src.b1 +FROM (SELECT t.a AS a1, t.b AS b1, d.a AS a2, d.b AS b2 FROM t JOIN d USING (b)) src +WHERE trg.a = src.a1 + AND trg.a = 2; + QUERY PLAN +--------------------------------------------------------------- + Update on d trg + -> Explicit Redistribute Motion 3:3 (slice2; segments: 3) + -> Hash Join + Hash Cond: (t.b = d.b) + -> Broadcast Motion 3:3 (slice1; segments: 3) + -> Nested Loop + -> Seq Scan on d trg + Filter: (a = 2) + -> Seq Scan on t + Filter: (a = 2) + -> Hash + -> Seq Scan on d + Optimizer: Postgres query optimizer +(13 rows) + +-- Use Nested Loop to change left tree with the right tree, to swap the extra +-- scan we don't indend to detect with the real one. +SET enable_hashjoin = off; +SET enable_nestloop = on; +EXPLAIN (costs off) update d trg +SET b = src.b1 +FROM (SELECT t.a AS a1, t.b AS b1, d.a AS a2, d.b AS b2 FROM t JOIN d USING (b)) src +WHERE trg.a = src.a1 + AND trg.a = 2; + QUERY PLAN +--------------------------------------------------------------------------- + Update on d trg + -> Explicit Redistribute Motion 3:3 (slice3; segments: 3) + -> Nested Loop + -> Nested Loop + Join Filter: (t.b = d.b) + -> Seq Scan on d + -> Materialize + -> Broadcast Motion 3:3 (slice1; segments: 3) + -> Seq Scan on t + Filter: (a = 2) + -> Materialize + -> Broadcast Motion 3:3 (slice2; segments: 3) + -> Seq Scan on d trg + Filter: (a = 2) + Optimizer: Postgres query optimizer +(15 rows) + +RESET enable_hashjoin; +RESET enable_nestloop; +DROP TABLE t; +DROP TABLE d; +-- Explicit Redistribute Motion should be elided for every partition that does +-- not have any motions above the scan on the table/partition we are going to +-- update. +CREATE TABLE into_table (a int, b int, c int) DISTRIBUTED BY (b) + PARTITION BY RANGE(b) (start (1) end(5) every(1)); +NOTICE: CREATE TABLE will create partition "into_table_1_prt_1" for table "into_table" +NOTICE: CREATE TABLE will create partition "into_table_1_prt_2" for table "into_table" +NOTICE: CREATE TABLE will create partition "into_table_1_prt_3" for table "into_table" +NOTICE: CREATE TABLE will create partition "into_table_1_prt_4" for table "into_table" +CREATE TABLE from_table (a int, b int, c int) DISTRIBUTED BY (a); +INSERT INTO into_table SELECT i * 2, i, i * 3 FROM generate_series(1,4) i; +INSERT INTO from_table SELECT i, i * 2, i * 3 FROM generate_series(1,4) i; +-- These partitions will need to have Explicit Redistribute above them. +TRUNCATE into_table_1_prt_1; +TRUNCATE into_table_1_prt_3; +ANALYZE into_table_1_prt_1; +ANALYZE into_table_1_prt_3; +EXPLAIN (costs off) + UPDATE into_table SET c = from_table.b FROM from_table; + QUERY PLAN +--------------------------------------------------------------- + Update on into_table_1_prt_1 + -> Explicit Redistribute Motion 3:3 (slice2; segments: 3) + -> Nested Loop + -> Broadcast Motion 3:3 (slice1; segments: 3) + -> Seq Scan on into_table_1_prt_1 + -> Materialize + -> Seq Scan on from_table + -> Nested Loop + -> Seq Scan on into_table_1_prt_2 + -> Materialize + -> Broadcast Motion 3:3 (slice3; segments: 3) + -> Seq Scan on from_table + -> Explicit Redistribute Motion 3:3 (slice5; segments: 3) + -> Nested Loop + -> Broadcast Motion 3:3 (slice4; segments: 3) + -> Seq Scan on into_table_1_prt_3 + -> Materialize + -> Seq Scan on from_table + -> Nested Loop + -> Seq Scan on into_table_1_prt_4 + -> Materialize + -> Broadcast Motion 3:3 (slice6; segments: 3) + -> Seq Scan on from_table + Optimizer: Postgres query optimizer +(24 rows) + +DROP TABLE into_table; +DROP TABLE from_table; -- -- text types. We should support the following updates. -- diff --git a/src/test/regress/expected/update_gp_optimizer.out b/src/test/regress/expected/update_gp_optimizer.out index 538d8a7f8437..0f7e3b060a14 100644 --- a/src/test/regress/expected/update_gp_optimizer.out +++ b/src/test/regress/expected/update_gp_optimizer.out @@ -452,6 +452,112 @@ DELETE FROM t1 USING t2 WHERE t1.a = t2.a; DELETE FROM t1 USING t2 WHERE t1.a = t2.a; DROP TABLE t1; DROP TABLE t2; +-- Explicit Redistribute Motion should not be elided if we encounter a scan on +-- the same table that we are going to update, but with different range table +-- index. +CREATE TABLE t(a int, b int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +CREATE TABLE d(a int, b int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +INSERT INTO t SELECT a, a FROM generate_series(1, 100) a; +INSERT INTO d SELECT a, a FROM generate_series(1, 100) a; +EXPLAIN (costs off) update d trg +SET b = src.b1 +FROM (SELECT t.a AS a1, t.b AS b1, d.a AS a2, d.b AS b2 FROM t JOIN d USING (b)) src +WHERE trg.a = src.a1 + AND trg.a = 2; + QUERY PLAN +--------------------------------------------------------------------------------------- + Update + -> Split + -> Hash Join + Hash Cond: (d.a = t.a) + -> Seq Scan on d + Filter: (a = 2) + -> Hash + -> Broadcast Motion 3:3 (slice2; segments: 3) + -> Hash Join + Hash Cond: (d_1.b = t.b) + -> Seq Scan on d d_1 + -> Hash + -> Broadcast Motion 3:3 (slice1; segments: 3) + -> Seq Scan on t + Filter: (a = 2) + Optimizer: Pivotal Optimizer (GPORCA) +(16 rows) + +-- Use Nested Loop to change left tree with the right tree, to swap the extra +-- scan we don't indend to detect with the real one. +SET enable_hashjoin = off; +SET enable_nestloop = on; +EXPLAIN (costs off) update d trg +SET b = src.b1 +FROM (SELECT t.a AS a1, t.b AS b1, d.a AS a2, d.b AS b2 FROM t JOIN d USING (b)) src +WHERE trg.a = src.a1 + AND trg.a = 2; + QUERY PLAN +--------------------------------------------------------------------------------------- + Update + -> Split + -> Hash Join + Hash Cond: (d.a = t.a) + -> Seq Scan on d + Filter: (a = 2) + -> Hash + -> Broadcast Motion 3:3 (slice2; segments: 3) + -> Hash Join + Hash Cond: (d_1.b = t.b) + -> Seq Scan on d d_1 + -> Hash + -> Broadcast Motion 3:3 (slice1; segments: 3) + -> Seq Scan on t + Filter: (a = 2) + Optimizer: Pivotal Optimizer (GPORCA) +(16 rows) + +RESET enable_hashjoin; +RESET enable_nestloop; +DROP TABLE t; +DROP TABLE d; +-- Explicit Redistribute Motion should be elided for every partition that does +-- not have any motions above the scan on the table/partition we are going to +-- update. +CREATE TABLE into_table (a int, b int, c int) DISTRIBUTED BY (b) + PARTITION BY RANGE(b) (start (1) end(5) every(1)); +NOTICE: CREATE TABLE will create partition "into_table_1_prt_1" for table "into_table" +NOTICE: CREATE TABLE will create partition "into_table_1_prt_2" for table "into_table" +NOTICE: CREATE TABLE will create partition "into_table_1_prt_3" for table "into_table" +NOTICE: CREATE TABLE will create partition "into_table_1_prt_4" for table "into_table" +CREATE TABLE from_table (a int, b int, c int) DISTRIBUTED BY (a); +INSERT INTO into_table SELECT i * 2, i, i * 3 FROM generate_series(1,4) i; +INSERT INTO from_table SELECT i, i * 2, i * 3 FROM generate_series(1,4) i; +-- These partitions will need to have Explicit Redistribute above them. +TRUNCATE into_table_1_prt_1; +TRUNCATE into_table_1_prt_3; +ANALYZE into_table_1_prt_1; +ANALYZE into_table_1_prt_3; +EXPLAIN (costs off) + UPDATE into_table SET c = from_table.b FROM from_table; + QUERY PLAN +-------------------------------------------------------------------------------- + Update + -> Split + -> Nested Loop + Join Filter: true + -> Sequence + -> Partition Selector for into_table (dynamic scan id: 1) + Partitions selected: 4 (out of 4) + -> Dynamic Seq Scan on into_table (dynamic scan id: 1) + -> Materialize + -> Broadcast Motion 3:3 (slice1; segments: 3) + -> Seq Scan on from_table + Optimizer: Pivotal Optimizer (GPORCA) +(12 rows) + +DROP TABLE into_table; +DROP TABLE from_table; -- -- text types. We should support the following updates. -- diff --git a/src/test/regress/sql/update_gp.sql b/src/test/regress/sql/update_gp.sql index 191ff90e69a0..56931ceeb1d9 100644 --- a/src/test/regress/sql/update_gp.sql +++ b/src/test/regress/sql/update_gp.sql @@ -211,6 +211,60 @@ DELETE FROM t1 USING t2 WHERE t1.a = t2.a; DROP TABLE t1; DROP TABLE t2; +-- Explicit Redistribute Motion should not be elided if we encounter a scan on +-- the same table that we are going to update, but with different range table +-- index. +CREATE TABLE t(a int, b int); +CREATE TABLE d(a int, b int); +INSERT INTO t SELECT a, a FROM generate_series(1, 100) a; +INSERT INTO d SELECT a, a FROM generate_series(1, 100) a; + +EXPLAIN (costs off) update d trg +SET b = src.b1 +FROM (SELECT t.a AS a1, t.b AS b1, d.a AS a2, d.b AS b2 FROM t JOIN d USING (b)) src +WHERE trg.a = src.a1 + AND trg.a = 2; + +-- Use Nested Loop to change left tree with the right tree, to swap the extra +-- scan we don't indend to detect with the real one. +SET enable_hashjoin = off; +SET enable_nestloop = on; + +EXPLAIN (costs off) update d trg +SET b = src.b1 +FROM (SELECT t.a AS a1, t.b AS b1, d.a AS a2, d.b AS b2 FROM t JOIN d USING (b)) src +WHERE trg.a = src.a1 + AND trg.a = 2; + +RESET enable_hashjoin; +RESET enable_nestloop; + +DROP TABLE t; +DROP TABLE d; + +-- Explicit Redistribute Motion should be elided for every partition that does +-- not have any motions above the scan on the table/partition we are going to +-- update. +CREATE TABLE into_table (a int, b int, c int) DISTRIBUTED BY (b) + PARTITION BY RANGE(b) (start (1) end(5) every(1)); + +CREATE TABLE from_table (a int, b int, c int) DISTRIBUTED BY (a); + +INSERT INTO into_table SELECT i * 2, i, i * 3 FROM generate_series(1,4) i; +INSERT INTO from_table SELECT i, i * 2, i * 3 FROM generate_series(1,4) i; + +-- These partitions will need to have Explicit Redistribute above them. +TRUNCATE into_table_1_prt_1; +TRUNCATE into_table_1_prt_3; +ANALYZE into_table_1_prt_1; +ANALYZE into_table_1_prt_3; + +EXPLAIN (costs off) + UPDATE into_table SET c = from_table.b FROM from_table; + +DROP TABLE into_table; +DROP TABLE from_table; + -- -- text types. We should support the following updates. -- From 4b98b7b810cde252aa0dec38679435d8c6379e82 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Fri, 9 Feb 2024 15:10:49 +0300 Subject: [PATCH 48/83] Fix typos in tests, use the same name for tables --- src/test/regress/expected/update_gp.out | 106 ++++++++++-------- .../regress/expected/update_gp_optimizer.out | 98 +++++++++------- src/test/regress/sql/update_gp.sql | 55 +++++---- 3 files changed, 148 insertions(+), 111 deletions(-) diff --git a/src/test/regress/expected/update_gp.out b/src/test/regress/expected/update_gp.out index ec64995cddb4..0c5d8eb83c4e 100644 --- a/src/test/regress/expected/update_gp.out +++ b/src/test/regress/expected/update_gp.out @@ -209,8 +209,16 @@ DROP TABLE keo3; DROP TABLE keo4; DROP TABLE keo5; -- Explicit Redistribute Motion should be added only if there is a motion --- between the scan and the ModifyTable on the relation we are going to update. +-- between the scan and the ModifyTable on the relation we are going to modify. -- (test case not applicable to ORCA) +DROP TABLE IF EXISTS t1; +NOTICE: table "t1" does not exist, skipping +DROP TABLE IF EXISTS t2; +NOTICE: table "t2" does not exist, skipping +DROP TABLE IF EXISTS t_strewn; +NOTICE: table "t_strewn" does not exist, skipping +DROP TABLE IF EXISTS t_strewn2; +NOTICE: table "t_strewn2" does not exist, skipping CREATE TABLE t1 (i int, j int) DISTRIBUTED BY (i); CREATE TABLE t2 (i int) DISTRIBUTED BY (i); CREATE TABLE t_strewn (i int) DISTRIBUTED RANDOMLY; @@ -334,6 +342,12 @@ DROP TABLE t1; DROP TABLE t2; -- Explicit Redistribute Motion should not be mistakenly elided for inherited -- tables. (test case not applicable to ORCA) +DROP TABLE IF EXISTS i; +NOTICE: table "i" does not exist, skipping +DROP TABLE IF EXISTS foochild; +NOTICE: table "foochild" does not exist, skipping +DROP TABLE IF EXISTS foo; +NOTICE: table "foo" does not exist, skipping CREATE TABLE i (i int, j int) DISTRIBUTED BY (i); INSERT INTO i SELECT generate_series(1, 100), generate_series(1, 100) * 3; @@ -486,35 +500,35 @@ DELETE FROM t1 USING t2 WHERE t1.a = t2.a; DROP TABLE t1; DROP TABLE t2; -- Explicit Redistribute Motion should not be elided if we encounter a scan on --- the same table that we are going to update, but with different range table +-- the same table that we are going to modify, but with different range table -- index. -CREATE TABLE t(a int, b int); +CREATE TABLE t1 (a int, b int); NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table. HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. -CREATE TABLE d(a int, b int); +CREATE TABLE t2 (a int, b int); NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table. HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. -INSERT INTO t SELECT a, a FROM generate_series(1, 100) a; -INSERT INTO d SELECT a, a FROM generate_series(1, 100) a; -EXPLAIN (costs off) update d trg +INSERT INTO t1 SELECT a, a FROM generate_series(1, 100) a; +INSERT INTO t2 SELECT a, a FROM generate_series(1, 100) a; +EXPLAIN (costs off) update t2 trg SET b = src.b1 -FROM (SELECT t.a AS a1, t.b AS b1, d.a AS a2, d.b AS b2 FROM t JOIN d USING (b)) src +FROM (SELECT t1.a AS a1, t1.b AS b1, t2.a AS a2, t2.b AS b2 FROM t1 JOIN t2 USING (b)) src WHERE trg.a = src.a1 AND trg.a = 2; QUERY PLAN --------------------------------------------------------------- - Update on d trg + Update on t2 trg -> Explicit Redistribute Motion 3:3 (slice2; segments: 3) -> Hash Join - Hash Cond: (t.b = d.b) + Hash Cond: (t1.b = t2.b) -> Broadcast Motion 3:3 (slice1; segments: 3) -> Nested Loop - -> Seq Scan on d trg + -> Seq Scan on t2 trg Filter: (a = 2) - -> Seq Scan on t + -> Seq Scan on t1 Filter: (a = 2) -> Hash - -> Seq Scan on d + -> Seq Scan on t2 Optimizer: Postgres query optimizer (13 rows) @@ -522,83 +536,83 @@ WHERE trg.a = src.a1 -- scan we don't indend to detect with the real one. SET enable_hashjoin = off; SET enable_nestloop = on; -EXPLAIN (costs off) update d trg +EXPLAIN (costs off) update t2 trg SET b = src.b1 -FROM (SELECT t.a AS a1, t.b AS b1, d.a AS a2, d.b AS b2 FROM t JOIN d USING (b)) src +FROM (SELECT t1.a AS a1, t1.b AS b1, t2.a AS a2, t2.b AS b2 FROM t1 JOIN t2 USING (b)) src WHERE trg.a = src.a1 AND trg.a = 2; QUERY PLAN --------------------------------------------------------------------------- - Update on d trg + Update on t2 trg -> Explicit Redistribute Motion 3:3 (slice3; segments: 3) -> Nested Loop -> Nested Loop - Join Filter: (t.b = d.b) - -> Seq Scan on d + Join Filter: (t1.b = t2.b) + -> Seq Scan on t2 -> Materialize -> Broadcast Motion 3:3 (slice1; segments: 3) - -> Seq Scan on t + -> Seq Scan on t1 Filter: (a = 2) -> Materialize -> Broadcast Motion 3:3 (slice2; segments: 3) - -> Seq Scan on d trg + -> Seq Scan on t2 trg Filter: (a = 2) Optimizer: Postgres query optimizer (15 rows) RESET enable_hashjoin; RESET enable_nestloop; -DROP TABLE t; -DROP TABLE d; +DROP TABLE t1; +DROP TABLE t2; -- Explicit Redistribute Motion should be elided for every partition that does -- not have any motions above the scan on the table/partition we are going to -- update. -CREATE TABLE into_table (a int, b int, c int) DISTRIBUTED BY (b) +CREATE TABLE t1 (a int, b int, c int) DISTRIBUTED BY (b) PARTITION BY RANGE(b) (start (1) end(5) every(1)); -NOTICE: CREATE TABLE will create partition "into_table_1_prt_1" for table "into_table" -NOTICE: CREATE TABLE will create partition "into_table_1_prt_2" for table "into_table" -NOTICE: CREATE TABLE will create partition "into_table_1_prt_3" for table "into_table" -NOTICE: CREATE TABLE will create partition "into_table_1_prt_4" for table "into_table" -CREATE TABLE from_table (a int, b int, c int) DISTRIBUTED BY (a); -INSERT INTO into_table SELECT i * 2, i, i * 3 FROM generate_series(1,4) i; -INSERT INTO from_table SELECT i, i * 2, i * 3 FROM generate_series(1,4) i; +NOTICE: CREATE TABLE will create partition "t1_1_prt_1" for table "t1" +NOTICE: CREATE TABLE will create partition "t1_1_prt_2" for table "t1" +NOTICE: CREATE TABLE will create partition "t1_1_prt_3" for table "t1" +NOTICE: CREATE TABLE will create partition "t1_1_prt_4" for table "t1" +CREATE TABLE t2 (a int, b int, c int) DISTRIBUTED BY (a); +INSERT INTO t1 SELECT i * 2, i, i * 3 FROM generate_series(1,4) i; +INSERT INTO t2 SELECT i, i * 2, i * 3 FROM generate_series(1,4) i; -- These partitions will need to have Explicit Redistribute above them. -TRUNCATE into_table_1_prt_1; -TRUNCATE into_table_1_prt_3; -ANALYZE into_table_1_prt_1; -ANALYZE into_table_1_prt_3; +TRUNCATE t1_1_prt_1; +TRUNCATE t1_1_prt_3; +ANALYZE t1_1_prt_1; +ANALYZE t1_1_prt_3; EXPLAIN (costs off) - UPDATE into_table SET c = from_table.b FROM from_table; + UPDATE t1 SET c = t2.b FROM t2; QUERY PLAN --------------------------------------------------------------- - Update on into_table_1_prt_1 + Update on t1_1_prt_1 -> Explicit Redistribute Motion 3:3 (slice2; segments: 3) -> Nested Loop -> Broadcast Motion 3:3 (slice1; segments: 3) - -> Seq Scan on into_table_1_prt_1 + -> Seq Scan on t1_1_prt_1 -> Materialize - -> Seq Scan on from_table + -> Seq Scan on t2 -> Nested Loop - -> Seq Scan on into_table_1_prt_2 + -> Seq Scan on t1_1_prt_2 -> Materialize -> Broadcast Motion 3:3 (slice3; segments: 3) - -> Seq Scan on from_table + -> Seq Scan on t2 -> Explicit Redistribute Motion 3:3 (slice5; segments: 3) -> Nested Loop -> Broadcast Motion 3:3 (slice4; segments: 3) - -> Seq Scan on into_table_1_prt_3 + -> Seq Scan on t1_1_prt_3 -> Materialize - -> Seq Scan on from_table + -> Seq Scan on t2 -> Nested Loop - -> Seq Scan on into_table_1_prt_4 + -> Seq Scan on t1_1_prt_4 -> Materialize -> Broadcast Motion 3:3 (slice6; segments: 3) - -> Seq Scan on from_table + -> Seq Scan on t2 Optimizer: Postgres query optimizer (24 rows) -DROP TABLE into_table; -DROP TABLE from_table; +DROP TABLE t1; +DROP TABLE t2; -- -- text types. We should support the following updates. -- diff --git a/src/test/regress/expected/update_gp_optimizer.out b/src/test/regress/expected/update_gp_optimizer.out index 0f7e3b060a14..77f2bdc2a346 100644 --- a/src/test/regress/expected/update_gp_optimizer.out +++ b/src/test/regress/expected/update_gp_optimizer.out @@ -214,8 +214,16 @@ DROP TABLE keo3; DROP TABLE keo4; DROP TABLE keo5; -- Explicit Redistribute Motion should be added only if there is a motion --- between the scan and the ModifyTable on the relation we are going to update. +-- between the scan and the ModifyTable on the relation we are going to modify. -- (test case not applicable to ORCA) +DROP TABLE IF EXISTS t1; +NOTICE: table "t1" does not exist, skipping +DROP TABLE IF EXISTS t2; +NOTICE: table "t2" does not exist, skipping +DROP TABLE IF EXISTS t_strewn; +NOTICE: table "t_strewn" does not exist, skipping +DROP TABLE IF EXISTS t_strewn2; +NOTICE: table "t_strewn2" does not exist, skipping CREATE TABLE t1 (i int, j int) DISTRIBUTED BY (i); CREATE TABLE t2 (i int) DISTRIBUTED BY (i); CREATE TABLE t_strewn (i int) DISTRIBUTED RANDOMLY; @@ -343,6 +351,12 @@ DROP TABLE t1; DROP TABLE t2; -- Explicit Redistribute Motion should not be mistakenly elided for inherited -- tables. (test case not applicable to ORCA) +DROP TABLE IF EXISTS i; +NOTICE: table "i" does not exist, skipping +DROP TABLE IF EXISTS foochild; +NOTICE: table "foochild" does not exist, skipping +DROP TABLE IF EXISTS foo; +NOTICE: table "foo" does not exist, skipping CREATE TABLE i (i int, j int) DISTRIBUTED BY (i); INSERT INTO i SELECT generate_series(1, 100), generate_series(1, 100) * 3; @@ -453,19 +467,19 @@ DELETE FROM t1 USING t2 WHERE t1.a = t2.a; DROP TABLE t1; DROP TABLE t2; -- Explicit Redistribute Motion should not be elided if we encounter a scan on --- the same table that we are going to update, but with different range table +-- the same table that we are going to modify, but with different range table -- index. -CREATE TABLE t(a int, b int); +CREATE TABLE t1 (a int, b int); NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table. HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. -CREATE TABLE d(a int, b int); +CREATE TABLE t2 (a int, b int); NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table. HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. -INSERT INTO t SELECT a, a FROM generate_series(1, 100) a; -INSERT INTO d SELECT a, a FROM generate_series(1, 100) a; -EXPLAIN (costs off) update d trg +INSERT INTO t1 SELECT a, a FROM generate_series(1, 100) a; +INSERT INTO t2 SELECT a, a FROM generate_series(1, 100) a; +EXPLAIN (costs off) update t2 trg SET b = src.b1 -FROM (SELECT t.a AS a1, t.b AS b1, d.a AS a2, d.b AS b2 FROM t JOIN d USING (b)) src +FROM (SELECT t1.a AS a1, t1.b AS b1, t2.a AS a2, t2.b AS b2 FROM t1 JOIN t2 USING (b)) src WHERE trg.a = src.a1 AND trg.a = 2; QUERY PLAN @@ -473,17 +487,17 @@ WHERE trg.a = src.a1 Update -> Split -> Hash Join - Hash Cond: (d.a = t.a) - -> Seq Scan on d + Hash Cond: (t2.a = t1.a) + -> Seq Scan on t2 Filter: (a = 2) -> Hash -> Broadcast Motion 3:3 (slice2; segments: 3) -> Hash Join - Hash Cond: (d_1.b = t.b) - -> Seq Scan on d d_1 + Hash Cond: (t2_1.b = t1.b) + -> Seq Scan on t2 t2_1 -> Hash -> Broadcast Motion 3:3 (slice1; segments: 3) - -> Seq Scan on t + -> Seq Scan on t1 Filter: (a = 2) Optimizer: Pivotal Optimizer (GPORCA) (16 rows) @@ -492,9 +506,9 @@ WHERE trg.a = src.a1 -- scan we don't indend to detect with the real one. SET enable_hashjoin = off; SET enable_nestloop = on; -EXPLAIN (costs off) update d trg +EXPLAIN (costs off) update t2 trg SET b = src.b1 -FROM (SELECT t.a AS a1, t.b AS b1, d.a AS a2, d.b AS b2 FROM t JOIN d USING (b)) src +FROM (SELECT t1.a AS a1, t1.b AS b1, t2.a AS a2, t2.b AS b2 FROM t1 JOIN t2 USING (b)) src WHERE trg.a = src.a1 AND trg.a = 2; QUERY PLAN @@ -502,62 +516,62 @@ WHERE trg.a = src.a1 Update -> Split -> Hash Join - Hash Cond: (d.a = t.a) - -> Seq Scan on d + Hash Cond: (t2.a = t1.a) + -> Seq Scan on t2 Filter: (a = 2) -> Hash -> Broadcast Motion 3:3 (slice2; segments: 3) -> Hash Join - Hash Cond: (d_1.b = t.b) - -> Seq Scan on d d_1 + Hash Cond: (t2_1.b = t1.b) + -> Seq Scan on t2 t2_1 -> Hash -> Broadcast Motion 3:3 (slice1; segments: 3) - -> Seq Scan on t + -> Seq Scan on t1 Filter: (a = 2) Optimizer: Pivotal Optimizer (GPORCA) (16 rows) RESET enable_hashjoin; RESET enable_nestloop; -DROP TABLE t; -DROP TABLE d; +DROP TABLE t1; +DROP TABLE t2; -- Explicit Redistribute Motion should be elided for every partition that does -- not have any motions above the scan on the table/partition we are going to -- update. -CREATE TABLE into_table (a int, b int, c int) DISTRIBUTED BY (b) +CREATE TABLE t1 (a int, b int, c int) DISTRIBUTED BY (b) PARTITION BY RANGE(b) (start (1) end(5) every(1)); -NOTICE: CREATE TABLE will create partition "into_table_1_prt_1" for table "into_table" -NOTICE: CREATE TABLE will create partition "into_table_1_prt_2" for table "into_table" -NOTICE: CREATE TABLE will create partition "into_table_1_prt_3" for table "into_table" -NOTICE: CREATE TABLE will create partition "into_table_1_prt_4" for table "into_table" -CREATE TABLE from_table (a int, b int, c int) DISTRIBUTED BY (a); -INSERT INTO into_table SELECT i * 2, i, i * 3 FROM generate_series(1,4) i; -INSERT INTO from_table SELECT i, i * 2, i * 3 FROM generate_series(1,4) i; +NOTICE: CREATE TABLE will create partition "t1_1_prt_1" for table "t1" +NOTICE: CREATE TABLE will create partition "t1_1_prt_2" for table "t1" +NOTICE: CREATE TABLE will create partition "t1_1_prt_3" for table "t1" +NOTICE: CREATE TABLE will create partition "t1_1_prt_4" for table "t1" +CREATE TABLE t2 (a int, b int, c int) DISTRIBUTED BY (a); +INSERT INTO t1 SELECT i * 2, i, i * 3 FROM generate_series(1,4) i; +INSERT INTO t2 SELECT i, i * 2, i * 3 FROM generate_series(1,4) i; -- These partitions will need to have Explicit Redistribute above them. -TRUNCATE into_table_1_prt_1; -TRUNCATE into_table_1_prt_3; -ANALYZE into_table_1_prt_1; -ANALYZE into_table_1_prt_3; +TRUNCATE t1_1_prt_1; +TRUNCATE t1_1_prt_3; +ANALYZE t1_1_prt_1; +ANALYZE t1_1_prt_3; EXPLAIN (costs off) - UPDATE into_table SET c = from_table.b FROM from_table; - QUERY PLAN --------------------------------------------------------------------------------- + UPDATE t1 SET c = t2.b FROM t2; + QUERY PLAN +------------------------------------------------------------------------ Update -> Split -> Nested Loop Join Filter: true -> Sequence - -> Partition Selector for into_table (dynamic scan id: 1) + -> Partition Selector for t1 (dynamic scan id: 1) Partitions selected: 4 (out of 4) - -> Dynamic Seq Scan on into_table (dynamic scan id: 1) + -> Dynamic Seq Scan on t1 (dynamic scan id: 1) -> Materialize -> Broadcast Motion 3:3 (slice1; segments: 3) - -> Seq Scan on from_table + -> Seq Scan on t2 Optimizer: Pivotal Optimizer (GPORCA) (12 rows) -DROP TABLE into_table; -DROP TABLE from_table; +DROP TABLE t1; +DROP TABLE t2; -- -- text types. We should support the following updates. -- diff --git a/src/test/regress/sql/update_gp.sql b/src/test/regress/sql/update_gp.sql index 56931ceeb1d9..7d64523f560c 100644 --- a/src/test/regress/sql/update_gp.sql +++ b/src/test/regress/sql/update_gp.sql @@ -118,8 +118,13 @@ DROP TABLE keo4; DROP TABLE keo5; -- Explicit Redistribute Motion should be added only if there is a motion --- between the scan and the ModifyTable on the relation we are going to update. +-- between the scan and the ModifyTable on the relation we are going to modify. -- (test case not applicable to ORCA) +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; +DROP TABLE IF EXISTS t_strewn; +DROP TABLE IF EXISTS t_strewn2; + CREATE TABLE t1 (i int, j int) DISTRIBUTED BY (i); CREATE TABLE t2 (i int) DISTRIBUTED BY (i); CREATE TABLE t_strewn (i int) DISTRIBUTED RANDOMLY; @@ -159,6 +164,10 @@ DROP TABLE t2; -- Explicit Redistribute Motion should not be mistakenly elided for inherited -- tables. (test case not applicable to ORCA) +DROP TABLE IF EXISTS i; +DROP TABLE IF EXISTS foochild; +DROP TABLE IF EXISTS foo; + CREATE TABLE i (i int, j int) DISTRIBUTED BY (i); INSERT INTO i SELECT generate_series(1, 100), generate_series(1, 100) * 3; @@ -212,16 +221,16 @@ DROP TABLE t1; DROP TABLE t2; -- Explicit Redistribute Motion should not be elided if we encounter a scan on --- the same table that we are going to update, but with different range table +-- the same table that we are going to modify, but with different range table -- index. -CREATE TABLE t(a int, b int); -CREATE TABLE d(a int, b int); -INSERT INTO t SELECT a, a FROM generate_series(1, 100) a; -INSERT INTO d SELECT a, a FROM generate_series(1, 100) a; +CREATE TABLE t1 (a int, b int); +CREATE TABLE t2 (a int, b int); +INSERT INTO t1 SELECT a, a FROM generate_series(1, 100) a; +INSERT INTO t2 SELECT a, a FROM generate_series(1, 100) a; -EXPLAIN (costs off) update d trg +EXPLAIN (costs off) update t2 trg SET b = src.b1 -FROM (SELECT t.a AS a1, t.b AS b1, d.a AS a2, d.b AS b2 FROM t JOIN d USING (b)) src +FROM (SELECT t1.a AS a1, t1.b AS b1, t2.a AS a2, t2.b AS b2 FROM t1 JOIN t2 USING (b)) src WHERE trg.a = src.a1 AND trg.a = 2; @@ -230,40 +239,40 @@ WHERE trg.a = src.a1 SET enable_hashjoin = off; SET enable_nestloop = on; -EXPLAIN (costs off) update d trg +EXPLAIN (costs off) update t2 trg SET b = src.b1 -FROM (SELECT t.a AS a1, t.b AS b1, d.a AS a2, d.b AS b2 FROM t JOIN d USING (b)) src +FROM (SELECT t1.a AS a1, t1.b AS b1, t2.a AS a2, t2.b AS b2 FROM t1 JOIN t2 USING (b)) src WHERE trg.a = src.a1 AND trg.a = 2; RESET enable_hashjoin; RESET enable_nestloop; -DROP TABLE t; -DROP TABLE d; +DROP TABLE t1; +DROP TABLE t2; -- Explicit Redistribute Motion should be elided for every partition that does -- not have any motions above the scan on the table/partition we are going to -- update. -CREATE TABLE into_table (a int, b int, c int) DISTRIBUTED BY (b) +CREATE TABLE t1 (a int, b int, c int) DISTRIBUTED BY (b) PARTITION BY RANGE(b) (start (1) end(5) every(1)); -CREATE TABLE from_table (a int, b int, c int) DISTRIBUTED BY (a); +CREATE TABLE t2 (a int, b int, c int) DISTRIBUTED BY (a); -INSERT INTO into_table SELECT i * 2, i, i * 3 FROM generate_series(1,4) i; -INSERT INTO from_table SELECT i, i * 2, i * 3 FROM generate_series(1,4) i; +INSERT INTO t1 SELECT i * 2, i, i * 3 FROM generate_series(1,4) i; +INSERT INTO t2 SELECT i, i * 2, i * 3 FROM generate_series(1,4) i; -- These partitions will need to have Explicit Redistribute above them. -TRUNCATE into_table_1_prt_1; -TRUNCATE into_table_1_prt_3; -ANALYZE into_table_1_prt_1; -ANALYZE into_table_1_prt_3; +TRUNCATE t1_1_prt_1; +TRUNCATE t1_1_prt_3; +ANALYZE t1_1_prt_1; +ANALYZE t1_1_prt_3; EXPLAIN (costs off) - UPDATE into_table SET c = from_table.b FROM from_table; + UPDATE t1 SET c = t2.b FROM t2; -DROP TABLE into_table; -DROP TABLE from_table; +DROP TABLE t1; +DROP TABLE t2; -- -- text types. We should support the following updates. From d2c801af5384938368ce302e0dd9f47d3e5a86ba Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Fri, 9 Feb 2024 15:13:12 +0300 Subject: [PATCH 49/83] Simplify motion conditions --- src/backend/cdb/cdbmutate.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index ac32357fdd0f..937d0f407251 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -865,8 +865,7 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) { Motion *motion = (Motion *) node; - if (motion->motionType == MOTIONTYPE_HASH || - (motion->motionType == MOTIONTYPE_FIXED && motion->isBroadcast)) + if (motion->motionType == MOTIONTYPE_HASH || motion->isBroadcast) { context->mt.nMotionsAbove += 1; } @@ -1151,8 +1150,7 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) { Motion *motion = (Motion *) node; - if (motion->motionType == MOTIONTYPE_HASH || - (motion->motionType == MOTIONTYPE_FIXED && motion->isBroadcast)) + if (motion->motionType == MOTIONTYPE_HASH || motion->isBroadcast) { /* * We're going out of this motion node. From 11be3caa72cd0eeb1249f2c36c036f65618215af Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Fri, 9 Feb 2024 15:15:06 +0300 Subject: [PATCH 50/83] Fix comments --- src/backend/cdb/cdbmutate.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index 937d0f407251..e829fd3d1d55 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -824,8 +824,7 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) ListCell *lcr; /* - * Sanity check, since we don't allow multiple ModifyTable nodes - * in the same plan. + * Sanity check, since we don't allow multiple ModifyTable nodes. */ Assert(context->mt.resultRtis == NULL); Assert(context->mt.nMotionsAbove == 0); @@ -837,7 +836,7 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) * is a part of inheritance tree, ModifyTable node will have more * than one relation in resultRelations. * - * We make a list of resulting relations' indexes to compare them + * We make a set of resulting relations' indexes to compare them * later. */ foreach(lcr, mt->resultRelations) From 73aac4c16ee233673f714bbdd5be98f7cbe76ff6 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Fri, 9 Feb 2024 15:38:16 +0300 Subject: [PATCH 51/83] Ignore DROP TABLE --- src/test/regress/expected/update_gp.out | 4 ++++ src/test/regress/expected/update_gp_optimizer.out | 4 ++++ src/test/regress/sql/update_gp.sql | 5 +++++ 3 files changed, 13 insertions(+) diff --git a/src/test/regress/expected/update_gp.out b/src/test/regress/expected/update_gp.out index 0c5d8eb83c4e..fce7ba993ed0 100644 --- a/src/test/regress/expected/update_gp.out +++ b/src/test/regress/expected/update_gp.out @@ -211,6 +211,7 @@ DROP TABLE keo5; -- Explicit Redistribute Motion should be added only if there is a motion -- between the scan and the ModifyTable on the relation we are going to modify. -- (test case not applicable to ORCA) +-- start_ignore DROP TABLE IF EXISTS t1; NOTICE: table "t1" does not exist, skipping DROP TABLE IF EXISTS t2; @@ -219,6 +220,7 @@ DROP TABLE IF EXISTS t_strewn; NOTICE: table "t_strewn" does not exist, skipping DROP TABLE IF EXISTS t_strewn2; NOTICE: table "t_strewn2" does not exist, skipping +-- end_ignore CREATE TABLE t1 (i int, j int) DISTRIBUTED BY (i); CREATE TABLE t2 (i int) DISTRIBUTED BY (i); CREATE TABLE t_strewn (i int) DISTRIBUTED RANDOMLY; @@ -342,12 +344,14 @@ DROP TABLE t1; DROP TABLE t2; -- Explicit Redistribute Motion should not be mistakenly elided for inherited -- tables. (test case not applicable to ORCA) +-- start_ignore DROP TABLE IF EXISTS i; NOTICE: table "i" does not exist, skipping DROP TABLE IF EXISTS foochild; NOTICE: table "foochild" does not exist, skipping DROP TABLE IF EXISTS foo; NOTICE: table "foo" does not exist, skipping +-- end_ignore CREATE TABLE i (i int, j int) DISTRIBUTED BY (i); INSERT INTO i SELECT generate_series(1, 100), generate_series(1, 100) * 3; diff --git a/src/test/regress/expected/update_gp_optimizer.out b/src/test/regress/expected/update_gp_optimizer.out index 77f2bdc2a346..dbbb8d5bfe8a 100644 --- a/src/test/regress/expected/update_gp_optimizer.out +++ b/src/test/regress/expected/update_gp_optimizer.out @@ -216,6 +216,7 @@ DROP TABLE keo5; -- Explicit Redistribute Motion should be added only if there is a motion -- between the scan and the ModifyTable on the relation we are going to modify. -- (test case not applicable to ORCA) +-- start_ignore DROP TABLE IF EXISTS t1; NOTICE: table "t1" does not exist, skipping DROP TABLE IF EXISTS t2; @@ -224,6 +225,7 @@ DROP TABLE IF EXISTS t_strewn; NOTICE: table "t_strewn" does not exist, skipping DROP TABLE IF EXISTS t_strewn2; NOTICE: table "t_strewn2" does not exist, skipping +-- end_ignore CREATE TABLE t1 (i int, j int) DISTRIBUTED BY (i); CREATE TABLE t2 (i int) DISTRIBUTED BY (i); CREATE TABLE t_strewn (i int) DISTRIBUTED RANDOMLY; @@ -351,12 +353,14 @@ DROP TABLE t1; DROP TABLE t2; -- Explicit Redistribute Motion should not be mistakenly elided for inherited -- tables. (test case not applicable to ORCA) +-- start_ignore DROP TABLE IF EXISTS i; NOTICE: table "i" does not exist, skipping DROP TABLE IF EXISTS foochild; NOTICE: table "foochild" does not exist, skipping DROP TABLE IF EXISTS foo; NOTICE: table "foo" does not exist, skipping +-- end_ignore CREATE TABLE i (i int, j int) DISTRIBUTED BY (i); INSERT INTO i SELECT generate_series(1, 100), generate_series(1, 100) * 3; diff --git a/src/test/regress/sql/update_gp.sql b/src/test/regress/sql/update_gp.sql index 7d64523f560c..bcb38372179b 100644 --- a/src/test/regress/sql/update_gp.sql +++ b/src/test/regress/sql/update_gp.sql @@ -120,10 +120,12 @@ DROP TABLE keo5; -- Explicit Redistribute Motion should be added only if there is a motion -- between the scan and the ModifyTable on the relation we are going to modify. -- (test case not applicable to ORCA) +-- start_ignore DROP TABLE IF EXISTS t1; DROP TABLE IF EXISTS t2; DROP TABLE IF EXISTS t_strewn; DROP TABLE IF EXISTS t_strewn2; +-- end_ignore CREATE TABLE t1 (i int, j int) DISTRIBUTED BY (i); CREATE TABLE t2 (i int) DISTRIBUTED BY (i); @@ -164,9 +166,11 @@ DROP TABLE t2; -- Explicit Redistribute Motion should not be mistakenly elided for inherited -- tables. (test case not applicable to ORCA) +-- start_ignore DROP TABLE IF EXISTS i; DROP TABLE IF EXISTS foochild; DROP TABLE IF EXISTS foo; +-- end_ignore CREATE TABLE i (i int, j int) DISTRIBUTED BY (i); INSERT INTO i SELECT @@ -225,6 +229,7 @@ DROP TABLE t2; -- index. CREATE TABLE t1 (a int, b int); CREATE TABLE t2 (a int, b int); + INSERT INTO t1 SELECT a, a FROM generate_series(1, 100) a; INSERT INTO t2 SELECT a, a FROM generate_series(1, 100) a; From 053ea462c8c17284a168f6fca9dd17cecb7f559e Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Wed, 21 Feb 2024 12:10:22 +0300 Subject: [PATCH 52/83] Remove scan types that aren't used for UPDATE --- src/backend/cdb/cdbmutate.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index e829fd3d1d55..affecca49d9e 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -889,12 +889,6 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) case T_BitmapHeapScan: case T_DynamicBitmapHeapScan: case T_TidScan: - case T_SubqueryScan: - case T_FunctionScan: - case T_TableFunctionScan: - case T_ValuesScan: - case T_CteScan: - case T_WorkTableScan: case T_ForeignScan: { Scan *scan = (Scan *) node; From 289068b97edd226a40ad9bdf86e46703a1d832c4 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Wed, 21 Feb 2024 13:10:48 +0300 Subject: [PATCH 53/83] Count gather motions as well --- src/backend/cdb/cdbmutate.c | 32 +++++++++----------------------- 1 file changed, 9 insertions(+), 23 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index affecca49d9e..8ac8117f315c 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -810,10 +810,10 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) } /* - * For UPDATE/DELETE, we check if there's any Redistribute or Broadcast - * Motions before scan in the same subtree for the table we're going to - * modify. If we encounter the scan before any motions, then we can elide - * unneccessary Explicit Redistribute Motion. + * For UPDATE/DELETE, we check if there's any Redistribute, Broadcast + * or Gather Motions before scan in the same subtree for the table we're + * going to modify. If we encounter the scan before any motions, then we can + * elide unneccessary Explicit Redistribute Motion. */ if (IsA(node, ModifyTable)) { @@ -857,18 +857,11 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) { /* - * Remember if we are descending into a Redistribute or Broadcast - * Motion node. + * Remember if we are descending into a Redistribute, Broadcast or + * Gather Motion node. */ if (IsA(node, Motion)) - { - Motion *motion = (Motion *) node; - - if (motion->motionType == MOTIONTYPE_HASH || motion->isBroadcast) - { - context->mt.nMotionsAbove += 1; - } - } + context->mt.nMotionsAbove += 1; /* * If this is a scan and it's scanrelid matches ModifyTable's relid, @@ -1141,15 +1134,8 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) if (context->mt.isChecking && IsA(node, Motion)) { - Motion *motion = (Motion *) node; - - if (motion->motionType == MOTIONTYPE_HASH || motion->isBroadcast) - { - /* - * We're going out of this motion node. - */ - context->mt.nMotionsAbove -= 1; - } + /* We're going out of this motion node. */ + context->mt.nMotionsAbove -= 1; } return newnode; From 5f2299f617b8ed15b6f8828b2827a295551562de Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Wed, 21 Feb 2024 13:30:41 +0300 Subject: [PATCH 54/83] Add test for gather motion --- src/test/regress/expected/update_gp.out | 23 +++++++++++++++++++ .../regress/expected/update_gp_optimizer.out | 23 +++++++++++++++++++ src/test/regress/sql/update_gp.sql | 16 +++++++++++++ 3 files changed, 62 insertions(+) diff --git a/src/test/regress/expected/update_gp.out b/src/test/regress/expected/update_gp.out index fce7ba993ed0..3640cc2fe62d 100644 --- a/src/test/regress/expected/update_gp.out +++ b/src/test/regress/expected/update_gp.out @@ -617,6 +617,29 @@ EXPLAIN (costs off) DROP TABLE t1; DROP TABLE t2; +-- Explicit Redistribute Motion should not be elided if there's a Gather Motion +-- beneath the ModifyTable. (test case not applicable to ORCA) +CREATE TABLE t1 (a int) DISTRIBUTED BY (a); +INSERT INTO t1 SELECT i FROM generate_series(1,4) i; +-- "USING pg_class" forces a Gather Motion. +EXPLAIN (costs off) +DELETE FROM t1 +USING pg_class; + QUERY PLAN +------------------------------------------------------------------ + Delete on t1 + -> Explicit Redistribute Motion 1:3 (slice2) + -> Nested Loop + -> Seq Scan on pg_class + -> Materialize + -> Gather Motion 3:1 (slice1; segments: 3) + -> Seq Scan on t1 + Optimizer: Postgres query optimizer +(8 rows) + +DELETE FROM t1 +USING pg_class; +DROP TABLE t1; -- -- text types. We should support the following updates. -- diff --git a/src/test/regress/expected/update_gp_optimizer.out b/src/test/regress/expected/update_gp_optimizer.out index dbbb8d5bfe8a..73504c770ce1 100644 --- a/src/test/regress/expected/update_gp_optimizer.out +++ b/src/test/regress/expected/update_gp_optimizer.out @@ -576,6 +576,29 @@ EXPLAIN (costs off) DROP TABLE t1; DROP TABLE t2; +-- Explicit Redistribute Motion should not be elided if there's a Gather Motion +-- beneath the ModifyTable. (test case not applicable to ORCA) +CREATE TABLE t1 (a int) DISTRIBUTED BY (a); +INSERT INTO t1 SELECT i FROM generate_series(1,4) i; +-- "USING pg_class" forces a Gather Motion. +EXPLAIN (costs off) +DELETE FROM t1 +USING pg_class; + QUERY PLAN +------------------------------------------------------------------ + Delete on t1 + -> Explicit Redistribute Motion 1:3 (slice2) + -> Nested Loop + -> Seq Scan on pg_class + -> Materialize + -> Gather Motion 3:1 (slice1; segments: 3) + -> Seq Scan on t1 + Optimizer: Postgres query optimizer +(8 rows) + +DELETE FROM t1 +USING pg_class; +DROP TABLE t1; -- -- text types. We should support the following updates. -- diff --git a/src/test/regress/sql/update_gp.sql b/src/test/regress/sql/update_gp.sql index bcb38372179b..552942092eb0 100644 --- a/src/test/regress/sql/update_gp.sql +++ b/src/test/regress/sql/update_gp.sql @@ -279,6 +279,22 @@ EXPLAIN (costs off) DROP TABLE t1; DROP TABLE t2; +-- Explicit Redistribute Motion should not be elided if there's a Gather Motion +-- beneath the ModifyTable. (test case not applicable to ORCA) +CREATE TABLE t1 (a int) DISTRIBUTED BY (a); + +INSERT INTO t1 SELECT i FROM generate_series(1,4) i; + +-- "USING pg_class" forces a Gather Motion. +EXPLAIN (costs off) +DELETE FROM t1 +USING pg_class; + +DELETE FROM t1 +USING pg_class; + +DROP TABLE t1; + -- -- text types. We should support the following updates. -- From a270ca14a014dbdf47b6432a964d8c2b7489a180 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Tue, 27 Feb 2024 08:59:21 +0300 Subject: [PATCH 55/83] Fix comment --- src/backend/cdb/cdbmutate.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index 8ac8117f315c..ffdeda07c2cf 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -70,8 +70,9 @@ typedef struct ModifyTableMotionState Bitmapset *resultRtis; /* Indexes into rtable for relations to * be modified */ bool needExplicitMotion; - int nMotionsAbove; /* Number of Redistribute/Broadcast - * motions above the current node */ + int nMotionsAbove; /* Number of Gather, Redistribute and + * Broadcast motions above the current + * node */ bool isChecking; /* True if we encountered ModifyTable * node with UPDATE/DELETE and we plan * to insert Explicit Motions */ From 8d7fd1989554125ec20170758435cec9616c3e7d Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Wed, 6 Mar 2024 16:36:16 +0300 Subject: [PATCH 56/83] Remove extra break --- src/backend/cdb/cdbmutate.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index ffdeda07c2cf..54c8dc836693 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -900,7 +900,6 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) * subtree anymore. */ context->mt.isChecking = false; - break; } } break; From 2970372fcf696a8c1087f56a08f7137fc1bfe78e Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Wed, 6 Mar 2024 17:36:27 +0300 Subject: [PATCH 57/83] Clean up tests --- src/test/regress/expected/update_gp.out | 283 ++++++++++-------- .../regress/expected/update_gp_optimizer.out | 227 ++++++++------ src/test/regress/sql/update_gp.sql | 70 +++-- 3 files changed, 326 insertions(+), 254 deletions(-) diff --git a/src/test/regress/expected/update_gp.out b/src/test/regress/expected/update_gp.out index 3640cc2fe62d..1b670cb6c123 100644 --- a/src/test/regress/expected/update_gp.out +++ b/src/test/regress/expected/update_gp.out @@ -211,16 +211,6 @@ DROP TABLE keo5; -- Explicit Redistribute Motion should be added only if there is a motion -- between the scan and the ModifyTable on the relation we are going to modify. -- (test case not applicable to ORCA) --- start_ignore -DROP TABLE IF EXISTS t1; -NOTICE: table "t1" does not exist, skipping -DROP TABLE IF EXISTS t2; -NOTICE: table "t2" does not exist, skipping -DROP TABLE IF EXISTS t_strewn; -NOTICE: table "t_strewn" does not exist, skipping -DROP TABLE IF EXISTS t_strewn2; -NOTICE: table "t_strewn2" does not exist, skipping --- end_ignore CREATE TABLE t1 (i int, j int) DISTRIBUTED BY (i); CREATE TABLE t2 (i int) DISTRIBUTED BY (i); CREATE TABLE t_strewn (i int) DISTRIBUTED RANDOMLY; @@ -242,30 +232,25 @@ UPDATE t1 SET j = t_strewn.i FROM t_strewn WHERE t_strewn.i = t1.i; EXPLAIN (costs off) WITH CTE AS (DELETE FROM t1 RETURNING *) -SELECT count(*) AS a FROM t_strewn JOIN t2 USING (i) -WHERE 0 < ALL (SELECT i FROM cte); +SELECT count(*) AS a FROM t_strewn JOIN cte USING (i); QUERY PLAN ------------------------------------------------------------------------------ Aggregate -> Gather Motion 3:1 (slice3; segments: 3) -> Aggregate - -> Result - One-Time Filter: (SubPlan 1) - -> Hash Join - Hash Cond: (t_strewn.i = t2.i) - -> Redistribute Motion 3:3 (slice1; segments: 3) + -> Hash Join + Hash Cond: (cte.i = t_strewn.i) + -> Redistribute Motion 3:3 (slice1; segments: 3) + Hash Key: cte.i + -> Subquery Scan on cte + -> Delete on t1 + -> Seq Scan on t1 + -> Hash + -> Redistribute Motion 3:3 (slice2; segments: 3) Hash Key: t_strewn.i -> Seq Scan on t_strewn - -> Hash - -> Seq Scan on t2 - SubPlan 1 (slice3; segments: 3) - -> Materialize - -> Broadcast Motion 3:3 (slice2; segments: 3) - -> Subquery Scan on cte - -> Delete on t1 - -> Seq Scan on t1 Optimizer: Postgres query optimizer -(19 rows) +(15 rows) EXPLAIN (costs off, verbose) DELETE FROM t_strewn WHERE t_strewn.i = (SELECT t2.i FROM t2 WHERE t_strewn.i = t2.i); @@ -316,12 +301,13 @@ DROP TABLE t_strewn2; -- Plans but not in the main plan. (test case not applicable to ORCA) CREATE TABLE t1 (a int, b int) DISTRIBUTED RANDOMLY; CREATE TABLE t2 (a int, b int) DISTRIBUTED RANDOMLY; -EXPLAIN (costs off) WITH cte AS ( +EXPLAIN (costs off) +WITH cte AS ( SELECT count(*) AS c FROM t1, t2 WHERE t1.b = t2.b -) UPDATE t2 SET b = 10 WHERE a = (SELECT * FROM cte); +) DELETE FROM t2 WHERE a = (SELECT * FROM cte); QUERY PLAN -------------------------------------------------------------------------------------- - Update on t2 + Delete on t2 InitPlan 1 (returns $0) (slice4) -> Aggregate -> Gather Motion 3:1 (slice3; segments: 3) @@ -344,17 +330,9 @@ DROP TABLE t1; DROP TABLE t2; -- Explicit Redistribute Motion should not be mistakenly elided for inherited -- tables. (test case not applicable to ORCA) --- start_ignore -DROP TABLE IF EXISTS i; -NOTICE: table "i" does not exist, skipping -DROP TABLE IF EXISTS foochild; -NOTICE: table "foochild" does not exist, skipping -DROP TABLE IF EXISTS foo; -NOTICE: table "foo" does not exist, skipping --- end_ignore CREATE TABLE i (i int, j int) DISTRIBUTED BY (i); INSERT INTO i SELECT - generate_series(1, 100), generate_series(1, 100) * 3; + generate_series(1, 16), generate_series(1, 16) * 3; CREATE TABLE foo (f1 serial, f2 text, f3 int) DISTRIBUTED RANDOMLY; INSERT INTO foo (f2, f3) VALUES ('first', 1), ('second', 2), ('third', 3); @@ -401,7 +379,13 @@ DELETE FROM foo DELETE FROM foo USING i - WHERE foo.f1 = i.j; + WHERE foo.f1 = i.j + RETURNING *; + f1 | f2 | f3 | i | j +----+-------+----+---+--- + 3 | third | 3 | 1 | 3 +(1 row) + DROP TABLE i; DROP TABLE foochild; DROP TABLE foo; @@ -409,138 +393,149 @@ DROP TABLE foo; -- tables. (test case not applicable to ORCA) CREATE TABLE t1 (a int, b int) DISTRIBUTED BY (a) PARTITION BY - range(b) (start(1) end(101) every(50)); + range(b) (start(1) end(16) every(5)); NOTICE: CREATE TABLE will create partition "t1_1_prt_1" for table "t1" NOTICE: CREATE TABLE will create partition "t1_1_prt_2" for table "t1" +NOTICE: CREATE TABLE will create partition "t1_1_prt_3" for table "t1" CREATE TABLE t2 (a int, b int) DISTRIBUTED BY (b) PARTITION BY - range(a) (start(1) end(101) every(25), default partition def); + range(a) (start(1) end(16) every(10), default partition def); NOTICE: CREATE TABLE will create partition "t2_1_prt_def" for table "t2" NOTICE: CREATE TABLE will create partition "t2_1_prt_2" for table "t2" NOTICE: CREATE TABLE will create partition "t2_1_prt_3" for table "t2" -NOTICE: CREATE TABLE will create partition "t2_1_prt_4" for table "t2" -NOTICE: CREATE TABLE will create partition "t2_1_prt_5" for table "t2" INSERT INTO t1 SELECT - generate_series(1, 100) * 3, generate_series(1, 100); + generate_series(1, 4) * 3, generate_series(1, 4); INSERT INTO t2 SELECT - generate_series(1, 100), generate_series(1, 100) * 3; + generate_series(1, 4), generate_series(1, 4) * 3; INSERT INTO t2 VALUES - (generate_series(101, 111), NULL); + (generate_series(7, 11), NULL); EXPLAIN (costs off, verbose) -DELETE FROM t1 USING t2 WHERE t1.a = t2.a; +DELETE FROM t2 USING t1 WHERE t1.a = t2.a; QUERY PLAN ----------------------------------------------------------------------------------------------------------- - Delete on public.t1_1_prt_1 + Delete on public.t2_1_prt_def -> Explicit Redistribute Motion 3:3 (slice2; segments: 3) - Output: t1_1_prt_1.ctid, t1_1_prt_1.gp_segment_id, t2_1_prt_def.ctid, t2_1_prt_def.tableoid + Output: t2_1_prt_def.ctid, t2_1_prt_def.gp_segment_id, t1_1_prt_1.ctid, t1_1_prt_1.tableoid -> Hash Join - Output: t1_1_prt_1.ctid, t1_1_prt_1.gp_segment_id, t2_1_prt_def.ctid, t2_1_prt_def.tableoid - Hash Cond: (t2_1_prt_def.a = t1_1_prt_1.a) + Output: t2_1_prt_def.ctid, t2_1_prt_def.gp_segment_id, t1_1_prt_1.ctid, t1_1_prt_1.tableoid + Hash Cond: (t1_1_prt_1.a = t2_1_prt_def.a) -> Append - -> Result - Output: t2_1_prt_def.ctid, t2_1_prt_def.tableoid, t2_1_prt_def.a - One-Time Filter: PartSelected - -> Seq Scan on public.t2_1_prt_def - Output: t2_1_prt_def.ctid, t2_1_prt_def.tableoid, t2_1_prt_def.a - -> Result - Output: t2_1_prt_2.ctid, t2_1_prt_2.tableoid, t2_1_prt_2.a - One-Time Filter: PartSelected - -> Seq Scan on public.t2_1_prt_2 - Output: t2_1_prt_2.ctid, t2_1_prt_2.tableoid, t2_1_prt_2.a - -> Result - Output: t2_1_prt_3.ctid, t2_1_prt_3.tableoid, t2_1_prt_3.a - One-Time Filter: PartSelected - -> Seq Scan on public.t2_1_prt_3 - Output: t2_1_prt_3.ctid, t2_1_prt_3.tableoid, t2_1_prt_3.a - -> Result - Output: t2_1_prt_4.ctid, t2_1_prt_4.tableoid, t2_1_prt_4.a - One-Time Filter: PartSelected - -> Seq Scan on public.t2_1_prt_4 - Output: t2_1_prt_4.ctid, t2_1_prt_4.tableoid, t2_1_prt_4.a - -> Result - Output: t2_1_prt_5.ctid, t2_1_prt_5.tableoid, t2_1_prt_5.a - One-Time Filter: PartSelected - -> Seq Scan on public.t2_1_prt_5 - Output: t2_1_prt_5.ctid, t2_1_prt_5.tableoid, t2_1_prt_5.a + -> Seq Scan on public.t1_1_prt_1 + Output: t1_1_prt_1.ctid, t1_1_prt_1.tableoid, t1_1_prt_1.a + -> Seq Scan on public.t1_1_prt_2 + Output: t1_1_prt_2.ctid, t1_1_prt_2.tableoid, t1_1_prt_2.a + -> Seq Scan on public.t1_1_prt_3 + Output: t1_1_prt_3.ctid, t1_1_prt_3.tableoid, t1_1_prt_3.a -> Hash - Output: t1_1_prt_1.ctid, t1_1_prt_1.gp_segment_id, t1_1_prt_1.a - -> Partition Selector for t2 (dynamic scan id: 2) - Output: t1_1_prt_1.ctid, t1_1_prt_1.gp_segment_id, t1_1_prt_1.a - Filter: t1_1_prt_1.a - -> Broadcast Motion 3:3 (slice1; segments: 3) - Output: t1_1_prt_1.ctid, t1_1_prt_1.gp_segment_id, t1_1_prt_1.a - -> Seq Scan on public.t1_1_prt_1 - Output: t1_1_prt_1.ctid, t1_1_prt_1.gp_segment_id, t1_1_prt_1.a + Output: t2_1_prt_def.ctid, t2_1_prt_def.gp_segment_id, t2_1_prt_def.a + -> Redistribute Motion 3:3 (slice1; segments: 3) + Output: t2_1_prt_def.ctid, t2_1_prt_def.gp_segment_id, t2_1_prt_def.a + Hash Key: t2_1_prt_def.a + -> Seq Scan on public.t2_1_prt_def + Output: t2_1_prt_def.ctid, t2_1_prt_def.gp_segment_id, t2_1_prt_def.a -> Explicit Redistribute Motion 3:3 (slice4; segments: 3) - Output: t1_1_prt_2.ctid, t1_1_prt_2.gp_segment_id, t2_1_prt_def.ctid, t2_1_prt_def.tableoid + Output: t2_1_prt_2.ctid, t2_1_prt_2.gp_segment_id, t1_1_prt_1.ctid, t1_1_prt_1.tableoid -> Hash Join - Output: t1_1_prt_2.ctid, t1_1_prt_2.gp_segment_id, t2_1_prt_def.ctid, t2_1_prt_def.tableoid - Hash Cond: (t2_1_prt_def.a = t1_1_prt_2.a) + Output: t2_1_prt_2.ctid, t2_1_prt_2.gp_segment_id, t1_1_prt_1.ctid, t1_1_prt_1.tableoid + Hash Cond: (t1_1_prt_1.a = t2_1_prt_2.a) -> Append - -> Seq Scan on public.t2_1_prt_def - Output: t2_1_prt_def.ctid, t2_1_prt_def.tableoid, t2_1_prt_def.a - -> Seq Scan on public.t2_1_prt_2 - Output: t2_1_prt_2.ctid, t2_1_prt_2.tableoid, t2_1_prt_2.a - -> Seq Scan on public.t2_1_prt_3 - Output: t2_1_prt_3.ctid, t2_1_prt_3.tableoid, t2_1_prt_3.a - -> Seq Scan on public.t2_1_prt_4 - Output: t2_1_prt_4.ctid, t2_1_prt_4.tableoid, t2_1_prt_4.a - -> Seq Scan on public.t2_1_prt_5 - Output: t2_1_prt_5.ctid, t2_1_prt_5.tableoid, t2_1_prt_5.a + -> Seq Scan on public.t1_1_prt_1 + Output: t1_1_prt_1.ctid, t1_1_prt_1.tableoid, t1_1_prt_1.a + -> Seq Scan on public.t1_1_prt_2 + Output: t1_1_prt_2.ctid, t1_1_prt_2.tableoid, t1_1_prt_2.a + -> Seq Scan on public.t1_1_prt_3 + Output: t1_1_prt_3.ctid, t1_1_prt_3.tableoid, t1_1_prt_3.a -> Hash - Output: t1_1_prt_2.ctid, t1_1_prt_2.gp_segment_id, t1_1_prt_2.a - -> Partition Selector for t2 (dynamic scan id: 2) - Output: t1_1_prt_2.ctid, t1_1_prt_2.gp_segment_id, t1_1_prt_2.a - Filter: t1_1_prt_2.a - -> Broadcast Motion 3:3 (slice3; segments: 3) - Output: t1_1_prt_2.ctid, t1_1_prt_2.gp_segment_id, t1_1_prt_2.a - -> Seq Scan on public.t1_1_prt_2 - Output: t1_1_prt_2.ctid, t1_1_prt_2.gp_segment_id, t1_1_prt_2.a + Output: t2_1_prt_2.ctid, t2_1_prt_2.gp_segment_id, t2_1_prt_2.a + -> Redistribute Motion 3:3 (slice3; segments: 3) + Output: t2_1_prt_2.ctid, t2_1_prt_2.gp_segment_id, t2_1_prt_2.a + Hash Key: t2_1_prt_2.a + -> Seq Scan on public.t2_1_prt_2 + Output: t2_1_prt_2.ctid, t2_1_prt_2.gp_segment_id, t2_1_prt_2.a + -> Explicit Redistribute Motion 3:3 (slice6; segments: 3) + Output: t2_1_prt_3.ctid, t2_1_prt_3.gp_segment_id, t1_1_prt_1.ctid, t1_1_prt_1.tableoid + -> Hash Join + Output: t2_1_prt_3.ctid, t2_1_prt_3.gp_segment_id, t1_1_prt_1.ctid, t1_1_prt_1.tableoid + Hash Cond: (t1_1_prt_1.a = t2_1_prt_3.a) + -> Append + -> Seq Scan on public.t1_1_prt_1 + Output: t1_1_prt_1.ctid, t1_1_prt_1.tableoid, t1_1_prt_1.a + -> Seq Scan on public.t1_1_prt_2 + Output: t1_1_prt_2.ctid, t1_1_prt_2.tableoid, t1_1_prt_2.a + -> Seq Scan on public.t1_1_prt_3 + Output: t1_1_prt_3.ctid, t1_1_prt_3.tableoid, t1_1_prt_3.a + -> Hash + Output: t2_1_prt_3.ctid, t2_1_prt_3.gp_segment_id, t2_1_prt_3.a + -> Redistribute Motion 3:3 (slice5; segments: 3) + Output: t2_1_prt_3.ctid, t2_1_prt_3.gp_segment_id, t2_1_prt_3.a + Hash Key: t2_1_prt_3.a + -> Seq Scan on public.t2_1_prt_3 + Output: t2_1_prt_3.ctid, t2_1_prt_3.gp_segment_id, t2_1_prt_3.a Optimizer: Postgres query optimizer Settings: optimizer=off -(68 rows) +(60 rows) + +DELETE FROM t2 USING t1 WHERE t1.a = t2.a +RETURNING *; + a | b | a | b +---+---+---+--- + 9 | | 9 | 3 + 3 | 9 | 3 | 1 +(2 rows) -DELETE FROM t1 USING t2 WHERE t1.a = t2.a; DROP TABLE t1; DROP TABLE t2; -- Explicit Redistribute Motion should not be elided if we encounter a scan on -- the same table that we are going to modify, but with different range table --- index. +-- index. (test case not applicable to ORCA) CREATE TABLE t1 (a int, b int); NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table. HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. CREATE TABLE t2 (a int, b int); NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table. HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. -INSERT INTO t1 SELECT a, a FROM generate_series(1, 100) a; -INSERT INTO t2 SELECT a, a FROM generate_series(1, 100) a; -EXPLAIN (costs off) update t2 trg +INSERT INTO t1 SELECT a, a FROM generate_series(1, 4) a; +INSERT INTO t2 SELECT a, a FROM generate_series(1, 16) a; +EXPLAIN (costs off) UPDATE t2 trg SET b = src.b1 FROM (SELECT t1.a AS a1, t1.b AS b1, t2.a AS a2, t2.b AS b2 FROM t1 JOIN t2 USING (b)) src WHERE trg.a = src.a1 - AND trg.a = 2; - QUERY PLAN ---------------------------------------------------------------- + AND trg.a = 2; + QUERY PLAN +--------------------------------------------------------------------------- Update on t2 trg - -> Explicit Redistribute Motion 3:3 (slice2; segments: 3) - -> Hash Join - Hash Cond: (t1.b = t2.b) - -> Broadcast Motion 3:3 (slice1; segments: 3) - -> Nested Loop + -> Explicit Redistribute Motion 3:3 (slice3; segments: 3) + -> Nested Loop + -> Hash Join + Hash Cond: (t2.b = t1.b) + -> Seq Scan on t2 + -> Hash + -> Broadcast Motion 3:3 (slice1; segments: 3) + -> Seq Scan on t1 + Filter: (a = 2) + -> Materialize + -> Broadcast Motion 3:3 (slice2; segments: 3) -> Seq Scan on t2 trg Filter: (a = 2) - -> Seq Scan on t1 - Filter: (a = 2) - -> Hash - -> Seq Scan on t2 Optimizer: Postgres query optimizer -(13 rows) +(15 rows) + +UPDATE t2 trg +SET b = src.b1 +FROM (SELECT t1.a AS a1, t1.b AS b1, t2.a AS a2, t2.b AS b2 FROM t1 JOIN t2 USING (b)) src +WHERE trg.a = src.a1 + AND trg.a = 2 +RETURNING *; + a | b | a1 | b1 | a2 | b2 +---+---+----+----+----+---- + 2 | 2 | 2 | 2 | 2 | 2 +(1 row) -- Use Nested Loop to change left tree with the right tree, to swap the extra -- scan we don't indend to detect with the real one. SET enable_hashjoin = off; SET enable_nestloop = on; -EXPLAIN (costs off) update t2 trg +EXPLAIN (costs off) UPDATE t2 trg SET b = src.b1 FROM (SELECT t1.a AS a1, t1.b AS b1, t2.a AS a2, t2.b AS b2 FROM t1 JOIN t2 USING (b)) src WHERE trg.a = src.a1 @@ -564,13 +559,24 @@ WHERE trg.a = src.a1 Optimizer: Postgres query optimizer (15 rows) +UPDATE t2 trg +SET b = src.b1 +FROM (SELECT t1.a AS a1, t1.b AS b1, t2.a AS a2, t2.b AS b2 FROM t1 JOIN t2 USING (b)) src +WHERE trg.a = src.a1 + AND trg.a = 2 +RETURNING *; + a | b | a1 | b1 | a2 | b2 +---+---+----+----+----+---- + 2 | 2 | 2 | 2 | 2 | 2 +(1 row) + RESET enable_hashjoin; RESET enable_nestloop; DROP TABLE t1; DROP TABLE t2; -- Explicit Redistribute Motion should be elided for every partition that does -- not have any motions above the scan on the table/partition we are going to --- update. +-- update. (test case not applicable to ORCA) CREATE TABLE t1 (a int, b int, c int) DISTRIBUTED BY (b) PARTITION BY RANGE(b) (start (1) end(5) every(1)); NOTICE: CREATE TABLE will create partition "t1_1_prt_1" for table "t1" @@ -578,8 +584,8 @@ NOTICE: CREATE TABLE will create partition "t1_1_prt_2" for table "t1" NOTICE: CREATE TABLE will create partition "t1_1_prt_3" for table "t1" NOTICE: CREATE TABLE will create partition "t1_1_prt_4" for table "t1" CREATE TABLE t2 (a int, b int, c int) DISTRIBUTED BY (a); -INSERT INTO t1 SELECT i * 2, i, i * 3 FROM generate_series(1,4) i; -INSERT INTO t2 SELECT i, i * 2, i * 3 FROM generate_series(1,4) i; +INSERT INTO t1 SELECT i * 2, i, i * 3 FROM generate_series(1, 4) i; +INSERT INTO t2 SELECT i, i * 2, i * 3 FROM generate_series(1, 4) i; -- These partitions will need to have Explicit Redistribute above them. TRUNCATE t1_1_prt_1; TRUNCATE t1_1_prt_3; @@ -615,12 +621,20 @@ EXPLAIN (costs off) Optimizer: Postgres query optimizer (24 rows) +UPDATE t1 SET c = t2.b FROM t2 +RETURNING *; + a | b | c | a | b | c +---+---+---+---+---+--- + 4 | 2 | 4 | 2 | 4 | 6 + 8 | 4 | 4 | 2 | 4 | 6 +(2 rows) + DROP TABLE t1; DROP TABLE t2; -- Explicit Redistribute Motion should not be elided if there's a Gather Motion -- beneath the ModifyTable. (test case not applicable to ORCA) CREATE TABLE t1 (a int) DISTRIBUTED BY (a); -INSERT INTO t1 SELECT i FROM generate_series(1,4) i; +INSERT INTO t1 SELECT i FROM generate_series(1, 4) i; -- "USING pg_class" forces a Gather Motion. EXPLAIN (costs off) DELETE FROM t1 @@ -638,7 +652,16 @@ USING pg_class; (8 rows) DELETE FROM t1 -USING pg_class; +USING pg_class +RETURNING t1.*; + a +--- + 1 + 2 + 3 + 4 +(4 rows) + DROP TABLE t1; -- -- text types. We should support the following updates. diff --git a/src/test/regress/expected/update_gp_optimizer.out b/src/test/regress/expected/update_gp_optimizer.out index 73504c770ce1..bbd673eb5b1f 100644 --- a/src/test/regress/expected/update_gp_optimizer.out +++ b/src/test/regress/expected/update_gp_optimizer.out @@ -216,16 +216,6 @@ DROP TABLE keo5; -- Explicit Redistribute Motion should be added only if there is a motion -- between the scan and the ModifyTable on the relation we are going to modify. -- (test case not applicable to ORCA) --- start_ignore -DROP TABLE IF EXISTS t1; -NOTICE: table "t1" does not exist, skipping -DROP TABLE IF EXISTS t2; -NOTICE: table "t2" does not exist, skipping -DROP TABLE IF EXISTS t_strewn; -NOTICE: table "t_strewn" does not exist, skipping -DROP TABLE IF EXISTS t_strewn2; -NOTICE: table "t_strewn2" does not exist, skipping --- end_ignore CREATE TABLE t1 (i int, j int) DISTRIBUTED BY (i); CREATE TABLE t2 (i int) DISTRIBUTED BY (i); CREATE TABLE t_strewn (i int) DISTRIBUTED RANDOMLY; @@ -250,30 +240,25 @@ UPDATE t1 SET j = t_strewn.i FROM t_strewn WHERE t_strewn.i = t1.i; EXPLAIN (costs off) WITH CTE AS (DELETE FROM t1 RETURNING *) -SELECT count(*) AS a FROM t_strewn JOIN t2 USING (i) -WHERE 0 < ALL (SELECT i FROM cte); +SELECT count(*) AS a FROM t_strewn JOIN cte USING (i); QUERY PLAN ------------------------------------------------------------------------------ Aggregate -> Gather Motion 3:1 (slice3; segments: 3) -> Aggregate - -> Result - One-Time Filter: (SubPlan 1) - -> Hash Join - Hash Cond: (t_strewn.i = t2.i) - -> Redistribute Motion 3:3 (slice1; segments: 3) + -> Hash Join + Hash Cond: (cte.i = t_strewn.i) + -> Redistribute Motion 3:3 (slice1; segments: 3) + Hash Key: cte.i + -> Subquery Scan on cte + -> Delete on t1 + -> Seq Scan on t1 + -> Hash + -> Redistribute Motion 3:3 (slice2; segments: 3) Hash Key: t_strewn.i -> Seq Scan on t_strewn - -> Hash - -> Seq Scan on t2 - SubPlan 1 (slice3; segments: 3) - -> Materialize - -> Broadcast Motion 3:3 (slice2; segments: 3) - -> Subquery Scan on cte - -> Delete on t1 - -> Seq Scan on t1 Optimizer: Postgres query optimizer -(19 rows) +(15 rows) EXPLAIN (costs off, verbose) DELETE FROM t_strewn WHERE t_strewn.i = (SELECT t2.i FROM t2 WHERE t_strewn.i = t2.i); @@ -324,46 +309,38 @@ DROP TABLE t_strewn2; -- Plans but not in the main plan. (test case not applicable to ORCA) CREATE TABLE t1 (a int, b int) DISTRIBUTED RANDOMLY; CREATE TABLE t2 (a int, b int) DISTRIBUTED RANDOMLY; -EXPLAIN (costs off) WITH cte AS ( +EXPLAIN (costs off) +WITH cte AS ( SELECT count(*) AS c FROM t1, t2 WHERE t1.b = t2.b -) UPDATE t2 SET b = 10 WHERE a = (SELECT * FROM cte); - QUERY PLAN ---------------------------------------------------------------------------------------------------------- - Update - -> Explicit Redistribute Motion 1:3 (slice4; segments: 1) - -> Split - -> Result - -> Hash Join - Hash Cond: ((t2.a)::bigint = (count())) - -> Gather Motion 3:1 (slice1; segments: 3) - -> Seq Scan on t2 - -> Hash - -> Aggregate - -> Gather Motion 3:1 (slice3; segments: 3) - -> Hash Join - Hash Cond: (t1.b = t2_1.b) - -> Seq Scan on t1 - -> Hash - -> Broadcast Motion 3:3 (slice2; segments: 3) - -> Seq Scan on t2 t2_1 +) DELETE FROM t2 WHERE a = (SELECT * FROM cte); + QUERY PLAN +--------------------------------------------------------------------------------------------------- + Delete + -> Result + -> Explicit Redistribute Motion 1:3 (slice4; segments: 1) + -> Hash Join + Hash Cond: ((t2.a)::bigint = (count())) + -> Gather Motion 3:1 (slice1; segments: 3) + -> Seq Scan on t2 + -> Hash + -> Aggregate + -> Gather Motion 3:1 (slice3; segments: 3) + -> Hash Join + Hash Cond: (t1.b = t2_1.b) + -> Seq Scan on t1 + -> Hash + -> Broadcast Motion 3:3 (slice2; segments: 3) + -> Seq Scan on t2 t2_1 Optimizer: Pivotal Optimizer (GPORCA) -(18 rows) +(17 rows) DROP TABLE t1; DROP TABLE t2; -- Explicit Redistribute Motion should not be mistakenly elided for inherited -- tables. (test case not applicable to ORCA) --- start_ignore -DROP TABLE IF EXISTS i; -NOTICE: table "i" does not exist, skipping -DROP TABLE IF EXISTS foochild; -NOTICE: table "foochild" does not exist, skipping -DROP TABLE IF EXISTS foo; -NOTICE: table "foo" does not exist, skipping --- end_ignore CREATE TABLE i (i int, j int) DISTRIBUTED BY (i); INSERT INTO i SELECT - generate_series(1, 100), generate_series(1, 100) * 3; + generate_series(1, 16), generate_series(1, 16) * 3; CREATE TABLE foo (f1 serial, f2 text, f3 int) DISTRIBUTED RANDOMLY; INSERT INTO foo (f2, f3) VALUES ('first', 1), ('second', 2), ('third', 3); @@ -410,7 +387,13 @@ DELETE FROM foo DELETE FROM foo USING i - WHERE foo.f1 = i.j; + WHERE foo.f1 = i.j + RETURNING *; + f1 | f2 | f3 | i | j +----+-------+----+---+--- + 3 | third | 3 | 1 | 3 +(1 row) + DROP TABLE i; DROP TABLE foochild; DROP TABLE foo; @@ -418,74 +401,83 @@ DROP TABLE foo; -- tables. (test case not applicable to ORCA) CREATE TABLE t1 (a int, b int) DISTRIBUTED BY (a) PARTITION BY - range(b) (start(1) end(101) every(50)); + range(b) (start(1) end(16) every(5)); NOTICE: CREATE TABLE will create partition "t1_1_prt_1" for table "t1" NOTICE: CREATE TABLE will create partition "t1_1_prt_2" for table "t1" +NOTICE: CREATE TABLE will create partition "t1_1_prt_3" for table "t1" CREATE TABLE t2 (a int, b int) DISTRIBUTED BY (b) PARTITION BY - range(a) (start(1) end(101) every(25), default partition def); + range(a) (start(1) end(16) every(10), default partition def); NOTICE: CREATE TABLE will create partition "t2_1_prt_def" for table "t2" NOTICE: CREATE TABLE will create partition "t2_1_prt_2" for table "t2" NOTICE: CREATE TABLE will create partition "t2_1_prt_3" for table "t2" -NOTICE: CREATE TABLE will create partition "t2_1_prt_4" for table "t2" -NOTICE: CREATE TABLE will create partition "t2_1_prt_5" for table "t2" INSERT INTO t1 SELECT - generate_series(1, 100) * 3, generate_series(1, 100); + generate_series(1, 4) * 3, generate_series(1, 4); INSERT INTO t2 SELECT - generate_series(1, 100), generate_series(1, 100) * 3; + generate_series(1, 4), generate_series(1, 4) * 3; INSERT INTO t2 VALUES - (generate_series(101, 111), NULL); + (generate_series(7, 11), NULL); EXPLAIN (costs off, verbose) -DELETE FROM t1 USING t2 WHERE t1.a = t2.a; +DELETE FROM t2 USING t1 WHERE t1.a = t2.a; QUERY PLAN ---------------------------------------------------------------------------------------- Delete - Output: t1.a, t1.b, "outer".ColRef_0018, t1.ctid + Output: t2.a, t2.b, "outer".ColRef_0018, t2.ctid -> Result - Output: t1.a, t1.b, t1.ctid, t1.gp_segment_id, 0 - -> Hash Join - Output: t1.a, t1.b, t1.ctid, t1.gp_segment_id - Hash Cond: (t1.a = t2.a) - -> Sequence - Output: t1.a, t1.b, t1.ctid, t1.gp_segment_id - -> Partition Selector for t1 (dynamic scan id: 1) - Partitions selected: 2 (out of 2) - -> Dynamic Seq Scan on public.t1 (dynamic scan id: 1) - Output: t1.a, t1.b, t1.ctid, t1.gp_segment_id - -> Hash - Output: t2.a + Output: t2.a, t2.b, t2.ctid, t2.gp_segment_id, 0 + -> Redistribute Motion 3:3 (slice2; segments: 3) + Output: t2.a, t2.b, t2.ctid, t2.gp_segment_id + Hash Key: t2.b + -> Hash Join + Output: t2.a, t2.b, t2.ctid, t2.gp_segment_id + Hash Cond: (t2.a = t1.a) -> Redistribute Motion 3:3 (slice1; segments: 3) - Output: t2.a + Output: t2.a, t2.b, t2.ctid, t2.gp_segment_id Hash Key: t2.a -> Sequence - Output: t2.a - -> Partition Selector for t2 (dynamic scan id: 2) - Partitions selected: 5 (out of 5) - -> Dynamic Seq Scan on public.t2 (dynamic scan id: 2) - Output: t2.a + Output: t2.a, t2.b, t2.ctid, t2.gp_segment_id + -> Partition Selector for t2 (dynamic scan id: 1) + Partitions selected: 3 (out of 3) + -> Dynamic Seq Scan on public.t2 (dynamic scan id: 1) + Output: t2.a, t2.b, t2.ctid, t2.gp_segment_id + -> Hash + Output: t1.a + -> Sequence + Output: t1.a + -> Partition Selector for t1 (dynamic scan id: 2) + Partitions selected: 3 (out of 3) + -> Dynamic Seq Scan on public.t1 (dynamic scan id: 2) + Output: t1.a Optimizer: Pivotal Optimizer (GPORCA) Settings: optimizer=on -(26 rows) +(29 rows) + +DELETE FROM t2 USING t1 WHERE t1.a = t2.a +RETURNING *; + a | b | a | b +---+---+---+--- + 9 | | 9 | 3 + 3 | 9 | 3 | 1 +(2 rows) -DELETE FROM t1 USING t2 WHERE t1.a = t2.a; DROP TABLE t1; DROP TABLE t2; -- Explicit Redistribute Motion should not be elided if we encounter a scan on -- the same table that we are going to modify, but with different range table --- index. +-- index. (test case not applicable to ORCA) CREATE TABLE t1 (a int, b int); NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table. HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. CREATE TABLE t2 (a int, b int); NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table. HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. -INSERT INTO t1 SELECT a, a FROM generate_series(1, 100) a; -INSERT INTO t2 SELECT a, a FROM generate_series(1, 100) a; -EXPLAIN (costs off) update t2 trg +INSERT INTO t1 SELECT a, a FROM generate_series(1, 4) a; +INSERT INTO t2 SELECT a, a FROM generate_series(1, 16) a; +EXPLAIN (costs off) UPDATE t2 trg SET b = src.b1 FROM (SELECT t1.a AS a1, t1.b AS b1, t2.a AS a2, t2.b AS b2 FROM t1 JOIN t2 USING (b)) src WHERE trg.a = src.a1 - AND trg.a = 2; + AND trg.a = 2; QUERY PLAN --------------------------------------------------------------------------------------- Update @@ -506,11 +498,22 @@ WHERE trg.a = src.a1 Optimizer: Pivotal Optimizer (GPORCA) (16 rows) +UPDATE t2 trg +SET b = src.b1 +FROM (SELECT t1.a AS a1, t1.b AS b1, t2.a AS a2, t2.b AS b2 FROM t1 JOIN t2 USING (b)) src +WHERE trg.a = src.a1 + AND trg.a = 2 +RETURNING *; + a | b | a1 | b1 | a2 | b2 +---+---+----+----+----+---- + 2 | 2 | 2 | 2 | 2 | 2 +(1 row) + -- Use Nested Loop to change left tree with the right tree, to swap the extra -- scan we don't indend to detect with the real one. SET enable_hashjoin = off; SET enable_nestloop = on; -EXPLAIN (costs off) update t2 trg +EXPLAIN (costs off) UPDATE t2 trg SET b = src.b1 FROM (SELECT t1.a AS a1, t1.b AS b1, t2.a AS a2, t2.b AS b2 FROM t1 JOIN t2 USING (b)) src WHERE trg.a = src.a1 @@ -535,13 +538,24 @@ WHERE trg.a = src.a1 Optimizer: Pivotal Optimizer (GPORCA) (16 rows) +UPDATE t2 trg +SET b = src.b1 +FROM (SELECT t1.a AS a1, t1.b AS b1, t2.a AS a2, t2.b AS b2 FROM t1 JOIN t2 USING (b)) src +WHERE trg.a = src.a1 + AND trg.a = 2 +RETURNING *; + a | b | a1 | b1 | a2 | b2 +---+---+----+----+----+---- + 2 | 2 | 2 | 2 | 2 | 2 +(1 row) + RESET enable_hashjoin; RESET enable_nestloop; DROP TABLE t1; DROP TABLE t2; -- Explicit Redistribute Motion should be elided for every partition that does -- not have any motions above the scan on the table/partition we are going to --- update. +-- update. (test case not applicable to ORCA) CREATE TABLE t1 (a int, b int, c int) DISTRIBUTED BY (b) PARTITION BY RANGE(b) (start (1) end(5) every(1)); NOTICE: CREATE TABLE will create partition "t1_1_prt_1" for table "t1" @@ -549,8 +563,8 @@ NOTICE: CREATE TABLE will create partition "t1_1_prt_2" for table "t1" NOTICE: CREATE TABLE will create partition "t1_1_prt_3" for table "t1" NOTICE: CREATE TABLE will create partition "t1_1_prt_4" for table "t1" CREATE TABLE t2 (a int, b int, c int) DISTRIBUTED BY (a); -INSERT INTO t1 SELECT i * 2, i, i * 3 FROM generate_series(1,4) i; -INSERT INTO t2 SELECT i, i * 2, i * 3 FROM generate_series(1,4) i; +INSERT INTO t1 SELECT i * 2, i, i * 3 FROM generate_series(1, 4) i; +INSERT INTO t2 SELECT i, i * 2, i * 3 FROM generate_series(1, 4) i; -- These partitions will need to have Explicit Redistribute above them. TRUNCATE t1_1_prt_1; TRUNCATE t1_1_prt_3; @@ -574,12 +588,20 @@ EXPLAIN (costs off) Optimizer: Pivotal Optimizer (GPORCA) (12 rows) +UPDATE t1 SET c = t2.b FROM t2 +RETURNING *; + a | b | c | a | b | c +---+---+---+---+---+--- + 4 | 2 | 4 | 2 | 4 | 6 + 8 | 4 | 4 | 2 | 4 | 6 +(2 rows) + DROP TABLE t1; DROP TABLE t2; -- Explicit Redistribute Motion should not be elided if there's a Gather Motion -- beneath the ModifyTable. (test case not applicable to ORCA) CREATE TABLE t1 (a int) DISTRIBUTED BY (a); -INSERT INTO t1 SELECT i FROM generate_series(1,4) i; +INSERT INTO t1 SELECT i FROM generate_series(1, 4) i; -- "USING pg_class" forces a Gather Motion. EXPLAIN (costs off) DELETE FROM t1 @@ -597,7 +619,16 @@ USING pg_class; (8 rows) DELETE FROM t1 -USING pg_class; +USING pg_class +RETURNING t1.*; + a +--- + 1 + 2 + 3 + 4 +(4 rows) + DROP TABLE t1; -- -- text types. We should support the following updates. diff --git a/src/test/regress/sql/update_gp.sql b/src/test/regress/sql/update_gp.sql index 552942092eb0..48f418d29a4e 100644 --- a/src/test/regress/sql/update_gp.sql +++ b/src/test/regress/sql/update_gp.sql @@ -137,8 +137,7 @@ UPDATE t1 SET j = t_strewn.i FROM t_strewn WHERE t_strewn.i = t1.i; EXPLAIN (costs off) WITH CTE AS (DELETE FROM t1 RETURNING *) -SELECT count(*) AS a FROM t_strewn JOIN t2 USING (i) -WHERE 0 < ALL (SELECT i FROM cte); +SELECT count(*) AS a FROM t_strewn JOIN cte USING (i); EXPLAIN (costs off, verbose) DELETE FROM t_strewn WHERE t_strewn.i = (SELECT t2.i FROM t2 WHERE t_strewn.i = t2.i); @@ -157,9 +156,10 @@ DROP TABLE t_strewn2; CREATE TABLE t1 (a int, b int) DISTRIBUTED RANDOMLY; CREATE TABLE t2 (a int, b int) DISTRIBUTED RANDOMLY; -EXPLAIN (costs off) WITH cte AS ( +EXPLAIN (costs off) +WITH cte AS ( SELECT count(*) AS c FROM t1, t2 WHERE t1.b = t2.b -) UPDATE t2 SET b = 10 WHERE a = (SELECT * FROM cte); +) DELETE FROM t2 WHERE a = (SELECT * FROM cte); DROP TABLE t1; DROP TABLE t2; @@ -174,15 +174,13 @@ DROP TABLE IF EXISTS foo; CREATE TABLE i (i int, j int) DISTRIBUTED BY (i); INSERT INTO i SELECT - generate_series(1, 100), generate_series(1, 100) * 3; + generate_series(1, 16), generate_series(1, 16) * 3; CREATE TABLE foo (f1 serial, f2 text, f3 int) DISTRIBUTED RANDOMLY; - INSERT INTO foo (f2, f3) VALUES ('first', 1), ('second', 2), ('third', 3); CREATE TABLE foochild (fc int) INHERITS (foo); - INSERT INTO foochild VALUES(123, 'child', 999, -123); @@ -193,7 +191,8 @@ DELETE FROM foo DELETE FROM foo USING i - WHERE foo.f1 = i.j; + WHERE foo.f1 = i.j + RETURNING *; DROP TABLE i; DROP TABLE foochild; @@ -203,53 +202,68 @@ DROP TABLE foo; -- tables. (test case not applicable to ORCA) CREATE TABLE t1 (a int, b int) DISTRIBUTED BY (a) PARTITION BY - range(b) (start(1) end(101) every(50)); + range(b) (start(1) end(16) every(5)); CREATE TABLE t2 (a int, b int) DISTRIBUTED BY (b) PARTITION BY - range(a) (start(1) end(101) every(25), default partition def); + range(a) (start(1) end(16) every(10), default partition def); INSERT INTO t1 SELECT - generate_series(1, 100) * 3, generate_series(1, 100); + generate_series(1, 4) * 3, generate_series(1, 4); INSERT INTO t2 SELECT - generate_series(1, 100), generate_series(1, 100) * 3; + generate_series(1, 4), generate_series(1, 4) * 3; INSERT INTO t2 VALUES - (generate_series(101, 111), NULL); + (generate_series(7, 11), NULL); EXPLAIN (costs off, verbose) -DELETE FROM t1 USING t2 WHERE t1.a = t2.a; +DELETE FROM t2 USING t1 WHERE t1.a = t2.a; -DELETE FROM t1 USING t2 WHERE t1.a = t2.a; +DELETE FROM t2 USING t1 WHERE t1.a = t2.a +RETURNING *; DROP TABLE t1; DROP TABLE t2; -- Explicit Redistribute Motion should not be elided if we encounter a scan on -- the same table that we are going to modify, but with different range table --- index. +-- index. (test case not applicable to ORCA) CREATE TABLE t1 (a int, b int); CREATE TABLE t2 (a int, b int); -INSERT INTO t1 SELECT a, a FROM generate_series(1, 100) a; -INSERT INTO t2 SELECT a, a FROM generate_series(1, 100) a; +INSERT INTO t1 SELECT a, a FROM generate_series(1, 4) a; +INSERT INTO t2 SELECT a, a FROM generate_series(1, 16) a; -EXPLAIN (costs off) update t2 trg +EXPLAIN (costs off) UPDATE t2 trg SET b = src.b1 FROM (SELECT t1.a AS a1, t1.b AS b1, t2.a AS a2, t2.b AS b2 FROM t1 JOIN t2 USING (b)) src WHERE trg.a = src.a1 - AND trg.a = 2; + AND trg.a = 2; + +UPDATE t2 trg +SET b = src.b1 +FROM (SELECT t1.a AS a1, t1.b AS b1, t2.a AS a2, t2.b AS b2 FROM t1 JOIN t2 USING (b)) src +WHERE trg.a = src.a1 + AND trg.a = 2 +RETURNING *; -- Use Nested Loop to change left tree with the right tree, to swap the extra -- scan we don't indend to detect with the real one. SET enable_hashjoin = off; SET enable_nestloop = on; -EXPLAIN (costs off) update t2 trg +EXPLAIN (costs off) UPDATE t2 trg SET b = src.b1 FROM (SELECT t1.a AS a1, t1.b AS b1, t2.a AS a2, t2.b AS b2 FROM t1 JOIN t2 USING (b)) src WHERE trg.a = src.a1 AND trg.a = 2; +UPDATE t2 trg +SET b = src.b1 +FROM (SELECT t1.a AS a1, t1.b AS b1, t2.a AS a2, t2.b AS b2 FROM t1 JOIN t2 USING (b)) src +WHERE trg.a = src.a1 + AND trg.a = 2 +RETURNING *; + RESET enable_hashjoin; RESET enable_nestloop; @@ -258,14 +272,14 @@ DROP TABLE t2; -- Explicit Redistribute Motion should be elided for every partition that does -- not have any motions above the scan on the table/partition we are going to --- update. +-- update. (test case not applicable to ORCA) CREATE TABLE t1 (a int, b int, c int) DISTRIBUTED BY (b) PARTITION BY RANGE(b) (start (1) end(5) every(1)); CREATE TABLE t2 (a int, b int, c int) DISTRIBUTED BY (a); -INSERT INTO t1 SELECT i * 2, i, i * 3 FROM generate_series(1,4) i; -INSERT INTO t2 SELECT i, i * 2, i * 3 FROM generate_series(1,4) i; +INSERT INTO t1 SELECT i * 2, i, i * 3 FROM generate_series(1, 4) i; +INSERT INTO t2 SELECT i, i * 2, i * 3 FROM generate_series(1, 4) i; -- These partitions will need to have Explicit Redistribute above them. TRUNCATE t1_1_prt_1; @@ -276,6 +290,9 @@ ANALYZE t1_1_prt_3; EXPLAIN (costs off) UPDATE t1 SET c = t2.b FROM t2; +UPDATE t1 SET c = t2.b FROM t2 +RETURNING *; + DROP TABLE t1; DROP TABLE t2; @@ -283,7 +300,7 @@ DROP TABLE t2; -- beneath the ModifyTable. (test case not applicable to ORCA) CREATE TABLE t1 (a int) DISTRIBUTED BY (a); -INSERT INTO t1 SELECT i FROM generate_series(1,4) i; +INSERT INTO t1 SELECT i FROM generate_series(1, 4) i; -- "USING pg_class" forces a Gather Motion. EXPLAIN (costs off) @@ -291,7 +308,8 @@ DELETE FROM t1 USING pg_class; DELETE FROM t1 -USING pg_class; +USING pg_class +RETURNING t1.*; DROP TABLE t1; From 4bef5258d25b0037e812faa696c6f6171b2356c4 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Thu, 7 Mar 2024 10:43:19 +0300 Subject: [PATCH 58/83] Remove flaky ORCA test --- src/test/regress/expected/update_gp.out | 8 -------- src/test/regress/expected/update_gp_optimizer.out | 8 -------- src/test/regress/sql/update_gp.sql | 4 +--- 3 files changed, 1 insertion(+), 19 deletions(-) diff --git a/src/test/regress/expected/update_gp.out b/src/test/regress/expected/update_gp.out index 1b670cb6c123..1d6163b394c8 100644 --- a/src/test/regress/expected/update_gp.out +++ b/src/test/regress/expected/update_gp.out @@ -621,14 +621,6 @@ EXPLAIN (costs off) Optimizer: Postgres query optimizer (24 rows) -UPDATE t1 SET c = t2.b FROM t2 -RETURNING *; - a | b | c | a | b | c ----+---+---+---+---+--- - 4 | 2 | 4 | 2 | 4 | 6 - 8 | 4 | 4 | 2 | 4 | 6 -(2 rows) - DROP TABLE t1; DROP TABLE t2; -- Explicit Redistribute Motion should not be elided if there's a Gather Motion diff --git a/src/test/regress/expected/update_gp_optimizer.out b/src/test/regress/expected/update_gp_optimizer.out index bbd673eb5b1f..6f15983492f8 100644 --- a/src/test/regress/expected/update_gp_optimizer.out +++ b/src/test/regress/expected/update_gp_optimizer.out @@ -588,14 +588,6 @@ EXPLAIN (costs off) Optimizer: Pivotal Optimizer (GPORCA) (12 rows) -UPDATE t1 SET c = t2.b FROM t2 -RETURNING *; - a | b | c | a | b | c ----+---+---+---+---+--- - 4 | 2 | 4 | 2 | 4 | 6 - 8 | 4 | 4 | 2 | 4 | 6 -(2 rows) - DROP TABLE t1; DROP TABLE t2; -- Explicit Redistribute Motion should not be elided if there's a Gather Motion diff --git a/src/test/regress/sql/update_gp.sql b/src/test/regress/sql/update_gp.sql index 48f418d29a4e..2c02d2c87137 100644 --- a/src/test/regress/sql/update_gp.sql +++ b/src/test/regress/sql/update_gp.sql @@ -284,15 +284,13 @@ INSERT INTO t2 SELECT i, i * 2, i * 3 FROM generate_series(1, 4) i; -- These partitions will need to have Explicit Redistribute above them. TRUNCATE t1_1_prt_1; TRUNCATE t1_1_prt_3; + ANALYZE t1_1_prt_1; ANALYZE t1_1_prt_3; EXPLAIN (costs off) UPDATE t1 SET c = t2.b FROM t2; -UPDATE t1 SET c = t2.b FROM t2 -RETURNING *; - DROP TABLE t1; DROP TABLE t2; From 16fc4779a4014085737bd12f603037904619413c Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Thu, 7 Mar 2024 11:13:52 +0300 Subject: [PATCH 59/83] Remove scans that can't cause an Explicit Motion --- src/backend/cdb/cdbmutate.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index 54c8dc836693..10f535ee18cb 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -874,7 +874,6 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) { case T_SeqScan: case T_DynamicSeqScan: - case T_ExternalScan: case T_IndexScan: case T_DynamicIndexScan: case T_IndexOnlyScan: @@ -883,7 +882,6 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) case T_BitmapHeapScan: case T_DynamicBitmapHeapScan: case T_TidScan: - case T_ForeignScan: { Scan *scan = (Scan *) node; From de565c22d589cfcbe20070db06cec42a3758986e Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Thu, 7 Mar 2024 11:24:40 +0300 Subject: [PATCH 60/83] Add comment about the list of scan nodes --- src/backend/cdb/cdbmutate.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index 10f535ee18cb..e5d99fbdaf46 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -870,6 +870,12 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) */ else { + /* + * These are scan nodes that can be used to perform parallel + * UPDATE/DELETE on the relation they scan, possibly with motions + * above them. This list needs to be updated for other nodes if they + * are changed to support DML execution on segments. + */ switch (nodeTag(node)) { case T_SeqScan: From bf188a4180d7254981556076064e448a80ca4f81 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Thu, 7 Mar 2024 12:55:19 +0300 Subject: [PATCH 61/83] Adjust comments --- src/backend/cdb/cdbmutate.c | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index e5d99fbdaf46..0dd33bf13f32 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -70,8 +70,7 @@ typedef struct ModifyTableMotionState Bitmapset *resultRtis; /* Indexes into rtable for relations to * be modified */ bool needExplicitMotion; - int nMotionsAbove; /* Number of Gather, Redistribute and - * Broadcast motions above the current + int nMotionsAbove; /* Number of motions above the current * node */ bool isChecking; /* True if we encountered ModifyTable * node with UPDATE/DELETE and we plan @@ -811,10 +810,10 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) } /* - * For UPDATE/DELETE, we check if there's any Redistribute, Broadcast - * or Gather Motions before scan in the same subtree for the table we're - * going to modify. If we encounter the scan before any motions, then we can - * elide unneccessary Explicit Redistribute Motion. + * For UPDATE/DELETE, we check if there's any motions before scan in the + * same subtree for the table we're going to modify. If we encounter the + * scan before any motions, then we can elide unneccessary Explicit + * Redistribute Motion. */ if (IsA(node, ModifyTable)) { @@ -857,10 +856,7 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) if (context->mt.isChecking) { - /* - * Remember if we are descending into a Redistribute, Broadcast or - * Gather Motion node. - */ + /* Remember if we are descending into a motion node. */ if (IsA(node, Motion)) context->mt.nMotionsAbove += 1; From 267077fec6abe1c3a1b06f57a0e664faa3610441 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Thu, 7 Mar 2024 13:28:07 +0300 Subject: [PATCH 62/83] Remove the braces --- src/backend/cdb/cdbmutate.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index 0dd33bf13f32..f58b293c3b80 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -1132,11 +1132,9 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) plan->nMotionNodes = context->nextMotionID - saveNextMotionID; plan->nInitPlans = hash_get_num_entries(context->planid_subplans) - saveNumInitPlans; + /* We're going out of this motion node. */ if (context->mt.isChecking && IsA(node, Motion)) - { - /* We're going out of this motion node. */ context->mt.nMotionsAbove -= 1; - } return newnode; } /* apply_motion_mutator */ From 4ffca2aaf8f933d41905bd9bcf45d9634a553208 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Thu, 7 Mar 2024 13:01:40 +0300 Subject: [PATCH 63/83] Remove unneeded verbose, only check test cases with changed plans --- src/test/regress/expected/update_gp.out | 271 ++++++++---------- .../regress/expected/update_gp_optimizer.out | 266 ++++++++--------- src/test/regress/sql/update_gp.sql | 68 +++-- 3 files changed, 281 insertions(+), 324 deletions(-) diff --git a/src/test/regress/expected/update_gp.out b/src/test/regress/expected/update_gp.out index 1d6163b394c8..b88a31c9896d 100644 --- a/src/test/regress/expected/update_gp.out +++ b/src/test/regress/expected/update_gp.out @@ -215,6 +215,11 @@ CREATE TABLE t1 (i int, j int) DISTRIBUTED BY (i); CREATE TABLE t2 (i int) DISTRIBUTED BY (i); CREATE TABLE t_strewn (i int) DISTRIBUTED RANDOMLY; CREATE TABLE t_strewn2 (i int) DISTRIBUTED RANDOMLY; +INSERT INTO t1 SELECT + generate_series(1, 4) * 3, generate_series(1, 4); +INSERT INTO t2 SELECT generate_series(1, 4) * 3; +INSERT INTO t_strewn SELECT generate_series(1, 16); +INSERT INTO t_strewn2 SELECT generate_series(2, 17); EXPLAIN (costs off) UPDATE t1 SET j = t_strewn.i FROM t_strewn WHERE t_strewn.i = t1.i; QUERY PLAN @@ -230,49 +235,67 @@ UPDATE t1 SET j = t_strewn.i FROM t_strewn WHERE t_strewn.i = t1.i; Optimizer: Postgres query optimizer (9 rows) +UPDATE t1 SET j = t_strewn.i FROM t_strewn WHERE t_strewn.i = t1.i +RETURNING *; + i | j | i +----+----+---- + 9 | 9 | 9 + 6 | 6 | 6 + 12 | 12 | 12 + 3 | 3 | 3 +(4 rows) + EXPLAIN (costs off) WITH CTE AS (DELETE FROM t1 RETURNING *) SELECT count(*) AS a FROM t_strewn JOIN cte USING (i); - QUERY PLAN ------------------------------------------------------------------------------- + QUERY PLAN +--------------------------------------------------------------------------- Aggregate - -> Gather Motion 3:1 (slice3; segments: 3) + -> Gather Motion 3:1 (slice2; segments: 3) -> Aggregate -> Hash Join - Hash Cond: (cte.i = t_strewn.i) - -> Redistribute Motion 3:3 (slice1; segments: 3) - Hash Key: cte.i - -> Subquery Scan on cte - -> Delete on t1 - -> Seq Scan on t1 + Hash Cond: (t_strewn.i = cte.i) + -> Seq Scan on t_strewn -> Hash - -> Redistribute Motion 3:3 (slice2; segments: 3) - Hash Key: t_strewn.i - -> Seq Scan on t_strewn + -> Broadcast Motion 3:3 (slice1; segments: 3) + -> Subquery Scan on cte + -> Delete on t1 + -> Seq Scan on t1 Optimizer: Postgres query optimizer -(15 rows) +(12 rows) + +WITH CTE AS (DELETE FROM t1 RETURNING *) +SELECT count(*) AS a FROM t_strewn JOIN cte USING (i); + a +--- + 4 +(1 row) -EXPLAIN (costs off, verbose) +EXPLAIN (costs off) DELETE FROM t_strewn WHERE t_strewn.i = (SELECT t2.i FROM t2 WHERE t_strewn.i = t2.i); QUERY PLAN ----------------------------------------------------------------------- - Delete on public.t_strewn - -> Seq Scan on public.t_strewn - Output: t_strewn.ctid, t_strewn.gp_segment_id - Filter: (t_strewn.i = (SubPlan 1)) + Delete on t_strewn + -> Seq Scan on t_strewn + Filter: (i = (SubPlan 1)) SubPlan 1 (slice0; segments: 3) -> Result - Output: t2.i Filter: (t_strewn.i = t2.i) -> Materialize - Output: t2.i, t2.i -> Broadcast Motion 3:3 (slice1; segments: 3) - Output: t2.i, t2.i - -> Seq Scan on public.t2 - Output: t2.i, t2.i + -> Seq Scan on t2 Optimizer: Postgres query optimizer - Settings: optimizer=off -(16 rows) +(10 rows) + +DELETE FROM t_strewn WHERE t_strewn.i = (SELECT t2.i FROM t2 WHERE t_strewn.i = t2.i) +RETURNING *; + i +---- + 3 + 9 + 6 + 12 +(4 rows) EXPLAIN (costs off) UPDATE t_strewn SET i = t_strewn2.i @@ -293,6 +316,24 @@ FROM t_strewn2 WHERE t_strewn.i = t_strewn2.i; Optimizer: Postgres query optimizer (12 rows) +UPDATE t_strewn SET i = t_strewn2.i +FROM t_strewn2 WHERE t_strewn.i = t_strewn2.i +RETURNING *; + i | i +----+---- + 5 | 5 + 13 | 13 + 15 | 15 + 10 | 10 + 11 | 11 + 2 | 2 + 7 | 7 + 8 | 8 + 14 | 14 + 4 | 4 + 16 | 16 +(11 rows) + DROP TABLE t1; DROP TABLE t2; DROP TABLE t_strewn; @@ -301,30 +342,42 @@ DROP TABLE t_strewn2; -- Plans but not in the main plan. (test case not applicable to ORCA) CREATE TABLE t1 (a int, b int) DISTRIBUTED RANDOMLY; CREATE TABLE t2 (a int, b int) DISTRIBUTED RANDOMLY; +INSERT INTO t1 SELECT + generate_series(1, 4) * 3, generate_series(1, 4); +INSERT INTO t2 SELECT + generate_series(1, 32), generate_series(1, 32) * 3; +ANALYZE t1; +ANALYZE t2; EXPLAIN (costs off) WITH cte AS ( SELECT count(*) AS c FROM t1, t2 WHERE t1.b = t2.b ) DELETE FROM t2 WHERE a = (SELECT * FROM cte); - QUERY PLAN --------------------------------------------------------------------------------------- + QUERY PLAN +----------------------------------------------------------------------------------- Delete on t2 - InitPlan 1 (returns $0) (slice4) + InitPlan 1 (returns $0) (slice3) -> Aggregate - -> Gather Motion 3:1 (slice3; segments: 3) + -> Gather Motion 3:1 (slice2; segments: 3) -> Aggregate -> Hash Join - Hash Cond: (t1.b = t2_1.b) - -> Redistribute Motion 3:3 (slice1; segments: 3) - Hash Key: t1.b - -> Seq Scan on t1 + Hash Cond: (t2_1.b = t1.b) + -> Seq Scan on t2 t2_1 -> Hash - -> Redistribute Motion 3:3 (slice2; segments: 3) - Hash Key: t2_1.b - -> Seq Scan on t2 t2_1 + -> Broadcast Motion 3:3 (slice1; segments: 3) + -> Seq Scan on t1 -> Seq Scan on t2 Filter: (a = $0) Optimizer: Postgres query optimizer -(17 rows) +(14 rows) + +WITH cte AS ( + SELECT count(*) AS c FROM t1, t2 WHERE t1.b = t2.b +) DELETE FROM t2 WHERE a = (SELECT * FROM cte) +RETURNING *; + a | b +---+--- + 1 | 3 +(1 row) DROP TABLE t1; DROP TABLE t2; @@ -340,51 +393,29 @@ CREATE TABLE foochild (fc int) INHERITS (foo); NOTICE: table has parent, setting distribution columns to match parent table INSERT INTO foochild VALUES(123, 'child', 999, -123); -EXPLAIN (costs off, verbose) +EXPLAIN (costs off) DELETE FROM foo USING i WHERE foo.f1 = i.j; - QUERY PLAN --------------------------------------------------------------------------------------------- - Delete on public.foo + QUERY PLAN +--------------------------------------------------------------------- + Delete on foo -> Explicit Redistribute Motion 3:3 (slice2; segments: 3) - Output: foo.ctid, foo.gp_segment_id, i.ctid -> Hash Join - Output: foo.ctid, foo.gp_segment_id, i.ctid Hash Cond: (i.j = foo.f1) - -> Seq Scan on public.i - Output: i.ctid, i.j + -> Seq Scan on i -> Hash - Output: foo.ctid, foo.gp_segment_id, foo.f1 -> Broadcast Motion 3:3 (slice1; segments: 3) - Output: foo.ctid, foo.gp_segment_id, foo.f1 - -> Seq Scan on public.foo - Output: foo.ctid, foo.gp_segment_id, foo.f1 + -> Seq Scan on foo -> Explicit Redistribute Motion 3:3 (slice4; segments: 3) - Output: foochild.ctid, foochild.gp_segment_id, i.ctid -> Hash Join - Output: foochild.ctid, foochild.gp_segment_id, i.ctid Hash Cond: (i.j = foochild.f1) - -> Seq Scan on public.i - Output: i.ctid, i.j + -> Seq Scan on i -> Hash - Output: foochild.ctid, foochild.gp_segment_id, foochild.f1 -> Broadcast Motion 3:3 (slice3; segments: 3) - Output: foochild.ctid, foochild.gp_segment_id, foochild.f1 - -> Seq Scan on public.foochild - Output: foochild.ctid, foochild.gp_segment_id, foochild.f1 + -> Seq Scan on foochild Optimizer: Postgres query optimizer - Settings: optimizer=off -(29 rows) - -DELETE FROM foo - USING i - WHERE foo.f1 = i.j - RETURNING *; - f1 | f2 | f3 | i | j -----+-------+----+---+--- - 3 | third | 3 | 1 | 3 -(1 row) +(16 rows) DROP TABLE i; DROP TABLE foochild; @@ -409,79 +440,46 @@ INSERT INTO t2 SELECT generate_series(1, 4), generate_series(1, 4) * 3; INSERT INTO t2 VALUES (generate_series(7, 11), NULL); -EXPLAIN (costs off, verbose) +EXPLAIN (costs off) DELETE FROM t2 USING t1 WHERE t1.a = t2.a; - QUERY PLAN ------------------------------------------------------------------------------------------------------------ - Delete on public.t2_1_prt_def + QUERY PLAN +------------------------------------------------------------------------ + Delete on t2_1_prt_def -> Explicit Redistribute Motion 3:3 (slice2; segments: 3) - Output: t2_1_prt_def.ctid, t2_1_prt_def.gp_segment_id, t1_1_prt_1.ctid, t1_1_prt_1.tableoid -> Hash Join - Output: t2_1_prt_def.ctid, t2_1_prt_def.gp_segment_id, t1_1_prt_1.ctid, t1_1_prt_1.tableoid Hash Cond: (t1_1_prt_1.a = t2_1_prt_def.a) -> Append - -> Seq Scan on public.t1_1_prt_1 - Output: t1_1_prt_1.ctid, t1_1_prt_1.tableoid, t1_1_prt_1.a - -> Seq Scan on public.t1_1_prt_2 - Output: t1_1_prt_2.ctid, t1_1_prt_2.tableoid, t1_1_prt_2.a - -> Seq Scan on public.t1_1_prt_3 - Output: t1_1_prt_3.ctid, t1_1_prt_3.tableoid, t1_1_prt_3.a + -> Seq Scan on t1_1_prt_1 + -> Seq Scan on t1_1_prt_2 + -> Seq Scan on t1_1_prt_3 -> Hash - Output: t2_1_prt_def.ctid, t2_1_prt_def.gp_segment_id, t2_1_prt_def.a -> Redistribute Motion 3:3 (slice1; segments: 3) - Output: t2_1_prt_def.ctid, t2_1_prt_def.gp_segment_id, t2_1_prt_def.a Hash Key: t2_1_prt_def.a - -> Seq Scan on public.t2_1_prt_def - Output: t2_1_prt_def.ctid, t2_1_prt_def.gp_segment_id, t2_1_prt_def.a + -> Seq Scan on t2_1_prt_def -> Explicit Redistribute Motion 3:3 (slice4; segments: 3) - Output: t2_1_prt_2.ctid, t2_1_prt_2.gp_segment_id, t1_1_prt_1.ctid, t1_1_prt_1.tableoid -> Hash Join - Output: t2_1_prt_2.ctid, t2_1_prt_2.gp_segment_id, t1_1_prt_1.ctid, t1_1_prt_1.tableoid Hash Cond: (t1_1_prt_1.a = t2_1_prt_2.a) -> Append - -> Seq Scan on public.t1_1_prt_1 - Output: t1_1_prt_1.ctid, t1_1_prt_1.tableoid, t1_1_prt_1.a - -> Seq Scan on public.t1_1_prt_2 - Output: t1_1_prt_2.ctid, t1_1_prt_2.tableoid, t1_1_prt_2.a - -> Seq Scan on public.t1_1_prt_3 - Output: t1_1_prt_3.ctid, t1_1_prt_3.tableoid, t1_1_prt_3.a + -> Seq Scan on t1_1_prt_1 + -> Seq Scan on t1_1_prt_2 + -> Seq Scan on t1_1_prt_3 -> Hash - Output: t2_1_prt_2.ctid, t2_1_prt_2.gp_segment_id, t2_1_prt_2.a -> Redistribute Motion 3:3 (slice3; segments: 3) - Output: t2_1_prt_2.ctid, t2_1_prt_2.gp_segment_id, t2_1_prt_2.a Hash Key: t2_1_prt_2.a - -> Seq Scan on public.t2_1_prt_2 - Output: t2_1_prt_2.ctid, t2_1_prt_2.gp_segment_id, t2_1_prt_2.a + -> Seq Scan on t2_1_prt_2 -> Explicit Redistribute Motion 3:3 (slice6; segments: 3) - Output: t2_1_prt_3.ctid, t2_1_prt_3.gp_segment_id, t1_1_prt_1.ctid, t1_1_prt_1.tableoid -> Hash Join - Output: t2_1_prt_3.ctid, t2_1_prt_3.gp_segment_id, t1_1_prt_1.ctid, t1_1_prt_1.tableoid Hash Cond: (t1_1_prt_1.a = t2_1_prt_3.a) -> Append - -> Seq Scan on public.t1_1_prt_1 - Output: t1_1_prt_1.ctid, t1_1_prt_1.tableoid, t1_1_prt_1.a - -> Seq Scan on public.t1_1_prt_2 - Output: t1_1_prt_2.ctid, t1_1_prt_2.tableoid, t1_1_prt_2.a - -> Seq Scan on public.t1_1_prt_3 - Output: t1_1_prt_3.ctid, t1_1_prt_3.tableoid, t1_1_prt_3.a + -> Seq Scan on t1_1_prt_1 + -> Seq Scan on t1_1_prt_2 + -> Seq Scan on t1_1_prt_3 -> Hash - Output: t2_1_prt_3.ctid, t2_1_prt_3.gp_segment_id, t2_1_prt_3.a -> Redistribute Motion 3:3 (slice5; segments: 3) - Output: t2_1_prt_3.ctid, t2_1_prt_3.gp_segment_id, t2_1_prt_3.a Hash Key: t2_1_prt_3.a - -> Seq Scan on public.t2_1_prt_3 - Output: t2_1_prt_3.ctid, t2_1_prt_3.gp_segment_id, t2_1_prt_3.a + -> Seq Scan on t2_1_prt_3 Optimizer: Postgres query optimizer - Settings: optimizer=off -(60 rows) - -DELETE FROM t2 USING t1 WHERE t1.a = t2.a -RETURNING *; - a | b | a | b ----+---+---+--- - 9 | | 9 | 3 - 3 | 9 | 3 | 1 -(2 rows) +(35 rows) DROP TABLE t1; DROP TABLE t2; @@ -520,17 +518,6 @@ WHERE trg.a = src.a1 Optimizer: Postgres query optimizer (15 rows) -UPDATE t2 trg -SET b = src.b1 -FROM (SELECT t1.a AS a1, t1.b AS b1, t2.a AS a2, t2.b AS b2 FROM t1 JOIN t2 USING (b)) src -WHERE trg.a = src.a1 - AND trg.a = 2 -RETURNING *; - a | b | a1 | b1 | a2 | b2 ----+---+----+----+----+---- - 2 | 2 | 2 | 2 | 2 | 2 -(1 row) - -- Use Nested Loop to change left tree with the right tree, to swap the extra -- scan we don't indend to detect with the real one. SET enable_hashjoin = off; @@ -539,7 +526,7 @@ EXPLAIN (costs off) UPDATE t2 trg SET b = src.b1 FROM (SELECT t1.a AS a1, t1.b AS b1, t2.a AS a2, t2.b AS b2 FROM t1 JOIN t2 USING (b)) src WHERE trg.a = src.a1 - AND trg.a = 2; + AND trg.a = 2; QUERY PLAN --------------------------------------------------------------------------- Update on t2 trg @@ -559,17 +546,6 @@ WHERE trg.a = src.a1 Optimizer: Postgres query optimizer (15 rows) -UPDATE t2 trg -SET b = src.b1 -FROM (SELECT t1.a AS a1, t1.b AS b1, t2.a AS a2, t2.b AS b2 FROM t1 JOIN t2 USING (b)) src -WHERE trg.a = src.a1 - AND trg.a = 2 -RETURNING *; - a | b | a1 | b1 | a2 | b2 ----+---+----+----+----+---- - 2 | 2 | 2 | 2 | 2 | 2 -(1 row) - RESET enable_hashjoin; RESET enable_nestloop; DROP TABLE t1; @@ -592,7 +568,7 @@ TRUNCATE t1_1_prt_3; ANALYZE t1_1_prt_1; ANALYZE t1_1_prt_3; EXPLAIN (costs off) - UPDATE t1 SET c = t2.b FROM t2; + UPDATE t1 SET c = t2.b FROM t2; QUERY PLAN --------------------------------------------------------------- Update on t1_1_prt_1 @@ -643,17 +619,6 @@ USING pg_class; Optimizer: Postgres query optimizer (8 rows) -DELETE FROM t1 -USING pg_class -RETURNING t1.*; - a ---- - 1 - 2 - 3 - 4 -(4 rows) - DROP TABLE t1; -- -- text types. We should support the following updates. diff --git a/src/test/regress/expected/update_gp_optimizer.out b/src/test/regress/expected/update_gp_optimizer.out index 6f15983492f8..6691074f6845 100644 --- a/src/test/regress/expected/update_gp_optimizer.out +++ b/src/test/regress/expected/update_gp_optimizer.out @@ -220,6 +220,11 @@ CREATE TABLE t1 (i int, j int) DISTRIBUTED BY (i); CREATE TABLE t2 (i int) DISTRIBUTED BY (i); CREATE TABLE t_strewn (i int) DISTRIBUTED RANDOMLY; CREATE TABLE t_strewn2 (i int) DISTRIBUTED RANDOMLY; +INSERT INTO t1 SELECT + generate_series(1, 4) * 3, generate_series(1, 4); +INSERT INTO t2 SELECT generate_series(1, 4) * 3; +INSERT INTO t_strewn SELECT generate_series(1, 16); +INSERT INTO t_strewn2 SELECT generate_series(2, 17); EXPLAIN (costs off) UPDATE t1 SET j = t_strewn.i FROM t_strewn WHERE t_strewn.i = t1.i; QUERY PLAN @@ -238,68 +243,106 @@ UPDATE t1 SET j = t_strewn.i FROM t_strewn WHERE t_strewn.i = t1.i; Optimizer: Pivotal Optimizer (GPORCA) (12 rows) +UPDATE t1 SET j = t_strewn.i FROM t_strewn WHERE t_strewn.i = t1.i +RETURNING *; + i | j | i +----+----+---- + 3 | 3 | 3 + 12 | 12 | 12 + 6 | 6 | 6 + 9 | 9 | 9 +(4 rows) + EXPLAIN (costs off) WITH CTE AS (DELETE FROM t1 RETURNING *) SELECT count(*) AS a FROM t_strewn JOIN cte USING (i); - QUERY PLAN ------------------------------------------------------------------------------- + QUERY PLAN +--------------------------------------------------------------------------- Aggregate - -> Gather Motion 3:1 (slice3; segments: 3) + -> Gather Motion 3:1 (slice2; segments: 3) -> Aggregate -> Hash Join - Hash Cond: (cte.i = t_strewn.i) - -> Redistribute Motion 3:3 (slice1; segments: 3) - Hash Key: cte.i - -> Subquery Scan on cte - -> Delete on t1 - -> Seq Scan on t1 + Hash Cond: (t_strewn.i = cte.i) + -> Seq Scan on t_strewn -> Hash - -> Redistribute Motion 3:3 (slice2; segments: 3) - Hash Key: t_strewn.i - -> Seq Scan on t_strewn + -> Broadcast Motion 3:3 (slice1; segments: 3) + -> Subquery Scan on cte + -> Delete on t1 + -> Seq Scan on t1 Optimizer: Postgres query optimizer -(15 rows) +(12 rows) -EXPLAIN (costs off, verbose) +WITH CTE AS (DELETE FROM t1 RETURNING *) +SELECT count(*) AS a FROM t_strewn JOIN cte USING (i); + a +--- + 4 +(1 row) + +EXPLAIN (costs off) DELETE FROM t_strewn WHERE t_strewn.i = (SELECT t2.i FROM t2 WHERE t_strewn.i = t2.i); QUERY PLAN ----------------------------------------------------------------------------- Delete - Output: t_strewn.i, "outer".ColRef_0016, t_strewn.ctid -> Result - Output: t_strewn.i, t_strewn.ctid, t_strewn.gp_segment_id, 0 - -> Seq Scan on public.t_strewn - Output: t_strewn.i, t_strewn.ctid, t_strewn.gp_segment_id - Filter: (t_strewn.i = (SubPlan 1)) + -> Seq Scan on t_strewn + Filter: (i = (SubPlan 1)) SubPlan 1 (slice0; segments: 3) -> Result - Output: t2.i Filter: (t_strewn.i = t2.i) -> Materialize - Output: t2.i -> Broadcast Motion 3:3 (slice1; segments: 3) - Output: t2.i - -> Seq Scan on public.t2 - Output: t2.i + -> Seq Scan on t2 Optimizer: Pivotal Optimizer (GPORCA) - Settings: optimizer=on -(19 rows) +(11 rows) + +DELETE FROM t_strewn WHERE t_strewn.i = (SELECT t2.i FROM t2 WHERE t_strewn.i = t2.i) +RETURNING *; + i +---- + 6 + 3 + 9 + 12 +(4 rows) EXPLAIN (costs off) UPDATE t_strewn SET i = t_strewn2.i FROM t_strewn2 WHERE t_strewn.i = t_strewn2.i; - QUERY PLAN ---------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------ Update - -> Split - -> Hash Join - Hash Cond: (t_strewn.i = t_strewn2.i) - -> Seq Scan on t_strewn - -> Hash - -> Broadcast Motion 3:3 (slice1; segments: 3) - -> Seq Scan on t_strewn2 + -> Explicit Redistribute Motion 3:3 (slice3; segments: 3) + -> Split + -> Hash Join + Hash Cond: (t_strewn.i = t_strewn2.i) + -> Redistribute Motion 3:3 (slice1; segments: 3) + Hash Key: t_strewn.i + -> Seq Scan on t_strewn + -> Hash + -> Redistribute Motion 3:3 (slice2; segments: 3) + Hash Key: t_strewn2.i + -> Seq Scan on t_strewn2 Optimizer: Pivotal Optimizer (GPORCA) -(9 rows) +(13 rows) + +UPDATE t_strewn SET i = t_strewn2.i +FROM t_strewn2 WHERE t_strewn.i = t_strewn2.i +RETURNING *; + i | i +----+---- + 14 | 14 + 16 | 16 + 5 | 5 + 10 | 10 + 7 | 7 + 8 | 8 + 11 | 11 + 13 | 13 + 15 | 15 + 2 | 2 + 4 | 4 +(11 rows) DROP TABLE t1; DROP TABLE t2; @@ -309,30 +352,47 @@ DROP TABLE t_strewn2; -- Plans but not in the main plan. (test case not applicable to ORCA) CREATE TABLE t1 (a int, b int) DISTRIBUTED RANDOMLY; CREATE TABLE t2 (a int, b int) DISTRIBUTED RANDOMLY; +INSERT INTO t1 SELECT + generate_series(1, 4) * 3, generate_series(1, 4); +INSERT INTO t2 SELECT + generate_series(1, 32), generate_series(1, 32) * 3; +ANALYZE t1; +ANALYZE t2; EXPLAIN (costs off) WITH cte AS ( SELECT count(*) AS c FROM t1, t2 WHERE t1.b = t2.b ) DELETE FROM t2 WHERE a = (SELECT * FROM cte); - QUERY PLAN ---------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------ Delete -> Result - -> Explicit Redistribute Motion 1:3 (slice4; segments: 1) - -> Hash Join - Hash Cond: ((t2.a)::bigint = (count())) - -> Gather Motion 3:1 (slice1; segments: 3) - -> Seq Scan on t2 - -> Hash + -> Hash Join + Hash Cond: ((t2.a)::bigint = (count())) + -> Seq Scan on t2 + -> Hash + -> Broadcast Motion 1:3 (slice4; segments: 1) -> Aggregate -> Gather Motion 3:1 (slice3; segments: 3) -> Hash Join - Hash Cond: (t1.b = t2_1.b) - -> Seq Scan on t1 + Hash Cond: (t2_1.b = t1.b) + -> Redistribute Motion 3:3 (slice1; segments: 3) + Hash Key: t2_1.b + -> Seq Scan on t2 t2_1 -> Hash - -> Broadcast Motion 3:3 (slice2; segments: 3) - -> Seq Scan on t2 t2_1 + -> Redistribute Motion 3:3 (slice2; segments: 3) + Hash Key: t1.b + -> Seq Scan on t1 Optimizer: Pivotal Optimizer (GPORCA) -(17 rows) +(19 rows) + +WITH cte AS ( + SELECT count(*) AS c FROM t1, t2 WHERE t1.b = t2.b +) DELETE FROM t2 WHERE a = (SELECT * FROM cte) +RETURNING *; + a | b +---+--- + 1 | 3 +(1 row) DROP TABLE t1; DROP TABLE t2; @@ -348,51 +408,29 @@ CREATE TABLE foochild (fc int) INHERITS (foo); NOTICE: table has parent, setting distribution columns to match parent table INSERT INTO foochild VALUES(123, 'child', 999, -123); -EXPLAIN (costs off, verbose) +EXPLAIN (costs off) DELETE FROM foo USING i WHERE foo.f1 = i.j; - QUERY PLAN --------------------------------------------------------------------------------------------- - Delete on public.foo + QUERY PLAN +--------------------------------------------------------------------- + Delete on foo -> Explicit Redistribute Motion 3:3 (slice2; segments: 3) - Output: foo.ctid, foo.gp_segment_id, i.ctid -> Hash Join - Output: foo.ctid, foo.gp_segment_id, i.ctid Hash Cond: (i.j = foo.f1) - -> Seq Scan on public.i - Output: i.ctid, i.j + -> Seq Scan on i -> Hash - Output: foo.ctid, foo.gp_segment_id, foo.f1 -> Broadcast Motion 3:3 (slice1; segments: 3) - Output: foo.ctid, foo.gp_segment_id, foo.f1 - -> Seq Scan on public.foo - Output: foo.ctid, foo.gp_segment_id, foo.f1 + -> Seq Scan on foo -> Explicit Redistribute Motion 3:3 (slice4; segments: 3) - Output: foochild.ctid, foochild.gp_segment_id, i.ctid -> Hash Join - Output: foochild.ctid, foochild.gp_segment_id, i.ctid Hash Cond: (i.j = foochild.f1) - -> Seq Scan on public.i - Output: i.ctid, i.j + -> Seq Scan on i -> Hash - Output: foochild.ctid, foochild.gp_segment_id, foochild.f1 -> Broadcast Motion 3:3 (slice3; segments: 3) - Output: foochild.ctid, foochild.gp_segment_id, foochild.f1 - -> Seq Scan on public.foochild - Output: foochild.ctid, foochild.gp_segment_id, foochild.f1 + -> Seq Scan on foochild Optimizer: Postgres query optimizer - Settings: optimizer=on -(29 rows) - -DELETE FROM foo - USING i - WHERE foo.f1 = i.j - RETURNING *; - f1 | f2 | f3 | i | j -----+-------+----+---+--- - 3 | third | 3 | 1 | 3 -(1 row) +(16 rows) DROP TABLE i; DROP TABLE foochild; @@ -417,48 +455,29 @@ INSERT INTO t2 SELECT generate_series(1, 4), generate_series(1, 4) * 3; INSERT INTO t2 VALUES (generate_series(7, 11), NULL); -EXPLAIN (costs off, verbose) +EXPLAIN (costs off) DELETE FROM t2 USING t1 WHERE t1.a = t2.a; - QUERY PLAN ----------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------ Delete - Output: t2.a, t2.b, "outer".ColRef_0018, t2.ctid -> Result - Output: t2.a, t2.b, t2.ctid, t2.gp_segment_id, 0 -> Redistribute Motion 3:3 (slice2; segments: 3) - Output: t2.a, t2.b, t2.ctid, t2.gp_segment_id Hash Key: t2.b -> Hash Join - Output: t2.a, t2.b, t2.ctid, t2.gp_segment_id Hash Cond: (t2.a = t1.a) -> Redistribute Motion 3:3 (slice1; segments: 3) - Output: t2.a, t2.b, t2.ctid, t2.gp_segment_id Hash Key: t2.a -> Sequence - Output: t2.a, t2.b, t2.ctid, t2.gp_segment_id -> Partition Selector for t2 (dynamic scan id: 1) Partitions selected: 3 (out of 3) - -> Dynamic Seq Scan on public.t2 (dynamic scan id: 1) - Output: t2.a, t2.b, t2.ctid, t2.gp_segment_id + -> Dynamic Seq Scan on t2 (dynamic scan id: 1) -> Hash - Output: t1.a -> Sequence - Output: t1.a -> Partition Selector for t1 (dynamic scan id: 2) Partitions selected: 3 (out of 3) - -> Dynamic Seq Scan on public.t1 (dynamic scan id: 2) - Output: t1.a + -> Dynamic Seq Scan on t1 (dynamic scan id: 2) Optimizer: Pivotal Optimizer (GPORCA) - Settings: optimizer=on -(29 rows) - -DELETE FROM t2 USING t1 WHERE t1.a = t2.a -RETURNING *; - a | b | a | b ----+---+---+--- - 9 | | 9 | 3 - 3 | 9 | 3 | 1 -(2 rows) +(18 rows) DROP TABLE t1; DROP TABLE t2; @@ -498,17 +517,6 @@ WHERE trg.a = src.a1 Optimizer: Pivotal Optimizer (GPORCA) (16 rows) -UPDATE t2 trg -SET b = src.b1 -FROM (SELECT t1.a AS a1, t1.b AS b1, t2.a AS a2, t2.b AS b2 FROM t1 JOIN t2 USING (b)) src -WHERE trg.a = src.a1 - AND trg.a = 2 -RETURNING *; - a | b | a1 | b1 | a2 | b2 ----+---+----+----+----+---- - 2 | 2 | 2 | 2 | 2 | 2 -(1 row) - -- Use Nested Loop to change left tree with the right tree, to swap the extra -- scan we don't indend to detect with the real one. SET enable_hashjoin = off; @@ -517,7 +525,7 @@ EXPLAIN (costs off) UPDATE t2 trg SET b = src.b1 FROM (SELECT t1.a AS a1, t1.b AS b1, t2.a AS a2, t2.b AS b2 FROM t1 JOIN t2 USING (b)) src WHERE trg.a = src.a1 - AND trg.a = 2; + AND trg.a = 2; QUERY PLAN --------------------------------------------------------------------------------------- Update @@ -538,17 +546,6 @@ WHERE trg.a = src.a1 Optimizer: Pivotal Optimizer (GPORCA) (16 rows) -UPDATE t2 trg -SET b = src.b1 -FROM (SELECT t1.a AS a1, t1.b AS b1, t2.a AS a2, t2.b AS b2 FROM t1 JOIN t2 USING (b)) src -WHERE trg.a = src.a1 - AND trg.a = 2 -RETURNING *; - a | b | a1 | b1 | a2 | b2 ----+---+----+----+----+---- - 2 | 2 | 2 | 2 | 2 | 2 -(1 row) - RESET enable_hashjoin; RESET enable_nestloop; DROP TABLE t1; @@ -571,7 +568,7 @@ TRUNCATE t1_1_prt_3; ANALYZE t1_1_prt_1; ANALYZE t1_1_prt_3; EXPLAIN (costs off) - UPDATE t1 SET c = t2.b FROM t2; + UPDATE t1 SET c = t2.b FROM t2; QUERY PLAN ------------------------------------------------------------------------ Update @@ -610,17 +607,6 @@ USING pg_class; Optimizer: Postgres query optimizer (8 rows) -DELETE FROM t1 -USING pg_class -RETURNING t1.*; - a ---- - 1 - 2 - 3 - 4 -(4 rows) - DROP TABLE t1; -- -- text types. We should support the following updates. diff --git a/src/test/regress/sql/update_gp.sql b/src/test/regress/sql/update_gp.sql index 2c02d2c87137..ba4cb24e2f29 100644 --- a/src/test/regress/sql/update_gp.sql +++ b/src/test/regress/sql/update_gp.sql @@ -132,20 +132,39 @@ CREATE TABLE t2 (i int) DISTRIBUTED BY (i); CREATE TABLE t_strewn (i int) DISTRIBUTED RANDOMLY; CREATE TABLE t_strewn2 (i int) DISTRIBUTED RANDOMLY; +INSERT INTO t1 SELECT + generate_series(1, 4) * 3, generate_series(1, 4); +INSERT INTO t2 SELECT generate_series(1, 4) * 3; +INSERT INTO t_strewn SELECT generate_series(1, 16); +INSERT INTO t_strewn2 SELECT generate_series(2, 17); + EXPLAIN (costs off) UPDATE t1 SET j = t_strewn.i FROM t_strewn WHERE t_strewn.i = t1.i; +UPDATE t1 SET j = t_strewn.i FROM t_strewn WHERE t_strewn.i = t1.i +RETURNING *; + EXPLAIN (costs off) WITH CTE AS (DELETE FROM t1 RETURNING *) SELECT count(*) AS a FROM t_strewn JOIN cte USING (i); -EXPLAIN (costs off, verbose) +WITH CTE AS (DELETE FROM t1 RETURNING *) +SELECT count(*) AS a FROM t_strewn JOIN cte USING (i); + +EXPLAIN (costs off) DELETE FROM t_strewn WHERE t_strewn.i = (SELECT t2.i FROM t2 WHERE t_strewn.i = t2.i); +DELETE FROM t_strewn WHERE t_strewn.i = (SELECT t2.i FROM t2 WHERE t_strewn.i = t2.i) +RETURNING *; + EXPLAIN (costs off) UPDATE t_strewn SET i = t_strewn2.i FROM t_strewn2 WHERE t_strewn.i = t_strewn2.i; +UPDATE t_strewn SET i = t_strewn2.i +FROM t_strewn2 WHERE t_strewn.i = t_strewn2.i +RETURNING *; + DROP TABLE t1; DROP TABLE t2; DROP TABLE t_strewn; @@ -156,11 +175,24 @@ DROP TABLE t_strewn2; CREATE TABLE t1 (a int, b int) DISTRIBUTED RANDOMLY; CREATE TABLE t2 (a int, b int) DISTRIBUTED RANDOMLY; +INSERT INTO t1 SELECT + generate_series(1, 4) * 3, generate_series(1, 4); +INSERT INTO t2 SELECT + generate_series(1, 32), generate_series(1, 32) * 3; + +ANALYZE t1; +ANALYZE t2; + EXPLAIN (costs off) WITH cte AS ( SELECT count(*) AS c FROM t1, t2 WHERE t1.b = t2.b ) DELETE FROM t2 WHERE a = (SELECT * FROM cte); +WITH cte AS ( + SELECT count(*) AS c FROM t1, t2 WHERE t1.b = t2.b +) DELETE FROM t2 WHERE a = (SELECT * FROM cte) +RETURNING *; + DROP TABLE t1; DROP TABLE t2; @@ -184,16 +216,11 @@ CREATE TABLE foochild (fc int) INHERITS (foo); INSERT INTO foochild VALUES(123, 'child', 999, -123); -EXPLAIN (costs off, verbose) +EXPLAIN (costs off) DELETE FROM foo USING i WHERE foo.f1 = i.j; -DELETE FROM foo - USING i - WHERE foo.f1 = i.j - RETURNING *; - DROP TABLE i; DROP TABLE foochild; DROP TABLE foo; @@ -215,12 +242,9 @@ INSERT INTO t2 SELECT INSERT INTO t2 VALUES (generate_series(7, 11), NULL); -EXPLAIN (costs off, verbose) +EXPLAIN (costs off) DELETE FROM t2 USING t1 WHERE t1.a = t2.a; -DELETE FROM t2 USING t1 WHERE t1.a = t2.a -RETURNING *; - DROP TABLE t1; DROP TABLE t2; @@ -239,13 +263,6 @@ FROM (SELECT t1.a AS a1, t1.b AS b1, t2.a AS a2, t2.b AS b2 FROM t1 JOIN t2 USIN WHERE trg.a = src.a1 AND trg.a = 2; -UPDATE t2 trg -SET b = src.b1 -FROM (SELECT t1.a AS a1, t1.b AS b1, t2.a AS a2, t2.b AS b2 FROM t1 JOIN t2 USING (b)) src -WHERE trg.a = src.a1 - AND trg.a = 2 -RETURNING *; - -- Use Nested Loop to change left tree with the right tree, to swap the extra -- scan we don't indend to detect with the real one. SET enable_hashjoin = off; @@ -255,14 +272,7 @@ EXPLAIN (costs off) UPDATE t2 trg SET b = src.b1 FROM (SELECT t1.a AS a1, t1.b AS b1, t2.a AS a2, t2.b AS b2 FROM t1 JOIN t2 USING (b)) src WHERE trg.a = src.a1 - AND trg.a = 2; - -UPDATE t2 trg -SET b = src.b1 -FROM (SELECT t1.a AS a1, t1.b AS b1, t2.a AS a2, t2.b AS b2 FROM t1 JOIN t2 USING (b)) src -WHERE trg.a = src.a1 - AND trg.a = 2 -RETURNING *; + AND trg.a = 2; RESET enable_hashjoin; RESET enable_nestloop; @@ -289,7 +299,7 @@ ANALYZE t1_1_prt_1; ANALYZE t1_1_prt_3; EXPLAIN (costs off) - UPDATE t1 SET c = t2.b FROM t2; + UPDATE t1 SET c = t2.b FROM t2; DROP TABLE t1; DROP TABLE t2; @@ -305,10 +315,6 @@ EXPLAIN (costs off) DELETE FROM t1 USING pg_class; -DELETE FROM t1 -USING pg_class -RETURNING t1.*; - DROP TABLE t1; -- From 9d35370946237e684c6ef5a4aac7b9baaa8d2b9a Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Mon, 11 Mar 2024 14:28:22 +0300 Subject: [PATCH 64/83] Update comments --- src/backend/cdb/cdbmutate.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index f58b293c3b80..cc4a75d13afc 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -812,7 +812,7 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) /* * For UPDATE/DELETE, we check if there's any motions before scan in the * same subtree for the table we're going to modify. If we encounter the - * scan before any motions, then we can elide unneccessary Explicit + * scan before any motions, then we can elide unnecessary Explicit * Redistribute Motion. */ if (IsA(node, ModifyTable)) @@ -849,10 +849,6 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) } } - /* - * Elide Explicit Redistribute Motion if there's no motions between the - * scan and the ModifyTable. - */ if (context->mt.isChecking) { @@ -862,12 +858,12 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) /* * If this is a scan and it's scanrelid matches ModifyTable's relid, - * we encountered a scan before any Motions. + * we need to check if there were any motions above. */ else { /* - * These are scan nodes that can be used to perform parallel + * These are scan nodes that can be used to perform distributed * UPDATE/DELETE on the relation they scan, possibly with motions * above them. This list needs to be updated for other nodes if they * are changed to support DML execution on segments. From 54dec02f86b88f459d17ec4060ad0cf41db0a65a Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Mon, 11 Mar 2024 14:28:33 +0300 Subject: [PATCH 65/83] Stop checking if we're out of ModifyTable --- src/backend/cdb/cdbmutate.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index cc4a75d13afc..0c04e677e2ff 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -1132,6 +1132,9 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) if (context->mt.isChecking && IsA(node, Motion)) context->mt.nMotionsAbove -= 1; + if (context->mt.isChecking && IsA(node, ModifyTable)) + context->mt.isChecking = false; + return newnode; } /* apply_motion_mutator */ From a62805010b795a09f2ef809be065d4c08185ddda Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Mon, 11 Mar 2024 15:28:06 +0300 Subject: [PATCH 66/83] Squash if condition --- src/backend/cdb/cdbmutate.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index 0c04e677e2ff..846dd2a163c5 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -1128,12 +1128,14 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) plan->nMotionNodes = context->nextMotionID - saveNextMotionID; plan->nInitPlans = hash_get_num_entries(context->planid_subplans) - saveNumInitPlans; - /* We're going out of this motion node. */ - if (context->mt.isChecking && IsA(node, Motion)) - context->mt.nMotionsAbove -= 1; - - if (context->mt.isChecking && IsA(node, ModifyTable)) - context->mt.isChecking = false; + if (context->mt.isChecking) + { + /* We're going out of this motion node. */ + if (IsA(node, Motion)) + context->mt.nMotionsAbove -= 1; + else if (IsA(node, ModifyTable)) + context->mt.isChecking = false; + } return newnode; } /* apply_motion_mutator */ From 4b43f33c2c8bf22f44faa3c20ccc9428e1f37610 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Thu, 14 Mar 2024 14:51:31 +0300 Subject: [PATCH 67/83] Get rid of ModifyTableMotionState --- src/backend/cdb/cdbmutate.c | 77 +++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 41 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index 846dd2a163c5..6949b10fb3d4 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -65,18 +65,6 @@ #include "executor/executor.h" -typedef struct ModifyTableMotionState -{ - Bitmapset *resultRtis; /* Indexes into rtable for relations to - * be modified */ - bool needExplicitMotion; - int nMotionsAbove; /* Number of motions above the current - * node */ - bool isChecking; /* True if we encountered ModifyTable - * node with UPDATE/DELETE and we plan - * to insert Explicit Motions */ -} ModifyTableMotionState; - /* * An ApplyMotionState holds state for the recursive apply_motion_mutator(). * It is externalized here to make it shareable by helper code in other @@ -91,7 +79,15 @@ typedef struct ApplyMotionState HTAB *planid_subplans; /* hash table for InitPlanItem */ /* Context for ModifyTable to elide Explicit Redistribute Motion */ - ModifyTableMotionState mt; + Bitmapset *mtResultRtis; /* Indexes into rtable for relations to + * be modified. Only valid if isMtAbove is + * true. */ + bool needExplicitMotion; + int nMotionsAbove; /* Number of motions above the current + * node */ + bool isMtAbove; /* True if we encountered ModifyTable + * node with UPDATE/DELETE and we plan + * to insert Explicit Motions. */ } ApplyMotionState; typedef struct InitPlanItem @@ -419,11 +415,10 @@ apply_motion(PlannerInfo *root, Plan *plan, Query *query) * plan */ state.nextMotionID = 1; /* Start at 1 so zero will mean "unassigned". */ state.sliceDepth = 0; - - state.mt.resultRtis = NULL; - state.mt.needExplicitMotion = false; - state.mt.nMotionsAbove = 0; - state.mt.isChecking = false; + state.mtResultRtis = NULL; + state.needExplicitMotion = false; + state.nMotionsAbove = 0; + state.isMtAbove = false; memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(int); @@ -775,14 +770,14 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) if (!is_plan_node(node)) { /* - * mt.isChecking is true whether we are checking for motions underneath + * isChecking is true whether we are checking for motions underneath * to add Explicit Reditribute Motion, ignoring any in InitPlans. So if * we recurse into an InitPlan, save it and temporarily set it to false. */ if (IsA(node, SubPlan) &&((SubPlan *) node)->is_initplan) { bool found; - bool saveMtIsChecking = context->mt.isChecking; + bool saveMtIsChecking = context->isMtAbove; int saveSliceDepth = context->sliceDepth; SubPlan *subplan = (SubPlan *) node; /* @@ -797,10 +792,10 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) /* reset sliceDepth for each init plan */ context->sliceDepth = 0; - context->mt.isChecking = false; + context->isMtAbove = false; node = plan_tree_mutator(node, apply_motion_mutator, context); - context->mt.isChecking = saveMtIsChecking; + context->isMtAbove = saveMtIsChecking; context->sliceDepth = saveSliceDepth; return node; @@ -826,10 +821,10 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) /* * Sanity check, since we don't allow multiple ModifyTable nodes. */ - Assert(context->mt.resultRtis == NULL); - Assert(context->mt.nMotionsAbove == 0); - Assert(!context->mt.needExplicitMotion); - Assert(!context->mt.isChecking); + Assert(context->mtResultRtis == NULL); + Assert(context->nMotionsAbove == 0); + Assert(!context->needExplicitMotion); + Assert(!context->isMtAbove); /* * When UPDATE/DELETE occurs on a partitioned table, or a table that @@ -841,20 +836,20 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) */ foreach(lcr, mt->resultRelations) { - context->mt.resultRtis = bms_add_member(context->mt.resultRtis, - lfirst_int(lcr)); + context->mtResultRtis = bms_add_member(context->mtResultRtis, + lfirst_int(lcr)); } - context->mt.isChecking = true; + context->isMtAbove = true; } } - if (context->mt.isChecking) + if (context->isMtAbove) { /* Remember if we are descending into a motion node. */ if (IsA(node, Motion)) - context->mt.nMotionsAbove += 1; + context->nMotionsAbove += 1; /* * If this is a scan and it's scanrelid matches ModifyTable's relid, @@ -883,19 +878,19 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) { Scan *scan = (Scan *) node; - if (bms_is_member(scan->scanrelid, context->mt.resultRtis)) + if (bms_is_member(scan->scanrelid, context->mtResultRtis)) { /* * We need Explicit Redistribute Motion only if * there were any motions above. */ - context->mt.needExplicitMotion = context->mt.nMotionsAbove > 0; + context->needExplicitMotion = context->nMotionsAbove > 0; /* * We don't need to check other nodes in this * subtree anymore. */ - context->mt.isChecking = false; + context->isMtAbove = false; } } break; @@ -1056,7 +1051,7 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) /* * Were there any motions above the scan? */ - if (context->mt.needExplicitMotion) + if (context->needExplicitMotion) { newnode = (Node *) make_explicit_motion(plan, flow->segidColIdx, @@ -1078,9 +1073,9 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) * other subtree. So reset the state and continue checking in case * of another Explicit Redistribute Motion is needed. */ - context->mt.isChecking = true; - context->mt.needExplicitMotion = false; - context->mt.nMotionsAbove = 0; + context->isMtAbove = true; + context->needExplicitMotion = false; + context->nMotionsAbove = 0; break; case MOVEMENT_NONE: @@ -1128,13 +1123,13 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) plan->nMotionNodes = context->nextMotionID - saveNextMotionID; plan->nInitPlans = hash_get_num_entries(context->planid_subplans) - saveNumInitPlans; - if (context->mt.isChecking) + if (context->isMtAbove) { /* We're going out of this motion node. */ if (IsA(node, Motion)) - context->mt.nMotionsAbove -= 1; + context->nMotionsAbove -= 1; else if (IsA(node, ModifyTable)) - context->mt.isChecking = false; + context->isMtAbove = false; } return newnode; From d8ec1f0385bdecef1ec5d0e973de6881e4ba3fc2 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Thu, 14 Mar 2024 15:14:09 +0300 Subject: [PATCH 68/83] Reorder members and update comments --- src/backend/cdb/cdbmutate.c | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index 6949b10fb3d4..7362e10fa37b 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -79,15 +79,15 @@ typedef struct ApplyMotionState HTAB *planid_subplans; /* hash table for InitPlanItem */ /* Context for ModifyTable to elide Explicit Redistribute Motion */ + bool mtIsChecking; /* True if we encountered ModifyTable + * node with UPDATE/DELETE and we plan + * to insert Explicit Motions. */ Bitmapset *mtResultRtis; /* Indexes into rtable for relations to - * be modified. Only valid if isMtAbove is - * true. */ + * be modified. Only valid if mtIsChecking + * is true. */ bool needExplicitMotion; int nMotionsAbove; /* Number of motions above the current * node */ - bool isMtAbove; /* True if we encountered ModifyTable - * node with UPDATE/DELETE and we plan - * to insert Explicit Motions. */ } ApplyMotionState; typedef struct InitPlanItem @@ -418,7 +418,7 @@ apply_motion(PlannerInfo *root, Plan *plan, Query *query) state.mtResultRtis = NULL; state.needExplicitMotion = false; state.nMotionsAbove = 0; - state.isMtAbove = false; + state.mtIsChecking = false; memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(int); @@ -770,14 +770,14 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) if (!is_plan_node(node)) { /* - * isChecking is true whether we are checking for motions underneath + * mtIsChecking is true whether we are checking for motions underneath * to add Explicit Reditribute Motion, ignoring any in InitPlans. So if * we recurse into an InitPlan, save it and temporarily set it to false. */ if (IsA(node, SubPlan) &&((SubPlan *) node)->is_initplan) { bool found; - bool saveMtIsChecking = context->isMtAbove; + bool saveMtIsChecking = context->mtIsChecking; int saveSliceDepth = context->sliceDepth; SubPlan *subplan = (SubPlan *) node; /* @@ -792,10 +792,10 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) /* reset sliceDepth for each init plan */ context->sliceDepth = 0; - context->isMtAbove = false; + context->mtIsChecking = false; node = plan_tree_mutator(node, apply_motion_mutator, context); - context->isMtAbove = saveMtIsChecking; + context->mtIsChecking = saveMtIsChecking; context->sliceDepth = saveSliceDepth; return node; @@ -824,7 +824,7 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) Assert(context->mtResultRtis == NULL); Assert(context->nMotionsAbove == 0); Assert(!context->needExplicitMotion); - Assert(!context->isMtAbove); + Assert(!context->mtIsChecking); /* * When UPDATE/DELETE occurs on a partitioned table, or a table that @@ -840,11 +840,11 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) lfirst_int(lcr)); } - context->isMtAbove = true; + context->mtIsChecking = true; } } - if (context->isMtAbove) + if (context->mtIsChecking) { /* Remember if we are descending into a motion node. */ @@ -890,7 +890,7 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) * We don't need to check other nodes in this * subtree anymore. */ - context->isMtAbove = false; + context->mtIsChecking = false; } } break; @@ -1073,7 +1073,7 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) * other subtree. So reset the state and continue checking in case * of another Explicit Redistribute Motion is needed. */ - context->isMtAbove = true; + context->mtIsChecking = true; context->needExplicitMotion = false; context->nMotionsAbove = 0; break; @@ -1123,13 +1123,13 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) plan->nMotionNodes = context->nextMotionID - saveNextMotionID; plan->nInitPlans = hash_get_num_entries(context->planid_subplans) - saveNumInitPlans; - if (context->isMtAbove) + if (context->mtIsChecking) { /* We're going out of this motion node. */ if (IsA(node, Motion)) context->nMotionsAbove -= 1; else if (IsA(node, ModifyTable)) - context->isMtAbove = false; + context->mtIsChecking = false; } return newnode; From 63f63cdbebc5777acfe78973b96b56ad7b276a99 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Thu, 14 Mar 2024 16:44:20 +0300 Subject: [PATCH 69/83] Remove blank line --- src/backend/cdb/cdbmutate.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index 7362e10fa37b..b31d5164430a 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -64,7 +64,6 @@ #include "executor/executor.h" - /* * An ApplyMotionState holds state for the recursive apply_motion_mutator(). * It is externalized here to make it shareable by helper code in other From 4301bf6cb5b1980e6a016a9267046d162c4c131d Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Thu, 14 Mar 2024 18:03:29 +0300 Subject: [PATCH 70/83] Use unary increment and decrement --- src/backend/cdb/cdbmutate.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index b31d5164430a..e7ee083ab34b 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -848,7 +848,7 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) /* Remember if we are descending into a motion node. */ if (IsA(node, Motion)) - context->nMotionsAbove += 1; + context->nMotionsAbove++; /* * If this is a scan and it's scanrelid matches ModifyTable's relid, @@ -1126,7 +1126,7 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) { /* We're going out of this motion node. */ if (IsA(node, Motion)) - context->nMotionsAbove -= 1; + context->nMotionsAbove--; else if (IsA(node, ModifyTable)) context->mtIsChecking = false; } From 71dbb778199834e3af9883404e66a6247f84d731 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Fri, 15 Mar 2024 13:36:20 +0300 Subject: [PATCH 71/83] Force a motion in InitPlan test case --- src/test/regress/expected/update_gp.out | 25 +++++++------ .../regress/expected/update_gp_optimizer.out | 35 ++++++++++--------- src/test/regress/sql/update_gp.sql | 2 +- 3 files changed, 33 insertions(+), 29 deletions(-) diff --git a/src/test/regress/expected/update_gp.out b/src/test/regress/expected/update_gp.out index b88a31c9896d..f854db1a6c7c 100644 --- a/src/test/regress/expected/update_gp.out +++ b/src/test/regress/expected/update_gp.out @@ -343,7 +343,7 @@ DROP TABLE t_strewn2; CREATE TABLE t1 (a int, b int) DISTRIBUTED RANDOMLY; CREATE TABLE t2 (a int, b int) DISTRIBUTED RANDOMLY; INSERT INTO t1 SELECT - generate_series(1, 4) * 3, generate_series(1, 4); + generate_series(1, 16) * 3, generate_series(1, 4); INSERT INTO t2 SELECT generate_series(1, 32), generate_series(1, 32) * 3; ANALYZE t1; @@ -352,31 +352,34 @@ EXPLAIN (costs off) WITH cte AS ( SELECT count(*) AS c FROM t1, t2 WHERE t1.b = t2.b ) DELETE FROM t2 WHERE a = (SELECT * FROM cte); - QUERY PLAN ------------------------------------------------------------------------------------ + QUERY PLAN +-------------------------------------------------------------------------------------- Delete on t2 - InitPlan 1 (returns $0) (slice3) + InitPlan 1 (returns $0) (slice4) -> Aggregate - -> Gather Motion 3:1 (slice2; segments: 3) + -> Gather Motion 3:1 (slice3; segments: 3) -> Aggregate -> Hash Join Hash Cond: (t2_1.b = t1.b) - -> Seq Scan on t2 t2_1 + -> Redistribute Motion 3:3 (slice1; segments: 3) + Hash Key: t2_1.b + -> Seq Scan on t2 t2_1 -> Hash - -> Broadcast Motion 3:3 (slice1; segments: 3) + -> Redistribute Motion 3:3 (slice2; segments: 3) + Hash Key: t1.b -> Seq Scan on t1 -> Seq Scan on t2 Filter: (a = $0) Optimizer: Postgres query optimizer -(14 rows) +(17 rows) WITH cte AS ( SELECT count(*) AS c FROM t1, t2 WHERE t1.b = t2.b ) DELETE FROM t2 WHERE a = (SELECT * FROM cte) RETURNING *; - a | b ----+--- - 1 | 3 + a | b +---+---- + 4 | 12 (1 row) DROP TABLE t1; diff --git a/src/test/regress/expected/update_gp_optimizer.out b/src/test/regress/expected/update_gp_optimizer.out index 6691074f6845..a1c0c5ea51b3 100644 --- a/src/test/regress/expected/update_gp_optimizer.out +++ b/src/test/regress/expected/update_gp_optimizer.out @@ -353,7 +353,7 @@ DROP TABLE t_strewn2; CREATE TABLE t1 (a int, b int) DISTRIBUTED RANDOMLY; CREATE TABLE t2 (a int, b int) DISTRIBUTED RANDOMLY; INSERT INTO t1 SELECT - generate_series(1, 4) * 3, generate_series(1, 4); + generate_series(1, 16) * 3, generate_series(1, 4); INSERT INTO t2 SELECT generate_series(1, 32), generate_series(1, 32) * 3; ANALYZE t1; @@ -362,36 +362,37 @@ EXPLAIN (costs off) WITH cte AS ( SELECT count(*) AS c FROM t1, t2 WHERE t1.b = t2.b ) DELETE FROM t2 WHERE a = (SELECT * FROM cte); - QUERY PLAN ------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------------ Delete -> Result -> Hash Join - Hash Cond: ((t2.a)::bigint = (count())) + Hash Cond: ((t2.a)::bigint = (count((count())))) -> Seq Scan on t2 -> Hash -> Broadcast Motion 1:3 (slice4; segments: 1) -> Aggregate -> Gather Motion 3:1 (slice3; segments: 3) - -> Hash Join - Hash Cond: (t2_1.b = t1.b) - -> Redistribute Motion 3:3 (slice1; segments: 3) - Hash Key: t2_1.b - -> Seq Scan on t2 t2_1 - -> Hash - -> Redistribute Motion 3:3 (slice2; segments: 3) - Hash Key: t1.b - -> Seq Scan on t1 + -> Aggregate + -> Hash Join + Hash Cond: (t2_1.b = t1.b) + -> Redistribute Motion 3:3 (slice1; segments: 3) + Hash Key: t2_1.b + -> Seq Scan on t2 t2_1 + -> Hash + -> Redistribute Motion 3:3 (slice2; segments: 3) + Hash Key: t1.b + -> Seq Scan on t1 Optimizer: Pivotal Optimizer (GPORCA) -(19 rows) +(20 rows) WITH cte AS ( SELECT count(*) AS c FROM t1, t2 WHERE t1.b = t2.b ) DELETE FROM t2 WHERE a = (SELECT * FROM cte) RETURNING *; - a | b ----+--- - 1 | 3 + a | b +---+---- + 4 | 12 (1 row) DROP TABLE t1; diff --git a/src/test/regress/sql/update_gp.sql b/src/test/regress/sql/update_gp.sql index ba4cb24e2f29..79d3568f2f5d 100644 --- a/src/test/regress/sql/update_gp.sql +++ b/src/test/regress/sql/update_gp.sql @@ -176,7 +176,7 @@ CREATE TABLE t1 (a int, b int) DISTRIBUTED RANDOMLY; CREATE TABLE t2 (a int, b int) DISTRIBUTED RANDOMLY; INSERT INTO t1 SELECT - generate_series(1, 4) * 3, generate_series(1, 4); + generate_series(1, 16) * 3, generate_series(1, 4); INSERT INTO t2 SELECT generate_series(1, 32), generate_series(1, 32) * 3; From 5e69424430815faa47eaf0f4101d01996b6a1a62 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Fri, 15 Mar 2024 13:37:05 +0300 Subject: [PATCH 72/83] Replace bitmapset with a pointer to existing List and get rid of needExplicitMotion --- src/backend/cdb/cdbmutate.c | 80 +++++++++++++++---------------------- 1 file changed, 33 insertions(+), 47 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index e7ee083ab34b..5b14f5e2d8f9 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -78,15 +78,14 @@ typedef struct ApplyMotionState HTAB *planid_subplans; /* hash table for InitPlanItem */ /* Context for ModifyTable to elide Explicit Redistribute Motion */ - bool mtIsChecking; /* True if we encountered ModifyTable - * node with UPDATE/DELETE and we plan - * to insert Explicit Motions. */ - Bitmapset *mtResultRtis; /* Indexes into rtable for relations to - * be modified. Only valid if mtIsChecking - * is true. */ - bool needExplicitMotion; - int nMotionsAbove; /* Number of motions above the current - * node */ + bool mtIsChecking; /* True if we encountered ModifyTable + * node with UPDATE/DELETE and we plan + * to insert Explicit Motions. */ + List *mtResultRtis; /* Indexes into rtable for relations to + * be modified. Only valid if mtIsChecking + * is true. */ + int nMotionsAbove; /* Number of motions above the current + * node */ } ApplyMotionState; typedef struct InitPlanItem @@ -414,10 +413,9 @@ apply_motion(PlannerInfo *root, Plan *plan, Query *query) * plan */ state.nextMotionID = 1; /* Start at 1 so zero will mean "unassigned". */ state.sliceDepth = 0; - state.mtResultRtis = NULL; - state.needExplicitMotion = false; - state.nMotionsAbove = 0; state.mtIsChecking = false; + state.mtResultRtis = NIL; + state.nMotionsAbove = 0; memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(int); @@ -815,48 +813,38 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) if (mt->operation == CMD_UPDATE || mt->operation == CMD_DELETE) { - ListCell *lcr; - /* * Sanity check, since we don't allow multiple ModifyTable nodes. */ - Assert(context->mtResultRtis == NULL); - Assert(context->nMotionsAbove == 0); - Assert(!context->needExplicitMotion); Assert(!context->mtIsChecking); + Assert(context->mtResultRtis == NIL); + Assert(context->nMotionsAbove == 0); /* * When UPDATE/DELETE occurs on a partitioned table, or a table that * is a part of inheritance tree, ModifyTable node will have more * than one relation in resultRelations. * - * We make a set of resulting relations' indexes to compare them - * later. + * Remember resulting relations' indexes to compare them later. */ - foreach(lcr, mt->resultRelations) - { - context->mtResultRtis = bms_add_member(context->mtResultRtis, - lfirst_int(lcr)); - } - context->mtIsChecking = true; + context->mtResultRtis = mt->resultRelations; } } if (context->mtIsChecking) { - - /* Remember if we are descending into a motion node. */ + /* + * Remember if we are descending into a motion node. + */ if (IsA(node, Motion)) context->nMotionsAbove++; - - /* - * If this is a scan and it's scanrelid matches ModifyTable's relid, - * we need to check if there were any motions above. - */ else { /* + * If this is a scan and it's scanrelid matches ModifyTable's relid, + * we need to check if there were any motions above. + * * These are scan nodes that can be used to perform distributed * UPDATE/DELETE on the relation they scan, possibly with motions * above them. This list needs to be updated for other nodes if they @@ -875,24 +863,23 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) case T_DynamicBitmapHeapScan: case T_TidScan: { + ListCell *lcr; Scan *scan = (Scan *) node; - if (bms_is_member(scan->scanrelid, context->mtResultRtis)) + foreach(lcr, context->mtResultRtis) { - /* - * We need Explicit Redistribute Motion only if - * there were any motions above. - */ - context->needExplicitMotion = context->nMotionsAbove > 0; - - /* - * We don't need to check other nodes in this - * subtree anymore. - */ - context->mtIsChecking = false; + if (scan->scanrelid == (Index) lfirst_int(lcr)) + { + /* + * Freeze the motion counter. Also, we don't + * need to check other nodes in this subtree + * anymore. + */ + context->mtIsChecking = false; + break; + } } } - break; default: break; } @@ -1050,7 +1037,7 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) /* * Were there any motions above the scan? */ - if (context->needExplicitMotion) + if (context->nMotionsAbove > 0) { newnode = (Node *) make_explicit_motion(plan, flow->segidColIdx, @@ -1073,7 +1060,6 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) * of another Explicit Redistribute Motion is needed. */ context->mtIsChecking = true; - context->needExplicitMotion = false; context->nMotionsAbove = 0; break; From e32d271a6c8aad00ae23201ec04a5ec9526485cb Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Fri, 15 Mar 2024 13:55:47 +0300 Subject: [PATCH 73/83] Replace loop with search --- src/backend/cdb/cdbmutate.c | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index 5b14f5e2d8f9..76af94d9de08 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -863,21 +863,16 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) case T_DynamicBitmapHeapScan: case T_TidScan: { - ListCell *lcr; Scan *scan = (Scan *) node; - foreach(lcr, context->mtResultRtis) + if (list_member_int(context->mtResultRtis, scan->scanrelid)) { - if (scan->scanrelid == (Index) lfirst_int(lcr)) - { - /* - * Freeze the motion counter. Also, we don't - * need to check other nodes in this subtree - * anymore. - */ - context->mtIsChecking = false; - break; - } + /* + * Freeze the motion counter. Also, we don't + * need to check other nodes in this subtree + * anymore. + */ + context->mtIsChecking = false; } } default: From d927e62ebd1c418bbcde712026ce96a89da532c4 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Fri, 15 Mar 2024 14:11:59 +0300 Subject: [PATCH 74/83] Squash if condition --- src/backend/cdb/cdbmutate.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index 76af94d9de08..5d3b987019a9 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -863,9 +863,8 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) case T_DynamicBitmapHeapScan: case T_TidScan: { - Scan *scan = (Scan *) node; - - if (list_member_int(context->mtResultRtis, scan->scanrelid)) + if (list_member_int(context->mtResultRtis, + ((Scan *) node)->scanrelid)) { /* * Freeze the motion counter. Also, we don't From 2d4550c2d2c70993ee92c0a4c24aaefbd859ce41 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Fri, 15 Mar 2024 14:17:49 +0300 Subject: [PATCH 75/83] Squash if condition 2 --- src/backend/cdb/cdbmutate.c | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index 5d3b987019a9..5cda3b5f928c 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -862,17 +862,15 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) case T_BitmapHeapScan: case T_DynamicBitmapHeapScan: case T_TidScan: + if (list_member_int(context->mtResultRtis, + ((Scan *) node)->scanrelid)) { - if (list_member_int(context->mtResultRtis, - ((Scan *) node)->scanrelid)) - { - /* - * Freeze the motion counter. Also, we don't - * need to check other nodes in this subtree - * anymore. - */ - context->mtIsChecking = false; - } + /* + * Freeze the motion counter. Also, we don't + * need to check other nodes in this subtree + * anymore. + */ + context->mtIsChecking = false; } default: break; From 850408243906606bcfc3a9c44567263d92ff1804 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Fri, 15 Mar 2024 14:37:11 +0300 Subject: [PATCH 76/83] if -> else if --- src/backend/cdb/cdbmutate.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index 5cda3b5f928c..1d59c805c06f 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -831,8 +831,7 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) context->mtResultRtis = mt->resultRelations; } } - - if (context->mtIsChecking) + else if (context->mtIsChecking) { /* * Remember if we are descending into a motion node. From 3e6acf12c36527cd5667cf83a1865e3a1aa86065 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Fri, 15 Mar 2024 14:38:07 +0300 Subject: [PATCH 77/83] Fix alignment --- src/backend/cdb/cdbmutate.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index 1d59c805c06f..2f6739d33f3a 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -81,7 +81,7 @@ typedef struct ApplyMotionState bool mtIsChecking; /* True if we encountered ModifyTable * node with UPDATE/DELETE and we plan * to insert Explicit Motions. */ - List *mtResultRtis; /* Indexes into rtable for relations to + List *mtResultRtis; /* Indexes into rtable for relations to * be modified. Only valid if mtIsChecking * is true. */ int nMotionsAbove; /* Number of motions above the current From bc8996044075da22754514f32f987da9b04f637c Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Fri, 15 Mar 2024 14:47:13 +0300 Subject: [PATCH 78/83] Try to restore ABI --- src/backend/cdb/cdbmutate.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index 2f6739d33f3a..62b9e8fbc71a 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -75,6 +75,8 @@ typedef struct ApplyMotionState * plan_tree_walker/mutator */ int nextMotionID; int sliceDepth; + bool containMotionNodes; /* Obsolete and unused, see below. Kept for + * ABI compatability. */ HTAB *planid_subplans; /* hash table for InitPlanItem */ /* Context for ModifyTable to elide Explicit Redistribute Motion */ @@ -413,6 +415,7 @@ apply_motion(PlannerInfo *root, Plan *plan, Query *query) * plan */ state.nextMotionID = 1; /* Start at 1 so zero will mean "unassigned". */ state.sliceDepth = 0; + state.containMotionNodes = false; state.mtIsChecking = false; state.mtResultRtis = NIL; state.nMotionsAbove = 0; From 3f250bee3023bc3b1b64832060e4bb4eecbb5270 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Fri, 15 Mar 2024 15:10:38 +0300 Subject: [PATCH 79/83] Revert "Try to restore ABI" This reverts commit bc8996044075da22754514f32f987da9b04f637c. --- src/backend/cdb/cdbmutate.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index 62b9e8fbc71a..2f6739d33f3a 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -75,8 +75,6 @@ typedef struct ApplyMotionState * plan_tree_walker/mutator */ int nextMotionID; int sliceDepth; - bool containMotionNodes; /* Obsolete and unused, see below. Kept for - * ABI compatability. */ HTAB *planid_subplans; /* hash table for InitPlanItem */ /* Context for ModifyTable to elide Explicit Redistribute Motion */ @@ -415,7 +413,6 @@ apply_motion(PlannerInfo *root, Plan *plan, Query *query) * plan */ state.nextMotionID = 1; /* Start at 1 so zero will mean "unassigned". */ state.sliceDepth = 0; - state.containMotionNodes = false; state.mtIsChecking = false; state.mtResultRtis = NIL; state.nMotionsAbove = 0; From 24a0b49916e5ee3997fc66686af151a3bd3de922 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Fri, 15 Mar 2024 15:17:43 +0300 Subject: [PATCH 80/83] mtResultRtis -> mtResultRelations --- src/backend/cdb/cdbmutate.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index 2f6739d33f3a..5539403d48e0 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -78,14 +78,14 @@ typedef struct ApplyMotionState HTAB *planid_subplans; /* hash table for InitPlanItem */ /* Context for ModifyTable to elide Explicit Redistribute Motion */ - bool mtIsChecking; /* True if we encountered ModifyTable - * node with UPDATE/DELETE and we plan - * to insert Explicit Motions. */ - List *mtResultRtis; /* Indexes into rtable for relations to - * be modified. Only valid if mtIsChecking - * is true. */ - int nMotionsAbove; /* Number of motions above the current - * node */ + bool mtIsChecking; /* True if we encountered ModifyTable + * node with UPDATE/DELETE and we plan + * to insert Explicit Motions. */ + List *mtResultRelations; /* Indexes into rtable for relations to + * be modified. Only valid if mtIsChecking + * is true. */ + int nMotionsAbove; /* Number of motions above the current + * node */ } ApplyMotionState; typedef struct InitPlanItem @@ -414,7 +414,7 @@ apply_motion(PlannerInfo *root, Plan *plan, Query *query) state.nextMotionID = 1; /* Start at 1 so zero will mean "unassigned". */ state.sliceDepth = 0; state.mtIsChecking = false; - state.mtResultRtis = NIL; + state.mtResultRelations = NIL; state.nMotionsAbove = 0; memset(&ctl, 0, sizeof(ctl)); @@ -817,7 +817,7 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) * Sanity check, since we don't allow multiple ModifyTable nodes. */ Assert(!context->mtIsChecking); - Assert(context->mtResultRtis == NIL); + Assert(context->mtResultRelations == NIL); Assert(context->nMotionsAbove == 0); /* @@ -828,7 +828,7 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) * Remember resulting relations' indexes to compare them later. */ context->mtIsChecking = true; - context->mtResultRtis = mt->resultRelations; + context->mtResultRelations = mt->resultRelations; } } else if (context->mtIsChecking) @@ -861,7 +861,7 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) case T_BitmapHeapScan: case T_DynamicBitmapHeapScan: case T_TidScan: - if (list_member_int(context->mtResultRtis, + if (list_member_int(context->mtResultRelations, ((Scan *) node)->scanrelid)) { /* From f2b9cbdf00060e44d84592523d4c6d050087f2e8 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Mon, 18 Mar 2024 10:10:05 +0300 Subject: [PATCH 81/83] Get rid of extra logic and tests related to InitPlans --- src/backend/cdb/cdbmutate.c | 8 --- src/test/regress/expected/update_gp.out | 46 ----------------- .../regress/expected/update_gp_optimizer.out | 49 ------------------- src/test/regress/sql/update_gp.sql | 26 ---------- 4 files changed, 129 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index 5539403d48e0..d16326d63cf8 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -766,15 +766,9 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) /* An expression node might have subtrees containing plans to be mutated. */ if (!is_plan_node(node)) { - /* - * mtIsChecking is true whether we are checking for motions underneath - * to add Explicit Reditribute Motion, ignoring any in InitPlans. So if - * we recurse into an InitPlan, save it and temporarily set it to false. - */ if (IsA(node, SubPlan) &&((SubPlan *) node)->is_initplan) { bool found; - bool saveMtIsChecking = context->mtIsChecking; int saveSliceDepth = context->sliceDepth; SubPlan *subplan = (SubPlan *) node; /* @@ -789,10 +783,8 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) /* reset sliceDepth for each init plan */ context->sliceDepth = 0; - context->mtIsChecking = false; node = plan_tree_mutator(node, apply_motion_mutator, context); - context->mtIsChecking = saveMtIsChecking; context->sliceDepth = saveSliceDepth; return node; diff --git a/src/test/regress/expected/update_gp.out b/src/test/regress/expected/update_gp.out index f854db1a6c7c..f62e0ffc2d37 100644 --- a/src/test/regress/expected/update_gp.out +++ b/src/test/regress/expected/update_gp.out @@ -338,52 +338,6 @@ DROP TABLE t1; DROP TABLE t2; DROP TABLE t_strewn; DROP TABLE t_strewn2; --- Explicit Redistribute Motion should not be added if there are motions in Init --- Plans but not in the main plan. (test case not applicable to ORCA) -CREATE TABLE t1 (a int, b int) DISTRIBUTED RANDOMLY; -CREATE TABLE t2 (a int, b int) DISTRIBUTED RANDOMLY; -INSERT INTO t1 SELECT - generate_series(1, 16) * 3, generate_series(1, 4); -INSERT INTO t2 SELECT - generate_series(1, 32), generate_series(1, 32) * 3; -ANALYZE t1; -ANALYZE t2; -EXPLAIN (costs off) -WITH cte AS ( - SELECT count(*) AS c FROM t1, t2 WHERE t1.b = t2.b -) DELETE FROM t2 WHERE a = (SELECT * FROM cte); - QUERY PLAN --------------------------------------------------------------------------------------- - Delete on t2 - InitPlan 1 (returns $0) (slice4) - -> Aggregate - -> Gather Motion 3:1 (slice3; segments: 3) - -> Aggregate - -> Hash Join - Hash Cond: (t2_1.b = t1.b) - -> Redistribute Motion 3:3 (slice1; segments: 3) - Hash Key: t2_1.b - -> Seq Scan on t2 t2_1 - -> Hash - -> Redistribute Motion 3:3 (slice2; segments: 3) - Hash Key: t1.b - -> Seq Scan on t1 - -> Seq Scan on t2 - Filter: (a = $0) - Optimizer: Postgres query optimizer -(17 rows) - -WITH cte AS ( - SELECT count(*) AS c FROM t1, t2 WHERE t1.b = t2.b -) DELETE FROM t2 WHERE a = (SELECT * FROM cte) -RETURNING *; - a | b ----+---- - 4 | 12 -(1 row) - -DROP TABLE t1; -DROP TABLE t2; -- Explicit Redistribute Motion should not be mistakenly elided for inherited -- tables. (test case not applicable to ORCA) CREATE TABLE i (i int, j int) DISTRIBUTED BY (i); diff --git a/src/test/regress/expected/update_gp_optimizer.out b/src/test/regress/expected/update_gp_optimizer.out index a1c0c5ea51b3..bee8de0f2b42 100644 --- a/src/test/regress/expected/update_gp_optimizer.out +++ b/src/test/regress/expected/update_gp_optimizer.out @@ -348,55 +348,6 @@ DROP TABLE t1; DROP TABLE t2; DROP TABLE t_strewn; DROP TABLE t_strewn2; --- Explicit Redistribute Motion should not be added if there are motions in Init --- Plans but not in the main plan. (test case not applicable to ORCA) -CREATE TABLE t1 (a int, b int) DISTRIBUTED RANDOMLY; -CREATE TABLE t2 (a int, b int) DISTRIBUTED RANDOMLY; -INSERT INTO t1 SELECT - generate_series(1, 16) * 3, generate_series(1, 4); -INSERT INTO t2 SELECT - generate_series(1, 32), generate_series(1, 32) * 3; -ANALYZE t1; -ANALYZE t2; -EXPLAIN (costs off) -WITH cte AS ( - SELECT count(*) AS c FROM t1, t2 WHERE t1.b = t2.b -) DELETE FROM t2 WHERE a = (SELECT * FROM cte); - QUERY PLAN ------------------------------------------------------------------------------------------------------------- - Delete - -> Result - -> Hash Join - Hash Cond: ((t2.a)::bigint = (count((count())))) - -> Seq Scan on t2 - -> Hash - -> Broadcast Motion 1:3 (slice4; segments: 1) - -> Aggregate - -> Gather Motion 3:1 (slice3; segments: 3) - -> Aggregate - -> Hash Join - Hash Cond: (t2_1.b = t1.b) - -> Redistribute Motion 3:3 (slice1; segments: 3) - Hash Key: t2_1.b - -> Seq Scan on t2 t2_1 - -> Hash - -> Redistribute Motion 3:3 (slice2; segments: 3) - Hash Key: t1.b - -> Seq Scan on t1 - Optimizer: Pivotal Optimizer (GPORCA) -(20 rows) - -WITH cte AS ( - SELECT count(*) AS c FROM t1, t2 WHERE t1.b = t2.b -) DELETE FROM t2 WHERE a = (SELECT * FROM cte) -RETURNING *; - a | b ----+---- - 4 | 12 -(1 row) - -DROP TABLE t1; -DROP TABLE t2; -- Explicit Redistribute Motion should not be mistakenly elided for inherited -- tables. (test case not applicable to ORCA) CREATE TABLE i (i int, j int) DISTRIBUTED BY (i); diff --git a/src/test/regress/sql/update_gp.sql b/src/test/regress/sql/update_gp.sql index 79d3568f2f5d..8074176900e2 100644 --- a/src/test/regress/sql/update_gp.sql +++ b/src/test/regress/sql/update_gp.sql @@ -170,32 +170,6 @@ DROP TABLE t2; DROP TABLE t_strewn; DROP TABLE t_strewn2; --- Explicit Redistribute Motion should not be added if there are motions in Init --- Plans but not in the main plan. (test case not applicable to ORCA) -CREATE TABLE t1 (a int, b int) DISTRIBUTED RANDOMLY; -CREATE TABLE t2 (a int, b int) DISTRIBUTED RANDOMLY; - -INSERT INTO t1 SELECT - generate_series(1, 16) * 3, generate_series(1, 4); -INSERT INTO t2 SELECT - generate_series(1, 32), generate_series(1, 32) * 3; - -ANALYZE t1; -ANALYZE t2; - -EXPLAIN (costs off) -WITH cte AS ( - SELECT count(*) AS c FROM t1, t2 WHERE t1.b = t2.b -) DELETE FROM t2 WHERE a = (SELECT * FROM cte); - -WITH cte AS ( - SELECT count(*) AS c FROM t1, t2 WHERE t1.b = t2.b -) DELETE FROM t2 WHERE a = (SELECT * FROM cte) -RETURNING *; - -DROP TABLE t1; -DROP TABLE t2; - -- Explicit Redistribute Motion should not be mistakenly elided for inherited -- tables. (test case not applicable to ORCA) -- start_ignore From b19810b2770b2885809fbc4b841e29d4581a7a05 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Mon, 18 Mar 2024 16:05:50 +0300 Subject: [PATCH 82/83] Update a comment --- src/backend/cdb/cdbmutate.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index 8194d4f754f5..15d81fecae68 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -85,7 +85,8 @@ typedef struct ApplyMotionState * be modified. Only valid if mtIsChecking * is true. */ int nMotionsAbove; /* Number of motions above the current - * node */ + * node. Only valid if mtIsChecking is + * true. */ } ApplyMotionState; typedef struct InitPlanItem From d00b0e3598d4283d0a04d03d5bea01232053b4d0 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Tue, 19 Mar 2024 15:25:19 +0300 Subject: [PATCH 83/83] Revert "Get rid of extra logic and tests related to InitPlans" Skip InitPlans, but don't restore a useless test. This reverts commit bb559fd236bda017152005fa8eebfc0b58751695. --- src/backend/cdb/cdbmutate.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index 15d81fecae68..db1e9d7ab0f8 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -767,9 +767,16 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) /* An expression node might have subtrees containing plans to be mutated. */ if (!is_plan_node(node)) { + /* + * If we expect to elide the Explicit Redistribute Motion, we can + * disable mtIsChecking while we're in an InitPlan, since there will not + * be any scan nodes that perform a scan on the same range table entry + * as ModifyTable under InitPlans. + */ if (IsA(node, SubPlan) &&((SubPlan *) node)->is_initplan) { bool found; + bool saveMtIsChecking = context->mtIsChecking; int saveSliceDepth = context->sliceDepth; SubPlan *subplan = (SubPlan *) node; /* @@ -784,8 +791,10 @@ apply_motion_mutator(Node *node, ApplyMotionState *context) /* reset sliceDepth for each init plan */ context->sliceDepth = 0; + context->mtIsChecking = false; node = plan_tree_mutator(node, apply_motion_mutator, context); + context->mtIsChecking = saveMtIsChecking; context->sliceDepth = saveSliceDepth; return node;