From adc26df693b95b581141d0df4528e6a53255d72c Mon Sep 17 00:00:00 2001 From: ztword <35316898+ztword@users.noreply.github.com> Date: Thu, 7 Mar 2024 17:15:46 +0800 Subject: [PATCH 1/4] =?UTF-8?q?[=E5=88=9D=E7=BA=A7][=E7=BF=BB=E8=AF=91]V2.?= =?UTF-8?q?3.0=20=E5=8A=9F=E8=83=BD=20opentenbase=5Fsubscription=E6=96=87?= =?UTF-8?q?=E6=A1=A3=20#42?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [初级][翻译]V2.3.0 功能 opentenbase_subscription文档 #42 --- docs/guide/12-opentenbase_subscription.en.md | 166 +++++++++++++++++++ 1 file changed, 166 insertions(+) create mode 100644 docs/guide/12-opentenbase_subscription.en.md diff --git a/docs/guide/12-opentenbase_subscription.en.md b/docs/guide/12-opentenbase_subscription.en.md new file mode 100644 index 0000000..87cf890 --- /dev/null +++ b/docs/guide/12-opentenbase_subscription.en.md @@ -0,0 +1,166 @@ +# V2.3.0 Upgrade Feature opentenbase_subscription Usage Notes +## 1. Modify wal_level +Because multi-live is realized by logical replication, you need to configure the wal_level of all CN/DN nodes in all clusters of multi-live to be set to logical. +```shell script + vim postgres.comf + wal_level = logical + :wq + ``` +## 2. Install the opentenbase_subscription tool: +Connect the CNs of each of the 2 clusters configured for multi-activity (for any cluster, just choose any CN to execute, but both clusters need to be installed) to execute it: +```shell script +CREATE EXTENSION opentenbase_subscription; +``` +## 3. One-way synchronization configuration +Here we take 1CN+2DN on the publishing side and 1CN+2DN on the subscribing side as an example, and the configuration steps are similar for other cluster sizes. +### 3.1 Connect to any CN on the publish side to build a table. +```shell script +postgres=# create table users(id int,name varchar(30),f3 int,PRIMARY KEY(id)); +postgres=# insert into users select i, 'a', i+1 from generate_series(1, 399999) i; +INSERT 0 399999 +``` +### 3.2 Publishing Data on a Publisher DN +Note: If there is a publisher cluster with multiple DNs, each DN needs to be configured for publishing +Publisher DN1. +```shell script +CREATE PUBLICATION mypub_dn001 FOR ALL TABLES; +``` +Publisher DN2. +```shell script +CREATE PUBLICATION mypub_dn002 FOR ALL TABLES; +``` +### 3.3 Data subscription on the subscription side CN +Subscription end CN: +```shell script +create table users(id int,name varchar(30),f3 int,PRIMARY KEY(id)) ; + +CREATE OPENTENBASE SUBSCRIPTION sub_dn001 CONNECTION 'host=100.105.39.157 port=20008 user=jennyer dbname=postgres' PUBLICATION mypub_dn001 WITH (connect=true, enabled=true, create_slot=true, copy_data=true, synchronous_commit=on, ignore_pk_conflict = true, parallel_number=4); + +CREATE OPENTENBASE SUBSCRIPTION sub_dn002 CONNECTION 'host=100.105.39.157 port=20009 user=jennyer dbname=postgres' PUBLICATION mypub_dn002 WITH (connect=true, enabled=true, create_slot=true, copy_data=true, synchronous_commit=on, ignore_pk_conflict = true, parallel_number=4); +``` +Query validation of stock data synchronized from the publishing side + +CN on the subscription side: +```shell script +select count(*) from users; + count +-------- + 399999 +(1 row) + +``` +### 3.4 Verifying incremental data synchronization +Connect the publisher CN to insert incremental data: +```shell script +insert into users select i, 'a', i+1 from generate_series(400000, 500000) i; +INSERT 0 100001 +``` +Query to validate incremental data synchronized from the publish side +CN on the subscription side: +```shell script +select count(*) from users; + count +-------- + 500000 +(1 row) +``` +## 4. Bidirectional synchronization configuration +### 4.1 Dual-Active Cluster Individual Table Building +Cluster 1-CN: +```shell script +create table test(id int,name varchar(30),PRIMARY KEY(id)); +``` +Cluster 2-CN: +```shell script +create table test(id int,name varchar(30),PRIMARY KEY(id)); +``` +### 4.2 Cluster 1 is the most publishing end, with data publishing on all DNs +Cluster 1-DN1. +```shell script +CREATE PUBLICATION cluster1_dn001 FOR ALL TABLES; +``` +Cluster 1-DN2. +```shell script +CREATE PUBLICATION cluster1_dn002 FOR ALL TABLES; +``` +### 4.3 Cluster 2 is the most publishing end, with data publishing on all DNs +Cluster 2-DN1. +```shell script +CREATE PUBLICATION cluster2_dn001 FOR ALL TABLES; +``` +Cluster 2-DN2. +```shell script +CREATE PUBLICATION cluster2_dn002 FOR ALL TABLES; +``` +### 4.4 Cluster 1 is the most subscribed side, subscribing to data on any CN +Cluster 1-CN: +```shell script +CREATE OPENTENBASE SUBSCRIPTION sub_cluster2_dn001 CONNECTION 'host=100.105.50.198 port=20008 user=jennyer dbname=postgres' PUBLICATION cluster2_dn001 WITH (connect=true, enabled=true, create_slot=true, copy_data=true, synchronous_commit=on, ignore_pk_conflict = true, parallel_number=4); +CREATE OPENTENBASE SUBSCRIPTION sub_cluster2_dn002 CONNECTION 'host=100.105.50.198 port=20009 user=jennyer dbname=postgres' PUBLICATION cluster2_dn002 WITH (connect=true, enabled=true, create_slot=true, copy_data=true, synchronous_commit=on, ignore_pk_conflict = true, parallel_number=4); +``` +### 4.5 Cluster 2 is the most subscribed side, with data subscription on any CN +Cluster 2-CN: +```shell script +CREATE OPENTENBASE SUBSCRIPTION sub_cluster1_dn001 CONNECTION 'host=100.105.39.157 port=20008 user=jennyer dbname=postgres' PUBLICATION cluster1_dn001 WITH (connect=true, enabled=true, create_slot=true, copy_data=true, synchronous_commit=on, ignore_pk_conflict = true, parallel_number=4); +CREATE OPENTENBASE SUBSCRIPTION sub_cluster1_dn002 CONNECTION 'host=100.105.39.157 port=20009 user=jennyer dbname=postgres' PUBLICATION cluster1_dn002 WITH (connect=true, enabled=true, create_slot=true, copy_data=true, synchronous_commit=on, ignore_pk_conflict = true, parallel_number=4); +``` +### 4.6 Write partial data performed in cluster 1 +Cluster 1-CN: +```shell script +insert into test values(1,'a'),(3,'a'),(5,'a'),(7,'a'),(9,'a'),(11,'a'),(13,'a'),(15,'a'); +``` +### 4.7 Write partial data performed in cluster 2 +Cluster 2-CN: +```shell script +insert into test values(2,'b'),(4,'b'),(6,'b'),(8,'b'),(10,'b'),(12,'b'),(14,'b'),(16,'b'); +``` +### 4.8 Results of validating cluster 1 synchronization (data source local writes and remote synchronization) +Cluster 1-CN: +```shell script +select * from test; +postgres=# select * from test order by 1; + id | name +----+------ + 1 | a + 2 | b + 3 | a + 4 | b + 5 | a + 6 | b + 7 | a + 8 | b + 9 | a + 10 | b + 11 | a + 12 | b + 13 | a + 14 | b + 15 | a + 16 | b +(16 rows) +``` +### 4.9 Validate full volume results of cluster 2 synchronization (data sources local writes and remote synchronization) +Cluster 2-CN: +```shell script +postgres=# select * from test order by 1; + id | name +----+------ + 1 | a + 2 | b + 3 | a + 4 | b + 5 | a + 6 | b + 7 | a + 8 | b + 9 | a + 10 | b + 11 | a + 12 | b + 13 | a + 14 | b + 15 | a + 16 | b +(16 rows) +``` +Writes can be done on either cluster 1 or 2, and eventually both clusters 1 and 2 have the full amount of data. \ No newline at end of file From e3a0c6096b75eb66d7d0c605029b802cabc08049 Mon Sep 17 00:00:00 2001 From: ztword <35316898+ztword@users.noreply.github.com> Date: Fri, 8 Mar 2024 10:48:08 +0800 Subject: [PATCH 2/4] Create 10-pg_clean_en.md [Beginner][Translate] V2.3.0 Feature pg_clean Document #40 --- docs/guide/docs/guide/10-pg_clean_en.md | 62 +++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 docs/guide/docs/guide/10-pg_clean_en.md diff --git a/docs/guide/docs/guide/10-pg_clean_en.md b/docs/guide/docs/guide/10-pg_clean_en.md new file mode 100644 index 0000000..4617f12 --- /dev/null +++ b/docs/guide/docs/guide/10-pg_clean_en.md @@ -0,0 +1,62 @@ + +# V2.3.0 Upgrade Features pg_clean Instructions for use +## 1. Install the pg_clean utility: +Connect to any main CN: +``` +CREATE EXTENSION pg_clean; +``` +## 2. Construct two-phase transaction residual use cases: +session1—cn1 +``` +begin; +create table a (id int); +prepare transaction 'create_a'; +set xc_maintenance_mode = on; +execute direct on (dn002) 'rollback prepared ''create_a'''; +set xc_maintenance_mode = off; +\q +``` +In this use case we create a two-phase transaction with a gid of 'create_a' and then turn xc_maintenance_mode on so that the transaction only performs a rollback in dn002. +## 3. Find two-phase transaction residuals in the cluster: +session2—cn2 +``` +select * from pg_clean_check_txn(); +``` +![image](https://github.com/ztword/docs/assets/35316898/19104658-1850-4ac5-a1ca-963a625b5fac) + +The above figure prints out the two-phase transaction remaining in the cluster. Where gid denotes the global identifier of the transaction, global_transaction_status denotes the status of the transaction globally, and transaction_status_on_allnodes denotes the status of the transaction in all nodes. + +## 4. View 2PC residual file names +Query all 2PC residual file names under the data directory pg_2pc in node cn1 + +session1-cn1 session 1 - cn1 +``` + postgres=# select * from pgxc_get_record_list(); + pgxc_get_record_list + ---------------------- + create_a + (1 row) +``` +## 5. Clean up the residual two-phase transactions in the cluster: +session2-cn2 Session 2 - CN2 +``` +select * from pg_clean_execute(); +``` +![image](https://github.com/ztword/docs/assets/35316898/3a4f5971-6727-4077-ad1e-24e4d7d8b2f1) + +The above figure prints all the residual two-phase transactions, and the operations performed on them. Where operation indicates the operation performed on the transaction at each node and operation_status indicates whether the operation was performed successfully or not. Since the global transaction status of the transaction is ABORT, we go to each node with a status of prepare and perform a rollback operation on the transaction. + +## 6. Check for residual documentation of cleaned two-phase transactions: +session1-cn1 Session 1 - CN1 +``` + postgres=# select * from pgxc_get_record_list(); + pgxc_get_record_list + ---------------------- + + (1 row) +``` +Since in the previous step pg_clean_execute was executed successfully, the file records of the transaction 'create_a' that has been rolled back in all the nodes have been deleted, here we are looking at all the 2pc file records in cn1 and it shows that it is empty and the result is correct. + + + + From 5c17874310bfb5eb23db4b5e6c77f2105fe58bfa Mon Sep 17 00:00:00 2001 From: ztword <35316898+ztword@users.noreply.github.com> Date: Fri, 8 Mar 2024 10:49:55 +0800 Subject: [PATCH 3/4] Create 10-pg_clean_en.md [Beginner][Translate] V2.3.0 Feature pg_clean Document #40 --- docs/guide/10-pg_clean_en.md | 58 ++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 docs/guide/10-pg_clean_en.md diff --git a/docs/guide/10-pg_clean_en.md b/docs/guide/10-pg_clean_en.md new file mode 100644 index 0000000..ba14cca --- /dev/null +++ b/docs/guide/10-pg_clean_en.md @@ -0,0 +1,58 @@ + +# V2.3.0 Upgrade Features pg_clean Instructions for use +## 1. Install the pg_clean utility: +Connect to any main CN: +``` +CREATE EXTENSION pg_clean; +``` +## 2. Construct two-phase transaction residual use cases: +session1—cn1 +``` +begin; +create table a (id int); +prepare transaction 'create_a'; +set xc_maintenance_mode = on; +execute direct on (dn002) 'rollback prepared ''create_a'''; +set xc_maintenance_mode = off; +\q +``` +In this use case we create a two-phase transaction with a gid of 'create_a' and then turn xc_maintenance_mode on so that the transaction only performs a rollback in dn002. +## 3. Find two-phase transaction residuals in the cluster: +session2—cn2 +``` +select * from pg_clean_check_txn(); +``` +![image](https://github.com/ztword/docs/assets/35316898/19104658-1850-4ac5-a1ca-963a625b5fac) + +The above figure prints out the two-phase transaction remaining in the cluster. Where gid denotes the global identifier of the transaction, global_transaction_status denotes the status of the transaction globally, and transaction_status_on_allnodes denotes the status of the transaction in all nodes. + +## 4. View 2PC residual file names +Query all 2PC residual file names under the data directory pg_2pc in node cn1 + +session1-cn1 session 1 - cn1 +``` + postgres=# select * from pgxc_get_record_list(); + pgxc_get_record_list + ---------------------- + create_a + (1 row) +``` +## 5. Clean up the residual two-phase transactions in the cluster: +session2-cn2 Session 2 - CN2 +``` +select * from pg_clean_execute(); +``` +![image](https://github.com/ztword/docs/assets/35316898/3a4f5971-6727-4077-ad1e-24e4d7d8b2f1) + +The above figure prints all the residual two-phase transactions, and the operations performed on them. Where operation indicates the operation performed on the transaction at each node and operation_status indicates whether the operation was performed successfully or not. Since the global transaction status of the transaction is ABORT, we go to each node with a status of prepare and perform a rollback operation on the transaction. + +## 6. Check for residual documentation of cleaned two-phase transactions: +session1-cn1 Session 1 - CN1 +``` + postgres=# select * from pgxc_get_record_list(); + pgxc_get_record_list + ---------------------- + + (1 row) +``` +Since in the previous step pg_clean_execute was executed successfully, the file records of the transaction 'create_a' that has been rolled back in all the nodes have been deleted, here we are looking at all the 2pc file records in cn1 and it shows that it is empty and the result is correct. From e895798999590c42069323de3666d5846958730a Mon Sep 17 00:00:00 2001 From: ztword <35316898+ztword@users.noreply.github.com> Date: Fri, 8 Mar 2024 11:13:23 +0800 Subject: [PATCH 4/4] Delete docs/guide/docs/guide directory delete --- docs/guide/docs/guide/10-pg_clean_en.md | 62 ------------------------- 1 file changed, 62 deletions(-) delete mode 100644 docs/guide/docs/guide/10-pg_clean_en.md diff --git a/docs/guide/docs/guide/10-pg_clean_en.md b/docs/guide/docs/guide/10-pg_clean_en.md deleted file mode 100644 index 4617f12..0000000 --- a/docs/guide/docs/guide/10-pg_clean_en.md +++ /dev/null @@ -1,62 +0,0 @@ - -# V2.3.0 Upgrade Features pg_clean Instructions for use -## 1. Install the pg_clean utility: -Connect to any main CN: -``` -CREATE EXTENSION pg_clean; -``` -## 2. Construct two-phase transaction residual use cases: -session1—cn1 -``` -begin; -create table a (id int); -prepare transaction 'create_a'; -set xc_maintenance_mode = on; -execute direct on (dn002) 'rollback prepared ''create_a'''; -set xc_maintenance_mode = off; -\q -``` -In this use case we create a two-phase transaction with a gid of 'create_a' and then turn xc_maintenance_mode on so that the transaction only performs a rollback in dn002. -## 3. Find two-phase transaction residuals in the cluster: -session2—cn2 -``` -select * from pg_clean_check_txn(); -``` -![image](https://github.com/ztword/docs/assets/35316898/19104658-1850-4ac5-a1ca-963a625b5fac) - -The above figure prints out the two-phase transaction remaining in the cluster. Where gid denotes the global identifier of the transaction, global_transaction_status denotes the status of the transaction globally, and transaction_status_on_allnodes denotes the status of the transaction in all nodes. - -## 4. View 2PC residual file names -Query all 2PC residual file names under the data directory pg_2pc in node cn1 - -session1-cn1 session 1 - cn1 -``` - postgres=# select * from pgxc_get_record_list(); - pgxc_get_record_list - ---------------------- - create_a - (1 row) -``` -## 5. Clean up the residual two-phase transactions in the cluster: -session2-cn2 Session 2 - CN2 -``` -select * from pg_clean_execute(); -``` -![image](https://github.com/ztword/docs/assets/35316898/3a4f5971-6727-4077-ad1e-24e4d7d8b2f1) - -The above figure prints all the residual two-phase transactions, and the operations performed on them. Where operation indicates the operation performed on the transaction at each node and operation_status indicates whether the operation was performed successfully or not. Since the global transaction status of the transaction is ABORT, we go to each node with a status of prepare and perform a rollback operation on the transaction. - -## 6. Check for residual documentation of cleaned two-phase transactions: -session1-cn1 Session 1 - CN1 -``` - postgres=# select * from pgxc_get_record_list(); - pgxc_get_record_list - ---------------------- - - (1 row) -``` -Since in the previous step pg_clean_execute was executed successfully, the file records of the transaction 'create_a' that has been rolled back in all the nodes have been deleted, here we are looking at all the 2pc file records in cn1 and it shows that it is empty and the result is correct. - - - -