diff --git a/content/chinese/blog/quickstart.md b/content/chinese/blog/01-quickstart.md similarity index 96% rename from content/chinese/blog/quickstart.md rename to content/chinese/blog/01-quickstart.md index 82b4591..bed41db 100644 --- a/content/chinese/blog/quickstart.md +++ b/content/chinese/blog/01-quickstart.md @@ -1,7 +1,7 @@ --- title: "快速入门" date: 2023-07-20T08:00:36+08:00 -author: Bart Dong +author: OpenTenBase image_webp: images/blog/blog-post-1.webp image: images/blog/blog-post-1.jpg description : "The Quick Start of OpenTenBase" @@ -18,6 +18,7 @@ OpenTenBase 采用分布式集群架构（如下图）， 该架构分布式为 OpenTenBase架构图 + 下面简单解读一下OpenTenBase的三大模块 - **Coordinator：协调节点（简称CN）** @@ -44,15 +45,15 @@ OS: TencentOS 2, TencentOS 3, OpenCloudOS, CentOS 7, CentOS 8, Ubuntu ### 安装依赖 -` yum -y install gcc make readline-devel zlib-devel openssl-devel uuid-devel bison flex` +` yum -y install gcc make readline-devel zlib-devel openssl-devel uuid-devel bison flex git` 或 -` apt install -y gcc make libreadline-dev zlib1g-dev libssl-dev libossp-uuid-dev bison flex` +` apt install -y gcc make libreadline-dev zlib1g-dev libssl-dev libossp-uuid-dev bison flex git` - **创建opentenbase用户** - 注意：所有需要安装OpenTenBase集群的机器上都需要创建 + 注意：所有需要安装OpenTenBase集群的机器上都需要创建 ``` shell mkdir /data @@ -84,7 +85,7 @@ make install 本文的使用环境中，上述两个参数如下 -```shell +``` ${SOURCECODE_PATH}=/data/opentenbase/OpenTenBase ${INSTALL_PATH}=/data/opentenbase/install ``` @@ -159,10 +160,21 @@ export LC_ALL=C [opentenbase@localhost ~/pgxc_ctl]$ vim pgxc_ctl.conf ``` -如下，是结合上文描述的IP，端口，数据库目录，二进制目录等规划来写的pgxc_ctl.conf文件。具体实践中只需按照自己的实际情况配置好即可. +如下，是结合上文描述的IP，端口，数据库目录，二进制目录等规划来写的pgxc_ctl.conf文件。具体实践中只需按照自己的实际情况配置好即可。 + +亦可从此处下载，修改文件名为 ```pgxc_ctl.conf``` ，按照实际情况修改内容： + +[点击此处下载双节点配置](https://docs.opentenbase.org/guide/pgxc_ctl_double.conf) -``` yaml +[点击此处下载单节点配置](https://docs.opentenbase.org/guide/pgxc_ctl_single.conf) + + +``` shell #!/bin/bash +# Double Node Config + +IP_1=10.215.147.158 +IP_2=10.240.138.159 pgxcInstallDir=/data/opentenbase/install/opentenbase_bin_v2.0 pgxcOwner=opentenbase @@ -178,13 +190,13 @@ configBackupFile=pgxc_ctl.bak #---- GTM ---------- gtmName=gtm -gtmMasterServer=10.215.147.158 +gtmMasterServer=$IP_1 gtmMasterPort=50001 gtmMasterDir=/data/opentenbase/data/gtm gtmExtraConfig=none gtmMasterSpecificExtraConfig=none gtmSlave=y -gtmSlaveServer=10.240.138.159 +gtmSlaveServer=$IP_2 gtmSlavePort=50001 gtmSlaveDir=/data/opentenbase/data/gtm gtmSlaveSpecificExtraConfig=none @@ -197,7 +209,7 @@ coordNames=(cn001 cn002 ) coordPorts=(30004 30004 ) poolerPorts=(31110 31110 ) coordPgHbaEntries=(0.0.0.0/0) -coordMasterServers=(10.215.147.158 10.240.138.159) +coordMasterServers=($IP_1 $IP_2) coordMasterDirs=($coordMasterDir $coordMasterDir) coordMaxWALsernder=2 coordMaxWALSenders=($coordMaxWALsernder $coordMaxWALsernder ) @@ -252,7 +264,7 @@ EOF coordSpecificExtraPgHba=(none none) -coordAdditionalSlaves=n +coordAdditionalSlaves=n cad1_Sync=n #---- Datanodes --------------------- @@ -268,13 +280,13 @@ datanodeNames=(dn001 dn002) datanodePorts=(40004 40004) datanodePoolerPorts=(41110 41110) datanodePgHbaEntries=(0.0.0.0/0) -datanodeMasterServers=(10.215.147.158 10.240.138.159) +datanodeMasterServers=($IP_1 $IP_2) datanodeMasterDirs=($dn1MstrDir $dn2MstrDir) dnWALSndr=4 datanodeMaxWALSenders=($dnWALSndr $dnWALSndr) datanodeSlave=y -datanodeSlaveServers=(10.240.138.159 10.215.147.158) +datanodeSlaveServers=($IP_2 $IP_1) datanodeSlavePorts=(50004 54004) datanodeSlavePoolerPorts=(51110 51110) datanodeSlaveSync=n diff --git a/content/chinese/blog/02-access.md b/content/chinese/blog/02-access.md new file mode 100644 index 0000000..8804b55 --- /dev/null +++ b/content/chinese/blog/02-access.md @@ -0,0 +1,1480 @@ +--- +title: "应用接入指南" +date: 2023-09-25T08:00:36+08:00 +author: OpenTenBase +image_webp: images/blog/blog-post-1.webp +image: images/blog/blog-post-1.jpg +description : "How to access OpenTenBase" +--- + + +>在[快速入门](01-quickstart.md)文章中我们介绍了 OpenTenBase 的架构、源码编译安装、集群运行状态、启动停止等内容。 +> +>本篇将介绍应用程序如何连接OpenTenBase数据库进行建库、建表、数据导入、查询等操作。 + +OpenTenBase兼容所有支持Postgres协议的客户端连接，这里将详细介绍JAVA、C语音、shell语言、Python、PHP、Golang 这6种最常用的开发语言连接OpenTenBase的操作方法。 + +## 1、JAVA开发 +### 1.1、创建数据表 + +``` +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; + + +public class createtable { + public static void main( String args[] ) + { + Connection c = null; + Statement stmt = null; + try { + Class.forName("org.postgresql.Driver"); + c = DriverManager.getConnection("jdbc:postgresql://127.0.0.1:15432/postgres?currentSchema=public&binaryTransfer=false","opentenbase", "opentenbase"); + System.out.println("Opened database successfully"); + stmt = c.createStatement(); + String sql = "create table opentenbase(id int,nickname text) distribute by shard(id) to group default_group" ; + stmt.executeUpdate(sql); + stmt.close(); + c.close(); + } catch ( Exception e ) { + System.err.println( e.getClass().getName()+": "+ e.getMessage() ); + System.exit(0); + } + System.out.println("Table created successfully"); + } +} +``` + +说明： + +* 这里连接的节点为任意CN主节点，后面所有操作，没特别说明，都是连接到CN主节点进行操作。 + + + +### 1.2、使用普通协议插入数据 +``` +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; + +public class insert { + public static void main(String args[]) { + Connection c = null; + Statement stmt = null; + try { + Class.forName("org.postgresql.Driver"); + c = DriverManager.getConnection("jdbc:postgresql://127.0.0.1:15432/postgres?currentSchema=public&binaryTransfer=false","opentenbase", "opentenbase"); + c.setAutoCommit(false); + System.out.println("Opened database successfully"); + + stmt = c.createStatement(); + String sql = "INSERT INTO opentenbase (id,nickname) " + + "VALUES (1,'opentenbase');"; + stmt.executeUpdate(sql); + + sql = "INSERT INTO opentenbase (id,nickname) " + + "VALUES (2, 'pgxz' ),(3,'pgxc');"; + stmt.executeUpdate(sql); + stmt.close(); + c.commit(); + c.close(); + } catch (Exception e) { + System.err.println( e.getClass().getName()+": "+ e.getMessage() ); + System.exit(0); + } + System.out.println("Records created successfully"); + } +} +``` +### 1.3、使用扩展协议插入数据 +``` +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.*; +import java.util.Random; + +public class insert_prepared { + public static void main(String args[]) { + Connection c = null; + PreparedStatement stmt; + try { + Class.forName("org.postgresql.Driver"); + c = DriverManager.getConnection("jdbc:postgresql://127.0.0.1:15432/postgres?currentSchema=public&binaryTransfer=false","opentenbase", "opentenbase"); + c.setAutoCommit(false); + System.out.println("Opened database successfully"); + //插入数据 + String sql = "INSERT INTO opentenbase (id,nickname) VALUES (?,?)"; + stmt = c.prepareStatement(sql); + stmt.setInt(1, 9999); + stmt.setString(2, "opentenbase_prepared"); + stmt.executeUpdate(); + + //插入更新 + sql = "INSERT INTO opentenbase (id,nickname) VALUES (?,?) ON CONFLICT(id) DO UPDATE SET nickname=?"; + stmt = c.prepareStatement(sql); + stmt.setInt(1, 9999); + stmt.setString(2, "opentenbase_prepared"); + stmt.setString(3, "opentenbase_prepared_update"); + stmt.executeUpdate(); + + stmt.close(); + c.commit(); + c.close(); + } catch (Exception e) { + System.err.println( e.getClass().getName()+": "+ e.getMessage() ); + System.exit(0); + } + System.out.println("Records created successfully"); + } +} +``` +### 1.4、copy from 加载文件到表 +``` +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; +import org.postgresql.copy.CopyManager; +import org.postgresql.core.BaseConnection; +import java.io.*; + +public class copyfrom { + public static void main( String args[] ) + { + Connection c = null; + Statement stmt = null; + FileInputStream fs = null; + try { + Class.forName("org.postgresql.Driver"); + c = DriverManager.getConnection("jdbc:postgresql://127.0.0.1:15432/postgres?currentSchema=public&binaryTransfer=false","opentenbase", "opentenbase"); + System.out.println("Opened database successfully"); + CopyManager cm = new CopyManager((BaseConnection) c); + fs = new FileInputStream("/data/opentenbase/opentenbase.csv"); + String sql = "COPY opentenbase FROM STDIN DELIMITER AS ','"; + cm.copyIn(sql, fs); + c.close(); + fs.close(); + } catch ( Exception e ) { + System.err.println( e.getClass().getName()+": "+ e.getMessage() ); + System.exit(0); + } + System.out.println("Copy data successfully"); + } +} +``` +### 1.5、copy to 导出数据到文件 +``` +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; +import org.postgresql.copy.CopyManager; +import org.postgresql.core.BaseConnection; +import java.io.*; + +public class copyto { + public static void main( String args[] ) + { + Connection c = null; + Statement stmt = null; + FileOutputStream fs = null; + try { + Class.forName("org.postgresql.Driver"); + c = DriverManager.getConnection("jdbc:postgresql://127.0.0.1:15432/postgres?currentSchema=public&binaryTransfer=false","opentenbase", "opentenbase"); + System.out.println("Opened database successfully"); + CopyManager cm = new CopyManager((BaseConnection) c); + fs = new FileOutputStream("/data/opentenbase/opentenbase.csv"); + String sql = "COPY opentenbase TO STDOUT DELIMITER AS ','"; + cm.copyOut(sql, fs); + c.close(); + fs.close(); + } catch ( Exception e ) { + System.err.println( e.getClass().getName()+": "+ e.getMessage() ); + System.exit(0); + } + System.out.println("Copy data successfully"); + } +} +``` +### 1.6、jdbc包下载地址 +``` +https://jdbc.postgresql.org/download.html +``` + +## 2、C程序开发 +### 2.1、连接数据库 +``` +#include +#include +#include "libpq-fe.h" +int +main(int argc, char **argv){ + const char *conninfo; + PGconn *conn; + if (argc > 1){ + conninfo = argv[1]; + }else{ + conninfo = "dbname = postgres"; + } + conn = PQconnectdb(conninfo); + if (PQstatus(conn) != CONNECTION_OK){ + fprintf(stderr, "连接数据库失败: %s",PQerrorMessage(conn)); + }else{ + printf("连接数据库成功！\n"); + } + PQfinish(conn); + return 0; +} +``` + +编译 + +``` +gcc -c -I /usr/local/install/opentenbase_pgxz/include/ conn.c +gcc -o conn conn.o -L /usr/local/install/opentenbase_pgxz/lib/ -lpq +``` + +运行 + +``` +./conn "host=172.16.0.3 dbname=postgres port=11000" +连接数据库成功！ +``` + +``` +./conn "host=172.16.0.3 dbname=postgres port=15432 user=opentenbase" +连接数据库成功！ +``` + +### 2.2、建立数据表 +``` +#include +#include +#include "libpq-fe.h" +int +main(int argc, char **argv){ + const char *conninfo; + PGconn *conn; + PGresult *res; + const char *sql = "create table opentenbase(id int,nickname text) distribute by shard(id) to group default_group"; + if (argc > 1){ + conninfo = argv[1]; + }else{ + conninfo = "dbname = postgres"; + } + conn = PQconnectdb(conninfo); + if (PQstatus(conn) != CONNECTION_OK){ + fprintf(stderr, "连接数据库失败: %s",PQerrorMessage(conn)); + }else{ + printf("连接数据库成功！\n"); + } + res = PQexec(conn,sql); + if(PQresultStatus(res) != PGRES_COMMAND_OK){ + fprintf(stderr, "建立数据表失败: %s",PQresultErrorMessage(res)); + }else{ + printf("建立数据表成功！\n"); + } + PQclear(res); + PQfinish(conn); + return 0; +} +``` +编译 + +``` +gcc -c -I /usr/local/install/opentenbase_pgxz/include/ createtable.c +gcc -o createtable createtable.o -L /usr/local/install/opentenbase_pgxz/lib/ -lpq +``` +运行 + +``` +./createtable "port=11000 dbname=postgres" +连接数据库成功！ +建立数据表成功！ +``` + + +### 2.3、插入数据 +``` +#include +#include +#include "libpq-fe.h" +int +main(int argc, char **argv){ + const char *conninfo; + PGconn *conn; + PGresult *res; + const char *sql = "INSERT INTO opentenbase (id,nickname) values(1,'opentenbase'),(2,'pgxz')"; + if (argc > 1){ + conninfo = argv[1]; + }else{ + conninfo = "dbname = postgres"; + } + conn = PQconnectdb(conninfo); + if (PQstatus(conn) != CONNECTION_OK){ + fprintf(stderr, "连接数据库失败: %s",PQerrorMessage(conn)); + }else{ + printf("连接数据库成功！\n"); + } + res = PQexec(conn,sql); + if(PQresultStatus(res) != PGRES_COMMAND_OK){ + fprintf(stderr, "插入数据失败: %s",PQresultErrorMessage(res)); + }else{ + printf("插入数据成功！\n"); + } + PQclear(res); + PQfinish(conn); + return 0; +} +``` +编译 + +``` +gcc -c -I /usr/local/install/opentenbase_pgxz/include/ insert.c +gcc -o insert insert.o -L /usr/local/install/opentenbase_pgxz/lib/ -lpq +``` +运行 + +``` +./insert "dbname=postgres port=15432" +``` + + +### 2.4、查询数据 +``` +#include +#include +#include "libpq-fe.h" +int +main(int argc, char **argv){ + const char *conninfo; + PGconn *conn; + PGresult *res; + const char *sql = "select * from opentenbase"; + if (argc > 1){ + conninfo = argv[1]; + }else{ + conninfo = "dbname = postgres"; + } + conn = PQconnectdb(conninfo); + if (PQstatus(conn) != CONNECTION_OK){ + fprintf(stderr, "连接数据库失败: %s",PQerrorMessage(conn)); + }else{ + printf("连接数据库成功！\n"); + } + res = PQexec(conn,sql); + if(PQresultStatus(res) != PGRES_TUPLES_OK){ + fprintf(stderr, "插入数据失败: %s",PQresultErrorMessage(res)); + }else{ + printf("查询数据成功！\n"); + int rownum = PQntuples(res) ; + int colnum = PQnfields(res); + for(int j = 0;j< colnum; ++j){ + printf("%s\t",PQfname(res,j)); + } + printf("\n"); + for(int i = 0;i< rownum; ++i){ + for(int j = 0;j< colnum; ++j){ + printf("%s\t",PQgetvalue(res,i,j)); + } + printf("\n"); + } + } + PQclear(res); + PQfinish(conn); + return 0; +} +``` + +编译 + +``` +gcc -std=c99 -c -I /usr/local/install/opentenbase_pgxz/include/ select.c +gcc -o select select.o -L /usr/local/install/opentenbase_pgxz/lib/ -lpq +``` +运行 + +``` +./select "dbname=postgres port=15432" +连接数据库成功！ +查询数据成功！ +id nickname +1 opentenbase +2 pgxz + +``` + +### 2.5、流数据COPY入表 +``` +#include +#include +#include +#include "libpq-fe.h" +int +main(int argc, char **argv){ + const char *conninfo; + PGconn *conn; + PGresult *res; + const char *buffer = "1,opentenbase\n2,pgxz\n3,opentenbase牛"; + if (argc > 1){ + conninfo = argv[1]; + }else{ + conninfo = "dbname = postgres"; + } + conn = PQconnectdb(conninfo); + if (PQstatus(conn) != CONNECTION_OK){ + fprintf(stderr, "连接数据库失败: %s",PQerrorMessage(conn)); + }else{ + printf("连接数据库成功！\n"); + } + res=PQexec(conn,"COPY opentenbase FROM STDIN DELIMITER ',';"); + if(PQresultStatus(res) != PGRES_COPY_IN){ + fprintf(stderr, "copy数据出错1: %s",PQresultErrorMessage(res)); + }else{ + int len = strlen(buffer); + if(PQputCopyData(conn,buffer,len) == 1){ + if(PQputCopyEnd(conn,NULL) == 1){ + res = PQgetResult(conn); + if(PQresultStatus(res) == PGRES_COMMAND_OK){ + printf("copy数据成功！\n"); + }else{ + fprintf(stderr, "copy数据出错2: %s",PQerrorMessage(conn)); + } + }else{ + fprintf(stderr, "copy数据出错3: %s",PQerrorMessage(conn)); + } + }else{ + fprintf(stderr, "copy数据出错4: %s",PQerrorMessage(conn)); + } + } + PQclear(res); + PQfinish(conn); + return 0; +} +``` + +编译 + +``` +gcc -c -I /usr/local/install/opentenbase_pgxz/include/ copy.c +gcc -o copy copy.o -L /usr/local/install/opentenbase_pgxz/lib/ -lpq +``` + +执行 + +``` +./copy "dbname=postgres port=15432" +连接数据库成功！ +copy数据成功！ +``` + + + +### 3、shell脚本开发 +``` +#!/bin/sh + +if [ $# -ne 0 ] +then + echo "usage: $0 exec_sql" + exit 1 +fi + +exec_sql=$1 + +masters=`psql -h 172.16.0.29 -d postgres -p 15432 -t -c "select string_agg(node_host, ' ') from (select * from pgxc_node where node_type = 'D' order by node_name) t"` +port_list=`psql -h 172.16.0.29 -d postgres -p 15432 -t -c "select string_agg(node_port::text, ' ') from (select * from pgxc_node where node_type = 'D' order by node_name) t"` +node_cnt=`psql -h 172.16.0.29 -d postgres -p 15432 -t -c "select count(*) from pgxc_node where node_type = 'D'"` +masters=($masters) +ports=($port_list) + +echo $node_cnt + +flag=0 + +for((i=0;i<$node_cnt;i++)); +do + seq=$(($i+1)) + master=${masters[$i]} + port=${ports[$i]} + echo $master + echo $port + + psql -h $master -p $port postgres -c "$exec_sql" +done +``` + +## 4、python程序开发 +### 4.1、安装psycopg2模块 +``` +[root@VM_0_29_centos ~]# yum install python-psycopg2 +``` + +### 4.2、连接数据库 +``` +#coding=utf-8 +#!/usr/bin/python +import psycopg2 +try: + conn = psycopg2.connect(database="postgres", user="opentenbase", password="", host="172.16.0.29", port="15432") + print "连接数据库成功" + conn.close() +except psycopg2.Error,msg: + print "连接数据库出错，错误详细信息： %s" %(msg.args[0]) +``` + +运行 + +``` +[opentenbase@VM_0_29_centos python]$ python conn.py +连接数据库成功 +``` + + +### 4.3、创建数据表 +``` +#coding=utf-8 +#!/usr/bin/python +import psycopg2 +try: + conn = psycopg2.connect(database="postgres", user="opentenbase", password="", host="172.16.0.29", port="15432") + print "连接数据库成功" + cur = conn.cursor() + sql = """ + create table opentenbase + ( + id int, + nickname varchar(100) + )distribute by shard(id) to group default_group + """ + cur.execute(sql) + conn.commit() + print "建立数据表成功" + conn.close() +except psycopg2.Error,msg: + print "OpenTenBase Error %s" %(msg.args[0]) +``` +运行 + +``` +[opentenbase@VM_0_29_centos python]$ python createtable.py +连接数据库成功 +建立数据表成功 +``` +### 4.4、插入数据 +``` +#coding=utf-8 +#!/usr/bin/python +import psycopg2 +try: + conn = psycopg2.connect(database="postgres", user="opentenbase", password="", host="172.16.0.29", port="15432") + print "连接数据库成功" + cur = conn.cursor() + sql = "insert into opentenbase values(1,'opentenbase'),(2,'opentenbase');" + cur.execute(sql) + sql = "insert into opentenbase values(%s,%s)" + cur.execute(sql,(3,'pg')) + conn.commit() + print "插入数据成功" + conn.close() +except psycopg2.Error,msg: + print "操作数据库出库 %s" %(msg.args[0]) +``` +运行 + +``` +[opentenbase@VM_0_29_centos python]$ python insert.py +连接数据库成功 +插入数据成功 +``` + +### 4.5、查询数据 +``` +#coding=utf-8 +#!/usr/bin/python +import psycopg2 +try: + conn = psycopg2.connect(database="postgres", user="opentenbase", password="", host="172.16.0.29", port="15432") + print "连接数据库成功" + cur = conn.cursor() + sql = "select * from opentenbase" + cur.execute(sql) + rows = cur.fetchall() + for row in rows: + print "ID = ", row[0] + print "NICKNAME = ", row[1],"\n" + conn.close() +except psycopg2.Error,msg: + print "操作数据库出库 %s" %(msg.args[0]) +``` + +运行 + +``` +[opentenbase@VM_0_29_centos python]$ python select.py +连接数据库成功 +ID = 1 +NICKNAME = opentenbase + +ID = 2 +NICKNAME = pgxz + +ID = 3 +NICKNAME = pg +``` + +### 4.6、copy from 加载文件到表 +``` +#coding=utf-8 +#!/usr/bin/python +import psycopg2 +try: + conn = psycopg2.connect(database="postgres", user="opentenbase", password="", host="172.16.0.29", port="15432") + print "连接数据库成功" + cur = conn.cursor() + filename = "/data/opentenbase/opentenbase.txt" + cols = ('id','nickname') + tablename="public.opentenbase" + cur.copy_from(file=open(filename),table=tablename,columns=cols,sep=',') + conn.commit() + print "导入数据成功" + conn.close() +except psycopg2.Error,msg: + print "操作数据库出库 %s" %(msg.args[0]) +``` + +执行 + +``` +[opentenbase@VM_0_29_centos python]$ python copy_from.py +连接数据库成功 +导入数据成功 +``` + +## 5、PHP程序开发 +### 5.1、连接数据库 +``` +"; ; + exit; +}else{ + echo "连接数据库成功"."\n
"; +} +//关闭连接 +pg_close($conn); +?> +``` + +执行 + +``` +[root@VM_0_47_centos test]# curl http://127.0.0.1:8080/dbsta/test/conn.php +连接数据库成功 +``` + +### 5.2、创建数据表 +``` + +``` + +执行 + +``` +[root@VM_0_47_centos test]# curl http://127.0.0.1:8080/dbsta/test/createtable.php +连接数据库成功 +创建数据表成功 +``` + +### 5.3、插入数据 +``` + +``` + +执行 + +``` +[opentenbase@VM_0_47_centos test]$ curl http://127.0.0.1:8080/dbsta/test/insert.php +连接数据库成功 +插入数据成功 +``` + +### 5.4、查询数据 +``` + +``` +调用方法 + +``` +[root@VM_0_47_centos ~]# curl http://127.0.0.1:8080/dbsta/test/select.php +连接数据库成功 +插入数据成功 +返回记录数2 +记录数#1 +id：1 +nickname：opentenbase + +记录数#2 +id：2 +nickname：pgxz +``` + +### 5.5、流数据copy 入表 +``` + +``` + +调用方法 + +``` +curl http://127.0.0.1/dbsta/cron/php_copy_from.php +连接数据库成功 +copy成功 +``` + +### 5.6、copy to导出数据到一个数组中 +``` + +``` + +调用方法 + +``` +curl http://127.0.0.1/dbsta/cron/php_copy_to.php +连接数据库成功 +Array +( + [0] => 1,opentenbase + + [1] => 2,pgxz + +) +``` + +## 6、golang程序开发 +### 6.1、连接数据库 +``` +package main + +import ( + "fmt" + "time" + + "github.com/jackc/pgx" +) + +func main() { + var error_msg string + + //连接数据库 + conn, err := db_connect() + if err != nil { + error_msg = "连接数据库失败，详情：" + err.Error() + write_log("Error", error_msg) + return + } + //程序运行结束时关闭连接 + defer conn.Close() + write_log("Log", "连接数据库成功") + +} + +/* +功能描述：写入日志处理 + +参数说明： +log_level -- 日志级别，只能是是Error或Log +error_msg -- 日志内容 + +返回值说明：无 +*/ + +func write_log(log_level string, error_msg string) { + //打印错误信息 + fmt.Println("访问时间：", time.Now().Format("2006-01-02 15:04:05")) + fmt.Println("日志级别：", log_level) + fmt.Println("详细信息：", error_msg) +} + +/* +功能描述：连接数据库 + +参数说明：无 + +返回值说明： +conn *pgx.Conn -- 连接信息 +err error --错误信息 + +*/ + +func db_connect() (conn *pgx.Conn, err error) { + var config pgx.ConnConfig + config.Host = "127.0.0.1" //数据库主机host或ip + config.User = "opentenbase" //连接用户 + config.Password = "pgsql" //用户密码 + config.Database = "postgres" //连接数据库名 + config.Port = 15432 //端口号 + conn, err = pgx.Connect(config) + return conn, err +} +``` + +``` +[root@VM_0_29_centos opentenbase]# go run conn.go +访问时间： 2018-04-03 20:40:28 +日志级别： Log +详细信息： 连接数据库成功 +``` + +编译后运行 + +``` +[root@VM_0_29_centos opentenbase]# go build conn.go +[root@VM_0_29_centos opentenbase]# ./conn +访问时间： 2018-04-03 20:40:48 +日志级别： Log +详细信息： 连接数据库成功 +``` + +### 6.2、创建数据表 +``` +package main + +import ( + "fmt" + "time" + + "github.com/jackc/pgx" +) + +func main() { + var error_msg string + var sql string + + //连接数据库 + conn, err := db_connect() + if err != nil { + error_msg = "连接数据库失败，详情：" + err.Error() + write_log("Error", error_msg) + return + } + //程序运行结束时关闭连接 + defer conn.Close() + write_log("Log", "连接数据库成功") + + //建立数据表 + sql = "create table public.opentenbase(id varchar(20),nickname varchar(100)) distribute by shard(id) to group default_group;" + _, err = conn.Exec(sql) + if err != nil { + error_msg = "创建数据表失败,详情：" + err.Error() + write_log("Error", error_msg) + return + } else { + write_log("Log", "创建数据表成功") + } +} + +/* +功能描述：写入日志处理 + +参数说明： +log_level -- 日志级别，只能是是Error或Log +error_msg -- 日志内容 + +返回值说明：无 +*/ + +func write_log(log_level string, error_msg string) { + //打印错误信息 + fmt.Println("访问时间：", time.Now().Format("2006-01-02 15:04:05")) + fmt.Println("日志级别：", log_level) + fmt.Println("详细信息：", error_msg) +} + +/* +功能描述：连接数据库 + +参数说明：无 + +返回值说明： +conn *pgx.Conn -- 连接信息 +err error --错误信息 + +*/ + +func db_connect() (conn *pgx.Conn, err error) { + var config pgx.ConnConfig + config.Host = "127.0.0.1" //数据库主机host或ip + config.User = "opentenbase" //连接用户 + config.Password = "pgsql" //用户密码 + config.Database = "postgres" //连接数据库名 + config.Port = 15432 //端口号 + conn, err = pgx.Connect(config) + return conn, err +} +``` + +``` +[root@VM_0_29_centos opentenbase]# go run createtable.go +访问时间： 2018-04-03 20:50:24 +日志级别： Log +详细信息： 连接数据库成功 +访问时间： 2018-04-03 20:50:24 +日志级别： Log +详细信息： 创建数据表成功 +``` + +### 6.3、插入数据 +``` +package main + +import ( + "fmt" + "strings" + "time" + + "github.com/jackc/pgx" +) + +func main() { + var error_msg string + var sql string + var nickname string + + //连接数据库 + conn, err := db_connect() + if err != nil { + error_msg = "连接数据库失败，详情：" + err.Error() + write_log("Error", error_msg) + return + } + //程序运行结束时关闭连接 + defer conn.Close() + write_log("Log", "连接数据库成功") + + //插入数据 + sql = "insert into public.opentenbase values('1','opentenbase'),('2','pgxz');" + _, err = conn.Exec(sql) + if err != nil { + error_msg = "插入数据失败,详情：" + err.Error() + write_log("Error", error_msg) + return + } else { + write_log("Log", "插入数据成功") + } + + //绑定变量插入数据,不需要做防注入处理 + sql = "insert into public.opentenbase values($1,$2),($1,$3);" + _, err = conn.Exec(sql, "3", "postgresql", "postgres") + if err != nil { + error_msg = "插入数据失败,详情：" + err.Error() + write_log("Error", error_msg) + return + } else { + write_log("Log", "插入数据成功") + } + + //拼接sql语句插入数据,需要做防注入处理 + nickname = "OpenTenBase is ' good!" + sql = "insert into public.opentenbase values('1','" + sql_data_encode(nickname) + "')" + _, err = conn.Exec(sql) + if err != nil { + error_msg = "插入数据失败,详情：" + err.Error() + write_log("Error", error_msg) + return + } else { + write_log("Log", "插入数据成功") + } +} + +/* +功能描述：sql查询拼接字符串编码 + +参数说明： +str -- 要编码的字符串 + +返回值说明： +返回编码过的字符串 + +*/ + +func sql_data_encode(str string) string { + return strings.Replace(str, "'", "''", -1) +} + +/* +功能描述：写入日志处理 + +参数说明： +log_level -- 日志级别，只能是是Error或Log +error_msg -- 日志内容 + +返回值说明：无 +*/ + +func write_log(log_level string, error_msg string) { + //打印错误信息 + fmt.Println("访问时间：", time.Now().Format("2006-01-02 15:04:05")) + fmt.Println("日志级别：", log_level) + fmt.Println("详细信息：", error_msg) +} + +/* +功能描述：连接数据库 + +参数说明：无 + +返回值说明： +conn *pgx.Conn -- 连接信息 +err error --错误信息 + +*/ + +func db_connect() (conn *pgx.Conn, err error) { + var config pgx.ConnConfig + config.Host = "127.0.0.1" //数据库主机host或ip + config.User = "opentenbase" //连接用户 + config.Password = "pgsql" //用户密码 + config.Database = "postgres" //连接数据库名 + config.Port = 15432 //端口号 + conn, err = pgx.Connect(config) + return conn, err +} +``` + +``` +[root@VM_0_29_centos opentenbase]# go run insert.go +访问时间： 2018-04-03 21:05:51 +日志级别： Log +详细信息： 连接数据库成功 +访问时间： 2018-04-03 21:05:51 +日志级别： Log +详细信息： 插入数据成功 +访问时间： 2018-04-03 21:05:51 +日志级别： Log +详细信息： 插入数据成功 +访问时间： 2018-04-03 21:05:51 +日志级别： Log +详细信息： 插入数据成功 +``` + +### 6.4、查询数据 +``` +package main + +import ( + "fmt" + "strings" + "time" + + "github.com/jackc/pgx" +) + +func main() { + var error_msg string + var sql string + + //连接数据库 + conn, err := db_connect() + if err != nil { + error_msg = "连接数据库失败，详情：" + err.Error() + write_log("Error", error_msg) + return + } + //程序运行结束时关闭连接 + defer conn.Close() + write_log("Log", "连接数据库成功") + + sql = "SELECT id,nickname FROM public.opentenbase LIMIT 2" + rows, err := conn.Query(sql) + if err != nil { + error_msg = "查询数据失败,详情：" + err.Error() + write_log("Error", error_msg) + return + } else { + write_log("Log", "查询数据成功") + } + + var nickname string + var id string + + for rows.Next() { + err = rows.Scan(&id, &nickname) + if err != nil { + error_msg = "执行查询失败，详情：" + err.Error() + write_log("Error", error_msg) + return + } + error_msg = fmt.Sprintf("id：%s nickname：%s", id, nickname) + write_log("Log", error_msg) + } + rows.Close() + + nickname = "opentenbase" + + sql = "SELECT id,nickname FROM public.opentenbase WHERE nickname ='" + sql_data_encode(nickname) + "' " + rows, err = conn.Query(sql) + if err != nil { + error_msg = "查询数据失败,详情：" + err.Error() + write_log("Error", error_msg) + return + } else { + write_log("Log", "查询数据成功") + } + defer rows.Close() + + for rows.Next() { + err = rows.Scan(&id, &nickname) + if err != nil { + error_msg = "执行查询失败，详情：" + err.Error() + write_log("Error", error_msg) + return + } + error_msg = fmt.Sprintf("id：%s nickname：%s", id, nickname) + write_log("Log", error_msg) + } +} + +/* +功能描述：sql查询拼接字符串编码 + +参数说明： +str -- 要编码的字符串 + +返回值说明： +返回编码过的字符串 + +*/ + +func sql_data_encode(str string) string { + return strings.Replace(str, "'", "''", -1) +} + +/* +功能描述：写入日志处理 + +参数说明： +log_level -- 日志级别，只能是是Error或Log +error_msg -- 日志内容 + +返回值说明：无 +*/ + +func write_log(log_level string, error_msg string) { + //打印错误信息 + fmt.Println("访问时间：", time.Now().Format("2006-01-02 15:04:05")) + fmt.Println("日志级别：", log_level) + fmt.Println("详细信息：", error_msg) +} + +/* +功能描述：连接数据库 + +参数说明：无 + +返回值说明： +conn *pgx.Conn -- 连接信息 +err error --错误信息 + +*/ + +func db_connect() (conn *pgx.Conn, err error) { + var config pgx.ConnConfig + config.Host = "127.0.0.1" //数据库主机host或ip + config.User = "opentenbase" //连接用户 + config.Password = "pgsql" //用户密码 + config.Database = "postgres" //连接数据库名 + config.Port = 15432 //端口号 + conn, err = pgx.Connect(config) + return conn, err +} +``` +``` +[root@VM_0_29_centos opentenbase]# go run select.go +访问时间： 2018-04-09 10:35:50 +日志级别： Log +详细信息： 连接数据库成功 +访问时间： 2018-04-09 10:35:50 +日志级别： Log +详细信息： 查询数据成功 +访问时间： 2018-04-09 10:35:50 +日志级别： Log +详细信息： id：2 nickname：opentenbase +访问时间： 2018-04-09 10:35:50 +日志级别： Log +详细信息： id：3 nickname：postgresql +访问时间： 2018-04-09 10:35:50 +日志级别： Log +详细信息： 查询数据成功 +访问时间： 2018-04-09 10:35:50 +日志级别： Log +详细信息： id：1 nickname：opentenbase +``` + +### 6.5、流数据copy from入表 +``` +package main + +import ( + "fmt" + "math/rand" + "time" + + "github.com/jackc/pgx" +) + +func main() { + var error_msg string + + //连接数据库 + conn, err := db_connect() + if err != nil { + error_msg = "连接数据库失败，详情：" + err.Error() + write_log("Error", error_msg) + return + } + //程序运行结束时关闭连接 + defer conn.Close() + write_log("Log", "连接数据库成功") + + //构造5000行数据 + inputRows := [][]interface{}{} + var id string + var nickname string + for i := 0; i < 5000; i++ { + id = fmt.Sprintf("%d", rand.Intn(10000)) + nickname = fmt.Sprintf("%d", rand.Intn(10000)) + inputRows = append(inputRows, []interface{}{id, nickname}) + } + copyCount, err := conn.CopyFrom(pgx.Identifier{"opentenbase"}, []string{"id", "nickname"}, pgx.CopyFromRows(inputRows)) + if err != nil { + error_msg = "执行copyFrom失败,详情：" + err.Error() + write_log("Error", error_msg) + return + } + if copyCount != len(inputRows) { + error_msg = fmt.Sprintf("执行copyFrom失败，copy行数：%d 返回行数为：%d", len(inputRows), copyCount) + write_log("Error", error_msg) + return + } else { + error_msg = "Copy 记录成功" + write_log("Log", error_msg) + } + +} + +/* +功能描述：写入日志处理 + +参数说明： +log_level -- 日志级别，只能是是Error或Log +error_msg -- 日志内容 + +返回值说明：无 +*/ + +func write_log(log_level string, error_msg string) { + //打印错误信息 + fmt.Println("访问时间：", time.Now().Format("2006-01-02 15:04:05")) + fmt.Println("日志级别：", log_level) + fmt.Println("详细信息：", error_msg) +} + +/* +功能描述：连接数据库 + +参数说明：无 + +返回值说明： +conn *pgx.Conn -- 连接信息 +err error --错误信息 + +*/ + +func db_connect() (conn *pgx.Conn, err error) { + var config pgx.ConnConfig + config.Host = "127.0.0.1" //数据库主机host或ip + config.User = "opentenbase" //连接用户 + config.Password = "pgsql" //用户密码 + config.Database = "postgres" //连接数据库名 + config.Port = 15432 //端口号 + conn, err = pgx.Connect(config) + return conn, err +} +``` +``` +[root@VM_0_29_centos opentenbase]# go run copy_from.go +访问时间： 2018-04-09 10:36:40 +日志级别： Log +详细信息： 连接数据库成功 +访问时间： 2018-04-09 10:36:40 +日志级别： Log +详细信息： Copy 记录成功 +``` + +### 6.6、golang相关资源包 +需要git的资源包: +https://github.com/jackc/pgx +https://github.com/pkg/errors \ No newline at end of file diff --git a/content/chinese/blog/03-basic-use.md b/content/chinese/blog/03-basic-use.md new file mode 100644 index 0000000..eceadbf --- /dev/null +++ b/content/chinese/blog/03-basic-use.md @@ -0,0 +1,538 @@ +--- +title: "基本使用" +date: 2023-09-25T08:00:36+08:00 +author: OpenTenBase +image_webp: images/blog/blog-post-1.webp +image: images/blog/blog-post-1.jpg +description : "The Basic Usage of OpenTenBase" +--- + +>在[快速入门](01-quickstart.md)文章中我们介绍了 OpenTenBase 的架构、源码编译安装、集群运行状态、启动停止等内容。 +> +>在[应用接入](02-access.md)中我们介绍了应用程序连接 OpenTenBase 数据库进行建库、建表、数据导入、查询等操作。 +> +>本篇将介绍OpenTenBase中特有的shard表、冷热分区表、复制表的创建，和基本的DML操作。 + +## 1、创建数据表 +### 1.1、创建shard普通表 + + + OpenTenBase_shard普通表 + + + OpenTenBase_shard普通表续 + + + OpenTenBase_shard普通表说明 + +说明： + +- distribute by shard(x) 用于指定分布键，数据分布于那个节点就是根据这个字段值来计算分片。 +- to group xxx 用于指定存储组（每个存储组可以有多个节点）。 +- 分布键字段值不能修改，字段长度不能修改，字段类型不能修改。 + +### 1.2、创建shard普通分区表 + + + OpenTenBase_shard分区表 + + + OpenTenBase_shard分区表续 + +``` +[opentenbase@VM_0_37_centos shell]$ psql -h 172.16.0.42 -p 11387 -d postgres -U opentenbase +psql (PostgreSQL 10.0 opentenbase V2) +Type "help" for help. + +postgres=# create table public.t1_pt +( +f1 int not null, +f2 timestamp not null, +f3 varchar(20), +primary key(f1) +) +partition by range (f2) +begin (timestamp without time zone '2019-01-01 0:0:0') +step (interval '1 month') partitions (3) +distribute by shard(f1) +to group default_group; + +CREATE TABLE +postgres=# + +postgres=# \d+ public.t1_pt + Table "public.t1_pt" + Column | Type | Collation | Nullable | Default | Storage | Stats target | Description +--------+-----------------------------+-----------+----------+---------+----------+--------------+------------- + f1 | integer | | not null | | plain | | + f2 | timestamp without time zone | | not null | | plain | | + f3 | character varying(20) | | | | extended | | +Indexes: + "t1_pt_pkey" PRIMARY KEY, btree (f1) +Distribute By: SHARD(f1) +Location Nodes: ALL DATANODES +Partition By: RANGE(f2) + # Of Partitions: 3 + Start With: 2019-01-01 + Interval Of Partition: 1 MONTH + +postgres=# + +``` +说明： + +- partition by range (x) 用于指定分区键，支持timesamp,int类型，数据分布于那个子表就是根据这个字段值来计算分区。 +- begin( xxx )指定开始分区的时间点。 +- step(xxx)指定分区有周期 +- partions(xx)初始化时建立分区子表个数。 +- 增加分区子表的方法ALTER TABLE public.t1_pt ADD PARTITIONS 2; + +### 1.3、创建shard冷热分区表 + + + OpenTenBase_shard冷热分区表 + + + + OpenTenBase_shard冷热分区表续 + +``` +[opentenbase@VM_0_37_centos shell]$ psql -h 172.16.0.42 -p 11387 -d postgres -U opentenbase +psql (PostgreSQL 10.0 opentenbase V2) +Type "help" for help. + +postgres=# create table public.t1_cold_hot +( +f1 int not null, +f2 timestamp not null, +f3 varchar(20), +primary key(f1) +) +partition by range (f2) +begin (timestamp without time zone '2017-01-01 0:0:0') +step (interval '12 month') partitions (4) +distribute by shard(f1,f2) +to group default_group cold_group; +CREATE TABLE +postgres=# \d+ public.t1_cold_hot + Table "public.t1_cold_hot" + Column | Type | Collation | Nullable | Default | Storage | Stats target | Description +--------+-----------------------------+-----------+----------+---------+----------+--------------+------------- + f1 | integer | | not null | | plain | | + f2 | timestamp without time zone | | not null | | plain | | + f3 | character varying(20) | | | | extended | | +Indexes: + "t1_cold_hot_pkey" PRIMARY KEY, btree (f1) +Distribute By SHARD(f1,f2) + Hotnodes:dn001 Coldnodes:dn002 +Partition By: RANGE(f2) + # Of Partitions: 4 + Start With: 2017-01-01 + Interval Of Partition: 12 MONTH + +postgres=# + +``` + +说明： + +- Distribute By SHARD(f1,f2)，冷热分区表需要指定两个字段来做路由，分别是分布键和分区键。 +- to group default_group cold_group，需要指定两个存储组，第一个是热数据存储组，第二个是冷存储组。 + +创建时间范围冷热分区表需要有两个group，冷数据的cold_group对应的节点需要标识为冷节点，如下所示 + +``` + +[opentenbase@VM_0_37_centos shell]$ psql -h 172.16.0.42 -p 11000 -d postgres -U opentenbase +psql (PostgreSQL 10.0 opentenbase V2) +Type "help" for help. + +postgres=# select pg_set_node_cold_access(); + pg_set_node_cold_access +------------------------- + success +(1 row) + +``` + +冷热分区表需要在postgresql.conf中配置冷热分区时间参数和分区级别，如下所示 + +``` + +cold_hot_sepration_mode = 'year' +enable_cold_seperation = true +manual_hot_date = '2019-01-01' + +``` + +### 1.4、创建复制表 + + + OpenTenBase_shard冷热分区表 + +``` + +[opentenbase@VM_0_37_centos shell]$ psql -h 172.16.0.42 -p 11387 -d postgres -U opentenbase +psql (PostgreSQL 10.0 opentenbase V2) +Type "help" for help. + +postgres=# create table public.t1_rep +( +f1 int not null, +f2 varchar(20), +primary key(f1) +) +distribute by replication ; +to group default_group; +CREATE TABLE + +``` +说明： + +- 经常要跨库JOIN的小数据量表可以考虑使用复制表。 +- 复制表是所有节点都有全量数据，对于大数据量的数据表不适合。 +- 复制表更新性能较低。 + + +## 2、DML相关操作 +### 2.1、INSERT + +- 插入多条记录 + +``` +CREATE TABLE public.t1_insert_mul +( + f1 int not null, + f2 varchar(20), + primary key(f1) +) distribute by shard(f1) to group default_group; + +postgres=# INSERT INTO t1_insert_mul VALUES(1,'opentenbase'),(2,'pg'); +INSERT 0 2 + +``` + + +- 插入更新 + +``` +create table public.t1_conflict +( + f1 int not null, + f2 varchar(20), + primary key(f1) +) distribute by shard(f1) to group default_group; + +insert into t1_conflict values(1,'opentenbase') ON CONFLICT (f1) DO UPDATE SET f2 = 'opentenbase'; + +create table public.t1_conflict +( + f1 int not null, + f2 varchar(20) not null, + f3 int , + primary key(f1,f2) +) distribute by shard(f1) to group default_group; + +insert into t1_conflict values(1,'opentenbase',2) ON CONFLICT (f1,f2) DO UPDATE SET f3 = 2; + +``` + +- 插入返回 + +``` +create table public.t1_insert_return +( + f1 int not null, + f2 varchar(20) not null default 'opentenbase', + primary key(f1) +) distribute by shard(f1) to group default_group; + +postgres=# insert into t1_insert_return values(1) returning *; + + f1 | f2 +----+------- + 1 | opentenbase + (1 row) + INSERT 0 1 + +``` + +- INSERT更多的使用方法请参考Postgresql用法 + +``` +http://www.postgres.cn/docs/10/sql-insert.html +``` + +### 2.2、UPDATE + +- 基于分布键条件更新 + +``` +create table public.t1_update_pkey +( + f1 int not null, + f2 varchar(20) not null default 'opentenbase', + f3 varchar(32), + primary key(f1) +) distribute by shard(f1) to group default_group; + +postgres=# explain UPDATE t1_update_pkey SET f2='opentenbase' where f1=1; + + QUERY PLAN +---------------------------------------------------------------------------------- +Remote Fast Query Execution (cost=0.00..0.00 rows=0 width=0) + Node/s: dn001 + -> Update on t1_update_pkey (cost=0.15..4.17 rows=1 width=154) + -> Index Scan using t1_update_pkey_pkey on t1_update_pkey (cost=0.15..4.17 rows=1 width=154) + Index Cond: (f1 = 1) + +``` +性能最优，扩展性好 + +- 非分布键更新 + +``` +postgres=# explain UPDATE t1_update_pkey SET f2='opentenbase' where f3='pg'; QUERY PLAN +---------------------------------------------------------------------------------- +Remote Fast Query Execution (cost=0.00..0.00 rows=0 width=0) + Node/s: dn001, dn002 + -> Update on t1_update_pkey (cost=0.00..15.12 rows=2 width=154) + -> Seq Scan on t1_update_pkey (cost=0.00..15.12 rows=2 width=154) + Filter: ((f3)::text = 'pg'::text) +(5 rows) + +``` +更新语句发往所有节点 + +- 分区表带分区条件更新 + +``` +create table public.t1_pt_update +( f1 int not null,f2 timestamp not null,f3 varchar(20),primary key(f1) ) +partition by range (f2) begin (timestamp without time zone '2019-01-01 0:0:0') step (interval '1 month') partitions (2) distribute by shard(f1) to group default_group; + +postgres=# explain update t1_pt_update set f3='opentenbase' where f1=1 and f2>'2019-01-01' and f2<'2019-02-01'; QUERY PLAN +----------------------------------------------------------------------------------- +Remote Fast Query Execution (cost=0.00..0.00 rows=0 width=0) + Node/s: dn001 + -> Update on t1_pt_update_part_0 (cost=0.15..4.17 rows=1 width=80) + -> Index Scan using t1_pt_update_pkey_part_0 on t1_pt_update_part_0 (cost=0.15..4.17 rows=1 width=80) + Index Cond: (f1 = 1) + Filter: ((f2 > '2019-01-01 00:00:00'::timestamp without time zone) AND (f2 < '2019-02-01 00:00:00'::timestamp without time zone)) +``` + +带分区条件更新，性能最优，扩展性好 + +- 分区表不带分区条件更新 + +``` +postgres=# explain update t1_pt_update set f3='opentenbase' where f1=1; QUERY PLAN +------------------------------------------------------------------------------------ +Remote Fast Query Execution (cost=0.00..0.00 rows=0 width=0) + Node/s: dn001 + -> Update on t1_pt_update (cost=0.15..4.17 rows=1 width=80) + -> Index Scan using t1_pt_update_pkey_part_0 on t1_pt_update (partition sequence: 0, name: t1_pt_update_part_0) (cost=0.15..2.08 rows=0 width=80) + Index Cond: (f1 = 1) + -> Index Scan using t1_pt_update_pkey_part_1 on t1_pt_update (partition sequence: 1, name: t1_pt_update_part_1) (cost=0.15..2.08 rows=0 width=80) + Index Cond: (f1 = 1) +(7 rows) + +``` +需要扫描所有分区子表 + +- 关联表更新 + +``` +create table public.t1_update_join1 +( + f1 int not null,f2 varchar(20) not null default 'opentenbase',primary key(f1) +) +distribute by shard(f1) to group default_group; + +create table public.t1_update_join2 +( + f1 int not null,f2 varchar(20) not null default 'opentenbase',primary key(f1) +) +distribute by shard(f1) to group default_group; + +update t1_update_join1 set f2='pg' from t1_update_join2 where t1_update_join1.f1=t1_update_join2.f1; + +``` +表关联更新只能是基于分布键关联 + +- 分布键，分区键不能更新 + +``` +create table public.t1_update_pkey +( + f1 int not null,f2 varchar(20) not null default 'opentenbase', primary key(f1) +) distribute by shard(f1) to group default_group; + +postgres=# update t1_update_pkey set f1=2 where f1=1; +ERROR: Distributed column or partition column "f1" can't be updated in current version +Time: 0.910 ms. + +``` + +目前的解决办法“删除旧记录，再新增记录” + +- 更多的UPDATE使用方法请参考Postgresql用法 + +``` +http://www.postgres.cn/docs/10/sql-update.html +``` + +### 2.3、DELETE +- 删除返回记录 + +``` +create table public.t1_delete_return +( + f1 int not null,f2 varchar(20) not null default 'opentenbase',primary key(f1) +) +distribute by shard(f1) to group default_group; + +postgres=# insert into t1_delete_return values(1,'opentenbase'); +INSERT 0 1 + +postgres=# delete from t1_delete_return where f1=1 returning *; + + f1 | f2 +----+------- + 1 | opentenbase +(1 row) + +``` + +- UPDATE最优使用方法同样适合于DELETE + +- DELETE更多的使用方法见 + +``` +http://www.postgres.cn/docs/10/sql-delete.html +``` + +### 2.4、SELECT +- 基于分布键查询 + +``` +create table public.t1_select +( + f1 int not null,f2 varchar(20) not null default 'opentenbase',f3 varchar(32), primary key(f1) +) +distribute by shard(f1) to group default_group; + +postgres=# explain select * from t1_select where f1=1; QUERY PLAN +---------------------------------------------------------------------------------- +Remote Fast Query Execution (cost=0.00..0.00 rows=0 width=0) + Node/s: dn001 + -> Index Scan using t1_select_pkey on t1_select (cost=0.15..4.17 rows=1 width=144) + Index Cond: (f1 = 1) + +``` + +性能最优，扩展性好 + +- 非分布键查询 + +``` +postgres=# explain select * from t1_select where f1<3; + QUERY PLAN +------------------------------------------------------------------------------------- +Remote Fast Query Execution (cost=0.00..0.00 rows=0 width=0) + Node/s: dn001, dn002 + -> Bitmap Heap Scan on t1_select (cost=3.21..14.92 rows=137 width=144) + Recheck Cond: (f1 < 3) + -> Bitmap Index Scan on t1_select_pkey (cost=0.00..3.17 rows=137 width=0) + Index Cond: (f1 < 3) + +``` + +查询语句发往所有节点，然后在CN汇总 + +- 分布键JOIN查询 + +``` +create table public.t1_select_join1 +( f1 int not null,f2 int,primary key(f1) ) +distribute by shard(f1) to group default_group; + +create index t1_select_join1_f2_idx on t1_select_join1(f2); + +create table public.t1_select_join2 +( f1 int not null,f2 int,primary key(f1) ) +distribute by shard(f1) to group default_group; + +create index t1_select_join2_f2_idx on t1_select_join2(f2); + +postgres=# explain select t1_select_join1.* from t1_select_join1,t1_select_join2 where t1_select_join1.f1=t1_select_join2.f1 and t1_select_join1.f1=1; + QUERY PLAN -------------------------------------------------------------------------------------- +Remote Fast Query Execution (cost=0.00..0.00 rows=0 width=0) + Node/s: dn001 + -> Nested Loop (cost=0.30..8.35 rows=1 width=8) + -> Index Scan using t1_select_join1_pkey on t1_select_join1 (cost=0.15..4.17 rows=1 width=8) + Index Cond: (f1 = 1) + -> Index Only Scan using t1_select_join2_pkey on t1_select_join2 (cost=0.15..4.17 rows=1 width=4) + Index Cond: (f1 = 1) + +``` + +性能最优，扩展性好 + +- 非分布键JOIN查询 + +``` +postgres=# explain select * from t1_select_join1,t1_select_join2 where t1_select_join1.f1=t1_select_join2.f2 and t1_select_join1.f2=1 ; + QUERY PLAN ------------------------------------------------------------------------------------------ +Remote Subquery Scan on all (dn001,dn002) (cost=2.26..33.48 rows=7 width=16) + -> Nested Loop (cost=2.26..33.48 rows=7 width=16) + -> Bitmap Heap Scan on t1_select_join1 (cost=2.13..9.57 rows=7 width=8) Recheck Cond: (f2 = 1) + -> Bitmap Index Scan on t1_select_join1_f2_idx (cost=0.00..2.13 rows=7 width=0) + Index Cond: (f2 = 1) + -> Materialize (cost=100.12..103.45 rows=7 width=8) + -> Remote Subquery Scan on all (dn001,dn002) (cost=100.12..103.44 rows=7 width=8) + Distribute results by S: f2 + -> Index Scan using t1_select_join2_f2_idx on t1_select_join2 (cost=0.12..3.35 rows=7 width=8) + Index Cond: (f2 = t1_select_join1.f1) + +``` + +需要在DN做数据重分布 + +### 2.5、TRUNCATE +- 普通表truncate + +``` +create table public.t1_delete_truncate +( f1 int not null,f2 varchar(20) not null default 'opentenbase',primary key(f1) ) +distribute by shard(f1) to group default_group; + +insert into t1_delete_truncate select t,t::text from generate_series(1,1000000) as t; + +truncate table t1_delete_truncate; +``` + +- 分区表truncate + +``` +postgres=# create table public.t1_pt +( +f1 int not null, +f2 timestamp not null, +f3 varchar(20), +primary key(f1) +) +partition by range (f2) +begin (timestamp without time zone '2019-01-01 0:0:0') +step (interval '1 month') partitions (3) +distribute by shard(f1) +to group default_group; + +truncate public.t1_pt partition for ('2019-01-01' ::timestamp without time zone); +``` + + + + + + diff --git a/content/chinese/blog/images/1.1shard.jpg b/content/chinese/blog/images/1.1shard.jpg new file mode 100644 index 0000000..01624a1 Binary files /dev/null and b/content/chinese/blog/images/1.1shard.jpg differ diff --git a/content/chinese/blog/images/1.1shard_2.jpg b/content/chinese/blog/images/1.1shard_2.jpg new file mode 100644 index 0000000..bacf151 Binary files /dev/null and b/content/chinese/blog/images/1.1shard_2.jpg differ diff --git a/content/chinese/blog/images/1.1shard_3.jpg b/content/chinese/blog/images/1.1shard_3.jpg new file mode 100644 index 0000000..3598a90 Binary files /dev/null and b/content/chinese/blog/images/1.1shard_3.jpg differ diff --git a/content/chinese/blog/images/1.2shard_part.jpg b/content/chinese/blog/images/1.2shard_part.jpg new file mode 100644 index 0000000..20fed10 Binary files /dev/null and b/content/chinese/blog/images/1.2shard_part.jpg differ diff --git a/content/chinese/blog/images/1.2shard_part_2.jpg b/content/chinese/blog/images/1.2shard_part_2.jpg new file mode 100644 index 0000000..62c8624 Binary files /dev/null and b/content/chinese/blog/images/1.2shard_part_2.jpg differ diff --git a/content/chinese/blog/images/1.3shard_cold_hot.jpg b/content/chinese/blog/images/1.3shard_cold_hot.jpg new file mode 100644 index 0000000..92aab65 Binary files /dev/null and b/content/chinese/blog/images/1.3shard_cold_hot.jpg differ diff --git a/content/chinese/blog/images/1.3shard_cold_hot_2.jpg b/content/chinese/blog/images/1.3shard_cold_hot_2.jpg new file mode 100644 index 0000000..d71ae2f Binary files /dev/null and b/content/chinese/blog/images/1.3shard_cold_hot_2.jpg differ diff --git a/content/chinese/blog/images/1.4repilication.jpg b/content/chinese/blog/images/1.4repilication.jpg new file mode 100644 index 0000000..01d49c0 Binary files /dev/null and b/content/chinese/blog/images/1.4repilication.jpg differ diff --git a/content/chinese/blog/images/1.4replication_table.jpg b/content/chinese/blog/images/1.4replication_table.jpg new file mode 100644 index 0000000..76e306e Binary files /dev/null and b/content/chinese/blog/images/1.4replication_table.jpg differ diff --git a/content/english/blog/quickstart.md b/content/english/blog/01-quickstart.md similarity index 97% rename from content/english/blog/quickstart.md rename to content/english/blog/01-quickstart.md index 4ed0300..e61c0ad 100644 --- a/content/english/blog/quickstart.md +++ b/content/english/blog/01-quickstart.md @@ -14,6 +14,7 @@ OpenTenBase is a relational database cluster platform that provides write reliab OpenTenBase is a distributed cluster architecture (as shown in the figure below), which is a distributed share nothing architecture mode, each node is independent and processes its own data. After processing, the results may be summarized to the upper layer or transferred between nodes. Each processing unit communicates with each other through the network protocol, with better parallel processing and expansion ability, which also means that the OpenTenBase cluster can be deployed on the basic x86 server. + OpenTenBase architecture @@ -44,17 +45,17 @@ OS: TencentOS 2, TencentOS 3, OpenCloudOS, CentOS 7, CentOS 8, Ubuntu ### Dependence -` yum -y install gcc make readline-devel zlib-devel openssl-devel uuid-devel bison flex` +` yum -y install gcc make readline-devel zlib-devel openssl-devel uuid-devel bison flex git` or -` apt install -y gcc make libreadline-dev zlib1g-dev libssl-dev libossp-uuid-dev bison flex` +` apt install -y gcc make libreadline-dev zlib1g-dev libssl-dev libossp-uuid-dev bison flex git` - **create user** Note: all machines that need to install OpenTenBase cluster need to create -```shell +``` mkdir /data useradd -d /data/opentenbase -s /bin/bash -m opentenbase passwd opentenbase # set password @@ -62,13 +63,13 @@ passwd opentenbase # set password - **get source code** -```shell +``` git clone https://github.com/OpenTenBase/OpenTenBase ``` - **source code compilation** -```shell +``` cd ${SOURCECODE_PATH} rm -rf ${INSTALL_PATH}/opentenbase_bin_v2.0 chmod +x configure* @@ -84,7 +85,7 @@ make install In this paper, the above two parameters are as follows -```shell +``` ${SOURCECODE_PATH}=/data/opentenbase/OpenTenBase ${INSTALL_PATH}=/data/opentenbase/install ``` @@ -160,9 +161,19 @@ Above, the required basic environment has been configured, and you can enter the The following shows the pgxc\_ctl.conf file content written using the IP, port, database directory, binary directory and other planning values described above. In practice, we only need to configure it according to our own actual situation. +We can also download and rename it to ```pgxc_ctl.conf``` and configure it according to our own actual situation. + +[Download Double Node Conf](https://docs.opentenbase.org/guide/pgxc_ctl_double.conf) + +[Download Single Node Conf](https://docs.opentenbase.org/guide/pgxc_ctl_single.conf) + ```shell #!/bin/bash +# Double Node Config + +IP_1=10.215.147.158 +IP_2=10.240.138.159 pgxcInstallDir=/data/opentenbase/install/opentenbase_bin_v2.0 pgxcOwner=opentenbase @@ -178,13 +189,13 @@ configBackupFile=pgxc_ctl.bak #---- GTM ---------- gtmName=gtm -gtmMasterServer=10.215.147.158 +gtmMasterServer=$IP_1 gtmMasterPort=50001 gtmMasterDir=/data/opentenbase/data/gtm gtmExtraConfig=none gtmMasterSpecificExtraConfig=none gtmSlave=y -gtmSlaveServer=10.240.138.159 +gtmSlaveServer=$IP_2 gtmSlavePort=50001 gtmSlaveDir=/data/opentenbase/data/gtm gtmSlaveSpecificExtraConfig=none @@ -197,7 +208,7 @@ coordNames=(cn001 cn002 ) coordPorts=(30004 30004 ) poolerPorts=(31110 31110 ) coordPgHbaEntries=(0.0.0.0/0) -coordMasterServers=(10.215.147.158 10.240.138.159) +coordMasterServers=($IP_1 $IP_2) coordMasterDirs=($coordMasterDir $coordMasterDir) coordMaxWALsernder=2 coordMaxWALSenders=($coordMaxWALsernder $coordMaxWALsernder ) @@ -268,13 +279,13 @@ datanodeNames=(dn001 dn002) datanodePorts=(40004 40004) datanodePoolerPorts=(41110 41110) datanodePgHbaEntries=(0.0.0.0/0) -datanodeMasterServers=(10.215.147.158 10.240.138.159) +datanodeMasterServers=($IP_1 $IP_2) datanodeMasterDirs=($dn1MstrDir $dn2MstrDir) dnWALSndr=4 datanodeMaxWALSenders=($dnWALSndr $dnWALSndr) datanodeSlave=y -datanodeSlaveServers=(10.240.138.159 10.215.147.158) +datanodeSlaveServers=($IP_2 $IP_1) datanodeSlavePorts=(50004 54004) datanodeSlavePoolerPorts=(51110 51110) datanodeSlaveSync=n diff --git a/content/english/blog/03-basic-use.md b/content/english/blog/03-basic-use.md new file mode 100644 index 0000000..00411e2 --- /dev/null +++ b/content/english/blog/03-basic-use.md @@ -0,0 +1,535 @@ +--- +title: "Basic Usage" +date: 2023-09-25T08:00:36+08:00 +author: OpenTenBase +image_webp: images/blog/blog-post-1.webp +image: images/blog/blog-post-1.jpg +description : "The Basic Usage of OpenTenBase" +--- + +>In < Quick Start > article, we introduced opentenbase architecture, source code compilation and installation, cluster running status, startup and stop, etc. +> +>This article will introduce the creation of shard table, hot and cold partition table, replication table and basic DML operation in opentenbase. + +## 1、Create table +### 1.1、Create shard table + + +OpenTenBase_shard_table_1 + + +OpenTenBase_shard_table_2 + + +OpenTenBase_shard_table_3 + +Explain： + +- distribute by shard (x) is used to specify the distribution key. Based on the value of this field, calculate which node the partition data is distributed to . +- to group XXX is used to specify storage groups (each storage group can have multiple nodes). +- distribution key field value cannot be modified, field length cannot be modified, and field type cannot be modified. + +### 1.2、Create partition shard table + + +OpenTenBase_shard_partition_table_1 + + +OpenTenBase_shard_partition_table_2 + +``` +[opentenbase@VM_0_37_centos shell]$ psql -h 172.16.0.42 -p 11387 -d postgres -U opentenbase +psql (PostgreSQL 10.0 opentenbase V2) +Type "help" for help. + +postgres=# create table public.t1_pt +( +f1 int not null, +f2 timestamp not null, +f3 varchar(20), +primary key(f1) +) +partition by range (f2) +begin (timestamp without time zone '2019-01-01 0:0:0') +step (interval '1 month') partitions (3) +distribute by shard(f1) +to group default_group; + +CREATE TABLE +postgres=# + +postgres=# \d+ public.t1_pt + Table "public.t1_pt" + Column | Type | Collation | Nullable | Default | Storage | Stats target | Description +--------+-----------------------------+-----------+----------+---------+----------+--------------+------------- + f1 | integer | | not null | | plain | | + f2 | timestamp without time zone | | not null | | plain | | + f3 | character varying(20) | | | | extended | | +Indexes: + "t1_pt_pkey" PRIMARY KEY, btree (f1) +Distribute By: SHARD(f1) +Location Nodes: ALL DATANODES +Partition By: RANGE(f2) + # Of Partitions: 3 + Start With: 2019-01-01 + Interval Of Partition: 1 MONTH + +postgres=# + +``` +Explain： + +- partition by range (x) is used to specify the partition key. It supports the type of timesamp and int. according to this field value, it calculates which sub table the partition data is distributed in. +- begin( xxx ) is used to specifies the time point to start the partition。 +- step(xxx) is used to specified partition has period +- partions(xx) is used to establish the number of partition sub tables during initialization. +- Method of adding partition sub table: ALTER TABLE public.t1_pt ADD PARTITIONS 2; + +### 1.3、Create hot and cold partition shard table + + +OpenTenBase_shard_hot_and_cold_partition_table_1 + + +OpenTenBase_shard_hot_and_cold_partition_table_2 + +``` +[opentenbase@VM_0_37_centos shell]$ psql -h 172.16.0.42 -p 11387 -d postgres -U opentenbase +psql (PostgreSQL 10.0 opentenbase V2) +Type "help" for help. + +postgres=# create table public.t1_cold_hot +( +f1 int not null, +f2 timestamp not null, +f3 varchar(20), +primary key(f1) +) +partition by range (f2) +begin (timestamp without time zone '2017-01-01 0:0:0') +step (interval '12 month') partitions (4) +distribute by shard(f1,f2) +to group default_group cold_group; +CREATE TABLE +postgres=# \d+ public.t1_cold_hot + Table "public.t1_cold_hot" + Column | Type | Collation | Nullable | Default | Storage | Stats target | Description +--------+-----------------------------+-----------+----------+---------+----------+--------------+------------- + f1 | integer | | not null | | plain | | + f2 | timestamp without time zone | | not null | | plain | | + f3 | character varying(20) | | | | extended | | +Indexes: + "t1_cold_hot_pkey" PRIMARY KEY, btree (f1) +Distribute By SHARD(f1,f2) + Hotnodes:dn001 Coldnodes:dn002 +Partition By: RANGE(f2) + # Of Partitions: 4 + Start With: 2017-01-01 + Interval Of Partition: 12 MONTH + +postgres=# + +``` + +Explain： + +- distribute By SHARD(f1,f2) : The hot and cold partition table needs to specify two fields for routing, namely distribution key and partition key. +- to group default\_group cold\_group: You need to specify two storage groups, the first is hot data storage group and the second is cold storage group。 + +Two groups are required to create the time range cold and hot partition table, and the nodes contained in the cold\_group storing cold data need to be identified as cold nodes: + +``` + +[opentenbase@VM_0_37_centos shell]$ psql -h 172.16.0.42 -p 11000 -d postgres -U opentenbase +psql (PostgreSQL 10.0 opentenbase V2) +Type "help" for help. + +postgres=# select pg_set_node_cold_access(); + pg_set_node_cold_access +------------------------- + success +(1 row) + +``` + +The hot and cold partition table needs to configure the hot and cold partition time parameters and partition level in postgresql.conf: + +``` + +cold_hot_sepration_mode = 'year' +enable_cold_seperation = true +manual_hot_date = '2019-01-01' + +``` + +### 1.4、Create repilication table + + +OpenTenBase_replicaiton_table + +``` + +[opentenbase@VM_0_37_centos shell]$ psql -h 172.16.0.42 -p 11387 -d postgres -U opentenbase +psql (PostgreSQL 10.0 opentenbase V2) +Type "help" for help. + +postgres=# create table public.t1_rep +( +f1 int not null, +f2 varchar(20), +primary key(f1) +) +distribute by replication ; +to group default_group; +CREATE TABLE + +``` +Explain： + +- Replication tables can be considered for small data scales that often need to join across nodes +- Replication table means that every DN has full data, which is not suitable for large data tables. +- Replication table update performance is low. + + +## 2、DML related operations +### 2.1、INSERT + +- Insert multiple records + +``` +CREATE TABLE public.t1_insert_mul +( + f1 int not null, + f2 varchar(20), + primary key(f1) +) distribute by shard(f1) to group default_group; + +postgres=# INSERT INTO t1_insert_mul VALUES(1,'opentenbase'),(2,'pg'); +INSERT 0 2 + +``` + + +- Insert for updating + +``` +create table public.t1_conflict +( + f1 int not null, + f2 varchar(20), + primary key(f1) +) distribute by shard(f1) to group default_group; + +insert into t1_conflict values(1,'opentenbase') ON CONFLICT (f1) DO UPDATE SET f2 = 'opentenbase'; + +create table public.t1_conflict +( + f1 int not null, + f2 varchar(20) not null, + f3 int , + primary key(f1,f2) +) distribute by shard(f1) to group default_group; + +insert into t1_conflict values(1,'opentenbase',2) ON CONFLICT (f1,f2) DO UPDATE SET f3 = 2; + +``` + +- Insert for returning + +``` +create table public.t1_insert_return +( + f1 int not null, + f2 varchar(20) not null default 'opentenbase', + primary key(f1) +) distribute by shard(f1) to group default_group; + +postgres=# insert into t1_insert_return values(1) returning *; + + f1 | f2 +----+------- + 1 | opentenbase + (1 row) + INSERT 0 1 + +``` + +- For more usage of insert, please refer to PostgreSQL usage + +``` +http://www.postgres.cn/docs/10/sql-insert.html +``` + +### 2.2、UPDATE + +- Update based on distribution key condition + +``` +create table public.t1_update_pkey +( + f1 int not null, + f2 varchar(20) not null default 'opentenbase', + f3 varchar(32), + primary key(f1) +) distribute by shard(f1) to group default_group; + +postgres=# explain UPDATE t1_update_pkey SET f2='opentenbase' where f1=1; + + QUERY PLAN +---------------------------------------------------------------------------------- +Remote Fast Query Execution (cost=0.00..0.00 rows=0 width=0) + Node/s: dn001 + -> Update on t1_update_pkey (cost=0.15..4.17 rows=1 width=154) + -> Index Scan using t1_update_pkey_pkey on t1_update_pkey (cost=0.15..4.17 rows=1 width=154) + Index Cond: (f1 = 1) + +``` +Best performance, good scalability + +- Update based on non-distribution key condition + +``` +postgres=# explain UPDATE t1_update_pkey SET f2='opentenbase' where f3='pg'; QUERY PLAN +---------------------------------------------------------------------------------- +Remote Fast Query Execution (cost=0.00..0.00 rows=0 width=0) + Node/s: dn001, dn002 + -> Update on t1_update_pkey (cost=0.00..15.12 rows=2 width=154) + -> Seq Scan on t1_update_pkey (cost=0.00..15.12 rows=2 width=154) + Filter: ((f3)::text = 'pg'::text) +(5 rows) + +``` +UPDATE statement will send to all nodes. + +- Update partition table based on partition condition + +``` +create table public.t1_pt_update +( f1 int not null,f2 timestamp not null,f3 varchar(20),primary key(f1) ) +partition by range (f2) begin (timestamp without time zone '2019-01-01 0:0:0') step (interval '1 month') partitions (2) distribute by shard(f1) to group default_group; + +postgres=# explain update t1_pt_update set f3='opentenbase' where f1=1 and f2>'2019-01-01' and f2<'2019-02-01'; QUERY PLAN +----------------------------------------------------------------------------------- +Remote Fast Query Execution (cost=0.00..0.00 rows=0 width=0) + Node/s: dn001 + -> Update on t1_pt_update_part_0 (cost=0.15..4.17 rows=1 width=80) + -> Index Scan using t1_pt_update_pkey_part_0 on t1_pt_update_part_0 (cost=0.15..4.17 rows=1 width=80) + Index Cond: (f1 = 1) + Filter: ((f2 > '2019-01-01 00:00:00'::timestamp without time zone) AND (f2 < '2019-02-01 00:00:00'::timestamp without time zone)) +``` + +Update based on partition condition，best performance, good scalability + +- Update partition table without partition condition + +``` +postgres=# explain update t1_pt_update set f3='opentenbase' where f1=1; QUERY PLAN +------------------------------------------------------------------------------------ +Remote Fast Query Execution (cost=0.00..0.00 rows=0 width=0) + Node/s: dn001 + -> Update on t1_pt_update (cost=0.15..4.17 rows=1 width=80) + -> Index Scan using t1_pt_update_pkey_part_0 on t1_pt_update (partition sequence: 0, name: t1_pt_update_part_0) (cost=0.15..2.08 rows=0 width=80) + Index Cond: (f1 = 1) + -> Index Scan using t1_pt_update_pkey_part_1 on t1_pt_update (partition sequence: 1, name: t1_pt_update_part_1) (cost=0.15..2.08 rows=0 width=80) + Index Cond: (f1 = 1) +(7 rows) + +``` +All partition sub tables need to be scanned + +- Table association update + +``` +create table public.t1_update_join1 +( + f1 int not null,f2 varchar(20) not null default 'opentenbase',primary key(f1) +) +distribute by shard(f1) to group default_group; + +create table public.t1_update_join2 +( + f1 int not null,f2 varchar(20) not null default 'opentenbase',primary key(f1) +) +distribute by shard(f1) to group default_group; + +update t1_update_join1 set f2='pg' from t1_update_join2 where t1_update_join1.f1=t1_update_join2.f1; + +``` +Table Association updates can only be based on distribution key associations + +- Distribution key, partition key cannot be updated + +``` +create table public.t1_update_pkey +( + f1 int not null,f2 varchar(20) not null default 'opentenbase', primary key(f1) +) distribute by shard(f1) to group default_group; + +postgres=# update t1_update_pkey set f1=2 where f1=1; +ERROR: Distributed column or partition column "f1" can't be updated in current version +Time: 0.910 ms. + +``` + +The current solution is "delete old records and add new records" + +- Please refer to PostgreSQL for more usage of UPDATE + +``` +http://www.postgres.cn/docs/10/sql-update.html +``` + +### 2.3、DELETE +- Return deleted records when deleting + +``` +create table public.t1_delete_return +( + f1 int not null,f2 varchar(20) not null default 'opentenbase',primary key(f1) +) +distribute by shard(f1) to group default_group; + +postgres=# insert into t1_delete_return values(1,'opentenbase'); +INSERT 0 1 + +postgres=# delete from t1_delete_return where f1=1 returning *; + + f1 | f2 +----+------- + 1 | opentenbase +(1 row) + +``` + +- The optimal usage of update is also suitable for delete + +- Please refer to PostgreSQL for more usage of DELETE + +``` +http://www.postgres.cn/docs/10/sql-delete.html +``` + +### 2.4、SELECT +- Select based on distributed key + +``` +create table public.t1_select +( + f1 int not null,f2 varchar(20) not null default 'opentenbase',f3 varchar(32), primary key(f1) +) +distribute by shard(f1) to group default_group; + +postgres=# explain select * from t1_select where f1=1; QUERY PLAN +---------------------------------------------------------------------------------- +Remote Fast Query Execution (cost=0.00..0.00 rows=0 width=0) + Node/s: dn001 + -> Index Scan using t1_select_pkey on t1_select (cost=0.15..4.17 rows=1 width=144) + Index Cond: (f1 = 1) + +``` + +best performance, good scalability + +- Select based on non-distributed key + +``` +postgres=# explain select * from t1_select where f1<3; + QUERY PLAN +------------------------------------------------------------------------------------- +Remote Fast Query Execution (cost=0.00..0.00 rows=0 width=0) + Node/s: dn001, dn002 + -> Bitmap Heap Scan on t1_select (cost=3.21..14.92 rows=137 width=144) + Recheck Cond: (f1 < 3) + -> Bitmap Index Scan on t1_select_pkey (cost=0.00..3.17 rows=137 width=0) + Index Cond: (f1 < 3) + +``` + +The query statement will be sent to all nodes and then summarized in CN. + +- Join query based on distributed key + +``` +create table public.t1_select_join1 +( f1 int not null,f2 int,primary key(f1) ) +distribute by shard(f1) to group default_group; + +create index t1_select_join1_f2_idx on t1_select_join1(f2); + +create table public.t1_select_join2 +( f1 int not null,f2 int,primary key(f1) ) +distribute by shard(f1) to group default_group; + +create index t1_select_join2_f2_idx on t1_select_join2(f2); + +postgres=# explain select t1_select_join1.* from t1_select_join1,t1_select_join2 where t1_select_join1.f1=t1_select_join2.f1 and t1_select_join1.f1=1; + QUERY PLAN -------------------------------------------------------------------------------------- +Remote Fast Query Execution (cost=0.00..0.00 rows=0 width=0) + Node/s: dn001 + -> Nested Loop (cost=0.30..8.35 rows=1 width=8) + -> Index Scan using t1_select_join1_pkey on t1_select_join1 (cost=0.15..4.17 rows=1 width=8) + Index Cond: (f1 = 1) + -> Index Only Scan using t1_select_join2_pkey on t1_select_join2 (cost=0.15..4.17 rows=1 width=4) + Index Cond: (f1 = 1) + +``` + +best performance, good scalability + +- Join query based on non-distributed key + +``` +postgres=# explain select * from t1_select_join1,t1_select_join2 where t1_select_join1.f1=t1_select_join2.f2 and t1_select_join1.f2=1 ; + QUERY PLAN ------------------------------------------------------------------------------------------ +Remote Subquery Scan on all (dn001,dn002) (cost=2.26..33.48 rows=7 width=16) + -> Nested Loop (cost=2.26..33.48 rows=7 width=16) + -> Bitmap Heap Scan on t1_select_join1 (cost=2.13..9.57 rows=7 width=8) Recheck Cond: (f2 = 1) + -> Bitmap Index Scan on t1_select_join1_f2_idx (cost=0.00..2.13 rows=7 width=0) + Index Cond: (f2 = 1) + -> Materialize (cost=100.12..103.45 rows=7 width=8) + -> Remote Subquery Scan on all (dn001,dn002) (cost=100.12..103.44 rows=7 width=8) + Distribute results by S: f2 + -> Index Scan using t1_select_join2_f2_idx on t1_select_join2 (cost=0.12..3.35 rows=7 width=8) + Index Cond: (f2 = t1_select_join1.f1) + +``` + +Need to redistribute data in DN. + +### 2.5、TRUNCATE +- Truncate of non-partition table + +``` +create table public.t1_delete_truncate +( f1 int not null,f2 varchar(20) not null default 'opentenbase',primary key(f1) ) +distribute by shard(f1) to group default_group; + +insert into t1_delete_truncate select t,t::text from generate_series(1,1000000) as t; + +truncate table t1_delete_truncate; +``` + +- Truncate of partition table + +``` +postgres=# create table public.t1_pt +( +f1 int not null, +f2 timestamp not null, +f3 varchar(20), +primary key(f1) +) +partition by range (f2) +begin (timestamp without time zone '2019-01-01 0:0:0') +step (interval '1 month') partitions (3) +distribute by shard(f1) +to group default_group; + +truncate public.t1_pt partition for ('2019-01-01' ::timestamp without time zone); +``` + + + + + + diff --git a/content/english/blog/images/1.1shard.EN.png b/content/english/blog/images/1.1shard.EN.png new file mode 100644 index 0000000..2271161 Binary files /dev/null and b/content/english/blog/images/1.1shard.EN.png differ diff --git a/content/english/blog/images/1.1shard_2.EN.png b/content/english/blog/images/1.1shard_2.EN.png new file mode 100644 index 0000000..aa3b109 Binary files /dev/null and b/content/english/blog/images/1.1shard_2.EN.png differ diff --git a/content/english/blog/images/1.1shard_3.EN.png b/content/english/blog/images/1.1shard_3.EN.png new file mode 100644 index 0000000..13ca2d0 Binary files /dev/null and b/content/english/blog/images/1.1shard_3.EN.png differ diff --git a/content/english/blog/images/1.2shard_part.EN.png b/content/english/blog/images/1.2shard_part.EN.png new file mode 100644 index 0000000..bca3a14 Binary files /dev/null and b/content/english/blog/images/1.2shard_part.EN.png differ diff --git a/content/english/blog/images/1.2shard_part_2.EN.png b/content/english/blog/images/1.2shard_part_2.EN.png new file mode 100644 index 0000000..a42a7e1 Binary files /dev/null and b/content/english/blog/images/1.2shard_part_2.EN.png differ diff --git a/content/english/blog/images/1.3shard_cold_hot.EN.png b/content/english/blog/images/1.3shard_cold_hot.EN.png new file mode 100644 index 0000000..e86c90c Binary files /dev/null and b/content/english/blog/images/1.3shard_cold_hot.EN.png differ diff --git a/content/english/blog/images/1.3shard_cold_hot_2.EN.png b/content/english/blog/images/1.3shard_cold_hot_2.EN.png new file mode 100644 index 0000000..890df64 Binary files /dev/null and b/content/english/blog/images/1.3shard_cold_hot_2.EN.png differ diff --git a/content/english/blog/images/1.4repilication.EN.png b/content/english/blog/images/1.4repilication.EN.png new file mode 100644 index 0000000..97ea9a5 Binary files /dev/null and b/content/english/blog/images/1.4repilication.EN.png differ diff --git a/hugo_stats.json b/hugo_stats.json index 513e20e..1ca4934 100644 --- a/hugo_stats.json +++ b/hugo_stats.json @@ -3,6 +3,7 @@ "tags": [ "a", "article", + "blockquote", "body", "br", "button", @@ -27,6 +28,7 @@ "option", "p", "pre", + "quick", "script", "section", "select", @@ -136,7 +138,6 @@ "post-block2", "post-single-content", "rounded-0", - "rounded-circle", "row", "section", "section-bg", @@ -175,6 +176,59 @@ "wow" ], "ids": [ + "11create-shard-table", + "11创建shard普通表", + "11创建数据表", + "12create-partition-shard-table", + "12使用普通协议插入数据", + "12创建shard普通分区表", + "13create-hot-and-cold-partition-shard-table", + "13使用扩展协议插入数据", + "13创建shard冷热分区表", + "14copy-from-加载文件到表", + "14create-repilication-table", + "14创建复制表", + "15copy-to-导出数据到文件", + "16jdbc包下载地址", + "1create-table", + "1java开发", + "1创建数据表", + "21insert", + "21连接数据库", + "22update", + "22建立数据表", + "23delete", + "23插入数据", + "24select", + "24查询数据", + "25truncate", + "25流数据copy入表", + "2c程序开发", + "2dml-related-operations", + "2dml相关操作", + "3shell脚本开发", + "41安装psycopg2模块", + "42连接数据库", + "43创建数据表", + "44插入数据", + "45查询数据", + "46copy-from-加载文件到表", + "4python程序开发", + "51连接数据库", + "52创建数据表", + "53插入数据", + "54查询数据", + "55流数据copy-入表", + "56copy-to导出数据到一个数组中", + "5php程序开发", + "61连接数据库", + "62创建数据表", + "63插入数据", + "64查询数据", + "65流数据copy-from入表", + "66golang相关资源包", + "6golang程序开发", + "basic-usage", "blog", "body", "cn", @@ -194,6 +248,7 @@ "news", "opentenbase-source-code-compilation-and-installation", "opentenbase源码编译安装", + "quick-start", "select-language", "service", "services", @@ -206,7 +261,10 @@ "为-llm-创建云原生代理和扩展", "云原生存储的技术实践-hwameistor", "什么是opentenbase", + "基本使用", "安装依赖", + "应用接入指南", + "快速入门", "系统要求" ] }