ADH-5240

- implemented trino-adb plugin (for writing) - added docker compose file for starting trino with adb cluster
arenadata · Nov 18, 2024 · 3959401 · 3959401
1 parent 44a9648
commit 3959401
Show file tree

Hide file tree

Showing 96 changed files with 7,691 additions and 1 deletion.
diff --git a/core/docker/arenadata/coordinator/etc/catalog/adb.properties b/core/docker/arenadata/coordinator/etc/catalog/adb.properties
@@ -0,0 +1,4 @@
+connector.name=adb
+connection-url=jdbc:postgresql://adb:6000/postgres
+connection-user=gpadmin
+connection-password=gpadmin
diff --git a/core/docker/arenadata/docker-compose-with-adb.yml b/core/docker/arenadata/docker-compose-with-adb.yml
@@ -0,0 +1,184 @@
+version: "3.8"
+services:
+  postgres:
+    image: "docker.io/library/postgres:12.0"
+    restart: unless-stopped
+    container_name: postgres-db
+    hostname: postgres-db
+    environment:
+      POSTGRES_DB: '${POSTGRES_DB}'
+      POSTGRES_USER: '${POSTGRES_USER}'
+      POSTGRES_PASSWORD: '${POSTGRES_PASSWORD}'
+    ports:
+      - '5432:5432'
+    healthcheck:
+      test: psql -d ${POSTGRES_DB} -U ${POSTGRES_USER} -Atc 'SELECT 1;'
+      interval: 30s
+      timeout: 15s
+      retries: 3
+    networks:
+      - trino-with-adb
+
+  coordinator:
+    image: "trino:${TRINO_VERSION}"
+    ports:
+      - "8080:8080"
+      - "8990:8990"
+    container_name: "coordinator"
+    volumes:
+      - ./coordinator/etc:/etc/trino:rw
+    depends_on:
+      - postgres
+    networks:
+      - trino-with-adb
+
+  worker0:
+    image: "trino:${TRINO_VERSION}"
+    container_name: "worker0"
+    ports:
+      - "8081:8080"
+      - "8991:8990"
+    volumes:
+      - ./worker/etc:/etc/trino:rw
+    depends_on:
+      - postgres
+    networks:
+      - "trino-with-adb"
+
+  worker1:
+    image: "trino:${TRINO_VERSION}"
+    container_name: "worker1"
+    ports:
+      - "8082:8080"
+      - "8992:8990"
+    volumes:
+      - ./worker/etc:/etc/trino:rw
+    depends_on:
+      - postgres
+    networks:
+      - trino-with-adb
+
+  adb:
+    image: "hub.adsw.io/adbcc/gpdb6-adbm-distributed:it"
+    working_dir: /home/gpadmin
+    hostname: mdw
+    ports:
+      - "6000:6000"
+      - "38005:35285"
+    environment:
+      - ADB_CLUSTER_NAME=adb
+      - ADBM_EUREKA_CLIENT_HOSTNAME=mdw
+      - ADBM_EUREKA_CLIENT_IP_ADDRESS=mdw
+      - ADBM_EUREKA_CLIENT_PREFER_IP_ADDRESS=true
+      - ADBM_EUREKA_CLIENT_SERV_URL_DEF_ZONE=http://registry:8761/eureka
+      - AGENT_JAVA_OPTS=-XX:+HeapDumpOnOutOfMemoryError -XX:+UseStringDeduplication -XX:+OptimizeStringConcat -XX:+UseLWPSynchronization -XX:HeapDumpPath=/opt/adbm-agent/tmp/java_pid.hprof -Xlog:gc*,safepoint:gc.log:time,uptime:filecount=100,filesize=50M -Djava.library.path=/opt/adbm-agent/tmp -Djava.io.tmpdir=/opt/adbm-agent/tmp -Dlogging.config=/opt/adbm-agent/adbm-agent-logback.xml -Xmx2g -Xms256m -Dspring.config.location=file:///opt/adbm-agent/adbm-agent-application.yml
+      - DOCKER_GP_CLUSTER_HOSTS=mdw,smdw,sdw1,sdw2
+      - DOCKER_GP_MASTER_SERVER=mdw
+      - DOCKER_GP_STANDBY_SERVER=smdw
+      - DOCKER_GP_SEGMENT_SERVERS=sdw1,sdw2
+      - DOCKER_GP_PRIMARY_SEGMENTS_PER_HOST=2
+      - DOCKER_GP_WITH_MIRROR=true
+    depends_on:
+      - sdw1
+      - sdw2
+      - smdw
+    volumes:
+      - backup-data:/tmp/pgbackrest/:rw
+    privileged: true
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    sysctls:
+      kernel.sem: 500 1024000 200 4096
+    networks:
+      - trino-with-adb
+
+  sdw1:
+    image: "hub.adsw.io/adbcc/gpdb6-adbm-distributed:it"
+    privileged: true
+    hostname: sdw1
+    ports:
+      - "38006:35285"
+    environment:
+      - ADB_CLUSTER_NAME=adb
+      - ADBM_EUREKA_CLIENT_HOSTNAME=sdw1
+      - ADBM_EUREKA_CLIENT_IP_ADDRESS=sdw1
+      - ADBM_EUREKA_CLIENT_PREFER_IP_ADDRESS=true
+      - ADBM_EUREKA_CLIENT_SERV_URL_DEF_ZONE=http://registry:8761/eureka
+      - AGENT_JAVA_OPTS=-XX:+HeapDumpOnOutOfMemoryError -XX:+UseStringDeduplication -XX:+OptimizeStringConcat -XX:+UseLWPSynchronization -XX:HeapDumpPath=/opt/adbm-agent/tmp/java_pid.hprof -Xlog:gc*,safepoint:gc.log:time,uptime:filecount=100,filesize=50M -Djava.library.path=/opt/adbm-agent/tmp -Djava.io.tmpdir=/opt/adbm-agent/tmp -Dlogging.config=/opt/adbm-agent/adbm-agent-logback.xml -Xmx2g -Xms256m -Dspring.config.location=file:///opt/adbm-agent/adbm-agent-application.yml
+      - DOCKER_GP_CLUSTER_HOSTS=mdw,smdw,sdw1,sdw2
+      - DOCKER_GP_MASTER_SERVER=mdw
+      - DOCKER_GP_STANDBY_SERVER=smdw
+      - DOCKER_GP_SEGMENT_SERVERS=sdw1,sdw2
+      - DOCKER_GP_PRIMARY_SEGMENTS_PER_HOST=2
+      - DOCKER_GP_WITH_MIRROR=true
+    volumes:
+      - backup-data:/tmp/pgbackrest/:rw
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    sysctls:
+      kernel.sem: 500 1024000 200 4096
+    networks:
+      - trino-with-adb
+
+  sdw2:
+    image: "hub.adsw.io/adbcc/gpdb6-adbm-distributed:it"
+    privileged: true
+    hostname: sdw2
+    ports:
+      - "38007:35285"
+    environment:
+      - ADB_CLUSTER_NAME=adb
+      - ADBM_EUREKA_CLIENT_HOSTNAME=sdw2
+      - ADBM_EUREKA_CLIENT_IP_ADDRESS=sdw2
+      - ADBM_EUREKA_CLIENT_PREFER_IP_ADDRESS=true
+      - ADBM_EUREKA_CLIENT_SERV_URL_DEF_ZONE=http://registry:8761/eureka
+      - AGENT_JAVA_OPTS=-XX:+HeapDumpOnOutOfMemoryError -XX:+UseStringDeduplication -XX:+OptimizeStringConcat -XX:+UseLWPSynchronization -XX:HeapDumpPath=/opt/adbm-agent/tmp/java_pid.hprof -Xlog:gc*,safepoint:gc.log:time,uptime:filecount=100,filesize=50M -Djava.library.path=/opt/adbm-agent/tmp -Djava.io.tmpdir=/opt/adbm-agent/tmp -Dlogging.config=/opt/adbm-agent/adbm-agent-logback.xml -Xmx2g -Xms256m -Dspring.config.location=file:///opt/adbm-agent/adbm-agent-application.yml
+      - DOCKER_GP_CLUSTER_HOSTS=mdw,smdw,sdw1,sdw2
+      - DOCKER_GP_MASTER_SERVER=mdw
+      - DOCKER_GP_STANDBY_SERVER=smdw
+      - DOCKER_GP_SEGMENT_SERVERS=sdw1,sdw2
+      - DOCKER_GP_PRIMARY_SEGMENTS_PER_HOST=2
+      - DOCKER_GP_WITH_MIRROR=true
+    volumes:
+      - backup-data:/tmp/pgbackrest/:rw
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    sysctls:
+      kernel.sem: 500 1024000 200 4096
+    networks:
+      - trino-with-adb
+
+  smdw:
+    image: "hub.adsw.io/adbcc/gpdb6-adbm-distributed:it"
+    privileged: true
+    hostname: smdw
+    ports:
+      - "5433:6000"
+      - "38008:35285"
+    environment:
+      - ADB_CLUSTER_NAME=adb
+      - ADBM_EUREKA_CLIENT_HOSTNAME=smdw
+      - ADBM_EUREKA_CLIENT_IP_ADDRESS=smdw
+      - ADBM_EUREKA_CLIENT_PREFER_IP_ADDRESS=true
+      - ADBM_EUREKA_CLIENT_SERV_URL_DEF_ZONE=http://registry:8761/eureka
+      - AGENT_JAVA_OPTS=-XX:+HeapDumpOnOutOfMemoryError -XX:+UseStringDeduplication -XX:+OptimizeStringConcat -XX:+UseLWPSynchronization -XX:HeapDumpPath=/opt/adbm-agent/tmp/java_pid.hprof -Xlog:gc*,safepoint:gc.log:time,uptime:filecount=100,filesize=50M -Djava.library.path=/opt/adbm-agent/tmp -Djava.io.tmpdir=/opt/adbm-agent/tmp -Dlogging.config=/opt/adbm-agent/adbm-agent-logback.xml -Xmx2g -Xms256m -Dspring.config.location=file:///opt/adbm-agent/adbm-agent-application.yml
+      - DOCKER_GP_CLUSTER_HOSTS=mdw,smdw,sdw1,sdw2
+      - DOCKER_GP_MASTER_SERVER=mdw
+      - DOCKER_GP_STANDBY_SERVER=smdw
+      - DOCKER_GP_SEGMENT_SERVERS=sdw1,sdw2
+      - DOCKER_GP_PRIMARY_SEGMENTS_PER_HOST=2
+      - DOCKER_GP_WITH_MIRROR=true
+    volumes:
+      - backup-data:/tmp/pgbackrest/:rw
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    sysctls:
+      kernel.sem: 500 1024000 200 4096
+    networks:
+      - trino-with-adb
+
+networks:
+  trino-with-adb:
+volumes:
+  backup-data:
+    driver: local
diff --git a/core/docker/arenadata/worker/etc/catalog/adb.properties b/core/docker/arenadata/worker/etc/catalog/adb.properties
@@ -0,0 +1,4 @@
+connector.name=adb
+connection-url=jdbc:postgresql://adb:6000/postgres
+connection-user=gpadmin
+connection-password=gpadmin
diff --git a/core/trino-server/src/main/provisio/trino.xml b/core/trino-server/src/main/provisio/trino.xml
@@ -33,6 +33,12 @@
     </artifactSet>
 
     <!-- Plugins -->
+    <artifactSet to="plugin/adb">
+        <artifact id="${project.groupId}:trino-adb:zip:${project.version}">
+            <unpack />
+        </artifact>
+    </artifactSet>
+
     <artifactSet to="plugin/accumulo">
         <artifact id="${project.groupId}:trino-accumulo:zip:${project.version}">
             <unpack />