arenadata · RomaZe · Nov 12, 2024 · Nov 28, 2024 · Dec 3, 2024 · Dec 3, 2024
diff --git a/automation/arenadata/Dockerfile b/automation/arenadata/Dockerfile
@@ -43,9 +43,10 @@ RUN set -eux; \
            yum -y install /tmp/jdk-17.0.12_linux-x64_bin.rpm && \
            rm -rf /tmp/jdk-17.0.12_linux-x64_bin.rpm; \
          sed -i "s/JAVA_HOME=.*/JAVA_HOME=\$(readlink -f \/usr\/bin\/java | sed 's:bin\/java::')/g" /etc/profile.d/jdk_home.sh; \
+         yum install -y ksh; \
          ;; \
        ubuntu*) \
-         apt-get -y update && apt-get install -y unzip vim nano openjdk-17-jdk; \
+         apt-get -y update && apt-get install -y unzip vim nano openjdk-17-jdk ksh; \
          update-locale LANG=en_US.UTF-8; \
          ;; \
     esac;
@@ -82,6 +83,9 @@ ENV PXF_HOME=/usr/local/greenplum-db-devel/pxf
 RUN localedef -c -i ru_RU -f CP1251 ru_RU.CP1251
 RUN cp ${PXF_HOME}/templates/*-site.xml ${PXF_HOME}/servers/default/
 
+# Copy pxf-application.properties
+COPY ./automation/arenadata/conf/pxf-application.properties ${PXF_HOME}/conf/pxf-application.properties
+
 # Need to change ssh key to RSA for automation tests with Ubuntu
 RUN set -eux; \
     . /etc/os-release; \
@@ -98,6 +102,6 @@ RUN set -eux; \
 # Move libs to the destination folder
 RUN cp /tmp/libs/* ${PXF_HOME}/lib/
 
-RUN chmod a+x ./pxf_src/automation/arenadata/scripts/start_adb_cluster.sh
+RUN chmod -R a+x ./pxf_src/automation/arenadata/scripts
 RUN chown -R gpadmin:gpadmin /usr/local/greenplum-db-devel
 ENTRYPOINT ["/home/gpadmin/pxf_src/automation/arenadata/scripts/start_adb_cluster.sh"]
diff --git a/automation/arenadata/README.md b/automation/arenadata/README.md
@@ -49,3 +49,8 @@ docker-compose exec mdw sudo -H -u gpadmin bash -l -c 'pushd $TEST_HOME && make
 cd pxf/automation/arenadata/hadoop/
 docker build -f Dockerfile -t cloud-hub.adsw.io/library/pxf-hadoop:3.1.3 .
 ```
+
+### Run PXF with SSL
+```shell
+docker-compose -f docker-compose-ssl.yaml up -d
+```
diff --git a/automation/arenadata/build-images.sh b/automation/arenadata/build-images.sh
@@ -15,6 +15,11 @@ popd
 #docker build -f Dockerfile -t cloud-hub.adsw.io/library/pxf-hadoop:3.3.6 .
 #popd
 
+#echo "===================================="
+#echo "      Build Vault image      "
+#echo "===================================="
+docker build -f ./vault/Dockerfile -t hub.adsw.io/pxf/pxf-vault-test:it .
+
 echo "=============================="
 echo "Build PXF image for automation"
 echo "=============================="

diff --git a/automation/arenadata/conf/pxf-application.properties b/automation/arenadata/conf/pxf-application.properties
@@ -0,0 +1,26 @@
+##############################################################################
+# This file contains PXF properties that can be specified by users           #
+# to customize their deployments. The configuration is loaded by Spring Boot #
+# upon service start up.                                                     #
+#                                                                            #
+# To update a property, uncomment the line and provide a new value.          #
+##############################################################################
+
+# Server connection timeout (-1 for infinite timeout)
+# pxf.connection.timeout=5m
+# pxf.connection.upload-timeout=5m
+
+# Threads
+# pxf.max.threads=200
+# pxf.task.pool.allow-core-thread-timeout=false
+# pxf.task.pool.core-size=8
+# pxf.task.pool.queue-capacity=0
+# pxf.task.pool.max-size=200
+
+# Logging
+# To enable debug logging, uncomment and change `info` to `debug` here
+# pxf.log.level=info
+
+# Security
+# Specify IP address (or hostname) of network interface that PXF listens to, or set to 0.0.0.0 for all interfaces
+server.address=0.0.0.0
diff --git a/automation/arenadata/conf/ssl/certs/ca-cert b/automation/arenadata/conf/ssl/certs/ca-cert
@@ -0,0 +1,30 @@
+-----BEGIN CERTIFICATE-----
+MIIFFTCCAv2gAwIBAgIUHolZhRFmeXWGQDUfp2ky44DUR5IwDQYJKoZIhvcNAQEL
+BQAwGjEYMBYGA1UEAwwPU1NMLVNlY3VyaXR5LUNBMB4XDTI0MTExMTIzNDQ1NloX
+DTM0MTEwOTIzNDQ1NlowGjEYMBYGA1UEAwwPU1NMLVNlY3VyaXR5LUNBMIICIjAN
+BgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAqfkVkLBs7aFgu+HhNOPxi7LgSC9i
+GBNjxD9KD0ABtrARyRWsILkU38Zy2i7FOJ1DZ07ZcV6ELkAlNpc7Opja/x9zOrHa
+JQAvBkkllc31kcJ5mSHgkOlww+gt1cGLsrkeP1YoR96RqJ4vsK7FP+XBS8z5K37N
+xVj3tmODYZqTdjRX+ee9DL2lI54vNimkGHQznEomERK0xvpja6u7xr18CBipeMUe
+EsFhIU0PW3NxMl84TqsdJigReOJx/3Hzggbd367f5yonSiq60KQqw0Qo0JhMUmQZ
+jBeSK452eHoJSK3Uz7aTEA7Tc3YKvFrWBFENie423BLzErF6EwepzU+iVSr7Uc3g
+812lnUZTQfIXzGQ0FFgNdl+2gvJ+FvsLoFI/yppuyB9oSu9FRmwpqRZ8xcJ0bSDT
+6vu21oV5tfep/4cjDxWq38lk+8QemlVA9nrMk2QNyVQT+NcOeXS/ijTIZ8KwPqPm
+554BODxKr8TNMMf7KEARa7sTZOGsHy09YosvBcwg0C9+/17FHWxsb4BqpLcd0FfK
+6b2rG596+62M3iD3FXG27rJi9qN8hx37kzHo1herUCK+VMzkGN5MoF/KJDEy83xZ
+XPUDQ+UN0anheMY1p+yVYixhTFlHTG5MgLVTWjfJWWs2c9fa5CLut8+X/R3zFt9D
+ZArj3b2qxhrEN/0CAwEAAaNTMFEwHQYDVR0OBBYEFF56+WTUlez31PgtJiLoFpJB
+rJ7bMB8GA1UdIwQYMBaAFF56+WTUlez31PgtJiLoFpJBrJ7bMA8GA1UdEwEB/wQF
+MAMBAf8wDQYJKoZIhvcNAQELBQADggIBAJkSgPbPBa5G5oNiiBrK/WjBM5t1NOD7
+WMrpC8++DJDRT1u90zmdK+us8Ab9DvBXkbjVcO0XXNOoV6RsGJuVJdufcvxyF3O6
+UdjWREic26PV/+1mJpgu2suwGVjU0FOKUGlQZVwHvrG6YJcFAcyqeZVvCoMC2Omp
+aSQ0Z+MlA6T+3fZCNIScVsFwaV63IjYdEkRsrou+NuXDEqR2zHJoXsZsgN1lFUu6
+x1vlKnw2DLhNOIWRTyLMfU9DIXq02UrD/lZjW6yszrmVBoFm1W0gP/T16GLo91+S
+BY66R0LGBtzmIOzjjgvcfB2aTSO+wskr4S8UhWXp7O4kkFf69rg6ogZLZbR0XxSG
+EEYe7aD96RxlQ3QUq5hWLtZ/Toj5fwTO8RDhA57b5J0nsNL1AnmN96VUVUadaxug
+VHLQIwpjqNdCVGfcV/WMdAFY6TMGBAjUe5xZLo0cxodEkQ/lUP9nxLH8wIBcpSXp
+VD2gQSJmbsr6Nk/iklk1FKiYs7xMFPVuQi4+oahwWC50N2mdhW/Xn8UZDyLpUTWt
+M5ld5jjlOBEAtyP07pLQj+K1qv+vEBN4rLNtkSbKPJkkVgTqhFSwo2RLM8rFkAxr
+B3nFzk7SxAQkwYXlnH2OQNTluWnH0r1MjQeU1VvQOjUduFOHDKpXciI9ajPxSVSF
+EchTqpmNaBAY
+-----END CERTIFICATE-----
diff --git a/automation/arenadata/conf/ssl/certs/pxf-client.key b/automation/arenadata/conf/ssl/certs/pxf-client.key
@@ -0,0 +1,32 @@
+Bag Attributes
+    friendlyName: pxf
+    localKeyID: 54 69 6D 65 20 31 37 33 31 33 36 38 37 30 30 34 37 36 
+Key Attributes: <No Attributes>
+-----BEGIN PRIVATE KEY-----
+MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDgipdjFjENt7U+
+rCK0UMVEz7QO6U8Qq1VjhBOeXHf2vwkID3Vb/G5bmd9xaqezx0HploVnNMF8CzCn
+M/Z/GcjV0e/eooKNHkulQRU7lNwbAJvylFH2+UNhepDIGzzHsyPEtSl00KIkFzMi
+ocblTs4dHppBcFkDcOgo6glJuAeKyTkrexYfGdSjvIM2RyTmSdam2qJF1c3nOxPl
+IHdJYFXgF4uqCRWWUh9MOKAUb4QxectBr9EnkDPhja4RmlgQcYlTbH3RMyHTk25Y
+mdcf5DMi+8U5NkJTKWSZVqTnCwKAxbkbJXsGu6Z3I9sHS1meWwxELACctdGiqcxj
+UOfFj8V3AgMBAAECggEAAPuU97SNqL3gl7J2K8GHaNUwjbjSBUR9XlmiIlkiQfIK
+bpQZaI/SxdOHFFPUrGdCBRfeY+9prLV51DQt2dbkN6IFwZIxgxTvsidLmf6n9fWg
+jm+9ahjOMDDdTbA81FzsL6dhU+cOhpzpYYyWtm3hXg7tED3seHo33JH6/MJxmgG5
+rxiNMByCFAgihVE9UszopT+ZJG8UNp4JGdsUJux9u1Er6RjjAoC0ffV71e1Ks0wr
+xulgKR7Y5AIEMoMAmzNluNcL65kgFl+ZslEtdbfbUa/rChJT2nu81EYr5Tu4IlRa
+jwxhrPHc41h7d9fRJRbixoBbw14xV5hLqpFwz1mcPQKBgQDi/weTb4ZWpyfiUcf1
+5hR2nTnRuhz0HXSx9XUVTp9TTkalw0cQnTrLHu9TvbZdPZTDah/JPap2x5NCdVWi
+e60nW2Z11GTzgD7QmQkBXgtMxvZvsFIRWkftGEOFe5zB3R/7tZYwCkFFHLsgmeLy
+V6LkpQRFxagbnUnf1vCx4KhCiwKBgQD9O0POQS1HqDKOCzy3EnJDMDwPJCSQR2VZ
+2hbGFiUhFf42buWfxbqpadl45RzZq92GzSmgPOSv/fKAjyYcL8Mc4qlGoZSY+xo+
+UvD6cWNafp3mIS6iMQb4der1HFd2l5wFoNDXQ7IDdEI4IP40yNlMUKTj/Z6OMlK/
+5WHOjOpCRQKBgE8Xk1kwy7sXpRL7cdPhJhIqOEI2Oe5o8Qcswm35JhSNupn5T6Kf
+ViP+FF+2Q3t7gIESUh75aJLYeRv37C2gMDX9Z9b4eXBKqBhgqj2aug83aBiEdCPt
+tBANm0FAT99Qn9tBayI0AIgHOcBjPis4k95dDP8OnsN4a4wdxWIFKDt/AoGAK2Jn
+74JDG6+5WZSbRlImuVQeNgnfKcjRmF9A35cTrWNzNigxtvnNTRERbSDznGdRgLLN
+muJT28MKNPRRdyrHpr8+eRddugfrKxpSptWeS/a9Xpv4vVjg9Nhe9LdyVKMM2/n9
+TaQPb4APiEY+xxRSBj0awuTEsV/1xWKrKxQBXnECgYBnRX9QYHTMpQN+UuJPohKZ
+jUdzgXplDk5LxPtor4BLjmvXUI7AtO1JMMr/ZLtJXZenLSAOZUVTYXUqTjM5661a
+7Au053b7nsr2LvZXNcTa4MaqLRVlZziHufmITCMZ8xb9PqtnLLhiePybRdVWIhC/
+04df7EwLJTtu9o7aIYaIqg==
+-----END PRIVATE KEY-----
diff --git a/automation/arenadata/conf/ssl/certs/pxf-client.pem b/automation/arenadata/conf/ssl/certs/pxf-client.pem
@@ -0,0 +1,23 @@
+-----BEGIN CERTIFICATE-----
+MIID2jCCAcKgAwIBAgIUbcasatcUVVbJyvpwf0Z21bApocUwDQYJKoZIhvcNAQEL
+BQAwGjEYMBYGA1UEAwwPU1NMLVNlY3VyaXR5LUNBMB4XDTI0MTExMTIzNDQ1N1oX
+DTM0MTEwOTIzNDQ1N1owDjEMMAoGA1UEAxMDcHhmMIIBIjANBgkqhkiG9w0BAQEF
+AAOCAQ8AMIIBCgKCAQEA4IqXYxYxDbe1PqwitFDFRM+0DulPEKtVY4QTnlx39r8J
+CA91W/xuW5nfcWqns8dB6ZaFZzTBfAswpzP2fxnI1dHv3qKCjR5LpUEVO5TcGwCb
+8pRR9vlDYXqQyBs8x7MjxLUpdNCiJBczIqHG5U7OHR6aQXBZA3DoKOoJSbgHisk5
+K3sWHxnUo7yDNkck5knWptqiRdXN5zsT5SB3SWBV4BeLqgkVllIfTDigFG+EMXnL
+Qa/RJ5Az4Y2uEZpYEHGJU2x90TMh05NuWJnXH+QzIvvFOTZCUylkmVak5wsCgMW5
+GyV7BrumdyPbB0tZnlsMRCwAnLXRoqnMY1DnxY/FdwIDAQABoyQwIjAgBgNVHREE
+GTAXggNtZHeCBHNkdzGCBHNkdzKCBHNtZHcwDQYJKoZIhvcNAQELBQADggIBAFrO
+cClAsTHyLmxdgdTeM7a0AHg5ppndOoLj3TDf+eqQiZNzvCuIqq2Y4/BLRXpB8/L+
+ef81UT49mtYnImPW15saYidopfsdboD+w+bvFfhcgpPl0EzGZ3/+9VHxk8ygVQVi
+jaiG0wjF7am4IIDKpKw6lZ/69/Co98s/Uh9x/Yr1raXrFkxDBmD1nZnzsron7/40
+teYt9cX08eRBe2wmZiTHr3H1s5VHlY2oojpATkBLUFBqYmY9cTnmgllue83qF6+D
+wvNAJBKqY2hSnom4SDo6PeepNUvDSj/jYc9rzzafU52P8n/rRQfGu82ZWGY+jen2
+eXnsCMhv/U6PnZAiYyJpbJrumJWtsbdtk99BopiilUjqoVWy39WAqrUkAPz1aY1j
+uNS7Wk/mkKVT0Haa2jh/hvaeFa1f0ZgpNXg/d5GPP7lxOuM3L+jQPYlXf8qmQpTX
+7LSiWHJGMGWzZ5BDtYRSVJQ57X7TimSrjMusiFA2z4MSDO7Ykw7RrmnPIrYPt1kt
+bhlV9vfKu1ldRI9dPkqme3fsT2ocIl1MWB5ZDvuRcIs4kE7NKVkix8pvqWpgcNlu
+1WrDeWWUHXg2ZAkmxHGI0PPTCfARUHo5pixqAtsVcazAHGfq/wI7jkfpxwdXPF8g
+lJsxcAVnz8O87DRzOnesVvE4WAXSipD8H/YdlBZ/
+-----END CERTIFICATE-----
diff --git a/automation/arenadata/conf/ssl/certs/pxf.jks b/automation/arenadata/conf/ssl/certs/pxf.jks
diff --git a/automation/arenadata/docker-compose-ssl.yaml b/automation/arenadata/docker-compose-ssl.yaml
@@ -0,0 +1,138 @@
+version: "3"
+services:
+  vault:
+    image: "hub.adsw.io/pxf/pxf-vault-test:it"
+    restart: on-failure
+    ports:
+      - "8200:8200"
+    cap_add:
+      - IPC_LOCK
+    volumes:
+      - ./vault/certs:/certs:ro
+      - ./vault/scripts:/scripts:ro
+      - ./vault/secrets:/secrets:ro
+      - vault-env:/env:rw
+    entrypoint: [ "bash", "-c", "/scripts/workflow-vault.sh" ]
+
+  mdw:
+    image: "gpdb6_pxf_automation:it"
+    restart: unless-stopped
+    working_dir: /home/gpadmin
+    hostname: mdw
+    ports:
+      - "5435:5432"
+      - "5005:5005"
+    environment:
+      - HOSTNAME=mdw
+      - DOCKER_GP_CLUSTER_HOSTS=mdw,sdw1,sdw2
+      - DOCKER_GP_MASTER_SERVER=mdw
+      - DOCKER_GP_SEGMENT_SERVERS=sdw1,sdw2
+      - DOCKER_GP_PRIMARY_SEGMENTS_PER_HOST=3
+      - DOCKER_GP_WITH_MIRROR=false
+      - PXF_PROTOCOL=https
+      - PXF_HOST=mdw
+      - PXF_VAULT_ENABLED=true
+      - PXF_VAULT_SECRET_PATH=adb/adb-it/service/pxf
+      - PXF_SSL_ENABLED=true
+      - PXF_SSL_KEY_STORE_PATH=/opt/ssl/certs/pxf.jks
+      - PXF_SSL_TRUST_STORE_PATH=/opt/ssl/certs/pxf.jks
+      - PXF_SSL_CLIENT_AUTH=NEED
+      - PXF_SSL_CACERT_PATH=/opt/ssl/certs/ca-cert
+      - PXF_SSL_CERT=/opt/ssl/certs/pxf-client.pem
+      - PXF_SSL_KEY=/opt/ssl/certs/pxf-client.key
+    volumes:
+      - "vault-env:/vault/env:ro"
+      - "./conf/ssl/certs:/opt/ssl/certs:ro"
+    healthcheck:
+      test:  sudo -H -u gpadmin bash -l -c "psql -d postgres -U gpadmin -Atc 'SELECT 1;'"
+      interval: 30s
+      timeout: 15s
+      retries: 3
+    depends_on:
+      - sdw1
+    privileged: true
+    sysctls:
+      kernel.sem: 500 1024000 200 4096
+      net.unix.max_dgram_qlen: 4096
+
+  sdw1:
+    image: "gpdb6_pxf_automation:it"
+    restart: unless-stopped
+    privileged: true
+    hostname: sdw1
+    ports:
+      - "8001:8000"
+    environment:
+      - HOSTNAME=sdw1
+      - DOCKER_GP_CLUSTER_HOSTS=mdw,sdw1,sdw2
+      - DOCKER_GP_MASTER_SERVER=mdw
+      - DOCKER_GP_SEGMENT_SERVERS=sdw1,sdw2
+      - DOCKER_GP_PRIMARY_SEGMENTS_PER_HOST=3
+      - DOCKER_GP_WITH_MIRROR=false
+      - PXF_PROTOCOL=https
+      - PXF_HOST=sdw1
+      - PXF_VAULT_ENABLED=true
+      - PXF_VAULT_SECRET_PATH=adb/adb-it/service/pxf
+      - PXF_SSL_ENABLED=true
+      - PXF_SSL_KEY_STORE_PATH=/opt/ssl/certs/pxf.jks
+      - PXF_SSL_TRUST_STORE_PATH=/opt/ssl/certs/pxf.jks
+      - PXF_SSL_CLIENT_AUTH=NEED
+      - PXF_SSL_CACERT_PATH=/opt/ssl/certs/ca-cert
+      - PXF_SSL_CERT=/opt/ssl/certs/pxf-client.pem
+      - PXF_SSL_KEY=/opt/ssl/certs/pxf-client.key
+    volumes:
+      - "vault-env:/vault/env:ro"
+      - "./conf/ssl/certs:/opt/ssl/certs:ro"
+    healthcheck:
+      test:  netstat -an | grep 5888 > /dev/null; if [ 0 != $$? ]; then exit 1; fi;
+      interval: 30s
+      timeout: 15s
+      retries: 3
+    sysctls:
+      kernel.sem: 500 1024000 200 4096
+      net.unix.max_dgram_qlen: 4096
+
+  sdw2:
+    image: "gpdb6_pxf_automation:it"
+    restart: unless-stopped
+    privileged: true
+    hostname: sdw2
+    ports:
+      - "8002:8000"
+    environment:
+      - HOSTNAME=sdw2
+      - DOCKER_GP_CLUSTER_HOSTS=mdw,sdw1,sdw2
+      - DOCKER_GP_MASTER_SERVER=mdw
+      - DOCKER_GP_SEGMENT_SERVERS=sdw1,sdw2
+      - DOCKER_GP_PRIMARY_SEGMENTS_PER_HOST=3
+      - DOCKER_GP_WITH_MIRROR=false
+      - PXF_PROTOCOL=https
+      - PXF_HOST=sdw2
+      - PXF_VAULT_ENABLED=true
+      - PXF_VAULT_SECRET_PATH=adb/adb-it/service/pxf
+      - PXF_SSL_ENABLED=true
+      - PXF_SSL_KEY_STORE_PATH=/opt/ssl/certs/pxf.jks
+      - PXF_SSL_TRUST_STORE_PATH=/opt/ssl/certs/pxf.jks
+      - PXF_SSL_CLIENT_AUTH=NEED
+      - PXF_SSL_CACERT_PATH=/opt/ssl/certs/ca-cert
+      - PXF_SSL_CERT=/opt/ssl/certs/pxf-client.pem
+      - PXF_SSL_KEY=/opt/ssl/certs/pxf-client.key
+    volumes:
+      - "vault-env:/vault/env:ro"
+      - "./conf/ssl/certs:/opt/ssl/certs:ro"
+    healthcheck:
+      test: netstat -an | grep 5888 > /dev/null; if [ 0 != $$? ]; then exit 1; fi;
+      interval: 30s
+      timeout: 15s
+      retries: 3
+    sysctls:
+      kernel.sem: 500 1024000 200 4096
+      net.unix.max_dgram_qlen: 4096
+
+networks:
+  default:
+    name: pxf-automation
+
+volumes:
+  m2:
+  vault-env: