Citus packages test infrastructure (#196)

.github/workflows/package-tests.yml

@@ -0,0 +1,72 @@
+name: Citus package tests
+
+on:
+  push:
+    branches:
+      - "**"
+
+  workflow_dispatch:
+    inputs:
+      prj_ver:
+        description: "The version to be tested"
+        required: true
+
+jobs:
+  metadata:
+    runs-on: ubuntu-latest
+    outputs:
+      pg_versions: ${{ steps.generate-postgres.outputs.pg_versions }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
+      - name: Install dependencies
+        run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources
+      - name: Install python requirements
+        run: python -m pip install -r packaging_automation/requirements.txt
+      - name: generate postgres
+        id: generate-postgres
+        run: |
+          export PROJECT_VERSION="${{ github.event.inputs.name }}"
+          [ -z ${PROJECT_VERSION} ] && export PROJECT_VERSION=10.2.1
+          POSTGRES_VERSIONS=$(python -m packaging_automation.get_postgres_versions --prj_ver ${PROJECT_VERSION})
+          echo "Postgres Version: ${POSTGRES_VERSIONS}"
+          echo "::set-output name=pg_versions::${POSTGRES_VERSIONS}"
+  test_execution:
+    runs-on: ubuntu-latest
+    needs: metadata
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - centos/8
+          - centos/7
+          - ol/7
+          - debian/stretch
+          - debian/buster
+          - debian/bullseye
+          - ubuntu/bionic
+          - ubuntu/focal
+        pg: ${{ fromJson(needs.metadata.outputs.pg_versions) }}
+    env:
+      PLATFORM: ${{ matrix.platform }}
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Install dependencies
+        run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources
+
+      - name: Install python requirements
+        run: python -m pip install -r packaging_automation/requirements.txt
+
+      - name: Citus package tests
+        run: |
+          export PROJECT_VERSION="${{ github.event.inputs.name }}"
+          [ -z ${PROJECT_VERSION} ]&& export PROJECT_VERSION=10.2.1
+          python -m  packaging_automation.test_citus_package \
+          --prj_ver "${PROJECT_VERSION}" \
+          --os_release ${{ matrix.platform }} \
+          --pg_major_version ${{ matrix.pg }}

packaging_automation/citus_package.py

@@ -17,7 +17,9 @@
                                   get_gpg_fingerprints_by_name,
                                   get_supported_postgres_nightly_versions,
                                   get_supported_postgres_release_versions,
+                                  platform_names,
                                   run_with_output, str_array_to_str,
+                                  supported_platforms,
                                   transform_key_into_base64_str)
 from .packaging_warning_handler import validate_output
 
@@ -26,23 +28,6 @@
 POSTGRES_VERSION_FILE = "supported-postgres"
 POSTGRES_MATRIX_FILE_NAME = "postgres-matrix.yml"
 
-supported_platforms = {
-    "debian": ["bullseye", "buster", "stretch", "jessie", "wheezy"],
-    "el": ["8", "7", "6"],
-    "ol": ["7", "8"],
-    "ubuntu": ["focal", "bionic", "xenial", "trusty"]
-}
-
-
-def platform_names() -> List[str]:
-    platforms = []
-    for platform_os, platform_releases in supported_platforms.items():
-        for platform_release in platform_releases:
-            platforms.append(f"{platform_os}/{platform_release}")
-    platforms.append("pgxn")
-    return platforms
-
-
 docker_image_names = {
     "debian": "debian",
     "el": "centos",

packaging_automation/common_tool_methods.py

@@ -36,6 +36,21 @@
 # all resources after the code execution.
 referenced_repos: List[Repo] = []
 
+supported_platforms = {
+    "debian": ["bullseye", "buster", "stretch", "jessie", "wheezy"],
+    "el": ["8", "7", "6"],
+    "ol": ["7", "8"],
+    "ubuntu": ["focal", "bionic", "xenial", "trusty"]
+}
+
+def platform_names() -> List[str]:
+    platforms = []
+    for platform_os, platform_releases in supported_platforms.items():
+        for platform_release in platform_releases:
+            platforms.append(f"{platform_os}/{platform_release}")
+    platforms.append("pgxn")
+    return platforms
+
 
 def get_new_repo(working_dir: str) -> Repo:
     repo = Repo(working_dir)

packaging_automation/get_postgres_versions.py

@@ -0,0 +1,12 @@
+import argparse
+import json
+
+from .test_citus_package import (get_postgres_versions_from_matrix_file)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--prj_ver', required=True)
+
+    args = parser.parse_args()
+    postgres_versions = get_postgres_versions_from_matrix_file(args.prj_ver)
+    print(json.dumps(postgres_versions))

packaging_automation/test_citus_package.py

@@ -0,0 +1,107 @@
+import argparse
+import os
+import subprocess
+import shlex
+import requests
+from enum import Enum
+import sys
+from typing import List
+
+from .common_tool_methods import (get_supported_postgres_release_versions, get_minor_version)
+
+POSTGRES_MATRIX_FILE = "postgres-matrix.yml"
+POSTGRES_MATRIX_WEB_ADDRESS = "https://raw.githubusercontent.com/citusdata/packaging/all-citus-unit-tests/postgres-matrix.yml"
+
+
+def run_command(command: str) -> int:
+    with subprocess.Popen(shlex.split(command), stdout=subprocess.PIPE, stderr=subprocess.STDOUT) as process:
+        for line in iter(process.stdout.readline, b''):  # b'\n'-separated lines
+            print(line.decode("utf-8"), end=" ")
+    exitcode = process.wait()
+    return exitcode
+
+
+class TestPlatform(Enum):
+    el_7 = {"name": "el/7", "docker_image_name": "el-7"}
+    el_8 = {"name": "el/8", "docker_image_name": "el-8"}
+    centos_8 = {"name": "centos/8", "docker_image_name": "centos-8"}
+    centos_7 = {"name": "centos/7", "docker_image_name": "centos-7"}
+    ol_7 = {"name": "ol/7", "docker_image_name": "ol-7"}
+    ol_8 = {"name": "ol/8", "docker_image_name": "ol-8"}
+    debian_buster = {"name": "debian/buster", "docker_image_name": "debian-buster", }
+    debian_bullseye = {"name": "debian/bullseye", "docker_image_name": "debian-bullseye"}
+    debian_stretch = {"name": "debian/stretch", "docker_image_name": "debian-stretch"}
+    ubuntu_bionic = {"name": "ubuntu/bionic", "docker_image_name": "ubuntu-bionic"}
+    ubuntu_focal = {"name": "ubuntu/focal", "docker_image_name": "ubuntu-focal"}
+    undefined = {"name": "undefined", "docker_image_name": "undefined"}
+
+
+def get_test_platform_for_os_release(os_release: str) -> TestPlatform:
+    result = TestPlatform.undefined
+    for tp in TestPlatform:
+        if tp.value["name"] == os_release:
+            result = tp
+    return result
+
+
+def get_postgres_versions_from_matrix_file(project_version: str) -> List[str]:
+    r = requests.get(POSTGRES_MATRIX_WEB_ADDRESS, allow_redirects=True)
+
+    with open(POSTGRES_MATRIX_FILE, 'wb') as writer:
+        writer.write(r.content)
+    pg_versions = get_supported_postgres_release_versions(POSTGRES_MATRIX_FILE, project_version)
+
+    return pg_versions
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--prj_ver', required=True)
+    parser.add_argument('--pg_major_version')
+    parser.add_argument("--os_release", choices=[t.value["name"] for t in TestPlatform])
+
+    args = parser.parse_args()
+    test_platform = get_test_platform_for_os_release(args.os_release)
+    minor_prj_ver = get_minor_version(args.prj_ver)
+
+    platform = args.os_release
+
+    postgres_versions = get_postgres_versions_from_matrix_file(args.prj_ver)
+
+    print(f'This version of Citus supports following pg versions: {postgres_versions}')
+
+    os.chdir("test-images")
+    return_codes = {}
+
+    if args.pg_major_version:
+        postgres_versions = [p for p in postgres_versions if p == args.pg_major_version]
+
+
+    if len(postgres_versions) == 0:
+        raise ValueError("At least one supported postgres version is required")
+
+    for postgres_version in postgres_versions:
+        print(f'Testing package for following pg version: {postgres_version}')
+        docker_image_name = f"test:{test_platform.value['docker_image_name']}-{postgres_version}"
+        build_command = (f"docker build -t {docker_image_name} "
+                         f"-f {test_platform.value['docker_image_name']}/Dockerfile "
+                         f"--build-arg CITUS_VERSION={args.prj_ver} --build-arg PG_MAJOR={postgres_version}   "
+                         f"--build-arg CITUS_MAJOR_VERSION={minor_prj_ver} .")
+        print(build_command)
+        return_build = run_command(build_command)
+        return_run = run_command(
+            f"docker run  -e POSTGRES_VERSION={postgres_version} {docker_image_name} ")
+        return_codes[f"{docker_image_name}-build"] = return_build
+        return_codes[f"{docker_image_name}-run"] = return_run
+
+    error_exists = False
+    print("-----------------Summary Report------------------")
+    for key, value in return_codes.items():
+        if value > 0:
+            error_exists = True
+        print(f"{key}: {'Success' if value == 0 else f'Fail. ErrorCode: {value}'}")
+    summary_error = 'FAILED :(' if error_exists else 'SUCCESS :)'
+    print(f'------------------------{summary_error}------------------------')
+
+    if error_exists:
+        sys.exit("Failed")

test-images/centos-7/Dockerfile

@@ -0,0 +1,71 @@
+FROM centos:7
+
+ARG CITUS_VERSION
+# Format should be XY and should not include dots e.g for 10.2.1=>102
+ARG CITUS_MAJOR_VERSION
+ARG PG_MAJOR
+ARG FANCY=1
+ARG HLL_VERSION=2.16.citus-1
+ARG TOPN_VERSION=2.4.0.citus-1
+ARG PACKAGE_RELEASE_SUFFIX=el7
+
+ENV CITUS_VERSION ${CITUS_VERSION}
+
+ENV PG_MAJOR ${PG_MAJOR}
+
+
+RUN yum update -y && \
+    yum install -y curl
+
+
+# TODO Parameterize Citus and postgres version
+RUN export CITUS_MAJOR_VER=${CITUS_MAJOR_VERSION//./} && \
+    curl https://install.citusdata.com/community/rpm.sh |  bash && \
+    yum install -y citus${CITUS_MAJOR_VER}_${PG_MAJOR}-${CITUS_VERSION}.citus-${FANCY}.${PACKAGE_RELEASE_SUFFIX} \
+           hll_${PG_MAJOR}-${HLL_VERSION}.${PACKAGE_RELEASE_SUFFIX} \
+           topn_${PG_MAJOR}-${TOPN_VERSION}.${PACKAGE_RELEASE_SUFFIX}
+
+
+
+ARG POSTGRES_HOME=/var/lib/pgsql
+ENV PATH=/usr/pgsql-${PG_MAJOR}/bin:${PATH}:${POSTGRES_HOME}
+
+WORKDIR ${POSTGRES_HOME}
+
+RUN mkdir citus && chown postgres citus
+
+
+
+
+
+USER postgres
+RUN cd ~ && initdb -D citus && echo "shared_preload_libraries = 'citus'" >> citus/postgresql.conf
+
+USER root
+# Install python 3.8 and its dependencies
+RUN yum install -y gcc make && \
+    yum -y install  libcurl-devel \
+                    openssl-devel \
+                    bzip2-devel \
+                    libffi-devel \
+                    xz-devel \
+                    python38-devel \
+                    openssl-devel &&\
+      curl  https://www.python.org/ftp/python/3.8.12/Python-3.8.12.tgz --output Python-3.8.12.tgz &&\
+      tar xvf Python-3.8.12.tgz &&\
+      cd Python-3.8.*/ && \
+      ./configure --enable-optimizations && \
+       make altinstall && \
+       python3.8 -m pip install pip-tools
+
+COPY scripts/* ./
+
+RUN pip-compile && python3.8 -m pip install -r requirements.txt
+
+
+USER postgres
+
+WORKDIR ${POSTGRES_HOME}
+
+CMD ["test_internal.sh"]
+

test-images/centos-8/Dockerfile

@@ -0,0 +1,71 @@
+FROM centos:8
+
+ARG CITUS_VERSION
+# Format should be XY and should not include dots e.g for 10.2.1=>102
+ARG CITUS_MAJOR_VERSION
+ARG PG_MAJOR
+ARG FANCY=1
+ARG HLL_VERSION=2.16.citus-1
+ARG TOPN_VERSION=2.4.0.citus-1
+ARG PACKAGE_RELEASE_SUFFIX=el8
+
+ENV CITUS_VERSION ${CITUS_VERSION}
+
+ENV PG_MAJOR ${PG_MAJOR}
+
+
+RUN yum update -y && \
+    yum install -y curl
+
+
+# TODO Parameterize Citus and postgres version
+RUN export CITUS_MAJOR_VER=${CITUS_MAJOR_VERSION//./} && \
+    curl https://install.citusdata.com/community/rpm.sh |  bash && \
+    yum install -y citus${CITUS_MAJOR_VER}_${PG_MAJOR}-${CITUS_VERSION}.citus-${FANCY}.${PACKAGE_RELEASE_SUFFIX} \
+           hll_${PG_MAJOR}-${HLL_VERSION}.${PACKAGE_RELEASE_SUFFIX} \
+           topn_${PG_MAJOR}-${TOPN_VERSION}.${PACKAGE_RELEASE_SUFFIX}
+
+
+
+ARG POSTGRES_HOME=/var/lib/pgsql
+ENV PATH=/usr/pgsql-${PG_MAJOR}/bin:${PATH}:${POSTGRES_HOME}
+
+WORKDIR ${POSTGRES_HOME}
+
+RUN mkdir citus && chown postgres citus
+
+
+
+
+
+USER postgres
+RUN cd ~ && initdb -D citus && echo "shared_preload_libraries = 'citus'" >> citus/postgresql.conf
+
+USER root
+# Install python 3.8 and its dependencies
+RUN yum install -y gcc make && \
+    yum -y install  libcurl-devel \
+                    openssl-devel \
+                    bzip2-devel \
+                    libffi-devel \
+                    xz-devel \
+                    python38-devel \
+                    openssl-devel &&\
+      curl  https://www.python.org/ftp/python/3.8.12/Python-3.8.12.tgz --output Python-3.8.12.tgz &&\
+      tar xvf Python-3.8.12.tgz &&\
+      cd Python-3.8.*/ && \
+      ./configure --enable-optimizations && \
+       make altinstall && \
+       python3.8 -m pip install pip-tools
+
+COPY scripts/* ./
+
+RUN pip-compile && python3.8 -m pip install -r requirements.txt
+
+
+USER postgres
+
+WORKDIR ${POSTGRES_HOME}
+
+CMD ["test_internal.sh"]
+