From ffad6ae4b558e446e6af1fd8b7e6ac44d7c67225 Mon Sep 17 00:00:00 2001 From: mrdrivingduck Date: Fri, 3 May 2024 12:46:45 +0800 Subject: [PATCH 1/3] feat: support DEB packaging --- .github/workflows/package.yml | 61 +++++++++++--- package/debian/build-deb.sh | 152 ++++++++++++++++++++++++++++++++++ package/debian/control | 17 ++++ package/rpm/PolarDB.spec | 1 - package/rpm/build-rpm.sh | 20 +++++ 5 files changed, 240 insertions(+), 11 deletions(-) create mode 100755 package/debian/build-deb.sh create mode 100644 package/debian/control diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index 32aafb19f30..a0edc52b4f3 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -8,13 +8,13 @@ on: pull_request: branches: [ POLARDB_11_DEV, POLARDB_11_STABLE ] paths: - - '.github/workflows/regression-test.yml' - - 'config/**' - - 'contrib/**' - - 'external/**' - - 'src/**' - - 'configure*' - - 'polardb_build.sh' + - '.github/workflows/regression-test.yml' + - 'config/**' + - 'contrib/**' + - 'external/**' + - 'src/**' + - 'configure*' + - 'polardb_build.sh' # trigger testing manually workflow_dispatch: @@ -60,19 +60,60 @@ jobs: run: | sudo chown -R runner:runner `pwd` - release-rpm: + build-and-publish-deb: + runs-on: ubuntu-latest + strategy: + matrix: + base_image: [ ubuntu20.04, ubuntu22.04, ubuntu24.04 ] + steps: + - name: Fetch source code + uses: actions/checkout@v4 + - name: Create and start the container + run: | + docker create \ + -t \ + --name polardb_${{ matrix.base_image }} \ + -v `pwd`:/home/postgres/PolarDB-for-PostgreSQL \ + polardb/polardb_pg_devel:${{ matrix.base_image }} \ + bash && \ + docker start polardb_${{ matrix.base_image }} + - name: Build deb package + run: | + docker exec \ + polardb_${{ matrix.base_image }} \ + bash -c "cd /home/postgres/PolarDB-for-PostgreSQL && \ + sudo chown -R postgres:postgres ./ && \ + cd ./package/debian && \ + if [ -f /etc/bashrc ]; then source /etc/bashrc; fi && \ + ./build-deb.sh" + - name: Get package name + run: | + cd ./package/debian && DEBNAME=`ls *.deb` && echo "debname=$DEBNAME" >> $GITHUB_ENV + - name: Upload artifact + if: startsWith(github.ref, 'refs/tags/') + uses: actions/upload-artifact@v4 + with: + name: ${{ env.debname }} + path: ./package/debian/${{ env.debname }} + compression-level: 0 + - name: Change ownership of the source code + run: | + sudo chown -R runner:runner `pwd` + + release-packages: if: startsWith(github.ref, 'refs/tags/') - needs: build-and-publish-rpm + needs: [ build-and-publish-rpm, build-and-publish-deb ] runs-on: ubuntu-latest steps: - name: Download all artifacts uses: actions/download-artifact@v4 with: pattern: | - *.rpm + *.{rpm,deb} merge-multiple: true - name: Upload artifacts to release uses: softprops/action-gh-release@v2 with: files: | *.rpm + *.deb diff --git a/package/debian/build-deb.sh b/package/debian/build-deb.sh new file mode 100755 index 00000000000..d56f878769d --- /dev/null +++ b/package/debian/build-deb.sh @@ -0,0 +1,152 @@ +#!/bin/bash + +# build-deb.sh +# Use shell script to build .deb packages. +# +# Copyright (c) 2024, Alibaba Group Holding Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# IDENTIFICATION +# package/debian/build-deb.sh + +build_date=${1:-$(date +"%Y%m%d%H%M%S")} +code_commit=$(git rev-parse HEAD || echo unknown) +code_branch=$(git rev-parse --abbrev-ref HEAD || echo unknown) +polar_commit=$(git rev-parse --short=8 HEAD || echo unknown) +pg_version=$(grep AC_INIT ../../configure.in | awk -F'[][]' '{print $4}') +polar_majorversion=2.0.$(echo ${pg_version} | awk -F'[^0-9]+' '{print $1}') +polar_minorversion=$(grep -A 1 '&polar_version' ../../src/backend/utils/misc/guc.c | awk 'NR==2{print}' | awk -F'[."]' '{print $4}').0 +polar_release_date=$(grep -A 1 '&polar_release_date' ../../src/backend/utils/misc/guc.c | awk 'NR==2{print}' | awk -F'[,"]' '{print $2}') +polar_version=${polar_majorversion}.${polar_minorversion} +polar_pg_version=${pg_version}.${polar_minorversion} + +package=PolarDB +debpkgname=$(grep 'Package:' ./control | awk '{print $2}') +distname=$(grep '^ID=' /etc/os-release | awk -F'=' '{print $2}') +distversion=$(grep 'VERSION_ID=' /etc/os-release | awk -F\" '{print $2}') +arch=$(dpkg --print-architecture) + +pkgname=${package}_${polar_pg_version}-${build_date}-${distname}${distversion}_${arch} +rm -rf ./${pkgname} +mkdir -p ${pkgname}/DEBIAN + +cat ./control >> ${pkgname}/DEBIAN/control +echo 'Version: '${polar_pg_version}'-'${build_date}'' >> ${pkgname}/DEBIAN/control +echo 'Architecture: '${arch}'' >> ${pkgname}/DEBIAN/control + +prefix=/u01/polardb_pg +buildroot=$(pwd) +cd ../../ +./polardb_build.sh --basedir=${buildroot}/${pkgname}${prefix} --debug=off --with-pfsd --noinit +cd ${buildroot} + +polar_install_dependency() +{ + target_dir=${1} + + # create link for lib inside lib + cd ${target_dir}/lib/ + ln -sf ../lib ./lib + + cd ${target_dir} + + # generate list of .so files + # collect all the executable binaries, scripts and shared libraries + binfiles=`find ${target_dir}/bin` + libfiles=`find ${target_dir}/lib` + filelist=${binfiles}$'\n'${libfiles} + exelist=`echo $filelist | xargs -r file | egrep -v ":.* (commands|script)" | \ + grep ":.*executable" | cut -d: -f1` + scriptlist=`echo $filelist | xargs -r file | \ + egrep ":.* (commands|script)" | cut -d: -f1` + liblist=`echo $filelist | xargs -r file | \ + grep ":.*shared object" | cut -d : -f1` + + # dependency list of the executable binaries and shared libraries + cp /dev/null mytmpfilelist + cp /dev/null mytmpfilelist2 + + # put PolarDB-PG libs before any other libs to let ldd take it first + export LD_LIBRARY_PATH=${target_dir}/lib:$LD_LIBRARY_PATH:/usr/lib + + # dependency list of all binaries and shared objects + for f in $liblist $exelist; do + ldd $f | awk '/=>/ { + if (X$3 != "X" && $3 !~ /libNoVersion.so/ && $3 !~ /4[um]lib.so/ && $3 !~ /libredhat-kernel.so/ && $3 !~ /libselinux.so/ && $3 !~ /\/u01\/polardb_pg/ && $3 !~ /libjvm.so/ && $3 ~ /\.so/) { + # printf "$s => $s\n", $1, $3 + print $3 + } + }' >> mytmpfilelist + done + + # deduplicate + cat mytmpfilelist | sort -u > mytmpfilelist2 + + for f in `cat mytmpfilelist2`; do + ldd $f | awk '/=>/ { + if (X$3 != "X" && $3 !~ /libNoVersion.so/ && $3 !~ /4[um]lib.so/ && $3 !~ /libredhat-kernel.so/ && $3 !~ /libselinux.so/ && $3 !~ /\/u01\/polardb_pg/ && $3 !~ /libjvm.so/ && $3 ~ /\.so/) { + # printf "$s => $s\n", $1, $3 + print $3 + } + }' >> mytmpfilelist + done + + # deduplicate + cat mytmpfilelist | sort -u > mytmpfilelist2 + + # copy libraries if necessary + for line in `cat mytmpfilelist2`; do + base=`basename $line` + dirpath=${target_dir}/lib + filepath=$dirpath/$base + + objdump -p $line | awk 'BEGIN { START=0; LIBNAME=""; } + /^$/ { START=0; } + /^Dynamic Section:$/ { START=1; } + (START==1) && /NEEDED/ { + print $2 ; + } + (START==2) && /^[A-Za-z]/ { START=3; } + /^Version References:$/ { START=2; } + (START==2) && /required from/ { + sub(/:/, "", $3); + LIBNAME=$3; + } + (START==2) && (LIBNAME!="") && ($4!="") && (($4~/^GLIBC_*/) || ($4~/^GCC_*/)) { + print LIBNAME "(" $4 ")"; + } + END { exit 0 } + ' > objdumpfile + + has_private= + if grep -q PRIVATE objdumpfile; then + has_private=true + fi + + if [[ ! -f $filepath ]]; then + if [[ $has_private != "true" ]]; then + cp $line $dirpath + echo $line $dirpath + fi + fi + done + + rm mytmpfilelist mytmpfilelist2 objdumpfile +} + +# install package dependencies +polar_install_dependency $(pwd)/${pkgname}${prefix} +cd ${buildroot} + +dpkg --build ./${pkgname} diff --git a/package/debian/control b/package/debian/control new file mode 100644 index 00000000000..c4309d327aa --- /dev/null +++ b/package/debian/control @@ -0,0 +1,17 @@ +Package: polardb-for-postgresql +Maintainer: mrdrivingduck +Depends: libicu66 | libicu70 | libicu74, libkrb5-3, libldap-2.4-2 | libldap-2.5-0 | libldap2 +Section: database +Priority: optional +Homepage: https://github.com/ApsaraDB/PolarDB-for-PostgreSQL +Description: PolarDB is an advanced Object-Relational database management system + (DBMS) that supports almost all SQL constructs (including + transactions, subselects and user-defined types and functions). The + PolarDB package includes the client programs and libraries that + you'll need to access a PolarDB DBMS server. These PolarDB + client programs are programs that directly manipulate the internal + structure of PolarDB databases on a PolarDB server. These client + programs can be located on the same machine with the PolarDB + server, or may be on a remote machine which accesses a PolarDB + server over a network connection. This package contains the command-line + utilities for managing PolarDB databases on a PolarDB server. diff --git a/package/rpm/PolarDB.spec b/package/rpm/PolarDB.spec index fd3c9f43d4f..8cec7e0dce4 100644 --- a/package/rpm/PolarDB.spec +++ b/package/rpm/PolarDB.spec @@ -103,7 +103,6 @@ polar_install_dependency() cp /dev/null mytmpfilelist2 # put PolarDB-PG libs before any other libs to let ldd take it first - # Add /usr/lib for libmxml.so needed by oss_fdw extension eval PERL_PATH=`cd /usr/lib64/perl[5-9]/CORE/ ; pwd` export LD_LIBRARY_PATH=%{buildroot}${target_dir}/lib:$PERL_PATH:$LD_LIBRARY_PATH:/usr/lib diff --git a/package/rpm/build-rpm.sh b/package/rpm/build-rpm.sh index 91f74e26873..5b85bbbbec6 100755 --- a/package/rpm/build-rpm.sh +++ b/package/rpm/build-rpm.sh @@ -1,5 +1,25 @@ #!/bin/bash +# build-rpm.sh +# Use shell script to build .rpm packages. +# +# Copyright (c) 2024, Alibaba Group Holding Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# IDENTIFICATION +# package/rpm/build-rpm.sh + RPM_MACROS=$HOME/.rpmmacros TOP_DIR=`pwd`/.self_rpm_create From 11da165b1779db8823917f17f0566e650eab6374 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Thu, 25 Nov 2021 14:19:22 +0100 Subject: [PATCH 2/3] Remove unneeded Python includes Inluding and has not been necessary since Python 2.4, since they are included via . Morever, is being removed in Python 3.11. So remove these includes. Reviewed-by: Tom Lane Discussion: https://www.postgresql.org/message-id/flat/84884.1637723223%40sss.pgh.pa.us --- src/pl/plpython/plpython.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/pl/plpython/plpython.h b/src/pl/plpython/plpython.h index 5c2e6a83f30..3177b56d1c2 100644 --- a/src/pl/plpython/plpython.h +++ b/src/pl/plpython/plpython.h @@ -121,9 +121,6 @@ typedef int Py_ssize_t; #undef TEXTDOMAIN #define TEXTDOMAIN PG_TEXTDOMAIN("plpython") -#include -#include - /* put back our snprintf and vsnprintf */ #ifdef USE_REPL_SNPRINTF #ifdef snprintf From b0670b3375f63bd05ed8ae89c0f5fc4016f840ac Mon Sep 17 00:00:00 2001 From: mrdrivingduck Date: Fri, 3 May 2024 17:55:43 +0800 Subject: [PATCH 3/3] feat: support building on Rocky 9 --- .github/workflows/package.yml | 2 +- package/rpm/PolarDB.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index a0edc52b4f3..dbbb81e9992 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -24,7 +24,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - base_image: [ centos7, anolis8, rocky8 ] + base_image: [ centos7, anolis8, rocky8, rocky9 ] steps: - name: Fetch source code uses: actions/checkout@v4 diff --git a/package/rpm/PolarDB.spec b/package/rpm/PolarDB.spec index 8cec7e0dce4..4bf5b45ad33 100644 --- a/package/rpm/PolarDB.spec +++ b/package/rpm/PolarDB.spec @@ -195,4 +195,4 @@ fi %defattr(-,root,root) %{prefix}/* -%{nil} +%define __os_install_post %{nil}