fix(github): fix redundant hadoop-bin while uploading artifact

.github/actions/rebuild_thirdparty_if_needed/action.yaml

@@ -46,4 +46,6 @@ runs:
         ../admin_tools/download_zk.sh zookeeper-bin
         rm -rf hadoop-bin/share/doc
         rm -rf zookeeper-bin/docs
+        mv hadoop-bin ..
+        mv zookeeper-bin ..
       shell: bash

.github/actions/unpack_prebuilt_thirdparties/action.yaml

@@ -27,4 +27,6 @@ runs:
         find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
         rm -rf ./thirdparty/hadoop-bin/share/doc
         rm -rf ./thirdparty/zookeeper-bin/docs
+        mv thirdparty/hadoop-bin ./
+        mv thirdparty/zookeeper-bin ./
       shell: bash

.github/actions/upload_artifact/action.yaml

@@ -21,8 +21,6 @@ runs:
   steps:
     - name: Tar files
       run: |
-        mv thirdparty/hadoop-bin ./
-        mv thirdparty/zookeeper-bin ./
         rm -rf thirdparty
         # The following operations are tricky, these directories and files don't exist if not build with '--test'.
         # When build binaries for client tests, it's not needed to add '--test'.

.github/workflows/lint_and_test_cpp.yaml

@@ -27,7 +27,7 @@ on:
       - ci-test # testing branch for github action
       - '*dev'
     paths:
-      - .github/actions
+      - .github/actions/**
       - .github/workflows/lint_and_test_cpp.yaml
       - .github/workflows/thirdparty-regular-push.yml
       - build_tools/pack_server.sh

.github/workflows/thirdparty-regular-push.yml

@@ -42,6 +42,15 @@ on:
 jobs:
   build_push_src_docker_images:
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        osversion:
+          - ubuntu1804
+          - ubuntu2004
+          - ubuntu2204
+          # TODO(wangdan): disable centos7 temporarily before image build-env-centos7-* is fixed.
+          # - centos7
     steps:
       - uses: actions/checkout@v4
       - name: Set up QEMU
@@ -60,11 +69,12 @@ jobs:
           file: ./docker/thirdparties-src/Dockerfile
           push: true
           tags: |
-            apache/pegasus:thirdparties-src-${{ github.ref_name }}
+            apache/pegasus:thirdparties-src-${{ matrix.osversion }}-${{ github.ref_name }}
           build-args: |
             GITHUB_BRANCH=${{ github.ref_name }}
-      - name: Image digest
-        run: echo ${{ steps.docker_build.outputs.digest }}
+            OS_VERSION=${{ matrix.osversion }}
+            HADOOP_BIN_PATH=hadoop-bin
+            ZOOKEEPER_BIN_PATH=zookeeper-bin
 
   build_push_bin_docker_images:
     runs-on: ubuntu-latest
@@ -82,7 +92,8 @@ jobs:
           - ubuntu1804
           - ubuntu2004
           - ubuntu2204
-          - centos7
+          # TODO(wangdan): disable centos7 temporarily before image build-env-centos7-* is fixed.
+          # - centos7
     steps:
       # The glibc version on ubuntu1804 and centos7 is lower than the actions/checkout@v4 required, so
       # we need to force to use actions/checkout@v3.
@@ -107,8 +118,8 @@ jobs:
           build-args: |
             GITHUB_BRANCH=${{ github.ref_name }}
             OS_VERSION=${{ matrix.osversion }}
-      - name: Image digest
-        run: echo ${{ steps.docker_build.outputs.digest }}
+            HADOOP_BIN_PATH=hadoop-bin
+            ZOOKEEPER_BIN_PATH=zookeeper-bin
 
   build_push_bin_jemalloc_docker_images:
     runs-on: ubuntu-latest
@@ -126,7 +137,8 @@ jobs:
           - ubuntu1804
           - ubuntu2004
           - ubuntu2204
-          - centos7
+          # TODO(wangdan): disable centos7 temporarily before image build-env-centos7-* is fixed.
+          # - centos7
     steps:
       # The glibc version on ubuntu1804 and centos7 is lower than the actions/checkout@v4 required, so
       # we need to force to use actions/checkout@v3.
@@ -152,8 +164,8 @@ jobs:
             GITHUB_BRANCH=${{ github.ref_name }}
             OS_VERSION=${{ matrix.osversion }}
             USE_JEMALLOC=ON
-      - name: Image digest
-        run: echo ${{ steps.docker_build.outputs.digest }}
+            HADOOP_BIN_PATH=hadoop-bin
+            ZOOKEEPER_BIN_PATH=zookeeper-bin
 
   build_push_bin_test_docker_images:
     runs-on: ubuntu-latest
@@ -188,8 +200,6 @@ jobs:
             ROCKSDB_PORTABLE=1
             HADOOP_BIN_PATH=hadoop-bin
             ZOOKEEPER_BIN_PATH=zookeeper-bin
-      - name: Image digest
-        run: echo ${{ steps.docker_build.outputs.digest }}
 
   build_push_bin_test_jemalloc_docker_images:
     runs-on: ubuntu-latest
@@ -225,5 +235,3 @@ jobs:
             USE_JEMALLOC=ON
             HADOOP_BIN_PATH=hadoop-bin
             ZOOKEEPER_BIN_PATH=zookeeper-bin
-      - name: Image digest
-        run: echo ${{ steps.docker_build.outputs.digest }}

admin_tools/download_hadoop.sh

@@ -21,15 +21,19 @@ set -e
 
 CWD=$(cd "$(dirname "$0")" && pwd)
 
-if [ $# -ge 1 ]; then
-    HADOOP_BIN_PATH=$1
+if [ $# -lt 1 ]; then
+    echo "Invalid arguments !"
+    echo "USAGE: $0 <HADOOP_BIN_PATH>"
+    exit 1
 fi
 
+HADOOP_BIN_PATH=$1
+
 HADOOP_VERSION="hadoop-3.3.6"
 arch_output=$(arch)
 if [ "$arch_output"x == "aarch64"x ]; then
-  HADOOP_PACKAGE_MD5="369f899194a920e0d1c3c3bc1718b3b5"
-  HADOOP_BASE_NAME=${HADOOP_VERSION}-"$(arch)"
+    HADOOP_PACKAGE_MD5="369f899194a920e0d1c3c3bc1718b3b5"
+    HADOOP_BASE_NAME=${HADOOP_VERSION}-"$(arch)"
 else
     if [ "$arch_output"x != "x86_64"x ]; then
         echo "WARNING: unrecognized CPU architecture '$arch_output', use 'x86_64' as default"

admin_tools/download_package.sh

@@ -19,20 +19,15 @@
 
 set -e
 
-if [ $# -lt 2 ]; then
+if [ $# -lt 3 ]; then
     echo "Invalid arguments !"
-    echo "USAGE: $0 <PACKAGE_BASE_NAME> <PACKAGE_MD5> [TARGET_PATH]"
+    echo "USAGE: $0 <PACKAGE_BASE_NAME> <PACKAGE_MD5> <TARGET_PATH>"
     exit 1
 fi
 
 PACKAGE_BASE_NAME=$1
 PACKAGE_MD5=$2
 
-if [ $# -lt 3 ]; then
-    echo "TARGET_PATH is not provided, thus do not try to download ${PACKAGE_BASE_NAME}"
-    exit 0
-fi
-
 TARGET_PATH=$3
 if [ -d "${TARGET_PATH}" ]; then
     echo "TARGET_PATH ${TARGET_PATH} has existed, thus do not try to download ${PACKAGE_BASE_NAME}"

admin_tools/download_zk.sh

@@ -21,10 +21,14 @@ set -e
 
 CWD=$(cd $(dirname $0) && pwd)
 
-if [ $# -ge 1 ]; then
-    ZOOKEEPER_BIN_PATH=$1
+if [ $# -lt 1 ]; then
+    echo "Invalid arguments !"
+    echo "USAGE: $0 <ZOOKEEPER_BIN_PATH>"
+    exit 1
 fi
 
+ZOOKEEPER_BIN_PATH=$1
+
 ZOOKEEPER_VERSION=3.7.0
 ZOOKEEPER_DIR_NAME=apache-zookeeper-${ZOOKEEPER_VERSION}-bin
 ZOOKEEPER_PACKAGE_MD5="8ffa97e7e6b0b2cf1d022e5156a7561a"

docker/README.md

@@ -51,7 +51,10 @@ It packages the downloaded sources into a zip in the container, so that
 other repos can easily extract third-parties from the container (via `docker cp`),
 without downloading from the cloud object storage.
 
-- `apache/pegasus:thirdparties-src-<branch>`
+- `apache/pegasus:thirdparties-src-centos7-<branch>`
+- `apache/pegasus:thirdparties-src-ubuntu1804-<branch>`
+- `apache/pegasus:thirdparties-src-ubuntu2004-<branch>`
+- `apache/pegasus:thirdparties-src-ubuntu2204-<branch>`
 
 ## thirdparties-bin
 

docker/pegasus-build-env/centos7/Dockerfile

@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-FROM centos:7.5.1804
+FROM centos:7.9.2009
 
 LABEL maintainer=wutao
 
@@ -24,12 +24,14 @@ RUN sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo && \
     sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo && \
     yum -y install centos-release-scl \
                    scl-utils \
-                   epel-release && \
+                   epel-release --exclude=aarch64 && \
     sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo && \
     sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo && \
     sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo && \
-    yum -y install devtoolset-7-gcc \
-                   devtoolset-7-gcc-c++ \
+    #yum --disablerepo=centos-sclo-rh --enablerepo=centos-sclo-rh-x86_64 --disablerepo=centos-sclo-sclo --enablerepo=centos-sclo-sclo-x86_64 -y install \
+    yum --skip-broken -y install \
+                   devtoolset-7-gcc.x86_64 \
+                   devtoolset-7-gcc-c++.x86_64 \
                    java-1.8.0-openjdk-devel.x86_64 \
                    python3 \
                    automake \
@@ -54,8 +56,8 @@ RUN sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo && \
                    flex \
                    krb5-devel \
                    cyrus-sasl-devel \
-                   patch && \
-    yum -y install ca-certificates && \
+                   patch --exclude=aarch64 && \
+    yum --disablerepo=centos-sclo-rh --disablerepo=centos-sclo-sclo -y install ca-certificates && \
     yum clean all && \
     rm -rf /var/cache/yum;
 

docker/thirdparties-bin/Dockerfile

@@ -17,7 +17,7 @@
 
 ARG GITHUB_BRANCH=master
 ARG OS_VERSION=centos7
-FROM apache/pegasus:thirdparties-src-${GITHUB_BRANCH} as builder
+FROM apache/pegasus:thirdparties-src-${OS_VERSION}-${GITHUB_BRANCH} as builder
 FROM apache/pegasus:build-env-${OS_VERSION}-${GITHUB_BRANCH}
 
 WORKDIR /root
@@ -28,8 +28,8 @@ ARG GITHUB_BRANCH
 ARG GITHUB_REPOSITORY_URL=https://github.com/apache/incubator-pegasus.git
 ARG ROCKSDB_PORTABLE=native
 ARG USE_JEMALLOC=OFF
-ARG HADOOP_BIN_PATH
-ARG ZOOKEEPER_BIN_PATH
+ARG HADOOP_BIN_PATH=hadoop-bin
+ARG ZOOKEEPER_BIN_PATH=zookeeper-bin
 RUN git clone --depth=1 --branch=${GITHUB_BRANCH} ${GITHUB_REPOSITORY_URL} \
     && cd incubator-pegasus/thirdparty \
     && unzip /root/thirdparties-src.zip -d . \

docker/thirdparties-src/Dockerfile

@@ -16,7 +16,8 @@
 # under the License.
 
 ARG GITHUB_BRANCH=master
-FROM apache/pegasus:build-env-centos7-${GITHUB_BRANCH} as builder
+ARG OS_VERSION=centos7
+FROM apache/pegasus:build-env-${OS_VERSION}-${GITHUB_BRANCH} as builder
 
 WORKDIR /root