Remove setup-circleci.sh and update setup-centos8.sh

scripts/circleci-container.dockfile

@@ -15,6 +15,8 @@
 #
 FROM quay.io/centos/centos:stream8
 ARG cpu_target
-COPY scripts/setup-circleci.sh /
 COPY scripts/setup-helper-functions.sh /
-RUN mkdir build && ( cd build && CPU_TARGET="$cpu_target" bash /setup-circleci.sh ) && rm -rf build
+COPY scripts/setup-centos8.sh /
+COPY scripts/setup-adapters.sh /
+RUN mkdir build && ( cd build && CPU_TARGET="$cpu_target" bash /setup-centos8.sh ) && rm -rf build
+RUN mkdir build && ( cd build && CPU_TARGET="$cpu_target" bash /setup-adapters.sh ) && rm -rf build

scripts/setup-adapters.sh

@@ -94,7 +94,7 @@ function install_azure-storage-sdk-cpp {
   cmake_install -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF
 }
 
-function install_libhdfs3 {
+function install_hdfs_deps {
   github_checkout apache/hawq master
   cd $DEPENDENCY_DIR/hawq/depends/libhdfs3
   if [[ "$OSTYPE" == darwin* ]]; then
@@ -105,6 +105,9 @@ function install_libhdfs3 {
   if [[ "$OSTYPE" == linux-gnu* ]]; then
     sed -i "/FIND_PACKAGE(GoogleTest REQUIRED)/d" ./CMakeLists.txt
     sed -i "s/dumpversion/dumpfullversion/" ./CMake/Platform.cmake
+    # Dependencies for Hadoop testing
+    wget_and_untar https://archive.apache.org/dist/hadoop/common/hadoop-2.10.1/hadoop-2.10.1.tar.gz hadoop
+    cp -a hadoop /usr/local/
   fi
   cmake_install
 }
@@ -181,7 +184,7 @@ if [ $install_aws -eq 1 ]; then
   install_aws-sdk-cpp
 fi
 if [ $install_hdfs -eq 1 ]; then
-  install_libhdfs3
+  install_hdfs_deps
 fi
 if [ $install_abfs -eq 1 ]; then
   install_azure-storage-sdk-cpp

scripts/setup-centos8.sh

@@ -23,52 +23,69 @@ NPROC=$(getconf _NPROCESSORS_ONLN)
 export CFLAGS=$(get_cxx_flags $CPU_TARGET)  # Used by LZO.
 export CXXFLAGS=$CFLAGS  # Used by boost.
 export CPPFLAGS=$CFLAGS  # Used by LZO.
+CMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE:-Release}"
 
 function dnf_install {
   dnf install -y -q --setopt=install_weak_deps=False "$@"
 }
 
 dnf_install epel-release dnf-plugins-core # For ccache, ninja
 dnf config-manager --set-enabled powertools
-dnf_install ninja-build ccache gcc-toolset-9 git wget which libevent-devel \
+dnf_install ninja-build cmake curl ccache gcc-toolset-9 git wget which libevent-devel \
   openssl-devel re2-devel libzstd-devel lz4-devel double-conversion-devel \
-  libdwarf-devel curl-devel cmake libicu-devel
+  libdwarf-devel curl-devel libicu-devel
 
 dnf remove -y gflags
 
 # Required for Thrift
-dnf_install autoconf automake libtool bison flex python3
-
-# Required for google-cloud-storage
-dnf_install curl-devel c-ares-devel
+dnf_install autoconf automake libtool bison flex python3 libsodium-devel
 
 dnf_install conda
 
+# install sphinx for doc gen
+pip3 install sphinx sphinx-tabs breathe sphinx_rtd_theme
+
 # Activate gcc9; enable errors on unset variables afterwards.
 source /opt/rh/gcc-toolset-9/enable || exit 1
 set -u
 
-function cmake_install_deps {
+function cmake_install {
   cmake -B "$1-build" -GNinja -DCMAKE_CXX_STANDARD=17 \
-    -DCMAKE_CXX_FLAGS="${CFLAGS}" -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_BUILD_TYPE=Release -Wno-dev "$@"
+    -DCMAKE_CXX_FLAGS="${CFLAGS}" -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" -Wno-dev "$@"
   ninja -C "$1-build" install
 }
 
 function wget_and_untar {
   local URL=$1
   local DIR=$2
   mkdir -p "${DIR}"
-  wget -q --max-redirect 3 -O - "${URL}" | tar -xz -C "${DIR}" --strip-components=1
+  pushd "${DIR}"
+  curl -L "${URL}" > $2.tar.gz
+  tar -xz --strip-components=1 -f $2.tar.gz
+  popd
 }
 
+# untar cmake binary release directly to /usr.
+wget_and_untar https://github.com/Kitware/CMake/releases/download/v3.17.5/cmake-3.17.5-Linux-x86_64.tar.gz /usr &
 
 # Fetch sources.
-wget_and_untar https://github.com/gflags/gflags/archive/v2.2.2.tar.gz gflags &
-wget_and_untar https://github.com/google/glog/archive/v0.4.0.tar.gz glog &
-wget_and_untar http://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz lzo &
-wget_and_untar https://boostorg.jfrog.io/artifactory/main/release/1.72.0/source/boost_1_72_0.tar.gz boost &
-wget_and_untar https://github.com/google/snappy/archive/1.1.8.tar.gz snappy &
-wget_and_untar https://github.com/fmtlib/fmt/archive/10.1.1.tar.gz fmt &
+wget_and_untar https://github.com/gflags/gflags/archive/v2.2.2.tar.gz gflags
+wget_and_untar https://github.com/google/glog/archive/v0.4.0.tar.gz glog
+wget_and_untar http://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz lzo
+wget_and_untar https://boostorg.jfrog.io/artifactory/main/release/1.72.0/source/boost_1_72_0.tar.gz boost
+wget_and_untar https://github.com/google/snappy/archive/1.1.8.tar.gz snappy
+wget_and_untar https://github.com/fmtlib/fmt/archive/10.1.1.tar.gz fmt
+
+#  wget_and_untar https://github.com/ericniebler/range-v3/archive/0.11.0.tar.gz ranges-v3
+wget_and_untar https://github.com/protocolbuffers/protobuf/releases/download/v21.4/protobuf-all-21.4.tar.gz protobuf
+
+FB_OS_VERSION="v2023.12.04.00"
+
+wget_and_untar https://github.com/facebookincubator/fizz/archive/refs/tags/${FB_OS_VERSION}.tar.gz fizz
+wget_and_untar https://github.com/facebook/folly/archive/refs/tags/${FB_OS_VERSION}.tar.gz folly
+wget_and_untar https://github.com/facebook/wangle/archive/refs/tags/${FB_OS_VERSION}.tar.gz wangle
+wget_and_untar https://github.com/facebook/fbthrift/archive/refs/tags/${FB_OS_VERSION}.tar.gz fbthrift
+wget_and_untar https://github.com/facebook/mvfst/archive/refs/tags/${FB_OS_VERSION}.tar.gz mvfst
 
 wait  # For cmake and source downloads to complete.
 
@@ -86,9 +103,27 @@ wait  # For cmake and source downloads to complete.
   ./b2 "-j$(nproc)" -d0 install threading=multi
 )
 
-cmake_install_deps gflags -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON -DBUILD_gflags_LIB=ON -DLIB_SUFFIX=64 -DCMAKE_INSTALL_PREFIX:PATH=/usr
-cmake_install_deps glog -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX:PATH=/usr
-cmake_install_deps snappy -DSNAPPY_BUILD_TESTS=OFF
-cmake_install_deps fmt -DFMT_TEST=OFF
+(
+  cd protobuf
+  ./configure --prefix=/usr
+  make "-j${NPROC}"
+  make install
+  ldconfig
+)
+
+cmake_install gflags -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON -DBUILD_gflags_LIB=ON -DLIB_SUFFIX=64 -DCMAKE_INSTALL_PREFIX:PATH=/usr
+cmake_install glog -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX:PATH=/usr
+cmake_install snappy -DSNAPPY_BUILD_TESTS=OFF
+cmake_install fmt -DFMT_TEST=OFF
+cmake_install folly -DFOLLY_HAVE_INT128_T=ON
+# remove in #7700
+sed -i 's/\[\[deprecated.*\]\]//g' /usr/local/include/folly/init/Init.h
+
+
+cmake_install fizz/fizz -DBUILD_TESTS=OFF
+cmake_install wangle/wangle -DBUILD_TESTS=OFF
+cmake_install mvfst -DBUILD_TESTS=OFF
+cmake_install fbthrift -Denable_tests=OFF
+# cmake_install ranges-v3
 
 dnf clean all

scripts/setup-helper-functions.sh

@@ -119,6 +119,16 @@ function get_cxx_flags {
 
 }
 
+function wget_and_untar {
+  local URL=$1
+  local DIR=$2
+  mkdir -p "${DIR}"
+  pushd "${DIR}"
+  curl -L "${URL}" > $2.tar.gz
+  tar -xz --strip-components=1 -f $2.tar.gz
+  popd
+}
+
 function cmake_install {
   local NAME=$(basename "$(pwd)")
   local BINARY_DIR=_build