From 546543ac7508cb2d52c45fca22ecb6d439f78af9 Mon Sep 17 00:00:00 2001 From: Deepak Majeti Date: Wed, 24 Jan 2024 22:45:57 -0500 Subject: [PATCH] Remove setup-circleci.sh and update setup-centos8.sh --- scripts/circleci-container.dockfile | 6 +- scripts/setup-adapters.sh | 7 +- scripts/setup-centos8.sh | 70 +++++++++++---- scripts/setup-circleci.sh | 131 ---------------------------- scripts/setup-helper-functions.sh | 10 +++ 5 files changed, 71 insertions(+), 153 deletions(-) delete mode 100755 scripts/setup-circleci.sh diff --git a/scripts/circleci-container.dockfile b/scripts/circleci-container.dockfile index b3a7ee96c4fed..628d5c4190945 100644 --- a/scripts/circleci-container.dockfile +++ b/scripts/circleci-container.dockfile @@ -15,6 +15,8 @@ # FROM quay.io/centos/centos:stream8 ARG cpu_target -COPY scripts/setup-circleci.sh / COPY scripts/setup-helper-functions.sh / -RUN mkdir build && ( cd build && CPU_TARGET="$cpu_target" bash /setup-circleci.sh ) && rm -rf build +COPY scripts/setup-centos8.sh / +COPY scripts/setup-adapters.sh / +RUN mkdir build && ( cd build && CPU_TARGET="$cpu_target" bash /setup-centos8.sh ) && rm -rf build +RUN mkdir build && ( cd build && CPU_TARGET="$cpu_target" bash /setup-adapters.sh ) && rm -rf build diff --git a/scripts/setup-adapters.sh b/scripts/setup-adapters.sh index a42c9f5480bac..0b6defb68a2d0 100755 --- a/scripts/setup-adapters.sh +++ b/scripts/setup-adapters.sh @@ -94,7 +94,7 @@ function install_azure-storage-sdk-cpp { cmake_install -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF } -function install_libhdfs3 { +function install_hdfs_deps { github_checkout apache/hawq master cd $DEPENDENCY_DIR/hawq/depends/libhdfs3 if [[ "$OSTYPE" == darwin* ]]; then @@ -105,6 +105,9 @@ function install_libhdfs3 { if [[ "$OSTYPE" == linux-gnu* ]]; then sed -i "/FIND_PACKAGE(GoogleTest REQUIRED)/d" ./CMakeLists.txt sed -i "s/dumpversion/dumpfullversion/" ./CMake/Platform.cmake + # Dependencies for Hadoop testing + wget_and_untar https://archive.apache.org/dist/hadoop/common/hadoop-2.10.1/hadoop-2.10.1.tar.gz hadoop + cp -a hadoop /usr/local/ fi cmake_install } @@ -181,7 +184,7 @@ if [ $install_aws -eq 1 ]; then install_aws-sdk-cpp fi if [ $install_hdfs -eq 1 ]; then - install_libhdfs3 + install_hdfs_deps fi if [ $install_abfs -eq 1 ]; then install_azure-storage-sdk-cpp diff --git a/scripts/setup-centos8.sh b/scripts/setup-centos8.sh index e6808097509f3..f82d2ff64606a 100755 --- a/scripts/setup-centos8.sh +++ b/scripts/setup-centos8.sh @@ -30,25 +30,25 @@ function dnf_install { dnf_install epel-release dnf-plugins-core # For ccache, ninja dnf config-manager --set-enabled powertools -dnf_install ninja-build ccache gcc-toolset-9 git wget which libevent-devel \ +dnf_install ninja-build cmake curl ccache gcc-toolset-9 git wget which libevent-devel \ openssl-devel re2-devel libzstd-devel lz4-devel double-conversion-devel \ - libdwarf-devel curl-devel cmake libicu-devel + libdwarf-devel curl-devel libicu-devel dnf remove -y gflags # Required for Thrift -dnf_install autoconf automake libtool bison flex python3 - -# Required for google-cloud-storage -dnf_install curl-devel c-ares-devel +dnf_install autoconf automake libtool bison flex python3 libsodium-devel dnf_install conda +# install sphinx for doc gen +pip3 install sphinx sphinx-tabs breathe sphinx_rtd_theme + # Activate gcc9; enable errors on unset variables afterwards. source /opt/rh/gcc-toolset-9/enable || exit 1 set -u -function cmake_install_deps { +function cmake_install { cmake -B "$1-build" -GNinja -DCMAKE_CXX_STANDARD=17 \ -DCMAKE_CXX_FLAGS="${CFLAGS}" -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_BUILD_TYPE=Release -Wno-dev "$@" ninja -C "$1-build" install @@ -58,17 +58,33 @@ function wget_and_untar { local URL=$1 local DIR=$2 mkdir -p "${DIR}" - wget -q --max-redirect 3 -O - "${URL}" | tar -xz -C "${DIR}" --strip-components=1 + pushd "${DIR}" + curl -L "${URL}" > $2.tar.gz + tar -xz --strip-components=1 -f $2.tar.gz + popd } +# untar cmake binary release directly to /usr. +wget_and_untar https://github.com/Kitware/CMake/releases/download/v3.17.5/cmake-3.17.5-Linux-x86_64.tar.gz /usr & # Fetch sources. -wget_and_untar https://github.com/gflags/gflags/archive/v2.2.2.tar.gz gflags & -wget_and_untar https://github.com/google/glog/archive/v0.4.0.tar.gz glog & -wget_and_untar http://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz lzo & -wget_and_untar https://boostorg.jfrog.io/artifactory/main/release/1.72.0/source/boost_1_72_0.tar.gz boost & -wget_and_untar https://github.com/google/snappy/archive/1.1.8.tar.gz snappy & -wget_and_untar https://github.com/fmtlib/fmt/archive/10.1.1.tar.gz fmt & +wget_and_untar https://github.com/gflags/gflags/archive/v2.2.2.tar.gz gflags +wget_and_untar https://github.com/google/glog/archive/v0.4.0.tar.gz glog +wget_and_untar http://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz lzo +wget_and_untar https://boostorg.jfrog.io/artifactory/main/release/1.72.0/source/boost_1_72_0.tar.gz boost +wget_and_untar https://github.com/google/snappy/archive/1.1.8.tar.gz snappy +wget_and_untar https://github.com/fmtlib/fmt/archive/10.1.1.tar.gz fmt + +# wget_and_untar https://github.com/ericniebler/range-v3/archive/0.11.0.tar.gz ranges-v3 +wget_and_untar https://github.com/protocolbuffers/protobuf/releases/download/v21.4/protobuf-all-21.4.tar.gz protobuf + +FB_OS_VERSION="v2023.12.04.00" + +wget_and_untar https://github.com/facebookincubator/fizz/archive/refs/tags/${FB_OS_VERSION}.tar.gz fizz +wget_and_untar https://github.com/facebook/folly/archive/refs/tags/${FB_OS_VERSION}.tar.gz folly +wget_and_untar https://github.com/facebook/wangle/archive/refs/tags/${FB_OS_VERSION}.tar.gz wangle +wget_and_untar https://github.com/facebook/fbthrift/archive/refs/tags/${FB_OS_VERSION}.tar.gz fbthrift +wget_and_untar https://github.com/facebook/mvfst/archive/refs/tags/${FB_OS_VERSION}.tar.gz mvfst wait # For cmake and source downloads to complete. @@ -86,9 +102,27 @@ wait # For cmake and source downloads to complete. ./b2 "-j$(nproc)" -d0 install threading=multi ) -cmake_install_deps gflags -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON -DBUILD_gflags_LIB=ON -DLIB_SUFFIX=64 -DCMAKE_INSTALL_PREFIX:PATH=/usr -cmake_install_deps glog -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX:PATH=/usr -cmake_install_deps snappy -DSNAPPY_BUILD_TESTS=OFF -cmake_install_deps fmt -DFMT_TEST=OFF +( + cd protobuf + ./configure --prefix=/usr + make "-j${NPROC}" + make install + ldconfig +) + +cmake_install gflags -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON -DBUILD_gflags_LIB=ON -DLIB_SUFFIX=64 -DCMAKE_INSTALL_PREFIX:PATH=/usr +cmake_install glog -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX:PATH=/usr +cmake_install snappy -DSNAPPY_BUILD_TESTS=OFF +cmake_install fmt -DFMT_TEST=OFF +cmake_install folly -DFOLLY_HAVE_INT128_T=ON +# remove in #7700 +sed -i 's/\[\[deprecated.*\]\]//g' /usr/local/include/folly/init/Init.h + + +cmake_install fizz/fizz -DBUILD_TESTS=OFF +cmake_install wangle/wangle -DBUILD_TESTS=OFF +cmake_install mvfst -DBUILD_TESTS=OFF +cmake_install fbthrift -Denable_tests=OFF +# cmake_install ranges-v3 dnf clean all diff --git a/scripts/setup-circleci.sh b/scripts/setup-circleci.sh deleted file mode 100755 index b49d21eaee95d..0000000000000 --- a/scripts/setup-circleci.sh +++ /dev/null @@ -1,131 +0,0 @@ -#!/bin/bash -# Copyright (c) Facebook, Inc. and its affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -efx -o pipefail -# Some of the packages must be build with the same compiler flags -# so that some low level types are the same size. Also, disable warnings. -SCRIPTDIR=$(dirname "${BASH_SOURCE[0]}") -source $SCRIPTDIR/setup-helper-functions.sh -CPU_TARGET="${CPU_TARGET:-avx}" -NPROC=$(getconf _NPROCESSORS_ONLN) -export CFLAGS=$(get_cxx_flags $CPU_TARGET) # Used by LZO. -export CXXFLAGS=$CFLAGS # Used by boost. -export CPPFLAGS=$CFLAGS # Used by LZO. - -function dnf_install { - dnf install -y -q --setopt=install_weak_deps=False "$@" -} - -dnf_install epel-release dnf-plugins-core # For ccache, ninja -dnf config-manager --set-enabled powertools -dnf_install ninja-build cmake curl ccache gcc-toolset-9 git wget which libevent-devel \ - openssl-devel re2-devel libzstd-devel lz4-devel double-conversion-devel \ - libdwarf-devel curl-devel libicu-devel - -dnf remove -y gflags - -# Required for Thrift -dnf_install autoconf automake libtool bison flex python3 libsodium-devel - -dnf_install conda - -# install sphinx for doc gen -pip3 install sphinx sphinx-tabs breathe sphinx_rtd_theme - -# Activate gcc9; enable errors on unset variables afterwards. -source /opt/rh/gcc-toolset-9/enable || exit 1 -set -u - -function cmake_install { - cmake -B "$1-build" -GNinja -DCMAKE_CXX_STANDARD=17 \ - -DCMAKE_CXX_FLAGS="${CFLAGS}" -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_BUILD_TYPE=Release -Wno-dev "$@" - ninja -C "$1-build" install -} - -function wget_and_untar { - local URL=$1 - local DIR=$2 - mkdir -p "${DIR}" - pushd "${DIR}" - curl -L "${URL}" > $2.tar.gz - tar -xz --strip-components=1 -f $2.tar.gz - popd -} - -# untar cmake binary release directly to /usr. -wget_and_untar https://github.com/Kitware/CMake/releases/download/v3.17.5/cmake-3.17.5-Linux-x86_64.tar.gz /usr & - -# Fetch sources. -wget_and_untar https://github.com/gflags/gflags/archive/v2.2.2.tar.gz gflags -wget_and_untar https://github.com/google/glog/archive/v0.4.0.tar.gz glog -wget_and_untar http://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz lzo -wget_and_untar https://boostorg.jfrog.io/artifactory/main/release/1.72.0/source/boost_1_72_0.tar.gz boost -wget_and_untar https://github.com/google/snappy/archive/1.1.8.tar.gz snappy -wget_and_untar https://github.com/fmtlib/fmt/archive/10.1.1.tar.gz fmt - -# wget_and_untar https://github.com/ericniebler/range-v3/archive/0.11.0.tar.gz ranges-v3 -wget_and_untar https://archive.apache.org/dist/hadoop/common/hadoop-2.10.1/hadoop-2.10.1.tar.gz hadoop -wget_and_untar https://github.com/protocolbuffers/protobuf/releases/download/v21.4/protobuf-all-21.4.tar.gz protobuf - -FB_OS_VERSION="v2023.12.04.00" - -wget_and_untar https://github.com/facebookincubator/fizz/archive/refs/tags/${FB_OS_VERSION}.tar.gz fizz -wget_and_untar https://github.com/facebook/folly/archive/refs/tags/${FB_OS_VERSION}.tar.gz folly -wget_and_untar https://github.com/facebook/wangle/archive/refs/tags/${FB_OS_VERSION}.tar.gz wangle -wget_and_untar https://github.com/facebook/fbthrift/archive/refs/tags/${FB_OS_VERSION}.tar.gz fbthrift -wget_and_untar https://github.com/facebook/mvfst/archive/refs/tags/${FB_OS_VERSION}.tar.gz mvfst - -wait # For cmake and source downloads to complete. - -cp -a hadoop /usr/local/ - -# Build & install. -( - cd lzo - ./configure --prefix=/usr --enable-shared --disable-static --docdir=/usr/share/doc/lzo-2.10 - make "-j$(nproc)" - make install -) - -( - cd boost - ./bootstrap.sh --prefix=/usr/local - ./b2 "-j$(nproc)" -d0 install threading=multi -) - -( - cd protobuf - ./configure --prefix=/usr - make "-j${NPROC}" - make install - ldconfig -) - -cmake_install gflags -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON -DBUILD_gflags_LIB=ON -DLIB_SUFFIX=64 -DCMAKE_INSTALL_PREFIX:PATH=/usr -cmake_install glog -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX:PATH=/usr -cmake_install snappy -DSNAPPY_BUILD_TESTS=OFF -cmake_install fmt -DFMT_TEST=OFF -cmake_install folly -DFOLLY_HAVE_INT128_T=ON -# remove in #7700 -sed -i 's/\[\[deprecated.*\]\]//g' /usr/local/include/folly/init/Init.h - - -cmake_install fizz/fizz -DBUILD_TESTS=OFF -cmake_install wangle/wangle -DBUILD_TESTS=OFF -cmake_install mvfst -DBUILD_TESTS=OFF -cmake_install fbthrift -Denable_tests=OFF -# cmake_install ranges-v3 - -dnf clean all diff --git a/scripts/setup-helper-functions.sh b/scripts/setup-helper-functions.sh index 8076edc4a1850..9495d42bfc367 100644 --- a/scripts/setup-helper-functions.sh +++ b/scripts/setup-helper-functions.sh @@ -119,6 +119,16 @@ function get_cxx_flags { } +function wget_and_untar { + local URL=$1 + local DIR=$2 + mkdir -p "${DIR}" + pushd "${DIR}" + curl -L "${URL}" > $2.tar.gz + tar -xz --strip-components=1 -f $2.tar.gz + popd +} + function cmake_install { local NAME=$(basename "$(pwd)") local BINARY_DIR=_build