From 1623b5ecfae60779648057e3616966166a85e036 Mon Sep 17 00:00:00 2001 From: "joey.ljy" Date: Thu, 21 Mar 2024 20:20:22 +0800 Subject: [PATCH 1/7] Download source code archive directly for ubuntu and macos setup --- scripts/setup-centos9.sh | 15 +++++----- scripts/setup-macos.sh | 65 ++++++++++++++++++++++++++++------------ scripts/setup-ubuntu.sh | 63 +++++++++++++++++++++++++------------- 3 files changed, 96 insertions(+), 47 deletions(-) diff --git a/scripts/setup-centos9.sh b/scripts/setup-centos9.sh index 81a9f4f23497..57ab15460f18 100755 --- a/scripts/setup-centos9.sh +++ b/scripts/setup-centos9.sh @@ -39,6 +39,11 @@ BUILD_DUCKDB="${BUILD_DUCKDB:-true}" export CC=/opt/rh/gcc-toolset-12/root/bin/gcc export CXX=/opt/rh/gcc-toolset-12/root/bin/g++ +FB_OS_VERSION="v2024.05.20.00" +FMT_VERSION="10.1.1" +BOOST_VERSION="boost-1.84.0" +ARROW_VERSION="15.0.0" + function dnf_install { dnf install -y -q --setopt=install_weak_deps=False "$@" } @@ -98,7 +103,7 @@ function install_lzo { } function install_boost { - wget_and_untar https://github.com/boostorg/boost/releases/download/boost-1.84.0/boost-1.84.0.tar.gz boost + wget_and_untar https://github.com/boostorg/boost/releases/download/${BOOST_VERSION}/${BOOST_VERSION}.tar.gz boost ( cd boost ./bootstrap.sh --prefix=/usr/local @@ -115,7 +120,7 @@ function install_snappy { } function install_fmt { - wget_and_untar https://github.com/fmtlib/fmt/archive/10.1.1.tar.gz fmt + wget_and_untar https://github.com/fmtlib/fmt/archive/${FMT_VERSION}.tar.gz fmt ( cd fmt cmake_install -DFMT_TEST=OFF @@ -133,8 +138,6 @@ function install_protobuf { ) } -FB_OS_VERSION="v2024.05.20.00" - function install_fizz { wget_and_untar https://github.com/facebookincubator/fizz/archive/refs/tags/${FB_OS_VERSION}.tar.gz fizz ( @@ -147,7 +150,7 @@ function install_folly { wget_and_untar https://github.com/facebook/folly/archive/refs/tags/${FB_OS_VERSION}.tar.gz folly ( cd folly - cmake_install -DFOLLY_HAVE_INT128_T=ON + cmake_install -DBUILD_TESTS=OFF -DFOLLY_HAVE_INT128_T=ON ) } @@ -186,8 +189,6 @@ function install_duckdb { fi } -ARROW_VERSION=15.0.0 - function install_arrow { wget_and_untar https://archive.apache.org/dist/arrow/arrow-${ARROW_VERSION}/apache-arrow-${ARROW_VERSION}.tar.gz arrow ( diff --git a/scripts/setup-macos.sh b/scripts/setup-macos.sh index 2aaa67ff437c..e30fab9c8b96 100755 --- a/scripts/setup-macos.sh +++ b/scripts/setup-macos.sh @@ -37,6 +37,7 @@ DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)} MACOS_VELOX_DEPS="flex bison protobuf@21 icu4c boost gflags glog libevent lz4 lzo snappy xz zstd openssl libsodium" MACOS_BUILD_DEPS="ninja cmake ccache" FB_OS_VERSION="v2024.05.20.00" +FMT_VERSION="10.1.1" function update_brew { DEFAULT_BREW_PATH=/usr/local/bin/brew @@ -81,49 +82,75 @@ function install_velox_deps_from_brew { } function install_fmt { - github_checkout fmtlib/fmt 10.1.1 - cmake_install -DFMT_TEST=OFF + wget_and_untar https://github.com/fmtlib/fmt/archive/${FMT_VERSION}.tar.gz fmt + ( + cd fmt + cmake_install -DFMT_TEST=OFF + ) } function install_folly { - github_checkout facebook/folly "${FB_OS_VERSION}" - cmake_install -DBUILD_TESTS=OFF -DFOLLY_HAVE_INT128_T=ON + wget_and_untar https://github.com/facebook/folly/archive/refs/tags/${FB_OS_VERSION}.tar.gz folly + ( + cd folly + cmake_install -DBUILD_TESTS=OFF -DFOLLY_HAVE_INT128_T=ON + ) } function install_fizz { - github_checkout facebookincubator/fizz "${FB_OS_VERSION}" - cmake_install -DBUILD_TESTS=OFF -S fizz + wget_and_untar https://github.com/facebookincubator/fizz/archive/refs/tags/${FB_OS_VERSION}.tar.gz fizz + ( + cd fizz/fizz + cmake_install -DBUILD_TESTS=OFF -S fizz + ) } function install_wangle { - github_checkout facebook/wangle "${FB_OS_VERSION}" - cmake_install -DBUILD_TESTS=OFF -S wangle + wget_and_untar https://github.com/facebook/wangle/archive/refs/tags/${FB_OS_VERSION}.tar.gz wangle + ( + cd wangle/wangle + cmake_install -DBUILD_TESTS=OFF -S wangle + ) } function install_mvfst { - github_checkout facebook/mvfst "${FB_OS_VERSION}" - cmake_install -DBUILD_TESTS=OFF + wget_and_untar https://github.com/facebook/mvfst/archive/refs/tags/${FB_OS_VERSION}.tar.gz mvfst + ( + cd mvfst + cmake_install -DBUILD_TESTS=OFF + ) } - function install_fbthrift { - github_checkout facebook/fbthrift "${FB_OS_VERSION}" - cmake_install -Denable_tests=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF + wget_and_untar https://github.com/facebook/fbthrift/archive/refs/tags/${FB_OS_VERSION}.tar.gz fbthrift + ( + cd fbthrift + cmake_install -Denable_tests=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF + ) } function install_double_conversion { - github_checkout google/double-conversion v3.1.5 - cmake_install -DBUILD_TESTING=OFF + wget_and_untar https://github.com/google/double-conversion/archive/refs/tags/v3.1.5.tar.gz double-conversion + ( + cd double-conversion + cmake_install -DBUILD_TESTING=OFF + ) } function install_ranges_v3 { - github_checkout ericniebler/range-v3 0.12.0 - cmake_install -DRANGES_ENABLE_WERROR=OFF -DRANGE_V3_TESTS=OFF -DRANGE_V3_EXAMPLES=OFF + wget_and_untar https://github.com/ericniebler/range-v3/archive/refs/tags/0.12.0.tar.gz ranges_v3 + ( + cd ranges_v3 + cmake_install -DRANGES_ENABLE_WERROR=OFF -DRANGE_V3_TESTS=OFF -DRANGE_V3_EXAMPLES=OFF + ) } function install_re2 { - github_checkout google/re2 2022-02-01 - cmake_install -DRE2_BUILD_TESTING=OFF + wget_and_untar https://github.com/google/re2/archive/refs/tags/2022-02-01.tar.gz re2 + ( + cd re2 + cmake_install -DRE2_BUILD_TESTING=OFF + ) } function install_velox_deps { diff --git a/scripts/setup-ubuntu.sh b/scripts/setup-ubuntu.sh index 8c39f79eaa75..1f14a9f54e85 100755 --- a/scripts/setup-ubuntu.sh +++ b/scripts/setup-ubuntu.sh @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# This script documents setting up a Centos8 host for Velox +# This script documents setting up a Ubuntu host for Velox # development. Running it should make you ready to compile. # # Environment variables: @@ -34,15 +34,17 @@ source $SCRIPTDIR/setup-helper-functions.sh # are the same size. COMPILER_FLAGS=$(get_cxx_flags) export COMPILER_FLAGS -FB_OS_VERSION=v2024.05.20.00 -FMT_VERSION=10.1.1 -BOOST_VERSION=boost-1.84.0 NPROC=$(getconf _NPROCESSORS_ONLN) DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)} BUILD_DUCKDB="${BUILD_DUCKDB:-true}" export CMAKE_BUILD_TYPE=Release SUDO="${SUDO:-"sudo --preserve-env"}" +FB_OS_VERSION="v2024.05.20.00" +FMT_VERSION="10.1.1" +BOOST_VERSION="boost-1.84.0" +ARROW_VERSION="15.0.0" + # Install packages required for build. function install_build_prerequisites { ${SUDO} apt update @@ -92,39 +94,60 @@ function install_velox_deps_from_apt { } function install_fmt { - github_checkout fmtlib/fmt "${FMT_VERSION}" - cmake_install -DFMT_TEST=OFF + wget_and_untar https://github.com/fmtlib/fmt/archive/${FMT_VERSION}.tar.gz fmt + ( + cd fmt + cmake_install -DFMT_TEST=OFF + ) } function install_boost { - github_checkout boostorg/boost "${BOOST_VERSION}" --recursive - ./bootstrap.sh --prefix=/usr/local - ${SUDO} ./b2 "-j$(nproc)" -d0 install threading=multi --without-python + wget_and_untar https://github.com/boostorg/boost/releases/download/${BOOST_VERSION}/${BOOST_VERSION}.tar.gz boost + ( + cd boost + ./bootstrap.sh --prefix=/usr/local + ${SUDO} ./b2 "-j$(nproc)" -d0 install threading=multi --without-python + ) } function install_folly { - github_checkout facebook/folly "${FB_OS_VERSION}" - cmake_install -DBUILD_TESTS=OFF -DFOLLY_HAVE_INT128_T=ON + wget_and_untar https://github.com/facebook/folly/archive/refs/tags/${FB_OS_VERSION}.tar.gz folly + ( + cd folly + cmake_install -DBUILD_TESTS=OFF -DFOLLY_HAVE_INT128_T=ON + ) } function install_fizz { - github_checkout facebookincubator/fizz "${FB_OS_VERSION}" - cmake_install -DBUILD_TESTS=OFF -S fizz + wget_and_untar https://github.com/facebookincubator/fizz/archive/refs/tags/${FB_OS_VERSION}.tar.gz fizz + ( + cd fizz/fizz + cmake_install -DBUILD_TESTS=OFF -S fizz + ) } function install_wangle { - github_checkout facebook/wangle "${FB_OS_VERSION}" - cmake_install -DBUILD_TESTS=OFF -S wangle + wget_and_untar https://github.com/facebook/wangle/archive/refs/tags/${FB_OS_VERSION}.tar.gz wangle + ( + cd wangle/wangle + cmake_install -DBUILD_TESTS=OFF -S wangle + ) } function install_mvfst { - github_checkout facebook/mvfst "${FB_OS_VERSION}" - cmake_install -DBUILD_TESTS=OFF + wget_and_untar https://github.com/facebook/mvfst/archive/refs/tags/${FB_OS_VERSION}.tar.gz mvfst + ( + cd mvfst + cmake_install -DBUILD_TESTS=OFF + ) } function install_fbthrift { - github_checkout facebook/fbthrift "${FB_OS_VERSION}" - cmake_install -Denable_tests=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF + wget_and_untar https://github.com/facebook/fbthrift/archive/refs/tags/${FB_OS_VERSION}.tar.gz fbthrift + ( + cd fbthrift + cmake_install -Denable_tests=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF + ) } function install_conda { @@ -155,8 +178,6 @@ function install_duckdb { fi } -ARROW_VERSION=15.0.0 - function install_arrow { wget_and_untar https://archive.apache.org/dist/arrow/arrow-${ARROW_VERSION}/apache-arrow-${ARROW_VERSION}.tar.gz arrow ( From 378e953bcd278b0d6d4f94b2dbe526ca60a18b07 Mon Sep 17 00:00:00 2001 From: "joey.ljy" Date: Thu, 21 Mar 2024 22:14:29 +0800 Subject: [PATCH 2/7] Remove -S --- scripts/setup-macos.sh | 4 ++-- scripts/setup-ubuntu.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/setup-macos.sh b/scripts/setup-macos.sh index e30fab9c8b96..349e9dbf4b50 100755 --- a/scripts/setup-macos.sh +++ b/scripts/setup-macos.sh @@ -101,7 +101,7 @@ function install_fizz { wget_and_untar https://github.com/facebookincubator/fizz/archive/refs/tags/${FB_OS_VERSION}.tar.gz fizz ( cd fizz/fizz - cmake_install -DBUILD_TESTS=OFF -S fizz + cmake_install -DBUILD_TESTS=OFF ) } @@ -109,7 +109,7 @@ function install_wangle { wget_and_untar https://github.com/facebook/wangle/archive/refs/tags/${FB_OS_VERSION}.tar.gz wangle ( cd wangle/wangle - cmake_install -DBUILD_TESTS=OFF -S wangle + cmake_install -DBUILD_TESTS=OFF ) } diff --git a/scripts/setup-ubuntu.sh b/scripts/setup-ubuntu.sh index 1f14a9f54e85..8ce1af06d6da 100755 --- a/scripts/setup-ubuntu.sh +++ b/scripts/setup-ubuntu.sh @@ -122,7 +122,7 @@ function install_fizz { wget_and_untar https://github.com/facebookincubator/fizz/archive/refs/tags/${FB_OS_VERSION}.tar.gz fizz ( cd fizz/fizz - cmake_install -DBUILD_TESTS=OFF -S fizz + cmake_install -DBUILD_TESTS=OFF ) } @@ -130,7 +130,7 @@ function install_wangle { wget_and_untar https://github.com/facebook/wangle/archive/refs/tags/${FB_OS_VERSION}.tar.gz wangle ( cd wangle/wangle - cmake_install -DBUILD_TESTS=OFF -S wangle + cmake_install -DBUILD_TESTS=OFF ) } From e0af9f2b04b99edbde83af35748fcce712701d1f Mon Sep 17 00:00:00 2001 From: "joey.ljy" Date: Thu, 28 Mar 2024 11:12:00 +0800 Subject: [PATCH 3/7] Add cd in cmake_install --- scripts/setup-adapters.sh | 34 +++++++++------------ scripts/setup-centos9.sh | 50 +++++++------------------------ scripts/setup-helper-functions.sh | 7 +++++ scripts/setup-macos.sh | 45 ++++++---------------------- scripts/setup-ubuntu.sh | 30 ++++--------------- 5 files changed, 46 insertions(+), 120 deletions(-) diff --git a/scripts/setup-adapters.sh b/scripts/setup-adapters.sh index 82a104906090..a8c832ea54c5 100755 --- a/scripts/setup-adapters.sh +++ b/scripts/setup-adapters.sh @@ -107,50 +107,44 @@ function install_azure-storage-sdk-cpp { github_checkout azure/azure-sdk-for-cpp azure-storage-files-datalake_12.8.0 sed -i "s/set(VCPKG_COMMIT_STRING .*)/set(VCPKG_COMMIT_STRING $vcpkg_commit_id)/" cmake-modules/AzureVcpkg.cmake - cd sdk/core/azure-core - if ! grep -q "baseline" vcpkg.json; then + azure_core_dir="sdk/core/azure-core" + if ! grep -q "baseline" $azure_core_dir/vcpkg.json; then # build and install azure-core with the version compatible with system pre-installed openssl openssl_version=$(openssl version -v | awk '{print $2}') if [[ "$openssl_version" == 1.1.1* ]]; then openssl_version="1.1.1n" fi - sed -i "s/\"version-string\"/\"builtin-baseline\": \"$vcpkg_commit_id\",\"version-string\"/" vcpkg.json - sed -i "s/\"version-string\"/\"overrides\": [{ \"name\": \"openssl\", \"version-string\": \"$openssl_version\" }],\"version-string\"/" vcpkg.json + sed -i "s/\"version-string\"/\"builtin-baseline\": \"$vcpkg_commit_id\",\"version-string\"/" $azure_core_dir/vcpkg.json + sed -i "s/\"version-string\"/\"overrides\": [{ \"name\": \"openssl\", \"version-string\": \"$openssl_version\" }],\"version-string\"/" $azure_core_dir/vcpkg.json fi - cmake_install -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS=OFF + cmake_install $azure_core_dir -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS=OFF - cd - # install azure-storage-common - cd sdk/storage/azure-storage-common - cmake_install -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS=OFF + cmake_install sdk/storage/azure-storage-common -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS=OFF - cd - # install azure-storage-blobs - cd sdk/storage/azure-storage-blobs - cmake_install -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS=OFF + cmake_install sdk/storage/azure-storage-blobs -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS=OFF - cd - # install azure-storage-files-datalake - cd sdk/storage/azure-storage-files-datalake - cmake_install -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS=OFF + cmake_install sdk/storage/azure-storage-files-datalake -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS=OFF } function install_hdfs_deps { github_checkout apache/hawq master - cd $DEPENDENCY_DIR/hawq/depends/libhdfs3 + libhdfs3_dir=$DEPENDENCY_DIR/hawq/depends/libhdfs3 if [[ "$OSTYPE" == darwin* ]]; then - sed -i '' -e "/FIND_PACKAGE(GoogleTest REQUIRED)/d" ./CMakeLists.txt - sed -i '' -e "s/dumpversion/dumpfullversion/" ./CMakeLists.txt + sed -i '' -e "/FIND_PACKAGE(GoogleTest REQUIRED)/d" $libhdfs3_dir/CMakeLists.txt + sed -i '' -e "s/dumpversion/dumpfullversion/" $libhdfs3_dir/CMakeLists.txt fi if [[ "$OSTYPE" == linux-gnu* ]]; then - sed -i "/FIND_PACKAGE(GoogleTest REQUIRED)/d" ./CMakeLists.txt - sed -i "s/dumpversion/dumpfullversion/" ./CMake/Platform.cmake + sed -i "/FIND_PACKAGE(GoogleTest REQUIRED)/d" $libhdfs3_dir/CMakeLists.txt + sed -i "s/dumpversion/dumpfullversion/" $libhdfs3_dir/CMake/Platform.cmake # Dependencies for Hadoop testing wget_and_untar https://archive.apache.org/dist/hadoop/common/hadoop-2.10.1/hadoop-2.10.1.tar.gz hadoop cp -a hadoop /usr/local/ fi - cmake_install + cmake_install $libhdfs3_dir } cd "${DEPENDENCY_DIR}" || exit diff --git a/scripts/setup-centos9.sh b/scripts/setup-centos9.sh index 57ab15460f18..3841f2568d2f 100755 --- a/scripts/setup-centos9.sh +++ b/scripts/setup-centos9.sh @@ -78,18 +78,12 @@ function install_gflags { # Remove an older version if present. dnf remove -y gflags wget_and_untar https://github.com/gflags/gflags/archive/v2.2.2.tar.gz gflags - ( - cd gflags - cmake_install -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON -DBUILD_gflags_LIB=ON -DLIB_SUFFIX=64 - ) + cmake_install gflags -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON -DBUILD_gflags_LIB=ON -DLIB_SUFFIX=64 } function install_glog { wget_and_untar https://github.com/google/glog/archive/v0.6.0.tar.gz glog - ( - cd glog - cmake_install -DBUILD_SHARED_LIBS=ON - ) + cmake_install glog -DBUILD_SHARED_LIBS=ON } function install_lzo { @@ -113,18 +107,12 @@ function install_boost { function install_snappy { wget_and_untar https://github.com/google/snappy/archive/1.1.8.tar.gz snappy - ( - cd snappy - cmake_install -DSNAPPY_BUILD_TESTS=OFF - ) + cmake_install snappy -DSNAPPY_BUILD_TESTS=OFF } function install_fmt { wget_and_untar https://github.com/fmtlib/fmt/archive/${FMT_VERSION}.tar.gz fmt - ( - cd fmt - cmake_install -DFMT_TEST=OFF - ) + cmake_install fmt -DFMT_TEST=OFF } function install_protobuf { @@ -140,52 +128,34 @@ function install_protobuf { function install_fizz { wget_and_untar https://github.com/facebookincubator/fizz/archive/refs/tags/${FB_OS_VERSION}.tar.gz fizz - ( - cd fizz/fizz - cmake_install -DBUILD_TESTS=OFF - ) + cmake_install fizz/fizz -DBUILD_TESTS=OFF } function install_folly { wget_and_untar https://github.com/facebook/folly/archive/refs/tags/${FB_OS_VERSION}.tar.gz folly - ( - cd folly - cmake_install -DBUILD_TESTS=OFF -DFOLLY_HAVE_INT128_T=ON - ) + cmake_install folly -DBUILD_TESTS=OFF -DFOLLY_HAVE_INT128_T=ON } function install_wangle { wget_and_untar https://github.com/facebook/wangle/archive/refs/tags/${FB_OS_VERSION}.tar.gz wangle - ( - cd wangle/wangle - cmake_install -DBUILD_TESTS=OFF - ) + cmake_install wangle/wangle -DBUILD_TESTS=OFF } function install_fbthrift { wget_and_untar https://github.com/facebook/fbthrift/archive/refs/tags/${FB_OS_VERSION}.tar.gz fbthrift - ( - cd fbthrift - cmake_install -Denable_tests=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF - ) + cmake_install fbthrift -Denable_tests=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF } function install_mvfst { wget_and_untar https://github.com/facebook/mvfst/archive/refs/tags/${FB_OS_VERSION}.tar.gz mvfst - ( - cd mvfst - cmake_install -DBUILD_TESTS=OFF - ) + cmake_install mvfst -DBUILD_TESTS=OFF } function install_duckdb { if $BUILD_DUCKDB ; then echo 'Building DuckDB' wget_and_untar https://github.com/duckdb/duckdb/archive/refs/tags/v0.8.1.tar.gz duckdb - ( - cd duckdb - cmake_install -DBUILD_UNITTESTS=OFF -DENABLE_SANITIZER=OFF -DENABLE_UBSAN=OFF -DBUILD_SHELL=OFF -DEXPORT_DLL_SYMBOLS=OFF -DCMAKE_BUILD_TYPE=Release - ) + cmake_install duckdb -DBUILD_UNITTESTS=OFF -DENABLE_SANITIZER=OFF -DENABLE_UBSAN=OFF -DBUILD_SHELL=OFF -DEXPORT_DLL_SYMBOLS=OFF -DCMAKE_BUILD_TYPE=Release fi } diff --git a/scripts/setup-helper-functions.sh b/scripts/setup-helper-functions.sh index a856dc9e1c6e..8ab1ec9eaff0 100755 --- a/scripts/setup-helper-functions.sh +++ b/scripts/setup-helper-functions.sh @@ -164,9 +164,16 @@ function wget_and_untar { } function cmake_install { + if [ -n "$1" ]; then + DIR="$1" + shift + else + DIR=$(pwd) + fi local NAME=$(basename "$(pwd)") local BINARY_DIR=_build SUDO="${SUDO:-""}" + cd "${DIR}" if [ -d "${BINARY_DIR}" ] && prompt "Do you want to rebuild ${NAME}?"; then ${SUDO} rm -rf "${BINARY_DIR}" fi diff --git a/scripts/setup-macos.sh b/scripts/setup-macos.sh index 349e9dbf4b50..8602a4932ae6 100755 --- a/scripts/setup-macos.sh +++ b/scripts/setup-macos.sh @@ -83,74 +83,47 @@ function install_velox_deps_from_brew { function install_fmt { wget_and_untar https://github.com/fmtlib/fmt/archive/${FMT_VERSION}.tar.gz fmt - ( - cd fmt - cmake_install -DFMT_TEST=OFF - ) + cmake_install fmt -DFMT_TEST=OFF } function install_folly { wget_and_untar https://github.com/facebook/folly/archive/refs/tags/${FB_OS_VERSION}.tar.gz folly - ( - cd folly - cmake_install -DBUILD_TESTS=OFF -DFOLLY_HAVE_INT128_T=ON - ) + cmake_install folly -DBUILD_TESTS=OFF -DFOLLY_HAVE_INT128_T=ON } function install_fizz { wget_and_untar https://github.com/facebookincubator/fizz/archive/refs/tags/${FB_OS_VERSION}.tar.gz fizz - ( - cd fizz/fizz - cmake_install -DBUILD_TESTS=OFF - ) + cmake_install fizz/fizz -DBUILD_TESTS=OFF } function install_wangle { wget_and_untar https://github.com/facebook/wangle/archive/refs/tags/${FB_OS_VERSION}.tar.gz wangle - ( - cd wangle/wangle - cmake_install -DBUILD_TESTS=OFF - ) + cmake_install wangle/wangle -DBUILD_TESTS=OFF } function install_mvfst { wget_and_untar https://github.com/facebook/mvfst/archive/refs/tags/${FB_OS_VERSION}.tar.gz mvfst - ( - cd mvfst - cmake_install -DBUILD_TESTS=OFF - ) + cmake_install mvfst -DBUILD_TESTS=OFF } function install_fbthrift { wget_and_untar https://github.com/facebook/fbthrift/archive/refs/tags/${FB_OS_VERSION}.tar.gz fbthrift - ( - cd fbthrift - cmake_install -Denable_tests=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF - ) + cmake_install fbthrift -Denable_tests=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF } function install_double_conversion { wget_and_untar https://github.com/google/double-conversion/archive/refs/tags/v3.1.5.tar.gz double-conversion - ( - cd double-conversion - cmake_install -DBUILD_TESTING=OFF - ) + cmake_install double-conversion -DBUILD_TESTING=OFF } function install_ranges_v3 { wget_and_untar https://github.com/ericniebler/range-v3/archive/refs/tags/0.12.0.tar.gz ranges_v3 - ( - cd ranges_v3 - cmake_install -DRANGES_ENABLE_WERROR=OFF -DRANGE_V3_TESTS=OFF -DRANGE_V3_EXAMPLES=OFF - ) + cmake_install ranges_v3 -DRANGES_ENABLE_WERROR=OFF -DRANGE_V3_TESTS=OFF -DRANGE_V3_EXAMPLES=OFF } function install_re2 { wget_and_untar https://github.com/google/re2/archive/refs/tags/2022-02-01.tar.gz re2 - ( - cd re2 - cmake_install -DRE2_BUILD_TESTING=OFF - ) + cmake_install re2 -DRE2_BUILD_TESTING=OFF } function install_velox_deps { diff --git a/scripts/setup-ubuntu.sh b/scripts/setup-ubuntu.sh index 8ce1af06d6da..d04675f46b8a 100755 --- a/scripts/setup-ubuntu.sh +++ b/scripts/setup-ubuntu.sh @@ -95,10 +95,7 @@ function install_velox_deps_from_apt { function install_fmt { wget_and_untar https://github.com/fmtlib/fmt/archive/${FMT_VERSION}.tar.gz fmt - ( - cd fmt - cmake_install -DFMT_TEST=OFF - ) + cmake_install fmt -DFMT_TEST=OFF } function install_boost { @@ -112,42 +109,27 @@ function install_boost { function install_folly { wget_and_untar https://github.com/facebook/folly/archive/refs/tags/${FB_OS_VERSION}.tar.gz folly - ( - cd folly - cmake_install -DBUILD_TESTS=OFF -DFOLLY_HAVE_INT128_T=ON - ) + cmake_install folly -DBUILD_TESTS=OFF -DFOLLY_HAVE_INT128_T=ON } function install_fizz { wget_and_untar https://github.com/facebookincubator/fizz/archive/refs/tags/${FB_OS_VERSION}.tar.gz fizz - ( - cd fizz/fizz - cmake_install -DBUILD_TESTS=OFF - ) + cmake_install fizz/fizz -DBUILD_TESTS=OFF } function install_wangle { wget_and_untar https://github.com/facebook/wangle/archive/refs/tags/${FB_OS_VERSION}.tar.gz wangle - ( - cd wangle/wangle - cmake_install -DBUILD_TESTS=OFF - ) + cmake_install wangle/wangle -DBUILD_TESTS=OFF } function install_mvfst { wget_and_untar https://github.com/facebook/mvfst/archive/refs/tags/${FB_OS_VERSION}.tar.gz mvfst - ( - cd mvfst - cmake_install -DBUILD_TESTS=OFF - ) + cmake_install mvfst -DBUILD_TESTS=OFF } function install_fbthrift { wget_and_untar https://github.com/facebook/fbthrift/archive/refs/tags/${FB_OS_VERSION}.tar.gz fbthrift - ( - cd fbthrift - cmake_install -Denable_tests=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF - ) + cmake_install fbthrift -Denable_tests=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF } function install_conda { From 3f071224b99a7b27abb6a2145fd525e9274b0ec7 Mon Sep 17 00:00:00 2001 From: "joey.ljy" Date: Sat, 20 Jul 2024 20:11:44 +0800 Subject: [PATCH 4/7] Use pushd and popd in cmake_install --- scripts/setup-helper-functions.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/setup-helper-functions.sh b/scripts/setup-helper-functions.sh index 8ab1ec9eaff0..c07e2daf8953 100755 --- a/scripts/setup-helper-functions.sh +++ b/scripts/setup-helper-functions.sh @@ -173,7 +173,7 @@ function cmake_install { local NAME=$(basename "$(pwd)") local BINARY_DIR=_build SUDO="${SUDO:-""}" - cd "${DIR}" + pushd "${DIR}" if [ -d "${BINARY_DIR}" ] && prompt "Do you want to rebuild ${NAME}?"; then ${SUDO} rm -rf "${BINARY_DIR}" fi @@ -193,5 +193,6 @@ function cmake_install { # Exit if the build fails. cmake --build "${BINARY_DIR}" || { echo 'build failed' ; exit 1; } ${SUDO} cmake --install "${BINARY_DIR}" + popd } From a30dcbf3aba96657fc8064c2ff3f90f3442d589a Mon Sep 17 00:00:00 2001 From: "joey.ljy" Date: Sat, 20 Jul 2024 22:32:00 +0800 Subject: [PATCH 5/7] fix duckdb build --- scripts/setup-ubuntu.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/scripts/setup-ubuntu.sh b/scripts/setup-ubuntu.sh index d04675f46b8a..061374d28d1b 100755 --- a/scripts/setup-ubuntu.sh +++ b/scripts/setup-ubuntu.sh @@ -153,10 +153,7 @@ function install_duckdb { if $BUILD_DUCKDB ; then echo 'Building DuckDB' wget_and_untar https://github.com/duckdb/duckdb/archive/refs/tags/v0.8.1.tar.gz duckdb - ( - cd duckdb - cmake_install -DBUILD_UNITTESTS=OFF -DENABLE_SANITIZER=OFF -DENABLE_UBSAN=OFF -DBUILD_SHELL=OFF -DEXPORT_DLL_SYMBOLS=OFF -DCMAKE_BUILD_TYPE=Release - ) + cmake_install duckdb -DBUILD_UNITTESTS=OFF -DENABLE_SANITIZER=OFF -DENABLE_UBSAN=OFF -DBUILD_SHELL=OFF -DEXPORT_DLL_SYMBOLS=OFF -DCMAKE_BUILD_TYPE=Release fi } From fa05a1210761ec2bfdfcd942e570188016f7076d Mon Sep 17 00:00:00 2001 From: "joey.ljy" Date: Sat, 20 Jul 2024 22:59:31 +0800 Subject: [PATCH 6/7] Use -d to check cmake install dir --- scripts/setup-helper-functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/setup-helper-functions.sh b/scripts/setup-helper-functions.sh index c07e2daf8953..bf5e7d9c5c55 100755 --- a/scripts/setup-helper-functions.sh +++ b/scripts/setup-helper-functions.sh @@ -164,7 +164,7 @@ function wget_and_untar { } function cmake_install { - if [ -n "$1" ]; then + if [ -d "$1" ]; then DIR="$1" shift else From 634687589a98f48097e43578d34870f85b83add2 Mon Sep 17 00:00:00 2001 From: "joey.ljy" Date: Sat, 20 Jul 2024 23:47:25 +0800 Subject: [PATCH 7/7] Fix conda cd --- scripts/setup-ubuntu.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scripts/setup-ubuntu.sh b/scripts/setup-ubuntu.sh index 061374d28d1b..2529716ddd56 100755 --- a/scripts/setup-ubuntu.sh +++ b/scripts/setup-ubuntu.sh @@ -143,10 +143,11 @@ function install_conda { echo "Unsupported architecture: $ARCH" exit 1 fi - - mkdir -p conda && cd conda - wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-$ARCH.sh -O Miniconda3-latest-Linux-$ARCH.sh - bash Miniconda3-latest-Linux-$ARCH.sh -b -p $MINICONDA_PATH + ( + mkdir -p conda && cd conda + wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-$ARCH.sh -O Miniconda3-latest-Linux-$ARCH.sh + bash Miniconda3-latest-Linux-$ARCH.sh -b -p $MINICONDA_PATH + ) } function install_duckdb {