Download source code archive directly for ubuntu and macos setup (#9198)

Summary: It is better to use wget_and_untar to download the source code archive directly, instead of using git to clone the source code. Because direct download is much faster than using git clone. The setup-centos8.sh script has been consistently downloading source code archive directly. Pull Request resolved: #9198 Reviewed By: kgpai Differential Revision: D60288820 Pulled By: Yuhta fbshipit-source-id: acf111d61673e8648adf0e82319375de92a2e942
facebookincubator · Jul 26, 2024 · b5713ef · b5713ef
1 parent cf5aa68
commit b5713ef
Show file tree

Hide file tree

Showing 5 changed files with 88 additions and 114 deletions.
diff --git a/scripts/setup-adapters.sh b/scripts/setup-adapters.sh
@@ -107,50 +107,44 @@ function install_azure-storage-sdk-cpp {
   github_checkout azure/azure-sdk-for-cpp azure-storage-files-datalake_12.8.0
   sed -i "s/set(VCPKG_COMMIT_STRING .*)/set(VCPKG_COMMIT_STRING $vcpkg_commit_id)/" cmake-modules/AzureVcpkg.cmake
 
-  cd sdk/core/azure-core
-  if ! grep -q "baseline" vcpkg.json; then
+  azure_core_dir="sdk/core/azure-core"
+  if ! grep -q "baseline" $azure_core_dir/vcpkg.json; then
     # build and install azure-core with the version compatible with system pre-installed openssl
     openssl_version=$(openssl version -v | awk '{print $2}')
     if [[ "$openssl_version" == 1.1.1* ]]; then
       openssl_version="1.1.1n"
     fi
-    sed -i "s/\"version-string\"/\"builtin-baseline\": \"$vcpkg_commit_id\",\"version-string\"/" vcpkg.json
-    sed -i "s/\"version-string\"/\"overrides\": [{ \"name\": \"openssl\", \"version-string\": \"$openssl_version\" }],\"version-string\"/" vcpkg.json
+    sed -i "s/\"version-string\"/\"builtin-baseline\": \"$vcpkg_commit_id\",\"version-string\"/" $azure_core_dir/vcpkg.json
+    sed -i "s/\"version-string\"/\"overrides\": [{ \"name\": \"openssl\", \"version-string\": \"$openssl_version\" }],\"version-string\"/" $azure_core_dir/vcpkg.json
   fi
-  cmake_install -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS=OFF
+  cmake_install $azure_core_dir -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS=OFF
 
-  cd -
   # install azure-storage-common
-  cd sdk/storage/azure-storage-common
-  cmake_install -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS=OFF
+  cmake_install sdk/storage/azure-storage-common -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS=OFF
 
-  cd -
   # install azure-storage-blobs
-  cd sdk/storage/azure-storage-blobs
-  cmake_install -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS=OFF
+  cmake_install sdk/storage/azure-storage-blobs -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS=OFF
 
-  cd -
   # install azure-storage-files-datalake
-  cd sdk/storage/azure-storage-files-datalake
-  cmake_install -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS=OFF
+  cmake_install sdk/storage/azure-storage-files-datalake -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS=OFF
 }
 
 function install_hdfs_deps {
   github_checkout apache/hawq master
-  cd $DEPENDENCY_DIR/hawq/depends/libhdfs3
+  libhdfs3_dir=$DEPENDENCY_DIR/hawq/depends/libhdfs3
   if [[ "$OSTYPE" == darwin* ]]; then
-     sed -i '' -e "/FIND_PACKAGE(GoogleTest REQUIRED)/d" ./CMakeLists.txt
-     sed -i '' -e "s/dumpversion/dumpfullversion/" ./CMakeLists.txt
+     sed -i '' -e "/FIND_PACKAGE(GoogleTest REQUIRED)/d" $libhdfs3_dir/CMakeLists.txt
+     sed -i '' -e "s/dumpversion/dumpfullversion/" $libhdfs3_dir/CMakeLists.txt
   fi
 
   if [[ "$OSTYPE" == linux-gnu* ]]; then
-    sed -i "/FIND_PACKAGE(GoogleTest REQUIRED)/d" ./CMakeLists.txt
-    sed -i "s/dumpversion/dumpfullversion/" ./CMake/Platform.cmake
+    sed -i "/FIND_PACKAGE(GoogleTest REQUIRED)/d" $libhdfs3_dir/CMakeLists.txt
+    sed -i "s/dumpversion/dumpfullversion/" $libhdfs3_dir/CMake/Platform.cmake
     # Dependencies for Hadoop testing
     wget_and_untar https://archive.apache.org/dist/hadoop/common/hadoop-2.10.1/hadoop-2.10.1.tar.gz hadoop
     cp -a hadoop /usr/local/
   fi
-  cmake_install
+  cmake_install $libhdfs3_dir
 }
 
 cd "${DEPENDENCY_DIR}" || exit

diff --git a/scripts/setup-centos9.sh b/scripts/setup-centos9.sh
@@ -39,6 +39,11 @@ BUILD_DUCKDB="${BUILD_DUCKDB:-true}"
 export CC=/opt/rh/gcc-toolset-12/root/bin/gcc
 export CXX=/opt/rh/gcc-toolset-12/root/bin/g++
 
+FB_OS_VERSION="v2024.05.20.00"
+FMT_VERSION="10.1.1"
+BOOST_VERSION="boost-1.84.0"
+ARROW_VERSION="15.0.0"
+
 function dnf_install {
   dnf install -y -q --setopt=install_weak_deps=False "$@"
 }
@@ -73,18 +78,12 @@ function install_gflags {
   # Remove an older version if present.
   dnf remove -y gflags
   wget_and_untar https://github.com/gflags/gflags/archive/v2.2.2.tar.gz gflags
-  (
-    cd gflags
-    cmake_install -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON -DBUILD_gflags_LIB=ON -DLIB_SUFFIX=64
-  )
+  cmake_install gflags -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON -DBUILD_gflags_LIB=ON -DLIB_SUFFIX=64
 }
 
 function install_glog {
   wget_and_untar https://github.com/google/glog/archive/v0.6.0.tar.gz glog
-  (
-    cd glog
-    cmake_install -DBUILD_SHARED_LIBS=ON
-  )
+  cmake_install glog -DBUILD_SHARED_LIBS=ON
 }
 
 function install_lzo {
@@ -98,7 +97,7 @@ function install_lzo {
 }
 
 function install_boost {
-  wget_and_untar https://github.com/boostorg/boost/releases/download/boost-1.84.0/boost-1.84.0.tar.gz boost
+  wget_and_untar https://github.com/boostorg/boost/releases/download/${BOOST_VERSION}/${BOOST_VERSION}.tar.gz boost
   (
    cd boost
    ./bootstrap.sh --prefix=/usr/local
@@ -108,18 +107,12 @@ function install_boost {
 
 function install_snappy {
   wget_and_untar https://github.com/google/snappy/archive/1.1.8.tar.gz snappy
-  (
-    cd snappy
-    cmake_install -DSNAPPY_BUILD_TESTS=OFF
-  )
+  cmake_install snappy -DSNAPPY_BUILD_TESTS=OFF
 }
 
 function install_fmt {
-  wget_and_untar https://github.com/fmtlib/fmt/archive/10.1.1.tar.gz fmt
-  (
-    cd fmt
-    cmake_install -DFMT_TEST=OFF
-  )
+  wget_and_untar https://github.com/fmtlib/fmt/archive/${FMT_VERSION}.tar.gz fmt
+  cmake_install fmt -DFMT_TEST=OFF
 }
 
 function install_protobuf {
@@ -133,61 +126,39 @@ function install_protobuf {
   )
 }
 
-FB_OS_VERSION="v2024.05.20.00"
-
 function install_fizz {
   wget_and_untar https://github.com/facebookincubator/fizz/archive/refs/tags/${FB_OS_VERSION}.tar.gz fizz
-  (
-    cd fizz/fizz
-    cmake_install -DBUILD_TESTS=OFF
-  )
+  cmake_install fizz/fizz -DBUILD_TESTS=OFF
 }
 
 function install_folly {
   wget_and_untar https://github.com/facebook/folly/archive/refs/tags/${FB_OS_VERSION}.tar.gz folly
-  (
-    cd folly
-    cmake_install -DFOLLY_HAVE_INT128_T=ON
-  )
+  cmake_install folly -DBUILD_TESTS=OFF -DFOLLY_HAVE_INT128_T=ON
 }
 
 function install_wangle {
   wget_and_untar https://github.com/facebook/wangle/archive/refs/tags/${FB_OS_VERSION}.tar.gz wangle
-  (
-    cd wangle/wangle
-    cmake_install -DBUILD_TESTS=OFF
-  )
+  cmake_install wangle/wangle -DBUILD_TESTS=OFF
 }
 
 function install_fbthrift {
   wget_and_untar https://github.com/facebook/fbthrift/archive/refs/tags/${FB_OS_VERSION}.tar.gz fbthrift
-  (
-    cd fbthrift
-    cmake_install -Denable_tests=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF
-  )
+  cmake_install fbthrift -Denable_tests=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF
 }
 
 function install_mvfst {
   wget_and_untar https://github.com/facebook/mvfst/archive/refs/tags/${FB_OS_VERSION}.tar.gz mvfst
-  (
-   cd mvfst
-   cmake_install -DBUILD_TESTS=OFF
-  )
+  cmake_install mvfst -DBUILD_TESTS=OFF
 }
 
 function install_duckdb {
   if $BUILD_DUCKDB ; then
     echo 'Building DuckDB'
     wget_and_untar https://github.com/duckdb/duckdb/archive/refs/tags/v0.8.1.tar.gz duckdb
-    (
-      cd duckdb
-      cmake_install -DBUILD_UNITTESTS=OFF -DENABLE_SANITIZER=OFF -DENABLE_UBSAN=OFF -DBUILD_SHELL=OFF -DEXPORT_DLL_SYMBOLS=OFF -DCMAKE_BUILD_TYPE=Release
-    )
+    cmake_install duckdb -DBUILD_UNITTESTS=OFF -DENABLE_SANITIZER=OFF -DENABLE_UBSAN=OFF -DBUILD_SHELL=OFF -DEXPORT_DLL_SYMBOLS=OFF -DCMAKE_BUILD_TYPE=Release
   fi
 }
 
-ARROW_VERSION=15.0.0
-
 function install_arrow {
   wget_and_untar https://archive.apache.org/dist/arrow/arrow-${ARROW_VERSION}/apache-arrow-${ARROW_VERSION}.tar.gz arrow
   (

diff --git a/scripts/setup-helper-functions.sh b/scripts/setup-helper-functions.sh
@@ -164,9 +164,16 @@ function wget_and_untar {
 }
 
 function cmake_install {
+  if [ -d "$1" ]; then
+    DIR="$1"
+    shift
+  else
+    DIR=$(pwd)
+  fi
   local NAME=$(basename "$(pwd)")
   local BINARY_DIR=_build
   SUDO="${SUDO:-""}"
+  pushd "${DIR}"
   if [ -d "${BINARY_DIR}" ] && prompt "Do you want to rebuild ${NAME}?"; then
     ${SUDO} rm -rf "${BINARY_DIR}"
   fi
@@ -186,5 +193,6 @@ function cmake_install {
   # Exit if the build fails.
   cmake --build "${BINARY_DIR}" || { echo 'build failed' ; exit 1; }
   ${SUDO} cmake --install "${BINARY_DIR}"
+  popd
 }
 
diff --git a/scripts/setup-macos.sh b/scripts/setup-macos.sh
@@ -37,6 +37,7 @@ DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)}
 MACOS_VELOX_DEPS="flex bison protobuf@21 icu4c boost gflags glog libevent lz4 lzo snappy xz zstd openssl libsodium"
 MACOS_BUILD_DEPS="ninja cmake ccache"
 FB_OS_VERSION="v2024.05.20.00"
+FMT_VERSION="10.1.1"
 
 function update_brew {
   DEFAULT_BREW_PATH=/usr/local/bin/brew
@@ -81,49 +82,48 @@ function install_velox_deps_from_brew {
 }
 
 function install_fmt {
-  github_checkout fmtlib/fmt 10.1.1
-  cmake_install -DFMT_TEST=OFF
+  wget_and_untar https://github.com/fmtlib/fmt/archive/${FMT_VERSION}.tar.gz fmt
+  cmake_install fmt -DFMT_TEST=OFF
 }
 
 function install_folly {
-  github_checkout facebook/folly "${FB_OS_VERSION}"
-  cmake_install -DBUILD_TESTS=OFF -DFOLLY_HAVE_INT128_T=ON
+  wget_and_untar https://github.com/facebook/folly/archive/refs/tags/${FB_OS_VERSION}.tar.gz folly
+  cmake_install folly -DBUILD_TESTS=OFF -DFOLLY_HAVE_INT128_T=ON
 }
 
 function install_fizz {
-  github_checkout facebookincubator/fizz "${FB_OS_VERSION}"
-  cmake_install -DBUILD_TESTS=OFF -S fizz
+  wget_and_untar https://github.com/facebookincubator/fizz/archive/refs/tags/${FB_OS_VERSION}.tar.gz fizz
+  cmake_install fizz/fizz -DBUILD_TESTS=OFF
 }
 
 function install_wangle {
-  github_checkout facebook/wangle "${FB_OS_VERSION}"
-  cmake_install -DBUILD_TESTS=OFF -S wangle
+  wget_and_untar https://github.com/facebook/wangle/archive/refs/tags/${FB_OS_VERSION}.tar.gz wangle
+  cmake_install wangle/wangle -DBUILD_TESTS=OFF
 }
 
 function install_mvfst {
-    github_checkout facebook/mvfst "${FB_OS_VERSION}"
-    cmake_install -DBUILD_TESTS=OFF
+  wget_and_untar https://github.com/facebook/mvfst/archive/refs/tags/${FB_OS_VERSION}.tar.gz mvfst
+  cmake_install mvfst -DBUILD_TESTS=OFF
 }
 
-
 function install_fbthrift {
-  github_checkout facebook/fbthrift "${FB_OS_VERSION}"
-  cmake_install -Denable_tests=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF
+  wget_and_untar https://github.com/facebook/fbthrift/archive/refs/tags/${FB_OS_VERSION}.tar.gz fbthrift
+  cmake_install fbthrift -Denable_tests=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF
 }
 
 function install_double_conversion {
-  github_checkout google/double-conversion v3.1.5
-  cmake_install -DBUILD_TESTING=OFF
+  wget_and_untar https://github.com/google/double-conversion/archive/refs/tags/v3.1.5.tar.gz double-conversion
+  cmake_install double-conversion -DBUILD_TESTING=OFF
 }
 
 function install_ranges_v3 {
-  github_checkout ericniebler/range-v3 0.12.0
-  cmake_install -DRANGES_ENABLE_WERROR=OFF -DRANGE_V3_TESTS=OFF -DRANGE_V3_EXAMPLES=OFF
+  wget_and_untar https://github.com/ericniebler/range-v3/archive/refs/tags/0.12.0.tar.gz ranges_v3
+  cmake_install ranges_v3 -DRANGES_ENABLE_WERROR=OFF -DRANGE_V3_TESTS=OFF -DRANGE_V3_EXAMPLES=OFF
 }
 
 function install_re2 {
-  github_checkout google/re2 2022-02-01
-  cmake_install -DRE2_BUILD_TESTING=OFF
+  wget_and_untar https://github.com/google/re2/archive/refs/tags/2022-02-01.tar.gz re2
+  cmake_install re2 -DRE2_BUILD_TESTING=OFF
 }
 
 function install_velox_deps {