Improve dependency install and build

.github/workflows/macos.yml

@@ -54,6 +54,7 @@ jobs:
       CCACHE_DIR: '${{ github.workspace }}/.ccache'
       # The arm runners have only 7GB RAM
       BUILD_TYPE: "${{ matrix.os ==  'macos-14' && 'Release' || 'Debug' }}"
+      INSTALL_PREFIX: "/tmp/deps-install"
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -66,6 +67,7 @@ jobs:
           source scripts/setup-macos.sh
           install_build_prerequisites
           install_velox_deps_from_brew
+          install_double_conversion
 
           echo "NJOBS=`sysctl -n hw.ncpu`" >> $GITHUB_ENV
           brew unlink protobuf || echo "protobuf not installed"

.gitignore

@@ -49,6 +49,8 @@ projects/*
 !projects/*.*
 !projects/Makefile
 .venv
+deps-install
+deps-download
 
 #==============================================================================#
 # Autotools artifacts

CMakeLists.txt

@@ -45,6 +45,14 @@ if(DEFINED ENV{CONDA_PREFIX})
   endif()
 endif()
 
+if(DEFINED ENV{INSTALL_PREFIX})
+  message(STATUS "Dependency install directory set to: $ENV{INSTALL_PREFIX}")
+  list(APPEND CMAKE_PREFIX_PATH "$ENV{INSTALL_PREFIX}")
+  # Allow installed package headers to be picked up before brew/system package
+  # headers
+  include_directories(BEFORE "$ENV{INSTALL_PREFIX}/include")
+endif()
+
 list(PREPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/CMake"
      "${PROJECT_SOURCE_DIR}/CMake/third-party")
 

README.md

@@ -85,19 +85,24 @@ dependencies for a given platform.
 ### Setting up dependencies
 
 The following setup scripts use the `DEPENDENCY_DIR` environment variable to set the
-location of the build packages. If you do not set this variable, it will default to
-the current working directory.
-
-```shell
-$ export DEPENDENCY_DIR=/path/to/your/dependencies
-```
+location to download and build packages. This defaults to `deps-download` in the current
+working directory. Use `INSTALL_PREFIX` to set the install directory of the packages.
+This defaults to `deps-install` in the current working directory on macOS and to the
+default install location (eg. `/usr/local`) on linux.
+Using the default install location `/usr/local` on macOS is discouraged since this
+location is used by certain Homebrew versions.
+
+You will have to manually add the `INSTALL_PREFIX` value in the IDE or bash environment,
+say `export INSTALL_PREFIX=/Users/$USERNAME/velox/deps-install` to `~/.zshrc` so that
+Velox builds can use the installed packages.
+You can also reuse this install for Velox clients such as Prestissimo by specifying a
+common location.
 
 ### Setting up on macOS
 
-On a MacOS machine (either Intel or Apple silicon) you can setup and then build like so:
+On a macOS machine (either Intel or Apple silicon) you can setup and then build like so:
 
 ```shell
-$ export INSTALL_PREFIX=/Users/$USERNAME/velox/velox_dependency_install
 $ ./scripts/setup-macos.sh
 $ make
 ```
@@ -136,7 +141,7 @@ $ ./scripts/setup-adapters.sh
 $ make
 ```
 
-Note that `setup-adapters.sh` supports MacOS and Ubuntu 20.04 or later.
+Note that `setup-adapters.sh` supports macOS and Ubuntu 20.04 or later.
 
 ### Using Clang on Linux
 

scripts/adapters.dockerfile

@@ -15,6 +15,7 @@
 ARG image=ghcr.io/facebookincubator/velox-dev:centos9
 FROM $image
 
+COPY scripts/setup-helper-functions.sh /
 COPY scripts/setup-adapters.sh /
 RUN mkdir build && ( cd build &&  source /opt/rh/gcc-toolset-12/enable && \
     bash /setup-adapters.sh ) && rm -rf build && dnf remove -y conda && dnf clean all

scripts/setup-adapters.sh

@@ -21,7 +21,7 @@ set -eufx -o pipefail
 
 SCRIPTDIR=$(dirname "${BASH_SOURCE[0]}")
 source $SCRIPTDIR/setup-helper-functions.sh
-DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)}
+DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)/deps-download}
 CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}"
 MACHINE=$(uname -m)
 
@@ -117,21 +117,21 @@ function install_azure-storage-sdk-cpp {
     sed -i "s/\"version-string\"/\"builtin-baseline\": \"$vcpkg_commit_id\",\"version-string\"/" $azure_core_dir/vcpkg.json
     sed -i "s/\"version-string\"/\"overrides\": [{ \"name\": \"openssl\", \"version-string\": \"$openssl_version\" }],\"version-string\"/" $azure_core_dir/vcpkg.json
   fi
-  cmake_install $azure_core_dir -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS=OFF
+  cmake_install_dir $azure_core_dir -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS=OFF
 
   # install azure-storage-common
-  cmake_install sdk/storage/azure-storage-common -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS=OFF
+  cmake_install_dir sdk/storage/azure-storage-common -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS=OFF
 
   # install azure-storage-blobs
-  cmake_install sdk/storage/azure-storage-blobs -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS=OFF
+  cmake_install_dir sdk/storage/azure-storage-blobs -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS=OFF
 
   # install azure-storage-files-datalake
-  cmake_install sdk/storage/azure-storage-files-datalake -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS=OFF
+  cmake_install_dir sdk/storage/azure-storage-files-datalake -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS=OFF
 }
 
 function install_hdfs_deps {
   github_checkout apache/hawq master
-  libhdfs3_dir=$DEPENDENCY_DIR/hawq/depends/libhdfs3
+  libhdfs3_dir=hawq/depends/libhdfs3
   if [[ "$OSTYPE" == darwin* ]]; then
      sed -i '' -e "/FIND_PACKAGE(GoogleTest REQUIRED)/d" $libhdfs3_dir/CMakeLists.txt
      sed -i '' -e "s/dumpversion/dumpfullversion/" $libhdfs3_dir/CMakeLists.txt
@@ -148,10 +148,10 @@ function install_hdfs_deps {
     yum install -y java-1.8.0-openjdk-devel
 
   fi
-  cmake_install $libhdfs3_dir
+  cmake_install_dir $libhdfs3_dir
 }
 
-cd "${DEPENDENCY_DIR}" || exit
+(mkdir -p "${DEPENDENCY_DIR}") || exit
 # aws-sdk-cpp missing dependencies
 
 if [[ "$OSTYPE" == "linux-gnu"* ]]; then

scripts/setup-centos9.sh

@@ -36,6 +36,7 @@ export CFLAGS=${CXXFLAGS//"-std=c++17"/} # Used by LZO.
 CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}"
 BUILD_DUCKDB="${BUILD_DUCKDB:-true}"
 USE_CLANG="${USE_CLANG:-false}"
+DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)/deps-download}
 
 FB_OS_VERSION="v2024.05.20.00"
 FMT_VERSION="10.1.1"
@@ -85,18 +86,18 @@ function install_gflags {
   # Remove an older version if present.
   dnf remove -y gflags
   wget_and_untar https://github.com/gflags/gflags/archive/v2.2.2.tar.gz gflags
-  cmake_install gflags -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON -DBUILD_gflags_LIB=ON -DLIB_SUFFIX=64
+  cmake_install_dir gflags -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON -DBUILD_gflags_LIB=ON -DLIB_SUFFIX=64
 }
 
 function install_glog {
   wget_and_untar https://github.com/google/glog/archive/v0.6.0.tar.gz glog
-  cmake_install glog -DBUILD_SHARED_LIBS=ON
+  cmake_install_dir glog -DBUILD_SHARED_LIBS=ON
 }
 
 function install_lzo {
   wget_and_untar http://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz lzo
   (
-    cd lzo
+    cd ${DEPENDENCY_DIR}/lzo
     ./configure --prefix=/usr --enable-shared --disable-static --docdir=/usr/share/doc/lzo-2.10
     make "-j$(nproc)"
     make install
@@ -106,7 +107,7 @@ function install_lzo {
 function install_boost {
   wget_and_untar https://github.com/boostorg/boost/releases/download/${BOOST_VERSION}/${BOOST_VERSION}.tar.gz boost
   (
-    cd boost
+    cd ${DEPENDENCY_DIR}/boost
     if [[ ${USE_CLANG} != "false" ]]; then
       ./bootstrap.sh --prefix=/usr/local --with-toolset="clang-15"
       # Switch the compiler from the clang-15 toolset which doesn't exist (clang-15.jam) to
@@ -123,18 +124,18 @@ function install_boost {
 
 function install_snappy {
   wget_and_untar https://github.com/google/snappy/archive/1.1.8.tar.gz snappy
-  cmake_install snappy -DSNAPPY_BUILD_TESTS=OFF
+  cmake_install_dir snappy -DSNAPPY_BUILD_TESTS=OFF
 }
 
 function install_fmt {
   wget_and_untar https://github.com/fmtlib/fmt/archive/${FMT_VERSION}.tar.gz fmt
-  cmake_install fmt -DFMT_TEST=OFF
+  cmake_install_dir fmt -DFMT_TEST=OFF
 }
 
 function install_protobuf {
   wget_and_untar https://github.com/protocolbuffers/protobuf/releases/download/v21.8/protobuf-all-21.8.tar.gz protobuf
   (
-    cd protobuf
+    cd ${DEPENDENCY_DIR}/protobuf
     ./configure --prefix=/usr
     make "-j${NPROC}"
     make install
@@ -144,60 +145,59 @@ function install_protobuf {
 
 function install_fizz {
   wget_and_untar https://github.com/facebookincubator/fizz/archive/refs/tags/${FB_OS_VERSION}.tar.gz fizz
-  cmake_install fizz/fizz -DBUILD_TESTS=OFF
+  cmake_install_dir fizz/fizz -DBUILD_TESTS=OFF
 }
 
 function install_folly {
   wget_and_untar https://github.com/facebook/folly/archive/refs/tags/${FB_OS_VERSION}.tar.gz folly
-  cmake_install folly -DBUILD_TESTS=OFF -DFOLLY_HAVE_INT128_T=ON
+  cmake_install_dir folly -DBUILD_TESTS=OFF -DFOLLY_HAVE_INT128_T=ON
 }
 
 function install_wangle {
   wget_and_untar https://github.com/facebook/wangle/archive/refs/tags/${FB_OS_VERSION}.tar.gz wangle
-  cmake_install wangle/wangle -DBUILD_TESTS=OFF
+  cmake_install_dir wangle/wangle -DBUILD_TESTS=OFF
 }
 
 function install_fbthrift {
   wget_and_untar https://github.com/facebook/fbthrift/archive/refs/tags/${FB_OS_VERSION}.tar.gz fbthrift
-  cmake_install fbthrift -Denable_tests=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF
+  cmake_install_dir fbthrift -Denable_tests=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF
 }
 
 function install_mvfst {
   wget_and_untar https://github.com/facebook/mvfst/archive/refs/tags/${FB_OS_VERSION}.tar.gz mvfst
-  cmake_install mvfst -DBUILD_TESTS=OFF
+  cmake_install_dir mvfst -DBUILD_TESTS=OFF
 }
 
 function install_duckdb {
   if $BUILD_DUCKDB ; then
     echo 'Building DuckDB'
     wget_and_untar https://github.com/duckdb/duckdb/archive/refs/tags/v0.8.1.tar.gz duckdb
-    cmake_install duckdb -DBUILD_UNITTESTS=OFF -DENABLE_SANITIZER=OFF -DENABLE_UBSAN=OFF -DBUILD_SHELL=OFF -DEXPORT_DLL_SYMBOLS=OFF -DCMAKE_BUILD_TYPE=Release
+    cmake_install_dir duckdb -DBUILD_UNITTESTS=OFF -DENABLE_SANITIZER=OFF -DENABLE_UBSAN=OFF -DBUILD_SHELL=OFF -DEXPORT_DLL_SYMBOLS=OFF -DCMAKE_BUILD_TYPE=Release
   fi
 }
 
 function install_arrow {
   wget_and_untar https://archive.apache.org/dist/arrow/arrow-${ARROW_VERSION}/apache-arrow-${ARROW_VERSION}.tar.gz arrow
-  (
-    cd arrow/cpp
-    cmake_install \
-      -DARROW_PARQUET=OFF \
-      -DARROW_WITH_THRIFT=ON \
-      -DARROW_WITH_LZ4=ON \
-      -DARROW_WITH_SNAPPY=ON \
-      -DARROW_WITH_ZLIB=ON \
-      -DARROW_WITH_ZSTD=ON \
-      -DARROW_JEMALLOC=OFF \
-      -DARROW_SIMD_LEVEL=NONE \
-      -DARROW_RUNTIME_SIMD_LEVEL=NONE \
-      -DARROW_WITH_UTF8PROC=OFF \
-      -DARROW_TESTING=ON \
-      -DCMAKE_INSTALL_PREFIX=/usr/local \
-      -DCMAKE_BUILD_TYPE=Release \
-      -DARROW_BUILD_STATIC=ON \
-      -DThrift_SOURCE=BUNDLED
+  cmake_install_dir arrow/cpp \
+    -DARROW_PARQUET=OFF \
+    -DARROW_WITH_THRIFT=ON \
+    -DARROW_WITH_LZ4=ON \
+    -DARROW_WITH_SNAPPY=ON \
+    -DARROW_WITH_ZLIB=ON \
+    -DARROW_WITH_ZSTD=ON \
+    -DARROW_JEMALLOC=OFF \
+    -DARROW_SIMD_LEVEL=NONE \
+    -DARROW_RUNTIME_SIMD_LEVEL=NONE \
+    -DARROW_WITH_UTF8PROC=OFF \
+    -DARROW_TESTING=ON \
+    -DCMAKE_INSTALL_PREFIX=/usr/local \
+    -DCMAKE_BUILD_TYPE=Release \
+    -DARROW_BUILD_STATIC=ON \
+    -DThrift_SOURCE=BUNDLED
 
+  (
     # Install thrift.
-    cd _build/thrift_ep-prefix/src/thrift_ep-build
+    cd ${DEPENDENCY_DIR}/arrow/cpp/_build/thrift_ep-prefix/src/thrift_ep-build
     cmake --install ./ --prefix /usr/local/
   )
 }

scripts/setup-helper-functions.sh

@@ -16,6 +16,9 @@
 # github_checkout $REPO $VERSION $GIT_CLONE_PARAMS clones or re-uses an existing clone of the
 # specified repo, checking out the requested version.
 
+DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)/deps-download}
+OS_CXXFLAGS=""
+
 function run_and_time {
   time "$@" || (echo "Failed to run $* ." ; exit 1 )
   { echo "+ Finished running $*"; } 2> /dev/null
@@ -156,29 +159,50 @@ function get_cxx_flags {
 function wget_and_untar {
   local URL=$1
   local DIR=$2
+  mkdir -p "${DEPENDENCY_DIR}"
+  pushd "${DEPENDENCY_DIR}"
+  SUDO="${SUDO:-""}"
+  if [ -d "${DIR}" ]; then
+    if prompt "${DIR} already exists. Delete?"; then
+      ${SUDO} rm -rf "${DIR}"
+    else
+      popd
+      return
+    fi
+  fi
   mkdir -p "${DIR}"
   pushd "${DIR}"
   curl -L "${URL}" > $2.tar.gz
   tar -xz --strip-components=1 -f $2.tar.gz
   popd
+  popd
 }
 
-function cmake_install {
-  if [ -d "$1" ]; then
-    DIR="$1"
-    shift
-  else
-    DIR=$(pwd)
-  fi
+function cmake_install_dir {
+  pushd "${DEPENDENCY_DIR}/$1"
   local NAME=$(basename "$(pwd)")
   local BINARY_DIR=_build
   SUDO="${SUDO:-""}"
-  pushd "${DIR}"
-  if [ -d "${BINARY_DIR}" ] && prompt "Do you want to rebuild ${NAME}?"; then
-    ${SUDO} rm -rf "${BINARY_DIR}"
+  if [ -d "${BINARY_DIR}" ]; then
+    if prompt "Do you want to rebuild ${NAME}?"; then
+      ${SUDO} rm -rf "${BINARY_DIR}"
+    else
+      popd
+      return
+    fi
   fi
+  # remove the directory argument
+  shift
+  cmake_install $@
+  popd
+}
+
+function cmake_install {
+  local BINARY_DIR=_build
   mkdir -p "${BINARY_DIR}"
   COMPILER_FLAGS=$(get_cxx_flags)
+  # Add platform specific CXX flags if any
+  COMPILER_FLAGS+=${OS_CXXFLAGS}
 
   # CMAKE_POSITION_INDEPENDENT_CODE is required so that Velox can be built into dynamic libraries \
   cmake -Wno-dev -B"${BINARY_DIR}" \
@@ -193,6 +217,5 @@ function cmake_install {
   # Exit if the build fails.
   cmake --build "${BINARY_DIR}" || { echo 'build failed' ; exit 1; }
   ${SUDO} cmake --install "${BINARY_DIR}"
-  popd
 }