Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Download source code archive directly for ubuntu and macos setup #9198

Closed
wants to merge 7 commits into from
Closed

Download source code archive directly for ubuntu and macos setup #9198

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
1623b5e
Download source code archive directly for ubuntu and macos setup
liujiayi771 Mar 21, 2024
378e953
Remove -S
liujiayi771 Mar 21, 2024
e0af9f2
Add cd in cmake_install
liujiayi771 Mar 28, 2024
3f07122
Use pushd and popd in cmake_install
liujiayi771 Jul 20, 2024
a30dcbf
fix duckdb build
liujiayi771 Jul 20, 2024
fa05a12
Use -d to check cmake install dir
liujiayi771 Jul 20, 2024
6346875
Fix conda cd
liujiayi771 Jul 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 14 additions & 20 deletions scripts/setup-adapters.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,50 +107,44 @@ function install_azure-storage-sdk-cpp {
github_checkout azure/azure-sdk-for-cpp azure-storage-files-datalake_12.8.0
sed -i "s/set(VCPKG_COMMIT_STRING .*)/set(VCPKG_COMMIT_STRING $vcpkg_commit_id)/" cmake-modules/AzureVcpkg.cmake

cd sdk/core/azure-core
if ! grep -q "baseline" vcpkg.json; then
azure_core_dir="sdk/core/azure-core"
if ! grep -q "baseline" $azure_core_dir/vcpkg.json; then
# build and install azure-core with the version compatible with system pre-installed openssl
openssl_version=$(openssl version -v | awk '{print $2}')
if [[ "$openssl_version" == 1.1.1* ]]; then
openssl_version="1.1.1n"
fi
sed -i "s/\"version-string\"/\"builtin-baseline\": \"$vcpkg_commit_id\",\"version-string\"/" vcpkg.json
sed -i "s/\"version-string\"/\"overrides\": [{ \"name\": \"openssl\", \"version-string\": \"$openssl_version\" }],\"version-string\"/" vcpkg.json
sed -i "s/\"version-string\"/\"builtin-baseline\": \"$vcpkg_commit_id\",\"version-string\"/" $azure_core_dir/vcpkg.json
sed -i "s/\"version-string\"/\"overrides\": [{ \"name\": \"openssl\", \"version-string\": \"$openssl_version\" }],\"version-string\"/" $azure_core_dir/vcpkg.json
fi
cmake_install -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS=OFF
cmake_install $azure_core_dir -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS=OFF

cd -
# install azure-storage-common
cd sdk/storage/azure-storage-common
cmake_install -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS=OFF
cmake_install sdk/storage/azure-storage-common -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS=OFF

cd -
# install azure-storage-blobs
cd sdk/storage/azure-storage-blobs
cmake_install -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS=OFF
cmake_install sdk/storage/azure-storage-blobs -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS=OFF

cd -
# install azure-storage-files-datalake
cd sdk/storage/azure-storage-files-datalake
cmake_install -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS=OFF
cmake_install sdk/storage/azure-storage-files-datalake -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS=OFF
}

function install_hdfs_deps {
github_checkout apache/hawq master
cd $DEPENDENCY_DIR/hawq/depends/libhdfs3
libhdfs3_dir=$DEPENDENCY_DIR/hawq/depends/libhdfs3
if [[ "$OSTYPE" == darwin* ]]; then
sed -i '' -e "/FIND_PACKAGE(GoogleTest REQUIRED)/d" ./CMakeLists.txt
sed -i '' -e "s/dumpversion/dumpfullversion/" ./CMakeLists.txt
sed -i '' -e "/FIND_PACKAGE(GoogleTest REQUIRED)/d" $libhdfs3_dir/CMakeLists.txt
sed -i '' -e "s/dumpversion/dumpfullversion/" $libhdfs3_dir/CMakeLists.txt
fi

if [[ "$OSTYPE" == linux-gnu* ]]; then
sed -i "/FIND_PACKAGE(GoogleTest REQUIRED)/d" ./CMakeLists.txt
sed -i "s/dumpversion/dumpfullversion/" ./CMake/Platform.cmake
sed -i "/FIND_PACKAGE(GoogleTest REQUIRED)/d" $libhdfs3_dir/CMakeLists.txt
sed -i "s/dumpversion/dumpfullversion/" $libhdfs3_dir/CMake/Platform.cmake
# Dependencies for Hadoop testing
wget_and_untar https://archive.apache.org/dist/hadoop/common/hadoop-2.10.1/hadoop-2.10.1.tar.gz hadoop
cp -a hadoop /usr/local/
fi
cmake_install
cmake_install $libhdfs3_dir
}

cd "${DEPENDENCY_DIR}" || exit
Expand Down
63 changes: 17 additions & 46 deletions scripts/setup-centos9.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,11 @@ BUILD_DUCKDB="${BUILD_DUCKDB:-true}"
export CC=/opt/rh/gcc-toolset-12/root/bin/gcc
export CXX=/opt/rh/gcc-toolset-12/root/bin/g++

FB_OS_VERSION="v2024.05.20.00"
FMT_VERSION="10.1.1"
BOOST_VERSION="boost-1.84.0"
ARROW_VERSION="15.0.0"

function dnf_install {
dnf install -y -q --setopt=install_weak_deps=False "$@"
}
Expand Down Expand Up @@ -73,18 +78,12 @@ function install_gflags {
# Remove an older version if present.
dnf remove -y gflags
wget_and_untar https://github.com/gflags/gflags/archive/v2.2.2.tar.gz gflags
(
cd gflags
cmake_install -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON -DBUILD_gflags_LIB=ON -DLIB_SUFFIX=64
)
cmake_install gflags -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON -DBUILD_gflags_LIB=ON -DLIB_SUFFIX=64
}

function install_glog {
wget_and_untar https://github.com/google/glog/archive/v0.6.0.tar.gz glog
(
cd glog
cmake_install -DBUILD_SHARED_LIBS=ON
)
cmake_install glog -DBUILD_SHARED_LIBS=ON
}

function install_lzo {
Expand All @@ -98,7 +97,7 @@ function install_lzo {
}

function install_boost {
wget_and_untar https://github.com/boostorg/boost/releases/download/boost-1.84.0/boost-1.84.0.tar.gz boost
wget_and_untar https://github.com/boostorg/boost/releases/download/${BOOST_VERSION}/${BOOST_VERSION}.tar.gz boost
(
cd boost
./bootstrap.sh --prefix=/usr/local
Expand All @@ -108,18 +107,12 @@ function install_boost {

function install_snappy {
wget_and_untar https://github.com/google/snappy/archive/1.1.8.tar.gz snappy
(
cd snappy
cmake_install -DSNAPPY_BUILD_TESTS=OFF
)
cmake_install snappy -DSNAPPY_BUILD_TESTS=OFF
}

function install_fmt {
wget_and_untar https://github.com/fmtlib/fmt/archive/10.1.1.tar.gz fmt
(
cd fmt
cmake_install -DFMT_TEST=OFF
)
wget_and_untar https://github.com/fmtlib/fmt/archive/${FMT_VERSION}.tar.gz fmt
cmake_install fmt -DFMT_TEST=OFF
}

function install_protobuf {
Expand All @@ -133,61 +126,39 @@ function install_protobuf {
)
}

FB_OS_VERSION="v2024.05.20.00"

function install_fizz {
wget_and_untar https://github.com/facebookincubator/fizz/archive/refs/tags/${FB_OS_VERSION}.tar.gz fizz
(
cd fizz/fizz
cmake_install -DBUILD_TESTS=OFF
)
cmake_install fizz/fizz -DBUILD_TESTS=OFF
}

function install_folly {
wget_and_untar https://github.com/facebook/folly/archive/refs/tags/${FB_OS_VERSION}.tar.gz folly
(
cd folly
cmake_install -DFOLLY_HAVE_INT128_T=ON
)
cmake_install folly -DBUILD_TESTS=OFF -DFOLLY_HAVE_INT128_T=ON
}

function install_wangle {
wget_and_untar https://github.com/facebook/wangle/archive/refs/tags/${FB_OS_VERSION}.tar.gz wangle
(
cd wangle/wangle
cmake_install -DBUILD_TESTS=OFF
)
cmake_install wangle/wangle -DBUILD_TESTS=OFF
}

function install_fbthrift {
wget_and_untar https://github.com/facebook/fbthrift/archive/refs/tags/${FB_OS_VERSION}.tar.gz fbthrift
(
cd fbthrift
cmake_install -Denable_tests=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF
)
cmake_install fbthrift -Denable_tests=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF
}

function install_mvfst {
wget_and_untar https://github.com/facebook/mvfst/archive/refs/tags/${FB_OS_VERSION}.tar.gz mvfst
(
cd mvfst
cmake_install -DBUILD_TESTS=OFF
)
cmake_install mvfst -DBUILD_TESTS=OFF
}

function install_duckdb {
if $BUILD_DUCKDB ; then
echo 'Building DuckDB'
wget_and_untar https://github.com/duckdb/duckdb/archive/refs/tags/v0.8.1.tar.gz duckdb
(
cd duckdb
cmake_install -DBUILD_UNITTESTS=OFF -DENABLE_SANITIZER=OFF -DENABLE_UBSAN=OFF -DBUILD_SHELL=OFF -DEXPORT_DLL_SYMBOLS=OFF -DCMAKE_BUILD_TYPE=Release
)
cmake_install duckdb -DBUILD_UNITTESTS=OFF -DENABLE_SANITIZER=OFF -DENABLE_UBSAN=OFF -DBUILD_SHELL=OFF -DEXPORT_DLL_SYMBOLS=OFF -DCMAKE_BUILD_TYPE=Release
fi
}

ARROW_VERSION=15.0.0

function install_arrow {
wget_and_untar https://archive.apache.org/dist/arrow/arrow-${ARROW_VERSION}/apache-arrow-${ARROW_VERSION}.tar.gz arrow
(
Expand Down
8 changes: 8 additions & 0 deletions scripts/setup-helper-functions.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,9 +164,16 @@ function wget_and_untar {
}

function cmake_install {
if [ -d "$1" ]; then
DIR="$1"
shift
else
DIR=$(pwd)
fi
local NAME=$(basename "$(pwd)")
local BINARY_DIR=_build
SUDO="${SUDO:-""}"
pushd "${DIR}"
if [ -d "${BINARY_DIR}" ] && prompt "Do you want to rebuild ${NAME}?"; then
${SUDO} rm -rf "${BINARY_DIR}"
fi
Expand All @@ -186,5 +193,6 @@ function cmake_install {
# Exit if the build fails.
cmake --build "${BINARY_DIR}" || { echo 'build failed' ; exit 1; }
${SUDO} cmake --install "${BINARY_DIR}"
popd
}

38 changes: 19 additions & 19 deletions scripts/setup-macos.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)}
MACOS_VELOX_DEPS="flex bison protobuf@21 icu4c boost gflags glog libevent lz4 lzo snappy xz zstd openssl libsodium"
MACOS_BUILD_DEPS="ninja cmake ccache"
FB_OS_VERSION="v2024.05.20.00"
FMT_VERSION="10.1.1"

function update_brew {
DEFAULT_BREW_PATH=/usr/local/bin/brew
Expand Down Expand Up @@ -81,49 +82,48 @@ function install_velox_deps_from_brew {
}

function install_fmt {
github_checkout fmtlib/fmt 10.1.1
cmake_install -DFMT_TEST=OFF
wget_and_untar https://github.com/fmtlib/fmt/archive/${FMT_VERSION}.tar.gz fmt
cmake_install fmt -DFMT_TEST=OFF
}

function install_folly {
github_checkout facebook/folly "${FB_OS_VERSION}"
cmake_install -DBUILD_TESTS=OFF -DFOLLY_HAVE_INT128_T=ON
wget_and_untar https://github.com/facebook/folly/archive/refs/tags/${FB_OS_VERSION}.tar.gz folly
cmake_install folly -DBUILD_TESTS=OFF -DFOLLY_HAVE_INT128_T=ON
}

function install_fizz {
github_checkout facebookincubator/fizz "${FB_OS_VERSION}"
cmake_install -DBUILD_TESTS=OFF -S fizz
wget_and_untar https://github.com/facebookincubator/fizz/archive/refs/tags/${FB_OS_VERSION}.tar.gz fizz
cmake_install fizz/fizz -DBUILD_TESTS=OFF
}

function install_wangle {
github_checkout facebook/wangle "${FB_OS_VERSION}"
cmake_install -DBUILD_TESTS=OFF -S wangle
wget_and_untar https://github.com/facebook/wangle/archive/refs/tags/${FB_OS_VERSION}.tar.gz wangle
cmake_install wangle/wangle -DBUILD_TESTS=OFF
}

function install_mvfst {
github_checkout facebook/mvfst "${FB_OS_VERSION}"
cmake_install -DBUILD_TESTS=OFF
wget_and_untar https://github.com/facebook/mvfst/archive/refs/tags/${FB_OS_VERSION}.tar.gz mvfst
cmake_install mvfst -DBUILD_TESTS=OFF
}


function install_fbthrift {
github_checkout facebook/fbthrift "${FB_OS_VERSION}"
cmake_install -Denable_tests=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF
wget_and_untar https://github.com/facebook/fbthrift/archive/refs/tags/${FB_OS_VERSION}.tar.gz fbthrift
cmake_install fbthrift -Denable_tests=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF
}

function install_double_conversion {
github_checkout google/double-conversion v3.1.5
cmake_install -DBUILD_TESTING=OFF
wget_and_untar https://github.com/google/double-conversion/archive/refs/tags/v3.1.5.tar.gz double-conversion
cmake_install double-conversion -DBUILD_TESTING=OFF
}

function install_ranges_v3 {
github_checkout ericniebler/range-v3 0.12.0
cmake_install -DRANGES_ENABLE_WERROR=OFF -DRANGE_V3_TESTS=OFF -DRANGE_V3_EXAMPLES=OFF
wget_and_untar https://github.com/ericniebler/range-v3/archive/refs/tags/0.12.0.tar.gz ranges_v3
cmake_install ranges_v3 -DRANGES_ENABLE_WERROR=OFF -DRANGE_V3_TESTS=OFF -DRANGE_V3_EXAMPLES=OFF
}

function install_re2 {
github_checkout google/re2 2022-02-01
cmake_install -DRE2_BUILD_TESTING=OFF
wget_and_untar https://github.com/google/re2/archive/refs/tags/2022-02-01.tar.gz re2
cmake_install re2 -DRE2_BUILD_TESTING=OFF
}

function install_velox_deps {
Expand Down
Loading
Loading