Skip to content

Commit

Permalink
Add setup functions for velox and build dependencies handled by packa…
Browse files Browse the repository at this point in the history 
…ge managers (facebookincubator#8917)

Summary:
In the current setup scripts, the packages handled by the package manager are always installed
in Centos8 and Ubuntu. This is unnecessary if we want to install a specific package.
Make these installs optional by wrapping them around a function.
Split the brew packages on MacOS based on the build vs Velox requirements.

Pull Request resolved: facebookincubator#8917

Reviewed By: xiaoxmeng

Differential Revision: D55506473

Pulled By: kgpai

fbshipit-source-id: 3e4fc31080faac6006dede1808ac9d76cb4b5f60
  • Loading branch information
@majetideepak @Real-Chen-Happy
majetideepak authored and Real-Chen-Happy committed Apr 2, 2024
1 parent b7c1f32 commit 7f35620
Show file tree
Hide file tree
Showing 3 changed files with 155 additions and 77 deletions.
64 changes: 47 additions & 17 deletions scripts/setup-centos8.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,18 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# This script documents setting up a Centos8 host for Velox
# development. Running it should make you ready to compile.
#
# Environment variables:
# * INSTALL_PREREQUISITES="N": Skip installation of packages for build.
# * PROMPT_ALWAYS_RESPOND="n": Automatically respond to interactive prompts.
# Use "n" to never wipe directories.
#
# You can also run individual functions below by specifying them as arguments:
# $ scripts/setup-centos8.sh install_googletest install_fmt
#

set -efx -o pipefail
# Some of the packages must be build with the same compiler flags
# so that some low level types are the same size. Also, disable warnings.
Expand All @@ -25,27 +37,32 @@ export CXXFLAGS=$CFLAGS # Used by boost.
export CPPFLAGS=$CFLAGS # Used by LZO.
CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}"
BUILD_DUCKDB="${BUILD_DUCKDB:-true}"
export CC=/opt/rh/gcc-toolset-9/root/bin/gcc
export CXX=/opt/rh/gcc-toolset-9/root/bin/g++

function dnf_install {
dnf install -y -q --setopt=install_weak_deps=False "$@"
}

dnf update -y
dnf_install epel-release dnf-plugins-core # For ccache, ninja
dnf config-manager --set-enabled powertools
dnf update -y
dnf_install ninja-build cmake curl ccache gcc-toolset-9 git wget which libevent-devel \
openssl-devel re2-devel libzstd-devel lz4-devel double-conversion-devel \
libdwarf-devel curl-devel libicu-devel

dnf_install autoconf automake libtool bison flex python3 libsodium-devel
# Install packages required for build.
function install_build_prerequisites {
dnf update -y
dnf_install epel-release dnf-plugins-core # For ccache, ninja
dnf config-manager --set-enabled powertools
dnf update -y
dnf_install ninja-build cmake curl ccache gcc-toolset-9 git wget which
dnf_install autoconf automake python39 python39-devel python39-pip libtool
}

# install sphinx for doc gen
pip3 install sphinx sphinx-tabs breathe sphinx_rtd_theme
# Install dependencies from the package managers.
function install_velox_deps_from_dnf {
dnf_install libevent-devel \
openssl-devel re2-devel libzstd-devel lz4-devel double-conversion-devel \
libdwarf-devel curl-devel libicu-devel bison flex libsodium-devel

# Activate gcc9; enable errors on unset variables afterwards.
source /opt/rh/gcc-toolset-9/enable || exit 1
set -u
# install sphinx for doc gen
pip3 install sphinx sphinx-tabs breathe sphinx_rtd_theme
}

function install_conda {
dnf_install conda
Expand Down Expand Up @@ -169,6 +186,7 @@ function install_duckdb {
}

function install_velox_deps {
run_and_time install_velox_deps_from_dnf
run_and_time install_conda
run_and_time install_gflags
run_and_time install_glog
Expand All @@ -189,14 +207,26 @@ function install_velox_deps {

(
if [[ $# -ne 0 ]]; then
# Activate gcc9; enable errors on unset variables afterwards.
source /opt/rh/gcc-toolset-9/enable || exit 1
set -u
for cmd in "$@"; do
run_and_time "${cmd}"
done
echo "All specified dependencies installed!"
else
if [ "${INSTALL_PREREQUISITES:-Y}" == "Y" ]; then
echo "Installing build dependencies"
run_and_time install_build_prerequisites
else
echo "Skipping installation of build dependencies since INSTALL_PREREQUISITES is not set"
fi
# Activate gcc9; enable errors on unset variables afterwards.
source /opt/rh/gcc-toolset-9/enable || exit 1
set -u
install_velox_deps
echo "All dependencies for Velox installed!"
dnf clean all
fi
)

echo "All dependencies for Velox installed!"

dnf clean all
57 changes: 36 additions & 21 deletions scripts/setup-macos.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@ source $SCRIPTDIR/setup-helper-functions.sh
NPROC=$(getconf _NPROCESSORS_ONLN)

DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)}
MACOS_DEPS="ninja flex bison cmake ccache protobuf@21 icu4c boost gflags glog libevent lz4 lzo snappy xz zstd openssl libsodium"

MACOS_VELOX_DEPS="flex bison protobuf@21 icu4c boost gflags glog libevent lz4 lzo snappy xz zstd openssl libsodium"
MACOS_BUILD_DEPS="ninja cmake ccache"
FB_OS_VERSION="v2024.02.26.00"

function update_brew {
Expand All @@ -49,26 +49,37 @@ function update_brew {
$BREW_PATH developer off
}

function install_from_brew {
pkg=$1
if [[ "${pkg}" =~ ^([0-9a-z-]*):([0-9](\.[0-9\])*)$ ]];
then
pkg=${BASH_REMATCH[1]}
ver=${BASH_REMATCH[2]}
echo "Installing '${pkg}' at '${ver}'"
tap="velox/local-${pkg}"
brew tap-new "${tap}"
brew extract "--version=${ver}" "${pkg}" "${tap}"
brew install "${tap}/${pkg}@${ver}" || ( echo "Failed to install ${tap}/${pkg}@${ver}" ; exit 1 )
else
( brew install --formula "${pkg}" && echo "Installation of ${pkg} is successful" || brew upgrade --formula "$pkg" ) || ( echo "Failed to install ${pkg}" ; exit 1 )
fi
}

function install_build_prerequisites {
for pkg in ${MACOS_DEPS}
for pkg in ${MACOS_BUILD_DEPS}
do
if [[ "${pkg}" =~ ^([0-9a-z-]*):([0-9](\.[0-9\])*)$ ]];
then
pkg=${BASH_REMATCH[1]}
ver=${BASH_REMATCH[2]}
echo "Installing '${pkg}' at '${ver}'"
tap="velox/local-${pkg}"
brew tap-new "${tap}"
brew extract "--version=${ver}" "${pkg}" "${tap}"
brew install "${tap}/${pkg}@${ver}" || ( echo "Failed to install ${tap}/${pkg}@${ver}" ; exit 1 )
else
( brew install --formula "${pkg}" && echo "Installation of ${pkg} is successful" || brew upgrade --formula "$pkg" ) || ( echo "Failed to install ${pkg}" ; exit 1 )
fi
install_from_brew ${pkg}
done

pip3 install --user cmake-format regex
}

function install_velox_deps_from_brew {
for pkg in ${MACOS_VELOX_DEPS}
do
install_from_brew ${pkg}
done
}

function install_fmt {
github_checkout fmtlib/fmt 10.1.1
cmake_install -DFMT_TEST=OFF
Expand Down Expand Up @@ -116,9 +127,7 @@ function install_re2 {
}

function install_velox_deps {
if [ "${INSTALL_PREREQUISITES:-Y}" == "Y" ]; then
run_and_time install_build_prerequisites
fi
run_and_time install_velox_deps_from_brew
run_and_time install_ranges_v3
run_and_time install_double_conversion
run_and_time install_re2
Expand All @@ -133,17 +142,23 @@ function install_velox_deps {
(return 2> /dev/null) && return # If script was sourced, don't run commands.

(
echo "Installing mac dependencies"
update_brew
if [[ $# -ne 0 ]]; then
for cmd in "$@"; do
run_and_time "${cmd}"
done
echo "All specified dependencies installed!"
else
if [ "${INSTALL_PREREQUISITES:-Y}" == "Y" ]; then
echo "Installing build dependencies"
run_and_time install_build_prerequisites
else
echo "Skipping installation of build dependencies since INSTALL_PREREQUISITES is not set"
fi
install_velox_deps
echo "All deps for Velox installed! Now try \"make\""
fi
)

echo "All deps for Velox installed! Now try \"make\""
echo 'To add cmake-format bin to your $PATH, consider adding this to your ~/.profile:'
echo 'export PATH=$HOME/bin:$HOME/Library/Python/3.7/bin:$PATH'
111 changes: 72 additions & 39 deletions scripts/setup-ubuntu.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,18 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# This script documents setting up a Centos8 host for Velox
# development. Running it should make you ready to compile.
#
# Environment variables:
# * INSTALL_PREREQUISITES="N": Skip installation of packages for build.
# * PROMPT_ALWAYS_RESPOND="n": Automatically respond to interactive prompts.
# Use "n" to never wipe directories.
#
# You can also run individual functions below by specifying them as arguments:
# $ scripts/setup-ubuntu.sh install_googletest install_fmt
#

# Minimal setup for Ubuntu 20.04.
set -eufx -o pipefail
SCRIPTDIR=$(dirname "${BASH_SOURCE[0]}")
Expand All @@ -31,43 +43,50 @@ DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)}
export CMAKE_BUILD_TYPE=Release
SUDO="${SUDO:-"sudo --preserve-env"}"

# Install all velox and folly dependencies.
# The is an issue on 22.04 where a version conflict prevents glog install,
# installing libunwind first fixes this.

${SUDO} apt update
${SUDO} apt install -y libunwind-dev
${SUDO} apt install -y \
g++ \
cmake \
ccache \
ninja-build \
checkinstall \
git \
libc-ares-dev \
libcurl4-openssl-dev \
libssl-dev \
libicu-dev \
libdouble-conversion-dev \
libgoogle-glog-dev \
libbz2-dev \
libgflags-dev \
libgmock-dev \
libevent-dev \
liblz4-dev \
libzstd-dev \
libre2-dev \
libsnappy-dev \
libsodium-dev \
libthrift-dev \
liblzo2-dev \
libelf-dev \
libdwarf-dev \
bison \
flex \
libfl-dev \
tzdata \
wget
# Install packages required for build.
function install_build_prerequisites {
${SUDO} apt update
# The is an issue on 22.04 where a version conflict prevents glog install,
# installing libunwind first fixes this.
${SUDO} apt install -y libunwind-dev
${SUDO} apt install -y \
build-essential \
cmake \
ccache \
ninja-build \
checkinstall \
git \
wget
}

# Install packages required for build.
function install_velox_deps_from_apt {
${SUDO} apt update
${SUDO} apt install -y \
libc-ares-dev \
libcurl4-openssl-dev \
libssl-dev \
libicu-dev \
libdouble-conversion-dev \
libgoogle-glog-dev \
libbz2-dev \
libgflags-dev \
libgmock-dev \
libevent-dev \
liblz4-dev \
libzstd-dev \
libre2-dev \
libsnappy-dev \
libsodium-dev \
libthrift-dev \
liblzo2-dev \
libelf-dev \
libdwarf-dev \
bison \
flex \
libfl-dev \
tzdata
}

function install_fmt {
github_checkout fmtlib/fmt "${FMT_VERSION}"
Expand Down Expand Up @@ -124,6 +143,7 @@ function install_conda {


function install_velox_deps {
run_and_time install_velox_deps_from_apt
run_and_time install_fmt
run_and_time install_boost
run_and_time install_folly
Expand All @@ -134,16 +154,29 @@ function install_velox_deps {
run_and_time install_conda
}

(return 2> /dev/null) && return # If script was sourced, don't run commands.
function install_apt_deps {
install_build_prerequisites
install_velox_deps_from_apt
}

# For backward compatibility, invoke install_apt_deps
(return 2> /dev/null) && install_apt_deps && return # If script was sourced, don't run commands.

(
if [[ $# -ne 0 ]]; then
for cmd in "$@"; do
run_and_time "${cmd}"
done
echo "All specified dependencies installed!"
else
if [ "${INSTALL_PREREQUISITES:-Y}" == "Y" ]; then
echo "Installing build dependencies"
run_and_time install_build_prerequisites
else
echo "Skipping installation of build dependencies since INSTALL_PREREQUISITES is not set"
fi
install_velox_deps
echo "All dependencies for Velox installed!"
fi
)

echo "All dependencies for Velox installed!"

0 comments on commit 7f35620

Please sign in to comment.