diff --git a/.circleci/dist_compile.yml b/.circleci/dist_compile.yml index 8b1fe7f551ed..d7cb2877fc91 100644 --- a/.circleci/dist_compile.yml +++ b/.circleci/dist_compile.yml @@ -413,11 +413,13 @@ jobs: - run: name: "Install Minio Server" command: | - set -xu - cd ~/adapter-deps/install/bin/ - wget https://dl.min.io/server/minio/release/linux-amd64/archive/minio-20220526054841.0.0.x86_64.rpm - rpm -i minio-20220526054841.0.0.x86_64.rpm - rm minio-20220526054841.0.0.x86_64.rpm + if ! rpm -q --quiet minio ; then + set -xu + cd ~/adapter-deps/install/bin/ + wget https://dl.min.io/server/minio/release/linux-amd64/archive/minio-20220526054841.0.0.x86_64.rpm + rpm -i minio-20220526054841.0.0.x86_64.rpm + rm minio-20220526054841.0.0.x86_64.rpm + fi - run: name: "Install Hadoop Dependency" command: | diff --git a/scripts/circleci-container.dockfile b/scripts/circleci-container.dockfile index b3a7ee96c4fe..af517edbc732 100644 --- a/scripts/circleci-container.dockfile +++ b/scripts/circleci-container.dockfile @@ -15,6 +15,11 @@ # FROM quay.io/centos/centos:stream8 ARG cpu_target -COPY scripts/setup-circleci.sh / +ENV CPU_TARGET=$cpu_target + COPY scripts/setup-helper-functions.sh / -RUN mkdir build && ( cd build && CPU_TARGET="$cpu_target" bash /setup-circleci.sh ) && rm -rf build +COPY scripts/setup-centos8.sh / +COPY scripts/setup-adapters.sh / +RUN mkdir build && ( cd build && bash /setup-centos8.sh ) && rm -rf build +RUN mkdir build && ( cd build && source /opt/rh/gcc-toolset-9/enable && \ + bash /setup-adapters.sh ) && rm -rf build diff --git a/scripts/setup-adapters.sh b/scripts/setup-adapters.sh index a42c9f5480ba..632dbd33b700 100755 --- a/scripts/setup-adapters.sh +++ b/scripts/setup-adapters.sh @@ -22,13 +22,35 @@ set -eufx -o pipefail SCRIPTDIR=$(dirname "${BASH_SOURCE[0]}") source $SCRIPTDIR/setup-helper-functions.sh DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)} +CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}" +MACHINE=$(uname -m) -function install_aws-sdk-cpp { +function install_aws_deps { local AWS_REPO_NAME="aws/aws-sdk-cpp" local AWS_SDK_VERSION="1.11.169" github_checkout $AWS_REPO_NAME $AWS_SDK_VERSION --depth 1 --recurse-submodules - cmake_install -DCMAKE_BUILD_TYPE=Debug -DBUILD_SHARED_LIBS:BOOL=OFF -DMINIMIZE_SIZE:BOOL=ON -DENABLE_TESTING:BOOL=OFF -DBUILD_ONLY:STRING="s3;identity-management" + cmake_install -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS:BOOL=OFF -DMINIMIZE_SIZE:BOOL=ON -DENABLE_TESTING:BOOL=OFF -DBUILD_ONLY:STRING="s3;identity-management" + # Dependencies for S3 testing + # We need this specific version of Minio for testing. + if [[ "$OSTYPE" == linux-gnu* ]]; then + wget https://dl.min.io/server/minio/release/linux-amd64/archive/minio-20220526054841.0.0.x86_64.rpm + rpm -i minio-20220526054841.0.0.x86_64.rpm + rm minio-20220526054841.0.0.x86_64.rpm + fi + # minio will have to approved under the Privacy & Security on MacOS on first use. + if [[ "$OSTYPE" == darwin* ]]; then + if [ "$MACHINE" = "x86_64" ]; then + wget https://dl.min.io/server/minio/release/darwin-arm64/archive/minio.RELEASE.2022-05-26T05-48-41Z -O minio + chmod +x ./minio + sudo mv ./minio /usr/local/bin/ + fi + if [ "$MACHINE" = "arm64" ]; then + wget https://dl.min.io/server/minio/release/darwin-arm64/archive/minio.RELEASE.2022-05-26T05-48-41Z -O minio + chmod +x ./minio + sudo mv ./minio /usr/local/bin/ + fi + fi } function install_gcs-sdk-cpp { @@ -76,25 +98,25 @@ function install_azure-storage-sdk-cpp { sed -i "s/\"version-string\"/\"builtin-baseline\": \"$vcpkg_commit_id\",\"version-string\"/" vcpkg.json sed -i "s/\"version-string\"/\"overrides\": [{ \"name\": \"openssl\", \"version-string\": \"$openssl_version\" }],\"version-string\"/" vcpkg.json fi - cmake_install -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF + cmake_install -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS=OFF cd - # install azure-storage-common cd sdk/storage/azure-storage-common - cmake_install -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF + cmake_install -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS=OFF cd - # install azure-storage-blobs cd sdk/storage/azure-storage-blobs - cmake_install -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF + cmake_install -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS=OFF cd - # install azure-storage-files-datalake cd sdk/storage/azure-storage-files-datalake - cmake_install -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF + cmake_install -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS=OFF } -function install_libhdfs3 { +function install_hdfs_deps { github_checkout apache/hawq master cd $DEPENDENCY_DIR/hawq/depends/libhdfs3 if [[ "$OSTYPE" == darwin* ]]; then @@ -105,6 +127,9 @@ function install_libhdfs3 { if [[ "$OSTYPE" == linux-gnu* ]]; then sed -i "/FIND_PACKAGE(GoogleTest REQUIRED)/d" ./CMakeLists.txt sed -i "s/dumpversion/dumpfullversion/" ./CMake/Platform.cmake + # Dependencies for Hadoop testing + wget_and_untar https://archive.apache.org/dist/hadoop/common/hadoop-2.10.1/hadoop-2.10.1.tar.gz hadoop + cp -a hadoop /usr/local/ fi cmake_install } @@ -178,10 +203,10 @@ if [ $install_gcs -eq 1 ]; then install_gcs-sdk-cpp fi if [ $install_aws -eq 1 ]; then - install_aws-sdk-cpp + install_aws_deps fi if [ $install_hdfs -eq 1 ]; then - install_libhdfs3 + install_hdfs_deps fi if [ $install_abfs -eq 1 ]; then install_azure-storage-sdk-cpp diff --git a/scripts/setup-centos8.sh b/scripts/setup-centos8.sh index e6808097509f..4e75aac660bb 100755 --- a/scripts/setup-centos8.sh +++ b/scripts/setup-centos8.sh @@ -23,6 +23,8 @@ NPROC=$(getconf _NPROCESSORS_ONLN) export CFLAGS=$(get_cxx_flags $CPU_TARGET) # Used by LZO. export CXXFLAGS=$CFLAGS # Used by boost. export CPPFLAGS=$CFLAGS # Used by LZO. +CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}" +BUILD_DUCKDB="${BUILD_DUCKDB:-true}" function dnf_install { dnf install -y -q --setopt=install_weak_deps=False "$@" @@ -30,45 +32,47 @@ function dnf_install { dnf_install epel-release dnf-plugins-core # For ccache, ninja dnf config-manager --set-enabled powertools -dnf_install ninja-build ccache gcc-toolset-9 git wget which libevent-devel \ +dnf_install ninja-build cmake curl ccache gcc-toolset-9 git wget which libevent-devel \ openssl-devel re2-devel libzstd-devel lz4-devel double-conversion-devel \ - libdwarf-devel curl-devel cmake libicu-devel + libdwarf-devel curl-devel libicu-devel dnf remove -y gflags # Required for Thrift -dnf_install autoconf automake libtool bison flex python3 - -# Required for google-cloud-storage -dnf_install curl-devel c-ares-devel +dnf_install autoconf automake libtool bison flex python3 libsodium-devel dnf_install conda +# install sphinx for doc gen +pip3 install sphinx sphinx-tabs breathe sphinx_rtd_theme + # Activate gcc9; enable errors on unset variables afterwards. source /opt/rh/gcc-toolset-9/enable || exit 1 set -u -function cmake_install_deps { +function cmake_install { cmake -B "$1-build" -GNinja -DCMAKE_CXX_STANDARD=17 \ - -DCMAKE_CXX_FLAGS="${CFLAGS}" -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_BUILD_TYPE=Release -Wno-dev "$@" + -DCMAKE_CXX_FLAGS="${CFLAGS}" -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" -Wno-dev "$@" ninja -C "$1-build" install } -function wget_and_untar { - local URL=$1 - local DIR=$2 - mkdir -p "${DIR}" - wget -q --max-redirect 3 -O - "${URL}" | tar -xz -C "${DIR}" --strip-components=1 -} +# Fetch sources. +wget_and_untar https://github.com/gflags/gflags/archive/v2.2.2.tar.gz gflags +wget_and_untar https://github.com/google/glog/archive/v0.4.0.tar.gz glog +wget_and_untar http://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz lzo +wget_and_untar https://boostorg.jfrog.io/artifactory/main/release/1.72.0/source/boost_1_72_0.tar.gz boost +wget_and_untar https://github.com/google/snappy/archive/1.1.8.tar.gz snappy +wget_and_untar https://github.com/fmtlib/fmt/archive/10.1.1.tar.gz fmt +wget_and_untar https://github.com/protocolbuffers/protobuf/releases/download/v21.4/protobuf-all-21.4.tar.gz protobuf -# Fetch sources. -wget_and_untar https://github.com/gflags/gflags/archive/v2.2.2.tar.gz gflags & -wget_and_untar https://github.com/google/glog/archive/v0.4.0.tar.gz glog & -wget_and_untar http://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz lzo & -wget_and_untar https://boostorg.jfrog.io/artifactory/main/release/1.72.0/source/boost_1_72_0.tar.gz boost & -wget_and_untar https://github.com/google/snappy/archive/1.1.8.tar.gz snappy & -wget_and_untar https://github.com/fmtlib/fmt/archive/10.1.1.tar.gz fmt & +FB_OS_VERSION="v2023.12.04.00" + +wget_and_untar https://github.com/facebookincubator/fizz/archive/refs/tags/${FB_OS_VERSION}.tar.gz fizz +wget_and_untar https://github.com/facebook/folly/archive/refs/tags/${FB_OS_VERSION}.tar.gz folly +wget_and_untar https://github.com/facebook/wangle/archive/refs/tags/${FB_OS_VERSION}.tar.gz wangle +wget_and_untar https://github.com/facebook/fbthrift/archive/refs/tags/${FB_OS_VERSION}.tar.gz fbthrift +wget_and_untar https://github.com/facebook/mvfst/archive/refs/tags/${FB_OS_VERSION}.tar.gz mvfst wait # For cmake and source downloads to complete. @@ -86,9 +90,43 @@ wait # For cmake and source downloads to complete. ./b2 "-j$(nproc)" -d0 install threading=multi ) -cmake_install_deps gflags -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON -DBUILD_gflags_LIB=ON -DLIB_SUFFIX=64 -DCMAKE_INSTALL_PREFIX:PATH=/usr -cmake_install_deps glog -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX:PATH=/usr -cmake_install_deps snappy -DSNAPPY_BUILD_TESTS=OFF -cmake_install_deps fmt -DFMT_TEST=OFF +( + cd protobuf + ./configure --prefix=/usr + make "-j${NPROC}" + make install + ldconfig +) + +cmake_install gflags -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON -DBUILD_gflags_LIB=ON -DLIB_SUFFIX=64 -DCMAKE_INSTALL_PREFIX:PATH=/usr +cmake_install glog -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX:PATH=/usr +cmake_install snappy -DSNAPPY_BUILD_TESTS=OFF +cmake_install fmt -DFMT_TEST=OFF +cmake_install folly -DFOLLY_HAVE_INT128_T=ON + +cmake_install fizz/fizz -DBUILD_TESTS=OFF +cmake_install wangle/wangle -DBUILD_TESTS=OFF +cmake_install mvfst -DBUILD_TESTS=OFF +cmake_install fbthrift -Denable_tests=OFF + +if $BUILD_DUCKDB ; then + echo 'Building DuckDB' + mkdir ~/duckdb-install && cd ~/duckdb-install + wget https://github.com/duckdb/duckdb/archive/refs/tags/v0.8.1.tar.gz + tar -xf v0.8.1.tar.gz + cd duckdb-0.8.1 + mkdir build && cd build + CMAKE_FLAGS=( + "-DBUILD_UNITTESTS=OFF" + "-DENABLE_SANITIZER=OFF" + "-DENABLE_UBSAN=OFF" + "-DBUILD_SHELL=OFF" + "-DEXPORT_DLL_SYMBOLS=OFF" + "-DCMAKE_BUILD_TYPE=Release" + ) + cmake ${CMAKE_FLAGS[*]} .. + make install -j 16 + rm -rf ~/duckdb-install +fi dnf clean all diff --git a/scripts/setup-circleci.sh b/scripts/setup-circleci.sh deleted file mode 100755 index f041ac089a4e..000000000000 --- a/scripts/setup-circleci.sh +++ /dev/null @@ -1,128 +0,0 @@ -#!/bin/bash -# Copyright (c) Facebook, Inc. and its affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -efx -o pipefail -# Some of the packages must be build with the same compiler flags -# so that some low level types are the same size. Also, disable warnings. -SCRIPTDIR=$(dirname "${BASH_SOURCE[0]}") -source $SCRIPTDIR/setup-helper-functions.sh -CPU_TARGET="${CPU_TARGET:-avx}" -NPROC=$(getconf _NPROCESSORS_ONLN) -export CFLAGS=$(get_cxx_flags $CPU_TARGET) # Used by LZO. -export CXXFLAGS=$CFLAGS # Used by boost. -export CPPFLAGS=$CFLAGS # Used by LZO. - -function dnf_install { - dnf install -y -q --setopt=install_weak_deps=False "$@" -} - -dnf_install epel-release dnf-plugins-core # For ccache, ninja -dnf config-manager --set-enabled powertools -dnf_install ninja-build cmake curl ccache gcc-toolset-9 git wget which libevent-devel \ - openssl-devel re2-devel libzstd-devel lz4-devel double-conversion-devel \ - libdwarf-devel curl-devel libicu-devel - -dnf remove -y gflags - -# Required for Thrift -dnf_install autoconf automake libtool bison flex python3 libsodium-devel - -dnf_install conda - -# install sphinx for doc gen -pip3 install sphinx sphinx-tabs breathe sphinx_rtd_theme - -# Activate gcc9; enable errors on unset variables afterwards. -source /opt/rh/gcc-toolset-9/enable || exit 1 -set -u - -function cmake_install { - cmake -B "$1-build" -GNinja -DCMAKE_CXX_STANDARD=17 \ - -DCMAKE_CXX_FLAGS="${CFLAGS}" -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_BUILD_TYPE=Release -Wno-dev "$@" - ninja -C "$1-build" install -} - -function wget_and_untar { - local URL=$1 - local DIR=$2 - mkdir -p "${DIR}" - pushd "${DIR}" - curl -L "${URL}" > $2.tar.gz - tar -xz --strip-components=1 -f $2.tar.gz - popd -} - -# untar cmake binary release directly to /usr. -wget_and_untar https://github.com/Kitware/CMake/releases/download/v3.17.5/cmake-3.17.5-Linux-x86_64.tar.gz /usr & - -# Fetch sources. -wget_and_untar https://github.com/gflags/gflags/archive/v2.2.2.tar.gz gflags -wget_and_untar https://github.com/google/glog/archive/v0.4.0.tar.gz glog -wget_and_untar http://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz lzo -wget_and_untar https://boostorg.jfrog.io/artifactory/main/release/1.72.0/source/boost_1_72_0.tar.gz boost -wget_and_untar https://github.com/google/snappy/archive/1.1.8.tar.gz snappy -wget_and_untar https://github.com/fmtlib/fmt/archive/10.1.1.tar.gz fmt - -# wget_and_untar https://github.com/ericniebler/range-v3/archive/0.11.0.tar.gz ranges-v3 -wget_and_untar https://archive.apache.org/dist/hadoop/common/hadoop-2.10.1/hadoop-2.10.1.tar.gz hadoop -wget_and_untar https://github.com/protocolbuffers/protobuf/releases/download/v21.4/protobuf-all-21.4.tar.gz protobuf - -FB_OS_VERSION="v2023.12.04.00" - -wget_and_untar https://github.com/facebookincubator/fizz/archive/refs/tags/${FB_OS_VERSION}.tar.gz fizz -wget_and_untar https://github.com/facebook/folly/archive/refs/tags/${FB_OS_VERSION}.tar.gz folly -wget_and_untar https://github.com/facebook/wangle/archive/refs/tags/${FB_OS_VERSION}.tar.gz wangle -wget_and_untar https://github.com/facebook/fbthrift/archive/refs/tags/${FB_OS_VERSION}.tar.gz fbthrift -wget_and_untar https://github.com/facebook/mvfst/archive/refs/tags/${FB_OS_VERSION}.tar.gz mvfst - -wait # For cmake and source downloads to complete. - -cp -a hadoop /usr/local/ - -# Build & install. -( - cd lzo - ./configure --prefix=/usr --enable-shared --disable-static --docdir=/usr/share/doc/lzo-2.10 - make "-j$(nproc)" - make install -) - -( - cd boost - ./bootstrap.sh --prefix=/usr/local - ./b2 "-j$(nproc)" -d0 install threading=multi -) - -( - cd protobuf - ./configure --prefix=/usr - make "-j${NPROC}" - make install - ldconfig -) - -cmake_install gflags -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON -DBUILD_gflags_LIB=ON -DLIB_SUFFIX=64 -DCMAKE_INSTALL_PREFIX:PATH=/usr -cmake_install glog -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX:PATH=/usr -cmake_install snappy -DSNAPPY_BUILD_TESTS=OFF -cmake_install fmt -DFMT_TEST=OFF -cmake_install folly -DFOLLY_HAVE_INT128_T=ON - -cmake_install fizz/fizz -DBUILD_TESTS=OFF -cmake_install wangle/wangle -DBUILD_TESTS=OFF -cmake_install mvfst -DBUILD_TESTS=OFF -cmake_install fbthrift -Denable_tests=OFF -# cmake_install ranges-v3 - -dnf clean all diff --git a/scripts/setup-helper-functions.sh b/scripts/setup-helper-functions.sh index 8076edc4a185..9495d42bfc36 100644 --- a/scripts/setup-helper-functions.sh +++ b/scripts/setup-helper-functions.sh @@ -119,6 +119,16 @@ function get_cxx_flags { } +function wget_and_untar { + local URL=$1 + local DIR=$2 + mkdir -p "${DIR}" + pushd "${DIR}" + curl -L "${URL}" > $2.tar.gz + tar -xz --strip-components=1 -f $2.tar.gz + popd +} + function cmake_install { local NAME=$(basename "$(pwd)") local BINARY_DIR=_build diff --git a/scripts/setup-macos.sh b/scripts/setup-macos.sh index 51dcfadd9a50..e0d727134f2a 100755 --- a/scripts/setup-macos.sh +++ b/scripts/setup-macos.sh @@ -34,7 +34,9 @@ source $SCRIPTDIR/setup-helper-functions.sh NPROC=$(getconf _NPROCESSORS_ONLN) DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)} -MACOS_DEPS="ninja flex bison cmake ccache protobuf@21 icu4c boost gflags glog libevent lz4 lzo snappy xz zstd openssl@1.1" +MACOS_DEPS="ninja flex bison cmake ccache protobuf@21 icu4c boost gflags glog libevent lz4 lzo snappy xz zstd openssl" + +FB_OS_VERSION="v2023.12.04.00" function run_and_time { time "$@" || (echo "Failed to run $* ." ; exit 1 ) @@ -93,11 +95,31 @@ function install_fmt { } function install_folly { - github_checkout facebook/folly "v2023.12.04.00" - OPENSSL_ROOT_DIR=$(brew --prefix openssl@1.1) \ + github_checkout facebook/folly "${FB_OS_VERSION}" cmake_install -DBUILD_TESTS=OFF -DFOLLY_HAVE_INT128_T=ON } +function install_fizz { + github_checkout facebookincubator/fizz "${FB_OS_VERSION}" + cmake_install -DBUILD_TESTS=OFF -S fizz +} + +function install_wangle { + github_checkout facebook/wangle "${FB_OS_VERSION}" + cmake_install -DBUILD_TESTS=OFF -S wangle +} + +function install_mvfst { + github_checkout facebook/mvfst "${FB_OS_VERSION}" + cmake_install -DBUILD_TESTS=OFF +} + + +function install_fbthrift { + github_checkout facebook/fbthrift "${FB_OS_VERSION}" + cmake_install -DBUILD_TESTS=OFF +} + function install_double_conversion { github_checkout google/double-conversion v3.1.5 cmake_install -DBUILD_TESTING=OFF @@ -118,9 +140,14 @@ function install_velox_deps { run_and_time install_build_prerequisites fi run_and_time install_ranges_v3 - run_and_time install_fmt run_and_time install_double_conversion run_and_time install_re2 + run_and_time install_fmt + run_and_time install_folly + run_and_time install_fizz + run_and_time install_wangle + run_and_time install_mvfst + run_and_time install_fbthrift } (return 2> /dev/null) && return # If script was sourced, don't run commands.