diff --git a/.github/workflows/velox_be.yml b/.github/workflows/velox_be.yml index 71618bb547ee..ecb44813f72a 100644 --- a/.github/workflows/velox_be.yml +++ b/.github/workflows/velox_be.yml @@ -63,7 +63,7 @@ jobs: $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' cd /opt/gluten/ep/build-velox/src && \ ./get_velox.sh --velox_home=/opt/velox && \ - ./build_velox.sh --run_setup_script=OFF --velox_home=/opt/velox --enable_ep_cache=ON --build_test_utils=ON' + ./build_velox.sh --run_setup_script=ON --velox_home=/opt/velox --enable_ep_cache=ON --build_test_utils=ON' - name: Build Gluten CPP library run: | $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' @@ -110,7 +110,7 @@ jobs: $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' cd /opt/gluten/ep/build-velox/src && \ ./get_velox.sh --velox_home=/opt/velox && \ - ./build_velox.sh --run_setup_script=OFF --velox_home=/opt/velox --enable_ep_cache=ON' + ./build_velox.sh --run_setup_script=ON --velox_home=/opt/velox --enable_ep_cache=ON' - name: Build Gluten CPP library run: | $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' @@ -149,7 +149,7 @@ jobs: $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' cd /opt/gluten/ep/build-velox/src && \ ./get_velox.sh --velox_home=/opt/velox && \ - ./build_velox.sh --run_setup_script=OFF --velox_home=/opt/velox --enable_ep_cache=ON' + ./build_velox.sh --run_setup_script=ON --velox_home=/opt/velox --enable_ep_cache=ON' - name: Build Gluten CPP library run: | $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' @@ -197,7 +197,7 @@ jobs: $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' cd /opt/gluten/ep/build-velox/src && \ ./get_velox.sh --velox_home=/opt/velox && \ - ./build_velox.sh --run_setup_script=OFF --velox_home=/opt/velox --enable_ep_cache=ON' + ./build_velox.sh --run_setup_script=ON --velox_home=/opt/velox --enable_ep_cache=ON' - name: Build Gluten CPP library run: | $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' @@ -228,7 +228,7 @@ jobs: $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' cd /opt/gluten/ep/build-velox/src && \ ./get_velox.sh --velox_home=/opt/velox && \ - ./build_velox.sh --run_setup_script=OFF --velox_home=/opt/velox --enable_ep_cache=ON' + ./build_velox.sh --run_setup_script=ON --velox_home=/opt/velox --enable_ep_cache=ON' - name: Build Gluten CPP library run: | $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' @@ -266,7 +266,7 @@ jobs: $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' cd /opt/gluten/ep/build-velox/src && \ ./get_velox.sh --velox_home=/opt/velox && \ - ./build_velox.sh --run_setup_script=OFF --velox_home=/opt/velox --enable_ep_cache=ON' + ./build_velox.sh --run_setup_script=ON --velox_home=/opt/velox --enable_ep_cache=ON' - name: Build Gluten CPP library run: | $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' @@ -297,7 +297,7 @@ jobs: $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' cd /opt/gluten/ep/build-velox/src && \ ./get_velox.sh --velox_home=/opt/velox --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON --enable_abfs=ON && \ - ./build_velox.sh --run_setup_script=OFF --velox_home=/opt/velox --enable_ep_cache=ON --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON --enable_abfs=ON' + ./build_velox.sh --run_setup_script=ON --velox_home=/opt/velox --enable_ep_cache=ON --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON --enable_abfs=ON' - name: Build Gluten CPP library run: | $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' @@ -349,7 +349,7 @@ jobs: $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' cd /opt/gluten/ep/build-velox/src && \ ./get_velox.sh --velox_home=/opt/velox --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON --enable_abfs=ON && \ - ./build_velox.sh --run_setup_script=OFF --velox_home=/opt/velox --enable_ep_cache=ON --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON --enable_abfs=ON' + ./build_velox.sh --run_setup_script=ON --velox_home=/opt/velox --enable_ep_cache=ON --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON --enable_abfs=ON' - name: Build Gluten CPP library run: | $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' @@ -406,7 +406,7 @@ jobs: sudo yum -y install patch && \ cd /opt/gluten/ep/build-velox/src && \ ./get_velox.sh --velox_home=/opt/velox --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON --enable_abfs=ON && \ - ./build_velox.sh --run_setup_script=OFF --velox_home=/opt/velox --enable_ep_cache=ON --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON --enable_abfs=ON' + ./build_velox.sh --run_setup_script=ON --velox_home=/opt/velox --enable_ep_cache=ON --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON --enable_abfs=ON' - name: Build Gluten CPP library run: | $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' diff --git a/cpp/compile.sh b/cpp/compile.sh index b14e48b16fac..655a081b8a00 100755 --- a/cpp/compile.sh +++ b/cpp/compile.sh @@ -142,5 +142,6 @@ cmake .. \ -DENABLE_GCS=${ENABLE_GCS} \ -DENABLE_S3=${ENABLE_S3} \ -DENABLE_HDFS=${ENABLE_HDFS} \ - -DENABLE_ABFS=${ENABLE_ABFS} -make -j$NPROC + -DENABLE_ABFS=${ENABLE_ABFS} \ + -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON +make -j$NPROC VERBOSE=1 diff --git a/cpp/velox/CMakeLists.txt b/cpp/velox/CMakeLists.txt index 41b848bfb637..b0e6bd8cff9c 100644 --- a/cpp/velox/CMakeLists.txt +++ b/cpp/velox/CMakeLists.txt @@ -286,6 +286,7 @@ macro(find_azure) find_package(LibXml2 REQUIRED) set (CMAKE_FIND_LIBRARY_SUFFIXES ".a") find_package(azure-storage-blobs-cpp CONFIG REQUIRED) + find_package(azure-storage-files-datalake-cpp CONFIG REQUIRED) endmacro() # Build Velox backend. @@ -425,4 +426,5 @@ if(ENABLE_ABFS) add_definitions(-DENABLE_ABFS) find_azure() target_link_libraries(velox PUBLIC Azure::azure-storage-blobs) + target_link_libraries(velox PUBLIC Azure::azure-storage-files-datalake) endif() diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 3350243f5b8c..4251ff4d24b5 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -16,8 +16,8 @@ set -exu -VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_02_08 +VELOX_REPO=https://github.com/FelixYBW/velox.git +VELOX_BRANCH=2024_02_09 VELOX_HOME="" #Set on run gluten on HDFS @@ -106,6 +106,8 @@ function process_setup_ubuntu { sed -i "s/apt install -y/sudo apt install -y/" ${VELOX_HOME}/scripts/setup-adapters.sh if [ $ENABLE_S3 == "ON" ]; then sed -i '/^ run_and_time install_folly/a \ \ '${VELOX_HOME}/scripts'/setup-adapters.sh aws' scripts/setup-ubuntu.sh + # it's used for velox CI + sed -i 's/rpm -i minio-20220526054841.0.0.x86_64.rpm/#rpm -i minio-20220526054841.0.0.x86_64.rpm/g' scripts/setup-adapters.sh fi if [ $ENABLE_GCS == "ON" ]; then sed -i '/^ run_and_time install_folly/a \ \ '${VELOX_HOME}/scripts'/setup-adapters.sh gcs' scripts/setup-ubuntu.sh