From ee9d4ddc80bce905713d2063ed2145e25f81c288 Mon Sep 17 00:00:00 2001 From: binwei Date: Fri, 9 Feb 2024 05:18:21 +0000 Subject: [PATCH 1/9] rebase 2024_02_09 remove #8678 remove #7218 --- ep/build-velox/src/get_velox.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 3350243f5b8c..9e80a05128a5 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -16,8 +16,8 @@ set -exu -VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_02_08 +VELOX_REPO=https://github.com/FelixYBW/velox.git +VELOX_BRANCH=2024_02_09 VELOX_HOME="" #Set on run gluten on HDFS From ab58669bac74b4aca346f6ef2d3686350067942b Mon Sep 17 00:00:00 2001 From: root Date: Sun, 11 Feb 2024 20:06:25 -0500 Subject: [PATCH 2/9] print cpp build cmd --- cpp/compile.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cpp/compile.sh b/cpp/compile.sh index b14e48b16fac..655a081b8a00 100755 --- a/cpp/compile.sh +++ b/cpp/compile.sh @@ -142,5 +142,6 @@ cmake .. \ -DENABLE_GCS=${ENABLE_GCS} \ -DENABLE_S3=${ENABLE_S3} \ -DENABLE_HDFS=${ENABLE_HDFS} \ - -DENABLE_ABFS=${ENABLE_ABFS} -make -j$NPROC + -DENABLE_ABFS=${ENABLE_ABFS} \ + -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON +make -j$NPROC VERBOSE=1 From 2927e04ecc7af149c963040fc4908f8d5ea29adc Mon Sep 17 00:00:00 2001 From: root Date: Sun, 11 Feb 2024 20:48:07 -0500 Subject: [PATCH 3/9] runsetup script --- .github/workflows/velox_be.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/velox_be.yml b/.github/workflows/velox_be.yml index 71618bb547ee..18b67b340a3c 100644 --- a/.github/workflows/velox_be.yml +++ b/.github/workflows/velox_be.yml @@ -63,7 +63,7 @@ jobs: $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' cd /opt/gluten/ep/build-velox/src && \ ./get_velox.sh --velox_home=/opt/velox && \ - ./build_velox.sh --run_setup_script=OFF --velox_home=/opt/velox --enable_ep_cache=ON --build_test_utils=ON' + ./build_velox.sh --run_setup_script=ON --velox_home=/opt/velox --enable_ep_cache=ON --build_test_utils=ON' - name: Build Gluten CPP library run: | $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' @@ -110,7 +110,7 @@ jobs: $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' cd /opt/gluten/ep/build-velox/src && \ ./get_velox.sh --velox_home=/opt/velox && \ - ./build_velox.sh --run_setup_script=OFF --velox_home=/opt/velox --enable_ep_cache=ON' + ./build_velox.sh --run_setup_script=ON --velox_home=/opt/velox --enable_ep_cache=ON' - name: Build Gluten CPP library run: | $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' From ae8f2c6b4c03d5f1c45a20e0119c1572c459b48d Mon Sep 17 00:00:00 2001 From: root Date: Mon, 12 Feb 2024 03:28:29 -0500 Subject: [PATCH 4/9] enabled setup env for velox build --- .github/workflows/velox_be.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/velox_be.yml b/.github/workflows/velox_be.yml index 18b67b340a3c..ecb44813f72a 100644 --- a/.github/workflows/velox_be.yml +++ b/.github/workflows/velox_be.yml @@ -149,7 +149,7 @@ jobs: $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' cd /opt/gluten/ep/build-velox/src && \ ./get_velox.sh --velox_home=/opt/velox && \ - ./build_velox.sh --run_setup_script=OFF --velox_home=/opt/velox --enable_ep_cache=ON' + ./build_velox.sh --run_setup_script=ON --velox_home=/opt/velox --enable_ep_cache=ON' - name: Build Gluten CPP library run: | $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' @@ -197,7 +197,7 @@ jobs: $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' cd /opt/gluten/ep/build-velox/src && \ ./get_velox.sh --velox_home=/opt/velox && \ - ./build_velox.sh --run_setup_script=OFF --velox_home=/opt/velox --enable_ep_cache=ON' + ./build_velox.sh --run_setup_script=ON --velox_home=/opt/velox --enable_ep_cache=ON' - name: Build Gluten CPP library run: | $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' @@ -228,7 +228,7 @@ jobs: $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' cd /opt/gluten/ep/build-velox/src && \ ./get_velox.sh --velox_home=/opt/velox && \ - ./build_velox.sh --run_setup_script=OFF --velox_home=/opt/velox --enable_ep_cache=ON' + ./build_velox.sh --run_setup_script=ON --velox_home=/opt/velox --enable_ep_cache=ON' - name: Build Gluten CPP library run: | $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' @@ -266,7 +266,7 @@ jobs: $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' cd /opt/gluten/ep/build-velox/src && \ ./get_velox.sh --velox_home=/opt/velox && \ - ./build_velox.sh --run_setup_script=OFF --velox_home=/opt/velox --enable_ep_cache=ON' + ./build_velox.sh --run_setup_script=ON --velox_home=/opt/velox --enable_ep_cache=ON' - name: Build Gluten CPP library run: | $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' @@ -297,7 +297,7 @@ jobs: $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' cd /opt/gluten/ep/build-velox/src && \ ./get_velox.sh --velox_home=/opt/velox --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON --enable_abfs=ON && \ - ./build_velox.sh --run_setup_script=OFF --velox_home=/opt/velox --enable_ep_cache=ON --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON --enable_abfs=ON' + ./build_velox.sh --run_setup_script=ON --velox_home=/opt/velox --enable_ep_cache=ON --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON --enable_abfs=ON' - name: Build Gluten CPP library run: | $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' @@ -349,7 +349,7 @@ jobs: $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' cd /opt/gluten/ep/build-velox/src && \ ./get_velox.sh --velox_home=/opt/velox --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON --enable_abfs=ON && \ - ./build_velox.sh --run_setup_script=OFF --velox_home=/opt/velox --enable_ep_cache=ON --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON --enable_abfs=ON' + ./build_velox.sh --run_setup_script=ON --velox_home=/opt/velox --enable_ep_cache=ON --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON --enable_abfs=ON' - name: Build Gluten CPP library run: | $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' @@ -406,7 +406,7 @@ jobs: sudo yum -y install patch && \ cd /opt/gluten/ep/build-velox/src && \ ./get_velox.sh --velox_home=/opt/velox --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON --enable_abfs=ON && \ - ./build_velox.sh --run_setup_script=OFF --velox_home=/opt/velox --enable_ep_cache=ON --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON --enable_abfs=ON' + ./build_velox.sh --run_setup_script=ON --velox_home=/opt/velox --enable_ep_cache=ON --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON --enable_abfs=ON' - name: Build Gluten CPP library run: | $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' From 747d015350b5b611f52ac784577ad4153c2ac101 Mon Sep 17 00:00:00 2001 From: root Date: Mon, 12 Feb 2024 15:24:18 -0500 Subject: [PATCH 5/9] fix compile bug --- ep/build-velox/src/get_velox.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 9e80a05128a5..e584e4514382 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -106,6 +106,8 @@ function process_setup_ubuntu { sed -i "s/apt install -y/sudo apt install -y/" ${VELOX_HOME}/scripts/setup-adapters.sh if [ $ENABLE_S3 == "ON" ]; then sed -i '/^ run_and_time install_folly/a \ \ '${VELOX_HOME}/scripts'/setup-adapters.sh aws' scripts/setup-ubuntu.sh + # it's used for velox CI + sed -i 's/^rpm -i minio-20220526054841.0.0.x86_64.rpm/#rpm -i minio-20220526054841.0.0.x86_64.rpm/g' script/setup-adapters.sh fi if [ $ENABLE_GCS == "ON" ]; then sed -i '/^ run_and_time install_folly/a \ \ '${VELOX_HOME}/scripts'/setup-adapters.sh gcs' scripts/setup-ubuntu.sh From 82f3be4f585b44dc6681264248fbf136d49af3d5 Mon Sep 17 00:00:00 2001 From: root Date: Mon, 12 Feb 2024 15:42:26 -0500 Subject: [PATCH 6/9] fix bug --- ep/build-velox/src/get_velox.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index e584e4514382..9a68637b79c5 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -107,7 +107,7 @@ function process_setup_ubuntu { if [ $ENABLE_S3 == "ON" ]; then sed -i '/^ run_and_time install_folly/a \ \ '${VELOX_HOME}/scripts'/setup-adapters.sh aws' scripts/setup-ubuntu.sh # it's used for velox CI - sed -i 's/^rpm -i minio-20220526054841.0.0.x86_64.rpm/#rpm -i minio-20220526054841.0.0.x86_64.rpm/g' script/setup-adapters.sh + sed -i 's/^rpm -i minio-20220526054841.0.0.x86_64.rpm/#rpm -i minio-20220526054841.0.0.x86_64.rpm/g' scripts/setup-adapters.sh fi if [ $ENABLE_GCS == "ON" ]; then sed -i '/^ run_and_time install_folly/a \ \ '${VELOX_HOME}/scripts'/setup-adapters.sh gcs' scripts/setup-ubuntu.sh From 42fc0f49de29f0d62f0f8460464c66bec7d7fde4 Mon Sep 17 00:00:00 2001 From: root Date: Mon, 12 Feb 2024 15:53:32 -0500 Subject: [PATCH 7/9] fix --- ep/build-velox/src/get_velox.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 9a68637b79c5..4251ff4d24b5 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -107,7 +107,7 @@ function process_setup_ubuntu { if [ $ENABLE_S3 == "ON" ]; then sed -i '/^ run_and_time install_folly/a \ \ '${VELOX_HOME}/scripts'/setup-adapters.sh aws' scripts/setup-ubuntu.sh # it's used for velox CI - sed -i 's/^rpm -i minio-20220526054841.0.0.x86_64.rpm/#rpm -i minio-20220526054841.0.0.x86_64.rpm/g' scripts/setup-adapters.sh + sed -i 's/rpm -i minio-20220526054841.0.0.x86_64.rpm/#rpm -i minio-20220526054841.0.0.x86_64.rpm/g' scripts/setup-adapters.sh fi if [ $ENABLE_GCS == "ON" ]; then sed -i '/^ run_and_time install_folly/a \ \ '${VELOX_HOME}/scripts'/setup-adapters.sh gcs' scripts/setup-ubuntu.sh From bac5f528d8fefd60c8164fc84a69997bf22752a6 Mon Sep 17 00:00:00 2001 From: root Date: Mon, 12 Feb 2024 18:16:43 -0500 Subject: [PATCH 8/9] fix link err --- cpp/velox/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/velox/CMakeLists.txt b/cpp/velox/CMakeLists.txt index 41b848bfb637..387c33d7f28e 100644 --- a/cpp/velox/CMakeLists.txt +++ b/cpp/velox/CMakeLists.txt @@ -425,4 +425,5 @@ if(ENABLE_ABFS) add_definitions(-DENABLE_ABFS) find_azure() target_link_libraries(velox PUBLIC Azure::azure-storage-blobs) + target_link_libraries(velox PUBLIC Azure::azure-storage-files-datalake) endif() From 44a19c1843c66d9eeb965b8a29bcfeddc552314a Mon Sep 17 00:00:00 2001 From: root Date: Mon, 12 Feb 2024 18:43:00 -0500 Subject: [PATCH 9/9] fix --- cpp/velox/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/velox/CMakeLists.txt b/cpp/velox/CMakeLists.txt index 387c33d7f28e..b0e6bd8cff9c 100644 --- a/cpp/velox/CMakeLists.txt +++ b/cpp/velox/CMakeLists.txt @@ -286,6 +286,7 @@ macro(find_azure) find_package(LibXml2 REQUIRED) set (CMAKE_FIND_LIBRARY_SUFFIXES ".a") find_package(azure-storage-blobs-cpp CONFIG REQUIRED) + find_package(azure-storage-files-datalake-cpp CONFIG REQUIRED) endmacro() # Build Velox backend.