From 1f809eec2af0b7924dc3a781a900de68d558a80e Mon Sep 17 00:00:00 2001 From: rui-mo Date: Tue, 17 Oct 2023 10:46:01 +0800 Subject: [PATCH] Fix CI compilation --- CMake/Findlz4.cmake | 27 ++++++++-------- CMakeLists.txt | 15 ++++++--- third_party/CMakeLists.txt | 2 ++ third_party/arrow_patches/helpers.patch | 32 +++++++++++++++++++ velox/common/process/tests/CMakeLists.txt | 2 +- .../hdfs/tests/HdfsMiniCluster.cpp | 2 +- .../parquet/writer/arrow/tests/CMakeLists.txt | 4 ++- 7 files changed, 63 insertions(+), 21 deletions(-) create mode 100644 third_party/arrow_patches/helpers.patch diff --git a/CMake/Findlz4.cmake b/CMake/Findlz4.cmake index d49115f12740f..1aaa8e532f9be 100644 --- a/CMake/Findlz4.cmake +++ b/CMake/Findlz4.cmake @@ -21,18 +21,19 @@ find_package_handle_standard_args(lz4 DEFAULT_MSG LZ4_LIBRARY LZ4_INCLUDE_DIR) mark_as_advanced(LZ4_LIBRARY LZ4_INCLUDE_DIR) -get_filename_component(liblz4_ext ${LZ4_LIBRARY} EXT) -if(liblz4_ext STREQUAL ".a") - set(liblz4_type STATIC) -else() - set(liblz4_type SHARED) -endif() - if(NOT TARGET lz4::lz4) - add_library(lz4::lz4 ${liblz4_type} IMPORTED) - set_target_properties(lz4::lz4 PROPERTIES INTERFACE_INCLUDE_DIRECTORIES - "${LZ4_INCLUDE_DIR}") - set_target_properties( - lz4::lz4 PROPERTIES IMPORTED_LINK_INTERFACE_LANGUAGES "C" - IMPORTED_LOCATION "${LZ4_LIBRARIES}") + add_library(lz4::lz4 UNKNOWN IMPORTED) + set_target_properties(lz4::lz4 PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${LZ4_INCLUDE_DIR}" + IMPORTED_LINK_INTERFACE_LANGUAGES "C" + IMPORTED_LOCATION_RELEASE "${LZ4_LIBRARY_RELEASE}") + set_property(TARGET lz4::lz4 APPEND PROPERTY + IMPORTED_CONFIGURATIONS RELEASE) + + if(LZ4_LIBRARY_DEBUG) + set_property(TARGET lz4::lz4 APPEND PROPERTY + IMPORTED_CONFIGURATIONS DEBUG) + set_property(TARGET lz4::lz4 PROPERTY + IMPORTED_LOCATION_DEBUG "${LZ4_LIBRARY_DEBUG}") + endif() endif() diff --git a/CMakeLists.txt b/CMakeLists.txt index 31df59045c044..dc96df3347682 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -213,10 +213,15 @@ if(VELOX_ENABLE_ABFS) endif() if(VELOX_ENABLE_HDFS) - find_library( - LIBHDFS3 - NAMES libhdfs3.so libhdfs3.dylib - HINTS "${CMAKE_SOURCE_DIR}/hawq/depends/libhdfs3/_build/src/" REQUIRED) + find_package(libhdfs3) + if(libhdfs3_FOUND AND TARGET HDFS::hdfs3) + set(LIBHDFS3 HDFS::hdfs3) + else() + find_library( + LIBHDFS3 + NAMES libhdfs3.so libhdfs3.dylib + HINTS "${CMAKE_SOURCE_DIR}/hawq/depends/libhdfs3/_build/src/" REQUIRED) + endif() add_definitions(-DVELOX_ENABLE_HDFS3) endif() @@ -382,7 +387,7 @@ resolve_dependency(Boost 1.66.0 COMPONENTS ${BOOST_INCLUDE_LIBRARIES}) # for reference. find_package(range-v3) set_source(gflags) -resolve_dependency(gflags COMPONENTS shared) +resolve_dependency(gflags) if(NOT TARGET gflags::gflags) # This is a bit convoluted, but we want to be able to use gflags::gflags as a # target even when velox is built as a subproject which uses diff --git a/third_party/CMakeLists.txt b/third_party/CMakeLists.txt index 570af4593e721..6b20e79084cb3 100644 --- a/third_party/CMakeLists.txt +++ b/third_party/CMakeLists.txt @@ -27,6 +27,7 @@ if(VELOX_ENABLE_ARROW) set(ARROW_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep") set(ARROW_CMAKE_ARGS -DARROW_PARQUET=ON + -DARROW_FILESYSTEM=ON -DARROW_WITH_LZ4=ON -DARROW_WITH_SNAPPY=ON -DARROW_WITH_ZLIB=ON @@ -68,6 +69,7 @@ if(VELOX_ENABLE_ARROW) arrow_ep PREFIX ${ARROW_PREFIX} URL ${VELOX_ARROW_SOURCE_URL} + PATCH_COMMAND patch -p1 < ${CMAKE_CURRENT_SOURCE_DIR}/arrow_patches/helpers.patch URL_HASH ${VELOX_ARROW_BUILD_SHA256_CHECKSUM} SOURCE_SUBDIR cpp CMAKE_ARGS ${ARROW_CMAKE_ARGS} diff --git a/third_party/arrow_patches/helpers.patch b/third_party/arrow_patches/helpers.patch new file mode 100644 index 0000000000000..a6d5e832c7794 --- /dev/null +++ b/third_party/arrow_patches/helpers.patch @@ -0,0 +1,32 @@ +diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake +index 94f926039..f7ebf9233 100644 +--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake ++++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake +@@ -2501,13 +2501,9 @@ if(ARROW_WITH_ZSTD) + if(ZSTD_VENDORED) + set(ARROW_ZSTD_LIBZSTD zstd::libzstd_static) + else() +- if(ARROW_ZSTD_USE_SHARED) +- set(ARROW_ZSTD_LIBZSTD zstd::libzstd_shared) +- else() +- set(ARROW_ZSTD_LIBZSTD zstd::libzstd_static) +- endif() ++ set(ARROW_ZSTD_LIBZSTD zstd::libzstd_shared) + if(NOT TARGET ${ARROW_ZSTD_LIBZSTD}) +- message(FATAL_ERROR "Zstandard target doesn't exist: ${ARROW_ZSTD_LIBZSTD}") ++ set(ARROW_ZSTD_LIBZSTD zstd::libzstd_static) + endif() + message(STATUS "Found Zstandard: ${ARROW_ZSTD_LIBZSTD}") + endif() +diff --git a/cpp/src/arrow/c/helpers.h b/cpp/src/arrow/c/helpers.h +index a24f272fe..e25f78c85 100644 +--- a/cpp/src/arrow/c/helpers.h ++++ b/cpp/src/arrow/c/helpers.h +@@ -17,6 +17,7 @@ + + #pragma once + ++#include + #include + #include + #include diff --git a/velox/common/process/tests/CMakeLists.txt b/velox/common/process/tests/CMakeLists.txt index d64466568a291..52927c4a5291c 100644 --- a/velox/common/process/tests/CMakeLists.txt +++ b/velox/common/process/tests/CMakeLists.txt @@ -17,4 +17,4 @@ add_executable(velox_process_test TraceContextTest.cpp) add_test(velox_process_test velox_process_test) target_link_libraries(velox_process_test PRIVATE velox_process fmt::fmt gtest - gtest_main) + gtest_main glog::glog gflags::gflags) diff --git a/velox/connectors/hive/storage_adapters/hdfs/tests/HdfsMiniCluster.cpp b/velox/connectors/hive/storage_adapters/hdfs/tests/HdfsMiniCluster.cpp index 10ee508ba638c..027a58ecc1911 100644 --- a/velox/connectors/hive/storage_adapters/hdfs/tests/HdfsMiniCluster.cpp +++ b/velox/connectors/hive/storage_adapters/hdfs/tests/HdfsMiniCluster.cpp @@ -72,7 +72,7 @@ HdfsMiniCluster::HdfsMiniCluster() { "Failed to find minicluster executable {}'", miniClusterExecutableName); } boost::filesystem::path hadoopHomeDirectory = exePath_; - hadoopHomeDirectory.remove_leaf().remove_leaf(); + hadoopHomeDirectory.remove_filename().remove_filename(); setupEnvironment(hadoopHomeDirectory.string()); } diff --git a/velox/dwio/parquet/writer/arrow/tests/CMakeLists.txt b/velox/dwio/parquet/writer/arrow/tests/CMakeLists.txt index ce1bc6782014c..3139314b47855 100644 --- a/velox/dwio/parquet/writer/arrow/tests/CMakeLists.txt +++ b/velox/dwio/parquet/writer/arrow/tests/CMakeLists.txt @@ -40,7 +40,9 @@ target_link_libraries( gtest_main parquet arrow - arrow_testing) + arrow_testing + glog::glog + gflags::gflags) add_library( velox_dwio_arrow_parquet_writer_test_lib