diff --git a/.github/workflows/build-thirdparty.yml b/.github/workflows/build-thirdparty.yml index 6f2504e855b656..6e7a888f416c08 100644 --- a/.github/workflows/build-thirdparty.yml +++ b/.github/workflows/build-thirdparty.yml @@ -132,6 +132,8 @@ jobs: export PATH="$(find /usr/lib/jvm/java-8-openjdk* -maxdepth 1 -type d -name 'bin'):${PATH}" export JAVA_HOME="$(find /usr/lib/jvm/java-8-openjdk* -maxdepth 0)" export DORIS_TOOLCHAIN=gcc + export CMAKE_POLICY_VERSION_MINIMUM=3.10 + export CUSTOM_CMAKE="/usr/local/bin/cmake" cd thirdparty ./build-thirdparty.sh -j "$(nproc)" @@ -190,6 +192,8 @@ jobs: - name: Build run: | export MACOSX_DEPLOYMENT_TARGET=12.0 + export CMAKE_POLICY_VERSION_MINIMUM=3.10 + export CUSTOM_CMAKE="/usr/local/bin/cmake" cd thirdparty ./build-thirdparty.sh -j "$(nproc)" @@ -248,6 +252,8 @@ jobs: - name: Build run: | export MACOSX_DEPLOYMENT_TARGET=12.0 + export CMAKE_POLICY_VERSION_MINIMUM=3.10 + export CUSTOM_CMAKE="/usr/local/bin/cmake" cd thirdparty ./build-thirdparty.sh -j "$(nproc)" diff --git a/be/test/CMakeLists.txt b/be/test/CMakeLists.txt index fda1c82b718bf8..88aa02f94c0c6b 100644 --- a/be/test/CMakeLists.txt +++ b/be/test/CMakeLists.txt @@ -113,7 +113,7 @@ add_executable(doris_be_test ${UT_FILES}) target_link_libraries(doris_be_test ${TEST_LINK_LIBS} -Wl,--whole-archive vector_search_test -Wl,--no-whole-archive) set_target_properties(doris_be_test PROPERTIES COMPILE_FLAGS "-fno-access-control") - +target_compile_options(doris_be_test PRIVATE -include gtest/gtest.h) if (OS_MACOSX AND ARCH_ARM) find_program(DSYMUTIL NAMES dsymutil) message(STATUS "dsymutil found: ${DSYMUTIL}") diff --git a/thirdparty/CHANGELOG.md b/thirdparty/CHANGELOG.md index 8f2a6d9beae8be..adffe708362557 100644 --- a/thirdparty/CHANGELOG.md +++ b/thirdparty/CHANGELOG.md @@ -2,6 +2,13 @@ This file contains version of the third-party dependency libraries in the build-env image. The docker build-env image is apache/doris, and the tag is `build-env-${version}` +## 20251021 + +- Modified: gtest 1.11.0 -> 1.12.1 +- Modified: zstd 1.5.5 -> 1.5.7 +- Modified: cctz 2.3 -> 2.5 +- Added: hadoop-libs 3.4.2.1 + ## 20250720 - Modified: zlib 1.2.11 -> 1.3.1 diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index 24bf81ee48dea4..7c9f75c138e744 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -349,7 +349,8 @@ build_libevent() { CFLAGS="-std=c99 -D_BSD_SOURCE -fno-omit-frame-pointer -g -ggdb -O2 -I${TP_INCLUDE_DIR}" \ CPPLAGS="-I${TP_INCLUDE_DIR}" \ LDFLAGS="-L${TP_LIB_DIR}" \ - "${CMAKE_CMD}" -G "${GENERATOR}" -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" -DEVENT__DISABLE_TESTS=ON \ + "${CMAKE_CMD}" -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -G "${GENERATOR}" -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" -DEVENT__DISABLE_TESTS=ON \ -DEVENT__DISABLE_SAMPLES=ON -DEVENT__DISABLE_REGRESS=ON .. "${BUILD_SYSTEM}" -j "${PARALLEL}" @@ -440,7 +441,8 @@ build_protobuf() { CXXFLAGS="-O2 -I${TP_INCLUDE_DIR}" \ LDFLAGS="${ldflags}" \ - "${CMAKE_CMD}" -DCMAKE_BUILD_TYPE=Release \ + "${CMAKE_CMD}" -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_PREFIX_PATH="${TP_INSTALL_DIR}" \ -Dprotobuf_USE_EXTERNAL_GTEST=ON \ -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ @@ -467,7 +469,8 @@ build_gflags() { rm -rf CMakeCache.txt CMakeFiles/ - "${CMAKE_CMD}" -G "${GENERATOR}" -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \ + "${CMAKE_CMD}" -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -G "${GENERATOR}" -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \ -DCMAKE_BUILD_TYPE=Release -DCMAKE_POSITION_INDEPENDENT_CODE=On ../ "${BUILD_SYSTEM}" -j "${PARALLEL}" @@ -493,13 +496,14 @@ build_glog() { elif [[ "${GLOG_SOURCE}" == "glog-0.6.0" ]]; then LDFLAGS="-L${TP_LIB_DIR}" \ "${CMAKE_CMD}" -S . -B build -G "Unix Makefiles" -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \ + -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ -DWITH_UNWIND=OFF \ -DBUILD_SHARED_LIBS=OFF \ -DWITH_TLS=OFF - cmake --build build --target install + "${CMAKE_CMD}" --build build --target install fi strip_lib libglog.a @@ -515,7 +519,8 @@ build_gtest() { cd "${BUILD_DIR}" rm -rf CMakeCache.txt CMakeFiles/ - "${CMAKE_CMD}" ../ -G "${GENERATOR}" -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" -DCMAKE_POSITION_INDEPENDENT_CODE=On + "${CMAKE_CMD}" ../ -G "${GENERATOR}" -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" -DCMAKE_POSITION_INDEPENDENT_CODE=On # -DCMAKE_CXX_FLAGS="$warning_uninitialized" "${BUILD_SYSTEM}" -j "${PARALLEL}" @@ -533,7 +538,8 @@ build_rapidjson() { rm -rf CMakeCache.txt CMakeFiles/ - "${CMAKE_CMD}" ../ -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" -DRAPIDJSON_BUILD_DOC=OFF \ + "${CMAKE_CMD}" ../ -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" -DRAPIDJSON_BUILD_DOC=OFF \ -DRAPIDJSON_BUILD_EXAMPLES=OFF -DRAPIDJSON_BUILD_TESTS=OFF make -j "${PARALLEL}" @@ -551,6 +557,7 @@ build_snappy() { rm -rf CMakeCache.txt CMakeFiles/ CFLAGS="-O3" CXXFLAGS="-O3" "${CMAKE_CMD}" -G "${GENERATOR}" -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \ + -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ -DCMAKE_INSTALL_INCLUDEDIR="${TP_INCLUDE_DIR}"/snappy \ -DSNAPPY_BUILD_TESTS=0 ../ @@ -627,7 +634,8 @@ build_zstd() { mkdir -p "${BUILD_DIR}" cd "${BUILD_DIR}" - "${CMAKE_CMD}" -G "${GENERATOR}" -DBUILD_TESTING=OFF -DZSTD_BUILD_TESTS=OFF -DZSTD_BUILD_STATIC=ON \ + "${CMAKE_CMD}" -G "${GENERATOR}" -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -DBUILD_TESTING=OFF -DZSTD_BUILD_TESTS=OFF -DZSTD_BUILD_STATIC=ON \ -DZSTD_BUILD_PROGRAMS=OFF -DZSTD_BUILD_SHARED=OFF -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" .. "${BUILD_SYSTEM}" -j "${PARALLEL}" install @@ -691,7 +699,8 @@ build_re2() { check_if_source_exist "${RE2_SOURCE}" cd "${TP_SOURCE_DIR}/${RE2_SOURCE}" - "${CMAKE_CMD}" -DCMAKE_BUILD_TYPE=Release -G "${GENERATOR}" -DBUILD_SHARED_LIBS=0 -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + "${CMAKE_CMD}" -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -DCMAKE_BUILD_TYPE=Release -G "${GENERATOR}" -DBUILD_SHARED_LIBS=0 -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ -DCMAKE_PREFIX_PATH="${TP_INSTALL_DIR}" -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" "${BUILD_SYSTEM}" -j "${PARALLEL}" install strip_lib libre2.a @@ -722,7 +731,8 @@ build_hyperscan() { cd "${BUILD_DIR}" CXXFLAGS="-D_HAS_AUTO_PTR_ETC=0" \ - "${CMAKE_CMD}" -G "${GENERATOR}" -DBUILD_SHARED_LIBS=0 -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + "${CMAKE_CMD}" -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -G "${GENERATOR}" -DBUILD_SHARED_LIBS=0 -DCMAKE_BUILD_TYPE=RelWithDebInfo \ -DBOOST_ROOT="${TP_INSTALL_DIR}" -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" -DBUILD_EXAMPLES=OFF .. "${BUILD_SYSTEM}" -j "${PARALLEL}" install strip_lib libhs.a @@ -772,7 +782,8 @@ build_mysql() { fi CFLAGS="${cflags}" CXXFLAGS="${cxxflags}" \ - "${CMAKE_CMD}" -G "${GENERATOR}" ../ -DCMAKE_LINK_SEARCH_END_STATIC=1 \ + "${CMAKE_CMD}" -G "${GENERATOR}" ../ -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -DCMAKE_LINK_SEARCH_END_STATIC=1 \ -DWITH_BOOST="$(pwd)/${BOOST_SOURCE}" -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}/mysql" \ -DWITHOUT_SERVER=1 -DWITH_ZLIB=1 -DZLIB_ROOT="${TP_INSTALL_DIR}" \ -DCMAKE_CXX_FLAGS_RELWITHDEBINFO="-O3 -g -fabi-version=2 -fno-omit-frame-pointer -fno-strict-aliasing -std=gnu++11" \ @@ -803,7 +814,8 @@ build_leveldb() { rm -rf CMakeCache.txt CMakeFiles/ - CXXFLAGS="-fPIC" "${CMAKE_CMD}" -G "${GENERATOR}" -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" -DLEVELDB_BUILD_BENCHMARKS=OFF \ + CXXFLAGS="-fPIC" "${CMAKE_CMD}" -G "${GENERATOR}" -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" -DLEVELDB_BUILD_BENCHMARKS=OFF \ -DLEVELDB_BUILD_TESTS=OFF .. "${BUILD_SYSTEM}" -j "${PARALLEL}" install strip_lib libleveldb.a @@ -838,6 +850,7 @@ build_brpc() { # glog must be enabled, otherwise error: `flag 'v' was defined more than once` (in files 'glog-0.6.0/src/vlog_is_on.cc' and 'brpc-1.6.0/src/butil/logging.cc') LDFLAGS="${ldflags}" \ "${CMAKE_CMD}" -G "${GENERATOR}" -DBUILD_SHARED_LIBS=ON -DWITH_GLOG=ON -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \ + -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ -DCMAKE_LIBRARY_PATH="${TP_INSTALL_DIR}/lib64" -DCMAKE_INCLUDE_PATH="${TP_INSTALL_DIR}/include" \ -DBUILD_BRPC_TOOLS=OFF \ -DPROTOBUF_PROTOC_EXECUTABLE="${TP_INSTALL_DIR}/bin/protoc" .. @@ -951,6 +964,7 @@ build_flatbuffers() { LDFLAGS="${ldflags}" \ "${CMAKE_CMD}" -G "${GENERATOR}" \ + -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ -DFLATBUFFERS_CXX_FLAGS="${warning_class_memaccess} ${warning_unused_but_set_variable}" \ -DFLATBUFFERS_BUILD_TESTS=OFF \ .. @@ -969,7 +983,8 @@ build_cares() { mkdir -p build cd build - cmake -DCMAKE_BUILD_TYPE=Release \ + cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -DCMAKE_BUILD_TYPE=Release \ -DCARES_STATIC=ON \ -DCARES_SHARED=OFF \ -DCARES_STATIC_PIC=ON \ @@ -986,7 +1001,8 @@ build_grpc() { mkdir -p cmake/build cd cmake/build - cmake -DgRPC_INSTALL=ON \ + "${CMAKE_CMD}" -DgRPC_INSTALL=ON \ + -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ -DgRPC_BUILD_TESTS=OFF \ -Dgrpc_csharp_plugin=OFF \ -Dgrpc_node_plugin=OFF \ @@ -1046,7 +1062,8 @@ build_arrow() { fi LDFLAGS="${ldflags}" \ - "${CMAKE_CMD}" -G "${GENERATOR}" -DARROW_PARQUET=ON -DARROW_IPC=ON -DARROW_BUILD_SHARED=OFF \ + "${CMAKE_CMD}" -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -G "${GENERATOR}" -DARROW_PARQUET=ON -DARROW_IPC=ON -DARROW_BUILD_SHARED=OFF \ -DARROW_BUILD_STATIC=ON -DARROW_WITH_BROTLI=ON -DARROW_WITH_LZ4=ON -DARROW_USE_GLOG=ON \ -DARROW_WITH_SNAPPY=ON -DARROW_WITH_ZLIB=ON -DARROW_WITH_ZSTD=ON -DARROW_JSON=ON \ -DARROW_WITH_UTF8PROC=OFF -DARROW_WITH_RE2=ON -DARROW_ORC=ON \ @@ -1105,7 +1122,8 @@ build_abseil() { cd "${TP_SOURCE_DIR}/${ABSEIL_SOURCE}" LDFLAGS="-L${TP_LIB_DIR}" \ - "${CMAKE_CMD}" -B "${BUILD_DIR}" -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \ + "${CMAKE_CMD}" -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -B "${BUILD_DIR}" -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \ -DABSL_ENABLE_INSTALL=ON \ -DBUILD_DEPS=ON \ -DCMAKE_BUILD_TYPE=Release \ @@ -1113,8 +1131,8 @@ build_abseil() { -DABSL_PROPAGATE_CXX_STD=ON \ -DBUILD_SHARED_LIBS=OFF - cmake --build "${BUILD_DIR}" -j "${PARALLEL}" - cmake --install "${BUILD_DIR}" --prefix "${TP_INSTALL_DIR}" + "${CMAKE_CMD}" --build "${BUILD_DIR}" -j "${PARALLEL}" + "${CMAKE_CMD}" --install "${BUILD_DIR}" --prefix "${TP_INSTALL_DIR}" } # s2 @@ -1128,7 +1146,8 @@ build_s2() { rm -rf CMakeCache.txt CMakeFiles/ LDFLAGS="-L${TP_LIB_DIR}" \ - ${CMAKE_CMD} -G "${GENERATOR}" -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \ + ${CMAKE_CMD} -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -G "${GENERATOR}" -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \ -DCMAKE_PREFIX_PATH="${TP_INSTALL_DIR}" \ -DBUILD_SHARED_LIBS=OFF \ -DWITH_GFLAGS=ON \ @@ -1240,7 +1259,8 @@ build_croaringbitmap() { CXXFLAGS="-O3" \ LDFLAGS="${ldflags}" \ - "${CMAKE_CMD}" -G "${GENERATOR}" ${avx_flag:+${avx_flag}} -DROARING_BUILD_STATIC=ON -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \ + "${CMAKE_CMD}" -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -G "${GENERATOR}" ${avx_flag:+${avx_flag}} -DROARING_BUILD_STATIC=ON -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \ -DENABLE_ROARING_TESTS=OFF .. "${BUILD_SYSTEM}" -j "${PARALLEL}" @@ -1257,7 +1277,8 @@ build_fmt() { rm -rf CMakeCache.txt CMakeFiles/ - "${CMAKE_CMD}" -G "${GENERATOR}" -DBUILD_SHARED_LIBS=FALSE -DFMT_TEST=OFF -DFMT_DOC=OFF -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" .. + "${CMAKE_CMD}" -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -G "${GENERATOR}" -DBUILD_SHARED_LIBS=FALSE -DFMT_TEST=OFF -DFMT_DOC=OFF -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" .. "${BUILD_SYSTEM}" -j"${PARALLEL}" "${BUILD_SYSTEM}" install } @@ -1295,6 +1316,7 @@ build_orc() { CXXFLAGS="-O3 -Wno-array-bounds ${warning_reserved_identifier} ${warning_suggest_override}" \ "${CMAKE_CMD}" -G "${GENERATOR}" ../ -DBUILD_JAVA=OFF \ + -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ -DPROTOBUF_HOME="${TP_INSTALL_DIR}" \ -DSNAPPY_HOME="${TP_INSTALL_DIR}" \ -DLZ4_HOME="${TP_INSTALL_DIR}" \ @@ -1322,7 +1344,10 @@ build_cctz() { rm -rf CMakeCache.txt CMakeFiles/ - "${CMAKE_CMD}" -G "${GENERATOR}" -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" -DBUILD_TESTING=OFF .. + # -Wno-elaborated-enum-base to make C++20 on MacOS happy + "${CMAKE_CMD}" -G "${GENERATOR}" \ + -DCMAKE_CXX_FLAGS="$CMAKE_CXX_FLAGS -Wno-elaborated-enum-base" \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" -DBUILD_TESTING=OFF .. "${BUILD_SYSTEM}" -j "${PARALLEL}" install } @@ -1361,6 +1386,7 @@ build_aws_sdk() { # -Wno-nonnull gcc-11 "${CMAKE_CMD}" -G "${GENERATOR}" -B"${BUILD_DIR}" -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \ + -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ -DCMAKE_PREFIX_PATH="${TP_INSTALL_DIR}" -DBUILD_SHARED_LIBS=OFF -DENABLE_TESTING=OFF \ -DCURL_LIBRARY_RELEASE="${TP_INSTALL_DIR}/lib/libcurl.a" -DZLIB_LIBRARY_RELEASE="${TP_INSTALL_DIR}/lib/libz.a" \ -DBUILD_ONLY="core;s3;s3-crt;transfer;identity-management;sts" \ @@ -1505,7 +1531,8 @@ build_hdfs3() { else SSE_OPTION='-DENABLE_SSE=OFF' fi - cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \ + "${CMAKE_CMD}" -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \ + -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ -DBUILD_STATIC_LIBS=ON -DBUILD_SHARED_LIBS=OFF -DBUILD_TEST=OFF "${SSE_OPTION}" \ -DProtobuf_PROTOC_EXECUTABLE="${TP_INSTALL_DIR}/bin/protoc" \ -DProtobuf_INCLUDE_DIR="${TP_INSTALL_DIR}/include" \ @@ -1586,7 +1613,7 @@ build_benchmark() { cd "${TP_SOURCE_DIR}/${BENCHMARK_SOURCE}" - cmake -E make_directory "build" + "${CMAKE_CMD}" -E make_directory "build" if [[ "${KERNEL}" != 'Darwin' ]]; then cxxflags='-lresolv -pthread -lrt' @@ -1615,6 +1642,7 @@ build_simdjson() { CXXFLAGS="-O3" CFLAGS="-O3" \ "${CMAKE_CMD}" -DSIMDJSON_EXCEPTIONS=OFF \ + -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ -DSIMDJSON_DEVELOPER_MODE=OFF -DSIMDJSON_BUILD_STATIC=ON \ -DSIMDJSON_JUST_LIBRARY=ON -DSIMDJSON_ENABLE_THREADS=ON .. "${CMAKE_CMD}" --build . --config Release @@ -1631,7 +1659,8 @@ build_nlohmann_json() { mkdir -p "${BUILD_DIR}" cd "${BUILD_DIR}" - "${CMAKE_CMD}" -G "${GENERATOR}" -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" -DCMAKE_PREFIX_PATH="${TP_INSTALL_DIR}" -DJSON_BuildTests=OFF .. + "${CMAKE_CMD}" -G "${GENERATOR}" -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" -DCMAKE_PREFIX_PATH="${TP_INSTALL_DIR}" -DJSON_BuildTests=OFF .. "${BUILD_SYSTEM}" -j "${PARALLEL}" "${BUILD_SYSTEM}" install @@ -1716,6 +1745,24 @@ build_hadoop_libs() { find ./hadoop-dist/target/hadoop-3.3.6/lib/native/ -type l -exec cp -P {} "${TP_INSTALL_DIR}/lib/hadoop_hdfs/native/" \; } +# hadoop_libs_3_4 +build_hadoop_libs_3_4() { + check_if_source_exist "${HADOOP_LIBS_3_4_SOURCE}" + cd "${TP_SOURCE_DIR}/${HADOOP_LIBS_3_4_SOURCE}" + echo "THIRDPARTY_INSTALLED=${TP_INSTALL_DIR}" >env.sh + ./build.sh + + rm -rf "${TP_INSTALL_DIR}/include/hadoop_hdfs_3_4/" + rm -rf "${TP_INSTALL_DIR}/lib/hadoop_hdfs_3_4/" + mkdir -p "${TP_INSTALL_DIR}/include/hadoop_hdfs_3_4/" + mkdir -p "${TP_INSTALL_DIR}/lib/hadoop_hdfs_3_4/" + cp -r ./hadoop-dist/target/hadoop-libhdfs-3.4.2/* "${TP_INSTALL_DIR}/lib/hadoop_hdfs_3_4/" + cp -r ./hadoop-dist/target/hadoop-libhdfs-3.4.2/include/hdfs.h "${TP_INSTALL_DIR}/include/hadoop_hdfs_3_4/" + rm -rf "${TP_INSTALL_DIR}/lib/hadoop_hdfs_3_4/native/*.a" + find ./hadoop-dist/target/hadoop-3.4.2/lib/native/ -type f ! -name '*.a' -exec cp {} "${TP_INSTALL_DIR}/lib/hadoop_hdfs_3_4/native/" \; + find ./hadoop-dist/target/hadoop-3.4.2/lib/native/ -type l -exec cp -P {} "${TP_INSTALL_DIR}/lib/hadoop_hdfs_3_4/native/" \; +} + # AvxToNeon build_avx2neon() { check_if_source_exist "${AVX2NEON_SOURCE}" @@ -1733,7 +1780,8 @@ build_libdeflate() { mkdir -p "${BUILD_DIR}" cd "${BUILD_DIR}" - "${CMAKE_CMD}" -G "${GENERATOR}" -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" -DCMAKE_BUILD_TYPE=Release .. + "${CMAKE_CMD}" -G "${GENERATOR}" -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" -DCMAKE_BUILD_TYPE=Release .. "${BUILD_SYSTEM}" -j "${PARALLEL}" "${BUILD_SYSTEM}" install } @@ -1747,7 +1795,8 @@ build_streamvbyte() { mkdir -p "${BUILD_DIR}" cd "${BUILD_DIR}" - "${CMAKE_CMD}" -G "${GENERATOR}" -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" -DCMAKE_BUILD_TYPE=Release .. + "${CMAKE_CMD}" -G "${GENERATOR}" -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" -DCMAKE_BUILD_TYPE=Release .. "${BUILD_SYSTEM}" -j "${PARALLEL}" "${BUILD_SYSTEM}" install } @@ -1759,7 +1808,8 @@ build_jsoncpp() { rm -rf "${BUILD_DIR}" mkdir -p "${BUILD_DIR}" cd "${BUILD_DIR}" - "${CMAKE_CMD}" -G "${GENERATOR}" -DJSONCPP_WITH_TESTS=OFF -DBUILD_STATIC_LIBS=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" .. + "${CMAKE_CMD}" -G "${GENERATOR}" -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -DJSONCPP_WITH_TESTS=OFF -DBUILD_STATIC_LIBS=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" .. "${BUILD_SYSTEM}" -j "${PARALLEL}" "${BUILD_SYSTEM}" install } @@ -1786,7 +1836,8 @@ build_ali_sdk() { CPPFLAGS="-I${TP_INCLUDE_DIR}" \ CXXFLAGS="-I${TP_INCLUDE_DIR}" \ LDFLAGS="-L${TP_LIB_DIR}" \ - "${CMAKE_CMD}" -G "${GENERATOR}" -DBUILD_PRODUCT=core -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \ + "${CMAKE_CMD}" -G "${GENERATOR}" -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -DBUILD_PRODUCT=core -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \ -DTP_INSTALL_DIR="${TP_INSTALL_DIR}" .. "${BUILD_SYSTEM}" -j "${PARALLEL}" "${BUILD_SYSTEM}" install @@ -1801,7 +1852,8 @@ build_base64() { mkdir -p "${BUILD_DIR}" cd "${BUILD_DIR}" - "${CMAKE_CMD}" -G "${GENERATOR}" -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" -DCMAKE_BUILD_TYPE=Release .. + "${CMAKE_CMD}" -G "${GENERATOR}" -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" -DCMAKE_BUILD_TYPE=Release .. MACHINE_TYPE="$(uname -m)" if [[ "${MACHINE_TYPE}" == "aarch64" || "${MACHINE_TYPE}" == 'arm64' ]]; then CFLAGS="--target=aarch64-linux-gnu -march=armv8-a+crc" NEON64_CFLAGS=" " @@ -1829,7 +1881,8 @@ build_azure() { AZURE_PORTS="vcpkg-custom-ports" AZURE_MANIFEST_DIR="." - "${CMAKE_CMD}" -G "${GENERATOR}" -DCMAKE_CXX_FLAGS="-Wno-maybe-uninitialized" -DDISABLE_RUST_IN_BUILD=ON -DVCPKG_MANIFEST_MODE=ON -DVCPKG_OVERLAY_PORTS="${azure_dir}/${AZURE_PORTS}" -DVCPKG_MANIFEST_DIR="${azure_dir}/${AZURE_MANIFEST_DIR}" -DWARNINGS_AS_ERRORS=FALSE -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" -DCMAKE_BUILD_TYPE=Release .. + "${CMAKE_CMD}" -G "${GENERATOR}" -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -DCMAKE_CXX_FLAGS="-Wno-maybe-uninitialized" -DDISABLE_RUST_IN_BUILD=ON -DVCPKG_MANIFEST_MODE=ON -DVCPKG_OVERLAY_PORTS="${azure_dir}/${AZURE_PORTS}" -DVCPKG_MANIFEST_DIR="${azure_dir}/${AZURE_MANIFEST_DIR}" -DWARNINGS_AS_ERRORS=FALSE -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" -DCMAKE_BUILD_TYPE=Release .. "${BUILD_SYSTEM}" -j "${PARALLEL}" "${BUILD_SYSTEM}" install fi @@ -1844,7 +1897,8 @@ build_dragonbox() { mkdir -p "${BUILD_DIR}" cd "${BUILD_DIR}" - "${CMAKE_CMD}" -G "${GENERATOR}" -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" -DDRAGONBOX_INSTALL_TO_CHARS=ON .. + "${CMAKE_CMD}" -G "${GENERATOR}" -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" -DDRAGONBOX_INSTALL_TO_CHARS=ON .. "${BUILD_SYSTEM}" -j "${PARALLEL}" "${BUILD_SYSTEM}" install @@ -1889,7 +1943,8 @@ build_pugixml() { mkdir -p "${BUILD_DIR}" cd "${BUILD_DIR}" - "${CMAKE_CMD}" -G "${GENERATOR}" -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" -DCMAKE_BUILD_TYPE=Release .. + "${CMAKE_CMD}" -G "${GENERATOR}" -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" -DCMAKE_BUILD_TYPE=Release .. "${BUILD_SYSTEM}" -j "${PARALLEL}" "${BUILD_SYSTEM}" install @@ -1973,6 +2028,7 @@ if [[ "${#packages[@]}" -eq 0 ]]; then read -r -a packages <<<"binutils gettext ${packages[*]}" elif [[ "$(uname -s)" == 'Linux' ]]; then read -r -a packages <<<"${packages[*]} hadoop_libs" + read -r -a packages <<<"${packages[*]} hadoop_libs_3_4" fi fi diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh index 4bf2183defc0e8..5ff6fe60a65d17 100755 --- a/thirdparty/download-thirdparty.sh +++ b/thirdparty/download-thirdparty.sh @@ -267,17 +267,6 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " GLOG " ]]; then echo "Finished patching ${GLOG_SOURCE}" fi -# gtest patch -if [[ " ${TP_ARCHIVES[*]} " =~ " GTEST " ]]; then - cd "${TP_SOURCE_DIR}/${GTEST_SOURCE}" - if [[ ! -f "${PATCHED_MARK}" ]]; then - patch -p1 <"${TP_PATCH_DIR}/googletest-release-1.11.0.patch" - touch "${PATCHED_MARK}" - fi - cd - - echo "Finished patching ${GTEST_SOURCE}" -fi - # mysql patch if [[ " ${TP_ARCHIVES[*]} " =~ " MYSQL " ]]; then cd "${TP_SOURCE_DIR}/${MYSQL_SOURCE}" @@ -289,19 +278,6 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " MYSQL " ]]; then echo "Finished patching ${MYSQL_SOURCE}" fi -# libevent patch -if [[ " ${TP_ARCHIVES[*]} " =~ " LIBEVENT " ]]; then - cd "${TP_SOURCE_DIR}/${LIBEVENT_SOURCE}" - if [[ ! -f "${PATCHED_MARK}" ]]; then - patch -p1 <"${TP_PATCH_DIR}/libevent.patch" - patch -p1 <"${TP_PATCH_DIR}/libevent-1532.patch" - patch -p1 <"${TP_PATCH_DIR}/libevent-keepalive-accepted-socket.patch" - touch "${PATCHED_MARK}" - fi - cd - - echo "Finished patching ${LIBEVENT_SOURCE}" -fi - # gsasl2 patch to fix link error such as mutilple func defination # when link target with kerberos if [[ " ${TP_ARCHIVES[*]} " =~ " GSASL " ]]; then diff --git a/thirdparty/patches/googletest-release-1.11.0.patch b/thirdparty/patches/googletest-release-1.11.0.patch deleted file mode 100644 index 976526775f4112..00000000000000 --- a/thirdparty/patches/googletest-release-1.11.0.patch +++ /dev/null @@ -1,71 +0,0 @@ -diff -u a/googletest/include/gtest/gtest-printers.h b/googletest/include/gtest/gtest-printers.h ---- a/googletest/include/gtest/gtest-printers.h -+++ b/googletest/include/gtest/gtest-printers.h -@@ -479,6 +479,12 @@ inline void PrintTo(char8_t c, ::std::ostream* os) { - } - #endif - -+// gcc/clang __{u,}int128_t -+#if defined(__SIZEOF_INT128__) -+GTEST_API_ void PrintTo(__uint128_t v, ::std::ostream* os); -+GTEST_API_ void PrintTo(__int128_t v, ::std::ostream* os); -+#endif // __SIZEOF_INT128__ -+ - // Overloads for C strings. - GTEST_API_ void PrintTo(const char* s, ::std::ostream* os); - inline void PrintTo(char* s, ::std::ostream* os) { -diff -u a/googletest/src/gtest-printers.cc b/googletest/src/gtest-printers.cc ---- a/googletest/src/gtest-printers.cc -+++ b/googletest/src/gtest-printers.cc -@@ -304,6 +304,51 @@ void PrintTo(char32_t c, ::std::ostream* os) { - << static_cast(c); - } - -+// gcc/clang __{u,}int128_t -+#if defined(__SIZEOF_INT128__) -+void PrintTo(__uint128_t v, ::std::ostream* os) { -+ if (v == 0) { -+ *os << "0"; -+ return; -+ } -+ -+ // Buffer large enough for ceil(log10(2^128))==39 and the null terminator -+ char buf[40]; -+ char* p = buf + sizeof(buf); -+ -+ // Some configurations have a __uint128_t, but no support for built in -+ // division. Do manual long division instead. -+ -+ uint64_t high = static_cast(v >> 64); -+ uint64_t low = static_cast(v); -+ -+ *--p = 0; -+ while (high != 0 || low != 0) { -+ uint64_t high_mod = high % 10; -+ high = high / 10; -+ // This is the long division algorithm specialized for a divisor of 10 and -+ // only two elements. -+ // Notable values: -+ // 2^64 / 10 == 1844674407370955161 -+ // 2^64 % 10 == 6 -+ const uint64_t carry = 6 * high_mod + low % 10; -+ low = low / 10 + high_mod * 1844674407370955161 + carry / 10; -+ -+ char digit = static_cast(carry % 10); -+ *--p = '0' + digit; -+ } -+ *os << p; -+} -+void PrintTo(__int128_t v, ::std::ostream* os) { -+ __uint128_t uv = static_cast<__uint128_t>(v); -+ if (v < 0) { -+ *os << "-"; -+ uv = -uv; -+ } -+ PrintTo(uv, os); -+} -+#endif // __SIZEOF_INT128__ -+ - // Prints the given array of characters to the ostream. CharType must be either - // char, char8_t, char16_t, char32_t, or wchar_t. - // The array starts at begin, the length is len, it may include '\0' characters diff --git a/thirdparty/patches/libevent-1532.patch b/thirdparty/patches/libevent-1532.patch deleted file mode 100644 index d05c99accc246f..00000000000000 --- a/thirdparty/patches/libevent-1532.patch +++ /dev/null @@ -1,188 +0,0 @@ -diff --git a/evutil.c b/evutil.c -index 9817f08..e1803de 100644 ---- a/evutil.c -+++ b/evutil.c -@@ -2763,3 +2763,137 @@ evutil_free_globals_(void) - evutil_free_secure_rng_globals_(); - evutil_free_sock_err_globals(); - } -+ -+int -+evutil_set_tcp_keepalive(evutil_socket_t fd, int on, int timeout) -+{ -+ int idle; -+ int intvl; -+ int cnt; -+ -+ /* Prevent compiler from complaining unused variables warnings. */ -+ (void) idle; -+ (void) intvl; -+ (void) cnt; -+ -+ if (timeout <= 0) -+ return 0; -+ -+ if (setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &on, sizeof(on))) -+ return -1; -+ if (!on) -+ return 0; -+ -+ /* Unlike Unix-like OS's, TCP keep-alive mechanism on Windows is kind of a mess, -+ * setting TCP_KEEPIDLE, TCP_KEEPINTVL and TCP_KEEPCNT on Windows could be a bit tricky. -+ * Check out https://learn.microsoft.com/en-us/windows/win32/winsock/sio-keepalive-vals, -+ * https://learn.microsoft.com/en-us/windows/win32/winsock/ipproto-tcp-socket-options. -+ * These three options are not available until Windows 10, version 1709 where we set them -+ * by `setsockopt` (slightly different from Unix-like OS's pattern), while on older Windows, -+ * we have to use `WSAIoctl` instead. -+ * Therefore, we skip setting those three options on Windows for now. -+ * TODO(panjf2000): enable the full TCP keep-alive mechanism on Windows when we find a feasible way to do it. -+ */ -+#ifndef _WIN32 -+ -+ /* The implementation of TCP keep-alive on Solaris/SmartOS is a bit unusual -+ * compared to other Unix-like systems. -+ * Thus, we need to specialize it on Solaris. -+ */ -+#ifdef __sun -+ /* There are two keep-alive mechanisms on Solaris: -+ * - By default, the first keep-alive probe is sent out after a TCP connection is idle for two hours. -+ * If the peer does not respond to the probe within eight minutes, the TCP connection is aborted. -+ * You can alter the interval for sending out the first probe using the socket option TCP_KEEPALIVE_THRESHOLD -+ * in milliseconds or TCP_KEEPIDLE in seconds. -+ * The system default is controlled by the TCP ndd parameter tcp_keepalive_interval. The minimum value is ten seconds. -+ * The maximum is ten days, while the default is two hours. If you receive no response to the probe, -+ * you can use the TCP_KEEPALIVE_ABORT_THRESHOLD socket option to change the time threshold for aborting a TCP connection. -+ * The option value is an unsigned integer in milliseconds. The value zero indicates that TCP should never time out and -+ * abort the connection when probing. The system default is controlled by the TCP ndd parameter tcp_keepalive_abort_interval. -+ * The default is eight minutes. -+ * -+ * - The second implementation is activated if socket option TCP_KEEPINTVL and/or TCP_KEEPCNT are set. -+ * The time between each consequent probes is set by TCP_KEEPINTVL in seconds. -+ * The minimum value is ten seconds. The maximum is ten days, while the default is two hours. -+ * The TCP connection will be aborted after certain amount of probes, which is set by TCP_KEEPCNT, without receiving response. -+ */ -+ -+ idle = timeout; -+ /* Kernel expects at least 10 seconds. */ -+ if (idle < 10) -+ idle = 10; -+ /* Kernel expects at most 10 days. */ -+ if (idle > 10*24*60*60) -+ idle = 10*24*60*60; -+ -+ /* `TCP_KEEPIDLE`, `TCP_KEEPINTVL`, and `TCP_KEEPCNT` were not available on Solaris -+ * until version 11.4, but let's gamble here. -+ */ -+#if defined(TCP_KEEPIDLE) && defined(TCP_KEEPINTVL) && defined(TCP_KEEPCNT) -+ if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPIDLE, &idle, sizeof(idle))) -+ return -1; -+ intvl = idle/3; -+ if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPINTVL, &intvl, sizeof(intvl))) -+ return -1; -+ cnt = 3; -+ if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPCNT, &cnt, sizeof(cnt))) -+ return -1; -+ return 0; -+#endif -+ -+ /* Fall back to the first implementation of tcp-alive mechanism for older Solaris, -+ * simulate the tcp-alive mechanism on other platforms via `TCP_KEEPALIVE_THRESHOLD` + `TCP_KEEPALIVE_ABORT_THRESHOLD`. -+ */ -+ idle *= 1000; /* kernel expects milliseconds */ -+ if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPALIVE_THRESHOLD, &idle, sizeof(idle))) -+ return -1; -+ -+ /* Note that the consequent probes will not be sent at equal intervals on Solaris, -+ * but will be sent using the exponential backoff algorithm. -+ */ -+ intvl = idle/3; -+ cnt = 3; -+ int time_to_abort = intvl * cnt; -+ if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPALIVE_ABORT_THRESHOLD, &time_to_abort, sizeof(time_to_abort))) -+ return -1; -+ -+ return 0; -+#endif -+ -+#ifdef TCP_KEEPIDLE -+ idle = timeout; -+ if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPIDLE, &idle, sizeof(idle))) -+ return -1; -+#elif defined(TCP_KEEPALIVE) -+ /* Darwin/macOS uses TCP_KEEPALIVE in place of TCP_KEEPIDLE. */ -+ idle = timeout; -+ if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPALIVE, &idle, sizeof(idle))) -+ return -1; -+#endif -+ -+#ifdef TCP_KEEPINTVL -+ /* Set the interval between individual keep-alive probes as timeout / 3 -+ * and the maximum number of keepalive probes as 3 to make it double timeout -+ * before aborting a dead connection. -+ */ -+ intvl = timeout/3; -+ if (intvl == 0) -+ intvl = 1; -+ if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPINTVL, &intvl, sizeof(intvl))) -+ return -1; -+#endif -+ -+#ifdef TCP_KEEPCNT -+ /* Set the maximum number of keepalive probes as 3 to collaborate with -+ * TCP_KEEPINTVL, see the previous comment. -+ */ -+ cnt = 3; -+ if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPCNT, &cnt, sizeof(cnt))) -+ return -1; -+#endif -+ -+#endif /* !_WIN32 */ -+ -+ return 0; -+} -diff --git a/http.c b/http.c -index 53951cb..1ad60f8 100644 ---- a/http.c -+++ b/http.c -@@ -4417,7 +4417,7 @@ create_bind_socket_nonblock(struct evutil_addrinfo *ai, int reuse) - { - evutil_socket_t fd; - -- int on = 1, r; -+ int r; - int serrno; - - /* Create listen socket */ -@@ -4428,7 +4428,8 @@ create_bind_socket_nonblock(struct evutil_addrinfo *ai, int reuse) - return (-1); - } - -- if (setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, (void *)&on, sizeof(on))<0) -+ /* TODO(panjf2000): make this TCP keep-alive value configurable */ -+ if (evutil_set_tcp_keepalive(fd, 1, 300) < 0) - goto out; - if (reuse) { - if (evutil_make_listen_socket_reuseable(fd) < 0) -diff --git a/include/event2/util.h b/include/event2/util.h -index 02aa7ba..688b641 100644 ---- a/include/event2/util.h -+++ b/include/event2/util.h -@@ -469,6 +469,18 @@ int evutil_closesocket(evutil_socket_t sock); - EVENT2_EXPORT_SYMBOL - int evutil_make_tcp_listen_socket_deferred(evutil_socket_t sock); - -+/** Do platform-specific operations to set/unset TCP keep-alive options -+ * TCP_KEEPIDLE, TCP_KEEPINTVL and TCP_KEEPCNT on a socket. -+ * -+ * @param sock The socket to be set TCP keep-alive -+ * @param on nonzero value to enable TCP keep-alive, 0 to disable -+ * @param timeout The timeout in seconds with no activity until -+ * the first keepalive probe is sent -+ * @return 0 on success, -1 on failure -+*/ -+EVENT2_EXPORT_SYMBOL -+int evutil_set_tcp_keepalive(evutil_socket_t sock, int on, int timeout); -+ - #ifdef _WIN32 - /** Return the most recent socket error. Not idempotent on all platforms. */ - #define EVUTIL_SOCKET_ERROR() WSAGetLastError() diff --git a/thirdparty/patches/libevent-keepalive-accepted-socket.patch b/thirdparty/patches/libevent-keepalive-accepted-socket.patch deleted file mode 100644 index 84a9d7b1ea633c..00000000000000 --- a/thirdparty/patches/libevent-keepalive-accepted-socket.patch +++ /dev/null @@ -1,17 +0,0 @@ -diff --git a/http.c b/http.c -index 1ad60f8..267fa1f 100644 ---- a/http.c -+++ b/http.c -@@ -4265,6 +4265,12 @@ evhttp_get_request_connection( - event_debug(("%s: new request from %s:%s on "EV_SOCK_FMT"\n", - __func__, hostname, portname, EV_SOCK_ARG(fd))); - -+ if (sa->sa_family != AF_UNIX) { -+ if (evutil_set_tcp_keepalive(fd, 1, 300) < 0) { -+ return (NULL); -+ } -+ } -+ - /* we need a connection object to put the http request on */ - if (http->bevcb != NULL) { - bev = (*http->bevcb)(http->base, http->bevcbarg); diff --git a/thirdparty/patches/libevent.patch b/thirdparty/patches/libevent.patch deleted file mode 100644 index a545897cf15149..00000000000000 --- a/thirdparty/patches/libevent.patch +++ /dev/null @@ -1,193 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt -index 676727f1..833fbf70 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -200,7 +200,7 @@ endif() - if (("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU") OR (${CLANG})) - set(GNUC 1) - endif() --if (("${CMAKE_C_COMPILER_ID}" STREQUAL "MSVC") OR (${CLANG})) -+if (("${CMAKE_C_COMPILER_ID}" STREQUAL "MSVC") OR ("${CMAKE_C_SIMULATE_ID}" STREQUAL "MSVC")) - set(MSVC 1) - endif() - -diff --git a/buffer.c b/buffer.c -index 3524b350..e5d97458 100644 ---- a/buffer.c -+++ b/buffer.c -@@ -2204,9 +2204,9 @@ evbuffer_expand(struct evbuffer *buf, size_t datlen) - #define IOV_LEN_TYPE unsigned long - #endif - #endif --#define NUM_READ_IOVEC 4 -+#define NUM_READ_IOVEC 8 - --#define EVBUFFER_MAX_READ 4096 -+#define EVBUFFER_MAX_READ (128 * 1024) - - /** Helper function to figure out which space to use for reading data into - an evbuffer. Internal use only. -diff --git a/bufferevent_async.c b/bufferevent_async.c -index 40c7c5e8..c1624878 100644 ---- a/bufferevent_async.c -+++ b/bufferevent_async.c -@@ -275,7 +275,7 @@ bev_async_consider_reading(struct bufferevent_async *beva) - } - at_most = read_high - cur_size; - } else { -- at_most = 16384; /* FIXME totally magic. */ -+ at_most = 128 * 1024; /* FIXME totally magic. */ - } - - /* XXXX This over-commits. */ -diff --git a/bufferevent_ratelim.c b/bufferevent_ratelim.c -index 25874968..9bc2b577 100644 ---- a/bufferevent_ratelim.c -+++ b/bufferevent_ratelim.c -@@ -179,7 +179,7 @@ ev_token_bucket_cfg_free(struct ev_token_bucket_cfg *cfg) - } - - /* Default values for max_single_read & max_single_write variables. */ --#define MAX_SINGLE_READ_DEFAULT 16384 -+#define MAX_SINGLE_READ_DEFAULT (128 * 1024) - #define MAX_SINGLE_WRITE_DEFAULT 16384 - - #define LOCK_GROUP(g) EVLOCK_LOCK((g)->lock, 0) -diff --git a/http-internal.h b/http-internal.h -index feaf436d..9f9b5ab5 100644 ---- a/http-internal.h -+++ b/http-internal.h -@@ -167,6 +167,8 @@ struct evhttp { - void *gencbarg; - struct bufferevent* (*bevcb)(struct event_base *, void *); - void *bevcbarg; -+ int (*newreqcb)(struct evhttp_request *req, void *); -+ void *newreqcbarg; - - struct event_base *base; - }; -diff --git a/http.c b/http.c -index 04f089bc..53951cba 100644 ---- a/http.c -+++ b/http.c -@@ -3975,6 +3975,14 @@ evhttp_set_bevcb(struct evhttp *http, - http->bevcbarg = cbarg; - } - -+void -+evhttp_set_newreqcb(struct evhttp *http, -+ int (*cb)(struct evhttp_request *, void *), void *cbarg) -+{ -+ http->newreqcb = cb; -+ http->newreqcbarg = cbarg; -+} -+ - /* - * Request related functions - */ -@@ -4036,6 +4044,8 @@ evhttp_request_free(struct evhttp_request *req) - req->flags |= EVHTTP_REQ_NEEDS_FREE; - return; - } -+ if (req->on_free_cb) -+ (*req->on_free_cb)(req, req->on_free_cb_arg); - - if (req->remote_host != NULL) - mm_free(req->remote_host); -@@ -4116,6 +4126,15 @@ evhttp_request_set_on_complete_cb(struct evhttp_request *req, - req->on_complete_cb_arg = cb_arg; - } - -+void -+evhttp_request_set_on_free_cb(struct evhttp_request *req, -+ void (*cb)(struct evhttp_request *, void *), void *cb_arg) -+{ -+ req->on_free_cb = cb; -+ req->on_free_cb_arg = cb_arg; -+} -+ -+ - /* - * Allows for inspection of the request URI - */ -@@ -4307,10 +4326,15 @@ evhttp_associate_new_request_with_connection(struct evhttp_connection *evcon) - */ - req->userdone = 1; - -- TAILQ_INSERT_TAIL(&evcon->requests, req, next); -- - req->kind = EVHTTP_REQUEST; - -+ if (http->newreqcb && http->newreqcb(req, http->newreqcbarg) == -1) { -+ evhttp_request_free(req); -+ return (-1); -+ } -+ -+ TAILQ_INSERT_TAIL(&evcon->requests, req, next); -+ - - evhttp_start_read_(evcon); - -diff --git a/include/event2/http.h b/include/event2/http.h -index 2a41303e..e80bab9a 100644 ---- a/include/event2/http.h -+++ b/include/event2/http.h -@@ -298,6 +298,20 @@ EVENT2_EXPORT_SYMBOL - void evhttp_set_bevcb(struct evhttp *http, - struct bufferevent *(*cb)(struct event_base *, void *), void *arg); - -+/** -+ Set a callback which allows the user to note or throttle incoming requests. -+ The requests are not populated with HTTP level information. They -+ are just associated to a connection. -+ If the callback returns -1, the associated connection is terminated -+ and the request is closed. -+ @param http the evhttp server object for which to set the callback -+ @param cb the callback to invoke for incoming connections -+ @param arg an context argument for the callback -+ */ -+EVENT2_EXPORT_SYMBOL -+void evhttp_set_newreqcb(struct evhttp *http, -+ int (*cb)(struct evhttp_request*, void *), void *arg); -+ - /** - Adds a virtual host to the http server. - -@@ -624,6 +638,20 @@ EVENT2_EXPORT_SYMBOL - void evhttp_request_set_on_complete_cb(struct evhttp_request *req, - void (*cb)(struct evhttp_request *, void *), void *cb_arg); - -+/** -+ * Set a callback to be called on request free. -+ * -+ * The callback function will be called just before the evhttp_request object -+ * is destroyed. -+ * -+ * @param req a request object -+ * @param cb callback function that will be called before request free -+ * @param cb_arg an additional context argument for the callback -+ */ -+EVENT2_EXPORT_SYMBOL -+void evhttp_request_set_on_free_cb(struct evhttp_request *req, -+ void (*cb)(struct evhttp_request *, void *), void *cb_arg); -+ - /** Frees the request object and removes associated events. */ - EVENT2_EXPORT_SYMBOL - void evhttp_request_free(struct evhttp_request *req); -diff --git a/include/event2/http_struct.h b/include/event2/http_struct.h -index 4bf5b1ff..0762cabd 100644 ---- a/include/event2/http_struct.h -+++ b/include/event2/http_struct.h -@@ -142,6 +142,12 @@ struct { - */ - void (*on_complete_cb)(struct evhttp_request *, void *); - void *on_complete_cb_arg; -+ -+ /* -+ * Free callback - called just before the request is freed. -+ */ -+ void (*on_free_cb)(struct evhttp_request *, void *); -+ void *on_free_cb_arg; - }; - - #ifdef __cplusplus diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh index 9463bf403796c0..873d5bde6a3f2d 100644 --- a/thirdparty/vars.sh +++ b/thirdparty/vars.sh @@ -55,10 +55,10 @@ export TP_JAR_DIR="${TP_INSTALL_DIR}/lib/jar" ##################################################### # libevent -LIBEVENT_DOWNLOAD="https://github.com/libevent/libevent/archive/release-2.1.12-stable.tar.gz" -LIBEVENT_NAME=libevent-release-2.1.12-stable.tar.gz -LIBEVENT_SOURCE=libevent-release-2.1.12-stable -LIBEVENT_MD5SUM="0d5a27436bf7ff8253420c8cf09f47ca" +LIBEVENT_DOWNLOAD="https://github.com/apache/doris-thirdparty/archive/refs/tags/libevent-2.1.12.1.tar.gz" +LIBEVENT_NAME=libevent-2.1.12.1.tar.gz +LIBEVENT_SOURCE=doris-thirdparty-libevent-2.1.12.1 +LIBEVENT_MD5SUM="af67af024d42e57e185eb0c5026f8ae6" # openssl OPENSSL_DOWNLOAD="https://github.com/openssl/openssl/archive/OpenSSL_1_1_1s.tar.gz" @@ -92,10 +92,10 @@ GLOG_SOURCE=glog-0.6.0 GLOG_MD5SUM="c98a6068bc9b8ad9cebaca625ca73aa2" # gtest -GTEST_DOWNLOAD="https://github.com/google/googletest/archive/release-1.11.0.tar.gz" -GTEST_NAME=googletest-release-1.11.0.tar.gz -GTEST_SOURCE=googletest-release-1.11.0 -GTEST_MD5SUM="e8a8df240b6938bb6384155d4c37d937" +GTEST_DOWNLOAD="https://github.com/google/googletest/archive/refs/tags/release-1.12.1.tar.gz" +GTEST_NAME=googletest-release-1.12.1.tar.gz +GTEST_SOURCE=googletest-release-1.12.1 +GTEST_MD5SUM="e82199374acdfda3f425331028eb4e2a" # snappy SNAPPY_DOWNLOAD="https://github.com/google/snappy/archive/1.1.8.tar.gz" @@ -222,10 +222,10 @@ LIBRDKAFKA_SOURCE=librdkafka-1.9.2 LIBRDKAFKA_MD5SUM="fe9624e905abbf8324b0f6be520d9c24" # zstd -ZSTD_DOWNLOAD="https://github.com/facebook/zstd/releases/download/v1.5.5/zstd-1.5.5.tar.gz" -ZSTD_NAME=zstd-1.5.5.tar.gz -ZSTD_SOURCE=zstd-1.5.5 -ZSTD_MD5SUM="63251602329a106220e0a5ad26ba656f" +ZSTD_DOWNLOAD="https://github.com/facebook/zstd/releases/download/v1.5.7/zstd-1.5.7.tar.gz" +ZSTD_NAME=zstd-1.5.7.tar.gz +ZSTD_SOURCE=zstd-1.5.7 +ZSTD_MD5SUM="780fc1896922b1bc52a4e90980cdda48" # brotli BROTLI_DOWNLOAD="https://github.com/google/brotli/archive/v1.0.9.tar.gz" @@ -321,10 +321,10 @@ LIBUNWIND_SOURCE="libunwind-1.6.2" LIBUNWIND_MD5SUM="f625b6a98ac1976116c71708a73dc44a" # cctz -CCTZ_DOWNLOAD="https://github.com/google/cctz/archive/v2.3.tar.gz" -CCTZ_NAME="cctz-2.3.tar.gz" -CCTZ_SOURCE="cctz-2.3" -CCTZ_MD5SUM="209348e50b24dbbdec6d961059c2fc92" +CCTZ_DOWNLOAD="https://github.com/google/cctz/archive/refs/tags/v2.5.tar.gz" +CCTZ_NAME="cctz-2.5.tar.gz" +CCTZ_SOURCE="cctz-2.5" +CCTZ_MD5SUM="cae0ee4a5da5399d591e770b7515a46d" # datatables, bootstrap 3 and jQuery 3 # The origin download url is always changing: https://datatables.net/download/builder?bs-3.3.7/jq-3.3.1/dt-1.10.25 @@ -468,6 +468,12 @@ HADOOP_LIBS_NAME="hadoop-3.3.6.6-for-doris.tar.gz" HADOOP_LIBS_SOURCE="doris-thirdparty-hadoop-3.3.6.6-for-doris" HADOOP_LIBS_MD5SUM="13b66d5f2abffd1740e692b65df5962e" +# libhdfs 3.4 +HADOOP_LIBS_3_4_DOWNLOAD="https://github.com/apache/doris-thirdparty/archive/refs/tags/hadoop-3.4.2.1-for-doris.tar.gz" +HADOOP_LIBS_3_4_NAME="hadoop-3.4.2.1-for-doris.tar.gz" +HADOOP_LIBS_3_4_SOURCE="doris-thirdparty-hadoop-3.4.2.1-for-doris" +HADOOP_LIBS_3_4_MD5SUM="4ca35773ecfae856962885413f6bbeac" + # AvxToNeon AVX2NEON_DOWNLOAD="https://github.com/kunpengcompute/AvxToNeon/archive/refs/tags/v1.0.0.tar.gz" AVX2NEON_NAME=v1.0.0.tar.gz @@ -608,6 +614,7 @@ export TP_ARCHIVES=( 'CONCURRENTQUEUE' 'FAST_FLOAT' 'HADOOP_LIBS' + 'HADOOP_LIBS_3_4' 'AVX2NEON' 'LIBDEFLATE' 'STREAMVBYTE'