From 471334dbf4a1282de3afd52faa16eefbca941adb Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 27 Oct 2023 09:51:00 +0900 Subject: [PATCH] GH-38424: [CI][C++] Use Fedora 38 instead of 35 (#38425) ### Rationale for this change Because Fedora 35 reached EOL. ### What changes are included in this PR? Use Fedora 38. ### Are these changes tested? Yes. ### Are there any user-facing changes? No. * Closes: #38424 Authored-by: Sutou Kouhei Signed-off-by: Sutou Kouhei --- .env | 2 +- ...pp.dockerfile => fedora-38-cpp.dockerfile} | 9 ++--- cpp/CMakeLists.txt | 6 +-- .../{FindORC.cmake => FindorcAlt.cmake} | 37 +++++++++++-------- cpp/cmake_modules/ThirdpartyToolchain.cmake | 34 ++++++++++------- cpp/src/arrow/adapters/orc/CMakeLists.txt | 2 +- dev/tasks/tasks.yml | 8 ++-- docker-compose.yml | 6 +-- 8 files changed, 57 insertions(+), 47 deletions(-) rename ci/docker/{fedora-35-cpp.dockerfile => fedora-38-cpp.dockerfile} (95%) rename cpp/cmake_modules/{FindORC.cmake => FindorcAlt.cmake} (68%) diff --git a/.env b/.env index a551e2120a6fb..6746892fd4ed8 100644 --- a/.env +++ b/.env @@ -49,7 +49,7 @@ ULIMIT_CORE=-1 ALMALINUX=8 ALPINE_LINUX=3.16 DEBIAN=11 -FEDORA=35 +FEDORA=38 UBUNTU=20.04 # Default versions for various dependencies diff --git a/ci/docker/fedora-35-cpp.dockerfile b/ci/docker/fedora-38-cpp.dockerfile similarity index 95% rename from ci/docker/fedora-35-cpp.dockerfile rename to ci/docker/fedora-38-cpp.dockerfile index aefa25663ba14..2dcc094ee20c5 100644 --- a/ci/docker/fedora-35-cpp.dockerfile +++ b/ci/docker/fedora-38-cpp.dockerfile @@ -16,7 +16,7 @@ # under the License. ARG arch -FROM ${arch}/fedora:35 +FROM ${arch}/fedora:38 ARG arch # install dependencies @@ -46,9 +46,9 @@ RUN dnf update -y && \ java-latest-openjdk-devel \ java-latest-openjdk-headless \ json-devel \ + liborc-devel \ libzstd-devel \ llvm-devel \ - llvm-static \ lz4-devel \ make \ ninja-build \ @@ -64,6 +64,7 @@ RUN dnf update -y && \ utf8proc-devel \ wget \ which \ + xsimd-devel \ zlib-devel COPY ci/scripts/install_minio.sh /arrow/ci/scripts/ @@ -100,8 +101,6 @@ ENV absl_SOURCE=BUNDLED \ CC=gcc \ CXX=g++ \ google_cloud_cpp_storage_SOURCE=BUNDLED \ - ORC_SOURCE=BUNDLED \ PARQUET_BUILD_EXAMPLES=ON \ PARQUET_BUILD_EXECUTABLES=ON \ - PATH=/usr/lib/ccache/:$PATH \ - xsimd_SOURCE=BUNDLED + PATH=/usr/lib/ccache/:$PATH diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 8566508406bd4..7138139b13ec0 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -770,10 +770,10 @@ if(ARROW_WITH_ZSTD) endif() if(ARROW_ORC) - list(APPEND ARROW_SHARED_LINK_LIBS orc::liborc ${ARROW_PROTOBUF_LIBPROTOBUF}) - list(APPEND ARROW_STATIC_LINK_LIBS orc::liborc ${ARROW_PROTOBUF_LIBPROTOBUF}) + list(APPEND ARROW_SHARED_LINK_LIBS orc::orc ${ARROW_PROTOBUF_LIBPROTOBUF}) + list(APPEND ARROW_STATIC_LINK_LIBS orc::orc ${ARROW_PROTOBUF_LIBPROTOBUF}) if(ORC_SOURCE STREQUAL "SYSTEM") - list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS orc::liborc + list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS orc::orc ${ARROW_PROTOBUF_LIBPROTOBUF}) endif() endif() diff --git a/cpp/cmake_modules/FindORC.cmake b/cpp/cmake_modules/FindorcAlt.cmake similarity index 68% rename from cpp/cmake_modules/FindORC.cmake rename to cpp/cmake_modules/FindorcAlt.cmake index aca915acc13d0..dc3b978cf4037 100644 --- a/cpp/cmake_modules/FindORC.cmake +++ b/cpp/cmake_modules/FindorcAlt.cmake @@ -15,13 +15,20 @@ # specific language governing permissions and limitations # under the License. -# - Find Apache ORC C++ (orc/orc-config.h, liborc.a) -# This module defines -# ORC_INCLUDE_DIR, directory containing headers -# ORC_STATIC_LIB, path to liborc.a -# ORC_FOUND, whether orc has been found +if(orcAlt_FOUND) + return() +endif() -if(ORC_FOUND) +set(find_package_args) +if(orcAlt_FIND_VERSION) + list(APPEND find_package_args ${orcAlt_FIND_VERSION}) +endif() +if(orcAlt_FIND_QUIETLY) + list(APPEND find_package_args QUIET) +endif() +find_package(orc ${find_package_args}) +if(orc_FOUND) + set(orcAlt_FOUND TRUE) return() endif() @@ -45,15 +52,13 @@ else() PATH_SUFFIXES ${ARROW_INCLUDE_PATH_SUFFIXES}) endif() -if(ORC_STATIC_LIB AND ORC_INCLUDE_DIR) - set(ORC_FOUND TRUE) - add_library(orc::liborc STATIC IMPORTED) - set_target_properties(orc::liborc - PROPERTIES IMPORTED_LOCATION "${ORC_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${ORC_INCLUDE_DIR}") -else() - if(ORC_FIND_REQUIRED) - message(FATAL_ERROR "ORC library was required in toolchain and unable to locate") +find_package_handle_standard_args(orcAlt REQUIRED_VARS ORC_STATIC_LIB ORC_INCLUDE_DIR) + +if(orcAlt_FOUND) + if(NOT TARGET orc::orc) + add_library(orc::orc STATIC IMPORTED) + set_target_properties(orc::orc + PROPERTIES IMPORTED_LOCATION "${ORC_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${ORC_INCLUDE_DIR}") endif() - set(ORC_FOUND FALSE) endif() diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 559ddf14f6a91..5de8ff9b1cb11 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -65,7 +65,7 @@ set(ARROW_THIRDPARTY_DEPENDENCIES lz4 nlohmann_json opentelemetry-cpp - ORC + orc re2 Protobuf RapidJSON @@ -94,6 +94,14 @@ if("${re2_SOURCE}" STREQUAL "" AND NOT "${RE2_SOURCE}" STREQUAL "") set(re2_SOURCE ${RE2_SOURCE}) endif() +# For backward compatibility. We use "ORC_SOURCE" if "orc_SOURCE" +# isn't specified and "ORC_SOURCE" is specified. +# We renamed "ORC" dependency name to "orc" in 15.0.0 because +# upstream uses "orc" not "ORC" as package name. +if("${orc_SOURCE}" STREQUAL "" AND NOT "${ORC_SOURCE}" STREQUAL "") + set(orc_SOURCE ${ORC_SOURCE}) +endif() + # For backward compatibility. We use "RE2_ROOT" if "re2_ROOT" # isn't specified and "RE2_ROOT" is specified. if("${re2_ROOT}" STREQUAL "" AND NOT "${RE2_ROOT}" STREQUAL "") @@ -193,7 +201,7 @@ macro(build_dependency DEPENDENCY_NAME) build_nlohmann_json() elseif("${DEPENDENCY_NAME}" STREQUAL "opentelemetry-cpp") build_opentelemetry() - elseif("${DEPENDENCY_NAME}" STREQUAL "ORC") + elseif("${DEPENDENCY_NAME}" STREQUAL "orc") build_orc() elseif("${DEPENDENCY_NAME}" STREQUAL "Protobuf") build_protobuf() @@ -4423,31 +4431,31 @@ macro(build_orc) set(ORC_VENDORED 1) - add_library(orc::liborc STATIC IMPORTED) - set_target_properties(orc::liborc PROPERTIES IMPORTED_LOCATION "${ORC_STATIC_LIB}") - target_include_directories(orc::liborc BEFORE INTERFACE "${ORC_INCLUDE_DIR}") - set(ORC_LINK_LIBRARIES LZ4::lz4 ZLIB::ZLIB ${ARROW_ZSTD_LIBZSTD} ${Snappy_TARGET}) + add_library(orc::orc STATIC IMPORTED) + set_target_properties(orc::orc PROPERTIES IMPORTED_LOCATION "${ORC_STATIC_LIB}") + target_include_directories(orc::orc BEFORE INTERFACE "${ORC_INCLUDE_DIR}") + target_link_libraries(orc::orc INTERFACE LZ4::lz4 ZLIB::ZLIB ${ARROW_ZSTD_LIBZSTD} + ${Snappy_TARGET}) # Protobuf generated files may use ABSL_DCHECK*() and # absl::log_internal_check_op is needed for them. if(TARGET absl::log_internal_check_op) - list(APPEND ORC_LINK_LIBRARIES absl::log_internal_check_op) + target_link_libraries(orc::orc INTERFACE absl::log_internal_check_op) endif() if(NOT MSVC) if(NOT APPLE AND ARROW_ENABLE_THREADING) - list(APPEND ORC_LINK_LIBRARIES Threads::Threads) + target_link_libraries(orc::orc INTERFACE Threads::Threads) endif() - list(APPEND ORC_LINK_LIBRARIES ${CMAKE_DL_LIBS}) + target_link_libraries(orc::orc INTERFACE ${CMAKE_DL_LIBS}) endif() - target_link_libraries(orc::liborc INTERFACE ${ORC_LINK_LIBRARIES}) add_dependencies(toolchain orc_ep) - add_dependencies(orc::liborc orc_ep) + add_dependencies(orc::orc orc_ep) - list(APPEND ARROW_BUNDLED_STATIC_LIBS orc::liborc) + list(APPEND ARROW_BUNDLED_STATIC_LIBS orc::orc) endmacro() if(ARROW_ORC) - resolve_dependency(ORC) + resolve_dependency(orc HAVE_ALT TRUE) message(STATUS "Found ORC static library: ${ORC_STATIC_LIB}") message(STATUS "Found ORC headers: ${ORC_INCLUDE_DIR}") endif() diff --git a/cpp/src/arrow/adapters/orc/CMakeLists.txt b/cpp/src/arrow/adapters/orc/CMakeLists.txt index e8ff69c191fb1..44719379994d9 100644 --- a/cpp/src/arrow/adapters/orc/CMakeLists.txt +++ b/cpp/src/arrow/adapters/orc/CMakeLists.txt @@ -32,7 +32,7 @@ else() set(ARROW_LIBRARIES_FOR_STATIC_TESTS arrow_testing_shared arrow_shared) endif() -set(ORC_STATIC_TEST_LINK_LIBS orc::liborc ${ARROW_LIBRARIES_FOR_STATIC_TESTS} +set(ORC_STATIC_TEST_LINK_LIBS orc::orc ${ARROW_LIBRARIES_FOR_STATIC_TESTS} ${ARROW_GTEST_GTEST_MAIN} ${ARROW_GTEST_GTEST}) add_arrow_test(adapter_test diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 17123ed2b8339..b620c9b8c8b0c 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -1182,12 +1182,12 @@ tasks: image: debian-cpp {% endfor %} - test-fedora-35-cpp: + test-fedora-38-cpp: ci: github template: docker-tests/github.linux.yml params: env: - FEDORA: 35 + FEDORA: 38 image: fedora-cpp {% for cpp_standard in [20] %} @@ -1295,12 +1295,12 @@ tasks: UBUNTU: 22.04 image: ubuntu-python - test-fedora-35-python-3: + test-fedora-38-python-3: ci: azure template: docker-tests/azure.linux.yml params: env: - FEDORA: 35 + FEDORA: 38 image: fedora-python test-r-linux-valgrind: diff --git a/docker-compose.yml b/docker-compose.yml index c23d3ff08e487..bbe4e65c10c43 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -655,7 +655,7 @@ services: # docker-compose run --rm fedora-cpp # Parameters: # ARCH: amd64, arm64v8, ... - # FEDORA: 33 + # FEDORA: 38 image: ${REPO}:${ARCH}-fedora-${FEDORA}-cpp build: context: . @@ -671,7 +671,6 @@ services: <<: [*common, *ccache, *sccache, *cpp] ARROW_ENABLE_TIMING_TESTS: # inherit ARROW_MIMALLOC: "ON" - Protobuf_SOURCE: "BUNDLED" # Need Protobuf >= 3.15 volumes: &fedora-volumes - .:/arrow:delegated - ${DOCKER_VOLUME_PREFIX}fedora-ccache:/ccache:delegated @@ -957,7 +956,7 @@ services: # docker-compose run --rm fedora-python # Parameters: # ARCH: amd64, arm64v8, ... - # FEDORA: 33 + # FEDORA: 38 image: ${REPO}:${ARCH}-fedora-${FEDORA}-python-3 build: context: . @@ -969,7 +968,6 @@ services: shm_size: *shm-size environment: <<: [*common, *ccache] - Protobuf_SOURCE: "BUNDLED" # Need Protobuf >= 3.15 volumes: *fedora-volumes command: *python-command