From 2d3722c6ac9fa20e97fbe3f57016b35b0d67b0c7 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Wed, 25 Oct 2023 04:16:54 +0200 Subject: [PATCH] GH-38402: [CI][Integration] Provide wrapper scripts for integration testing (#38403) ### Rationale for this change It is currently cumbersome to run integration tests from an other repository, as each tested implementation must be built explicitly. See explanatory comment at https://github.com/apache/arrow-rs/pull/4957#issuecomment-1771707301 ### What changes are included in this PR? Provide a wrapper script to automate building the implementations being tested. Allow usage of environment variables such as `ARROW_INTEGRATION_CPP`, etc. to selectively disable implementations. ### Are these changes tested? Yes, by construction. ### Are there any user-facing changes? No. * Closes: #38402 Authored-by: Antoine Pitrou Signed-off-by: Sutou Kouhei --- ci/scripts/go_build.sh | 4 +- ci/scripts/integration_arrow.sh | 26 ++++++++++--- ci/scripts/integration_arrow_build.sh | 55 +++++++++++++++++++++++++++ docker-compose.yml | 13 +------ 4 files changed, 79 insertions(+), 19 deletions(-) create mode 100755 ci/scripts/integration_arrow_build.sh diff --git a/ci/scripts/go_build.sh b/ci/scripts/go_build.sh index 7c5ca3230c96e..94f75e501ea0b 100755 --- a/ci/scripts/go_build.sh +++ b/ci/scripts/go_build.sh @@ -42,7 +42,9 @@ go install -v ./... popd -if [[ -n "${ARROW_GO_INTEGRATION}" ]]; then +: ${ARROW_INTEGRATION_GO:=ON} + +if [ "${ARROW_INTEGRATION_GO}" == "ON" ]; then pushd ${source_dir}/arrow/internal/cdata_integration case "$(uname)" in diff --git a/ci/scripts/integration_arrow.sh b/ci/scripts/integration_arrow.sh index 2861b1c09d479..b5a38f01412d4 100755 --- a/ci/scripts/integration_arrow.sh +++ b/ci/scripts/integration_arrow.sh @@ -20,11 +20,25 @@ set -ex arrow_dir=${1} +build_dir=${2} + gold_dir=$arrow_dir/testing/data/arrow-ipc-stream/integration +: ${ARROW_INTEGRATION_CPP:=ON} +: ${ARROW_INTEGRATION_CSHARP:=ON} +: ${ARROW_INTEGRATION_GO:=ON} +: ${ARROW_INTEGRATION_JAVA:=ON} +: ${ARROW_INTEGRATION_JS:=ON} + pip install -e $arrow_dir/dev/archery[integration] + # For C Data Interface testing -pip install jpype1 pythonnet +if [ "${ARROW_INTEGRATION_CSHARP}" == "ON" ]; then + pip install pythonnet +fi +if [ "${ARROW_INTEGRATION_JAVA}" == "ON" ]; then + pip install jpype1 +fi # Get more detailed context on crashes export PYTHONFAULTHANDLER=1 @@ -34,11 +48,11 @@ time archery integration \ --run-c-data \ --run-ipc \ --run-flight \ - --with-cpp=1 \ - --with-csharp=1 \ - --with-java=1 \ - --with-js=1 \ - --with-go=1 \ + --with-cpp=$([ "$ARROW_INTEGRATION_CPP" == "ON" ] && echo "1" || echo "0") \ + --with-csharp=$([ "$ARROW_INTEGRATION_CSHARP" == "ON" ] && echo "1" || echo "0") \ + --with-go=$([ "$ARROW_INTEGRATION_GO" == "ON" ] && echo "1" || echo "0") \ + --with-java=$([ "$ARROW_INTEGRATION_JAVA" == "ON" ] && echo "1" || echo "0") \ + --with-js=$([ "$ARROW_INTEGRATION_JS" == "ON" ] && echo "1" || echo "0") \ --gold-dirs=$gold_dir/0.14.1 \ --gold-dirs=$gold_dir/0.17.1 \ --gold-dirs=$gold_dir/1.0.0-bigendian \ diff --git a/ci/scripts/integration_arrow_build.sh b/ci/scripts/integration_arrow_build.sh new file mode 100755 index 0000000000000..02f593bf77b23 --- /dev/null +++ b/ci/scripts/integration_arrow_build.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -ex + +arrow_dir=${1} +build_dir=${2} + +: ${ARROW_INTEGRATION_CPP:=ON} +: ${ARROW_INTEGRATION_CSHARP:=ON} +: ${ARROW_INTEGRATION_GO:=ON} +: ${ARROW_INTEGRATION_JAVA:=ON} +: ${ARROW_INTEGRATION_JS:=ON} + +${arrow_dir}/ci/scripts/rust_build.sh ${arrow_dir} ${build_dir} + +if [ "${ARROW_INTEGRATION_CPP}" == "ON" ]; then + ${arrow_dir}/ci/scripts/cpp_build.sh ${arrow_dir} ${build_dir} +fi + +if [ "${ARROW_INTEGRATION_CSHARP}" == "ON" ]; then + ${arrow_dir}/ci/scripts/csharp_build.sh ${arrow_dir} ${build_dir} +fi + +if [ "${ARROW_INTEGRATION_GO}" == "ON" ]; then + ${arrow_dir}/ci/scripts/go_build.sh ${arrow_dir} ${build_dir} +fi + +if [ "${ARROW_INTEGRATION_JAVA}" == "ON" ]; then + export ARROW_JAVA_CDATA="ON" + export JAVA_JNI_CMAKE_ARGS="-DARROW_JAVA_JNI_ENABLE_DEFAULT=OFF -DARROW_JAVA_JNI_ENABLE_C=ON" + + ${arrow_dir}/ci/scripts/java_jni_build.sh ${arrow_dir} ${ARROW_HOME} ${build_dir} /tmp/dist/java/$(arch) + ${arrow_dir}/ci/scripts/java_build.sh ${arrow_dir} ${build_dir} /tmp/dist/java +fi + +if [ "${ARROW_INTEGRATION_JS}" == "ON" ]; then + ${arrow_dir}/ci/scripts/js_build.sh ${arrow_dir} ${build_dir} +fi diff --git a/docker-compose.yml b/docker-compose.yml index e54c609e54138..c23d3ff08e487 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1733,19 +1733,8 @@ services: ARCHERY_INTEGRATION_WITH_RUST: 0 # Tell Archery where the arrow C++ binaries are located ARROW_CPP_EXE_PATH: /build/cpp/debug - ARROW_GO_INTEGRATION: 1 - ARROW_JAVA_CDATA: "ON" - JAVA_JNI_CMAKE_ARGS: >- - -DARROW_JAVA_JNI_ENABLE_DEFAULT=OFF - -DARROW_JAVA_JNI_ENABLE_C=ON command: - ["/arrow/ci/scripts/rust_build.sh /arrow /build && - /arrow/ci/scripts/cpp_build.sh /arrow /build && - /arrow/ci/scripts/csharp_build.sh /arrow /build && - /arrow/ci/scripts/go_build.sh /arrow && - /arrow/ci/scripts/java_jni_build.sh /arrow $${ARROW_HOME} /build /tmp/dist/java/$$(arch) && - /arrow/ci/scripts/java_build.sh /arrow /build /tmp/dist/java && - /arrow/ci/scripts/js_build.sh /arrow /build && + ["/arrow/ci/scripts/integration_arrow_build.sh /arrow /build && /arrow/ci/scripts/integration_arrow.sh /arrow /build"] ################################ Docs #######################################